diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py index d8537ed20f..0b01c83846 100644 --- a/funasr/models/fsmn_vad_streaming/model.py +++ b/funasr/models/fsmn_vad_streaming/model.py @@ -353,6 +353,7 @@ def __init__( self.output_data_buf_offset = 0 self.frame_probs = [] self.max_end_sil_frame_cnt_thresh = max_end_sil_frame_cnt_thresh + self.silence_schedule = None self.speech_noise_thres = speech_noise_thres self.scores = None self.max_time_out = False @@ -1012,9 +1013,11 @@ def inference( "dynamic_silence", kwargs.get("max_end_silence_time") is None ) silence_schedule = kwargs.get("silence_schedule", DEFAULT_SILENCE_SCHEDULE) - speech_to_sil_ms = self.vad_opts.speech_to_sil_time_thres - accumulated_ms = cache.get("_dynamic_accumulated_ms", 0) - in_speech = cache.get("_dynamic_in_speech", False) + # The schedule is looked up per frame in DetectOneFrame from the current segment's duration. + if "stats" in cache: + cache["stats"].silence_schedule = silence_schedule if dynamic_silence else None + if dynamic_silence: + cache["stats"].speech_noise_thres = 0.5 frontend_inference_kwargs = {} if getattr(frontend, "supports_aligned_waveforms", False): frontend_inference_kwargs["return_waveform"] = True @@ -1023,21 +1026,6 @@ def inference( kwargs["is_final"] = _is_final and i == n - 1 audio_sample_i = audio_sample[i * chunk_stride_samples : (i + 1) * chunk_stride_samples] - # Apply dynamic silence threshold (only accumulate while in speech) - if dynamic_silence and "stats" in cache: - vad_state = cache["stats"].vad_state_machine - # VadStateMachine: 2=kVadInStateStartPointNotDetected, 3=kVadInStateInSpeechSegment - if vad_state.value == 2 or in_speech: # kVadInStateInSpeechSegment - accumulated_ms += chunk_size - in_speech = True - for limit_ms, silence_ms in silence_schedule: - if accumulated_ms <= limit_ms: - cache["stats"].max_end_sil_frame_cnt_thresh = max(silence_ms - speech_to_sil_ms, 0) - cache["stats"].speech_noise_thres = 0.5 - break - cache["_dynamic_accumulated_ms"] = accumulated_ms - cache["_dynamic_in_speech"] = in_speech - # extract fbank feats speech, speech_lengths = extract_fbank( [audio_sample_i], @@ -1074,11 +1062,6 @@ def inference( segments_i = self.forward(**batch) if len(segments_i) > 0: segments.extend(*segments_i) - if dynamic_silence: - accumulated_ms = 0 - in_speech = False - cache["_dynamic_accumulated_ms"] = 0 - cache["_dynamic_in_speech"] = False cache["prev_samples"] = audio_sample[-m:] if m > 0 else torch.empty(0) if _is_final: @@ -1264,6 +1247,17 @@ def DetectOneFrame( cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), cache=cache ) elif cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + silence_schedule = getattr(cache["stats"], "silence_schedule", None) + if silence_schedule is not None: + segment_ms = ( + cur_frm_idx - cache["stats"].confirmed_start_frame + 1 + ) * frm_shift_in_ms + for limit_ms, silence_ms in silence_schedule: + if segment_ms <= limit_ms: + cache["stats"].max_end_sil_frame_cnt_thresh = max( + silence_ms - self.vad_opts.speech_to_sil_time_thres, 0 + ) + break if ( cache["stats"].continous_silence_frame_count * frm_shift_in_ms >= cache["stats"].max_end_sil_frame_cnt_thresh diff --git a/tests/test_fsmn_vad_dynamic_silence.py b/tests/test_fsmn_vad_dynamic_silence.py index f396614f98..d8f39c345a 100644 --- a/tests/test_fsmn_vad_dynamic_silence.py +++ b/tests/test_fsmn_vad_dynamic_silence.py @@ -55,13 +55,83 @@ def test_explicit_max_end_silence_time_keeps_fixed_threshold_by_default(self): cache = self._run_inference(max_end_silence_time=300) self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200) - self.assertNotIn("_dynamic_accumulated_ms", cache) + self.assertIsNone(cache["stats"].silence_schedule) + self.assertEqual(cache["stats"].speech_noise_thres, 0.6) def test_explicit_dynamic_silence_still_enables_schedule(self): cache = self._run_inference(max_end_silence_time=300, dynamic_silence=True) - self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 1900) - self.assertEqual(cache["_dynamic_accumulated_ms"], 1000) + self.assertEqual(cache["stats"].silence_schedule, vad_model.DEFAULT_SILENCE_SCHEDULE) + self.assertEqual(cache["stats"].speech_noise_thres, 0.5) + # The threshold is no longer advanced per chunk; DetectOneFrame applies the schedule. + self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200) + self.assertNotIn("_dynamic_accumulated_ms", cache) + + +class TestFsmnVadDynamicSilencePerFrame(unittest.TestCase): + """Drive DetectOneFrame with scripted window-detector transitions.""" + + def _end_frames(self, speech_ms, silence_ms, schedule, start_frame=0, fixed_thresh_ms=650): + vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming) + torch.nn.Module.__init__(vad) + vad.vad_opts = vad_model.VADXOptions() + ends = [] + + def on_voice_end(end_frame, fake_result, is_last_frame, cache=None): + ends.append(end_frame) + + def reset_detection(cache=None): + cache["stats"].confirmed_start_frame = -1 + cache["stats"].vad_state_machine = vad_model.VadStateMachine.kVadInStateStartPointNotDetected + + vad.OnVoiceEnd = on_voice_end + vad.PopDataToOutputBuf = lambda *args, **kwargs: None + vad.PopDataBufTillFrame = lambda *args, **kwargs: None + vad.LatencyFrmNumAtStartPoint = lambda cache=None: 0 + vad.ResetDetection = reset_detection + + stats = vad_model.Stats( + sil_pdf_ids=[0], max_end_sil_frame_cnt_thresh=fixed_thresh_ms, speech_noise_thres=0.5 + ) + stats.silence_schedule = schedule + speech_frames, silence_frames = speech_ms // 10, silence_ms // 10 + changes = ( + [vad_model.AudioChangeState.kChangeStateSil2Speech] + + [vad_model.AudioChangeState.kChangeStateSpeech2Speech] * (speech_frames - 1) + + [vad_model.AudioChangeState.kChangeStateSpeech2Sil] + + [vad_model.AudioChangeState.kChangeStateSil2Sil] * silence_frames + ) + script = iter(changes) + cache = { + "stats": stats, + "windows_detector": SimpleNamespace(DetectOneFrame=lambda *a, **k: next(script)), + } + for i in range(len(changes)): + vad.DetectOneFrame(vad_model.FrameState.kFrameStateSpeech, start_frame + i, False, cache=cache) + return [e - start_frame for e in ends] + + def test_schedule_follows_current_segment_duration(self): + schedule = vad_model.DEFAULT_SILENCE_SCHEDULE + # Thresholds below are the schedule values minus speech_to_sil_time_thres (150 ms). + # 5 s into a segment: 2000 ms -> 1850 ms, so 900 ms of silence does not end it. + self.assertEqual(self._end_frames(5000, 900, schedule), []) + # 15 s into a segment: 1000 ms -> 850 ms, so 900 ms ends it and 700 ms does not. + self.assertEqual(len(self._end_frames(15000, 900, schedule)), 1) + self.assertEqual(self._end_frames(15000, 700, schedule), []) + # 55 s into a segment: 200 ms -> 50 ms, so 300 ms ends it. + self.assertEqual(len(self._end_frames(55000, 300, schedule)), 1) + + def test_schedule_does_not_depend_on_absolute_position(self): + schedule = vad_model.DEFAULT_SILENCE_SCHEDULE + for speech_ms, silence_ms in [(5000, 900), (15000, 900), (55000, 300)]: + at_zero = self._end_frames(speech_ms, silence_ms, schedule) + shifted = self._end_frames(speech_ms, silence_ms, schedule, start_frame=6000) + self.assertEqual(at_zero, shifted) + + def test_without_schedule_threshold_stays_fixed(self): + # 650 ms fixed threshold: 700 ms of silence ends a 5 s segment, 600 ms does not. + self.assertEqual(len(self._end_frames(5000, 700, None)), 1) + self.assertEqual(self._end_frames(5000, 600, None), []) if __name__ == "__main__":