From 76b5189e0afb62e83131f3a0fe1c2c23530a4217 Mon Sep 17 00:00:00 2001 From: Tanishq Date: Mon, 2 Nov 2026 11:31:51 +0530 Subject: [PATCH] fix(onnx): return one segment list per input from Fsmn_vad Fsmn_vad.__call__ started from [[]] * self.batch_size and added the segments of each batch to segments[batch_num] for every batch_num below batch_size. With the default batch_size=1, each file of a list input is its own batch, so the segments of all files ended up in the first list: model([a, b]) returned [a_segments + b_segments], and model([silence, b]) returned b's segments as the result of the silent file. With batch_size=2, a single file with speech raised IndexError, because E2EVadModel returns one segment list per waveform, and a file without speech returned two aliased empty lists. A batch of two or more files failed before that: np.array(waveform) could not stack files of different lengths, and int(feats_len) raised TypeError for files of equal length. E2EVadModel supports one waveform per call and prepare_cache builds the FSMN cache for one waveform, so score the inputs one at a time and add each input's segments to its own list. Fsmn_vad no longer uses batch_size; the argument is still accepted. A single input still returns [segments]. Add tests/test_funasr_onnx_vad_multiple_inputs.py, which builds the model without model files and stubs only the ONNX session, and run it in the funasr-onnx package workflow. --- .../workflows/test-funasr-onnx-package.yml | 5 + .../python/onnxruntime/funasr_onnx/vad_bin.py | 10 +- tests/test_funasr_onnx_vad_multiple_inputs.py | 104 ++++++++++++++++++ 3 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 tests/test_funasr_onnx_vad_multiple_inputs.py diff --git a/.github/workflows/test-funasr-onnx-package.yml b/.github/workflows/test-funasr-onnx-package.yml index dc0f886ff..8ef0e63f0 100644 --- a/.github/workflows/test-funasr-onnx-package.yml +++ b/.github/workflows/test-funasr-onnx-package.yml @@ -12,6 +12,7 @@ on: - "tests/test_funasr_onnx_paraformer_online_overlap.py" - "tests/test_funasr_onnx_paraformer_online_empty_final.py" - "tests/test_funasr_onnx_vad_online_short_chunks.py" + - "tests/test_funasr_onnx_vad_multiple_inputs.py" - "tests/test_rich_transcription_postprocess.py" - "tests/test_rich_transcription_event_segments.py" push: @@ -27,6 +28,7 @@ on: - "tests/test_funasr_onnx_paraformer_online_overlap.py" - "tests/test_funasr_onnx_paraformer_online_empty_final.py" - "tests/test_funasr_onnx_vad_online_short_chunks.py" + - "tests/test_funasr_onnx_vad_multiple_inputs.py" - "tests/test_rich_transcription_postprocess.py" - "tests/test_rich_transcription_event_segments.py" workflow_dispatch: @@ -110,6 +112,9 @@ jobs: - name: Verify streaming VAD short chunks run: python tests/test_funasr_onnx_vad_online_short_chunks.py + - name: Verify offline VAD results per input + run: python tests/test_funasr_onnx_vad_multiple_inputs.py + - name: Verify source distribution error paths run: | python -m pip install --force-reinstall --no-deps dist/funasr-onnx/*.tar.gz diff --git a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py index fc132d17c..cb9ba3d0d 100644 --- a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -96,10 +96,11 @@ def __call__(self, audio_in: Union[str, np.ndarray, List[str]], **kwargs) -> Lis waveform_list = self.load_data(audio_in, self.frontend.opts.frame_opts.samp_freq) waveform_nums = len(waveform_list) is_final = kwargs.get("kwargs", False) - segments = [[]] * self.batch_size - for beg_idx in range(0, waveform_nums, self.batch_size): + segments = [[] for _ in range(waveform_nums)] + # E2EVadModel and prepare_cache handle one waveform at a time, so batch_size is not used. + for beg_idx in range(0, waveform_nums): vad_scorer = E2EVadModel(self.vad_scorer_config) - end_idx = min(waveform_nums, beg_idx + self.batch_size) + end_idx = beg_idx + 1 waveform = waveform_list[beg_idx:end_idx] feats, feats_len = self.extract_feat(waveform) waveform = np.array(waveform) @@ -137,8 +138,7 @@ def __call__(self, audio_in: Union[str, np.ndarray, List[str]], **kwargs) -> Lis # segments = self.vad_scorer(scores, waveform[0][None, :], is_final=is_final, max_end_sil=self.max_end_sil) if segments_part: - for batch_num in range(0, self.batch_size): - segments[batch_num] += segments_part[batch_num] + segments[beg_idx] += segments_part[0] except ONNXRuntimeError: # logging.warning(traceback.format_exc()) diff --git a/tests/test_funasr_onnx_vad_multiple_inputs.py b/tests/test_funasr_onnx_vad_multiple_inputs.py new file mode 100644 index 000000000..fd2478114 --- /dev/null +++ b/tests/test_funasr_onnx_vad_multiple_inputs.py @@ -0,0 +1,104 @@ +"""Run against an installed funasr-onnx package.""" + +import importlib.util +import os +import tempfile +import unittest + +import numpy as np + +if importlib.util.find_spec("funasr_onnx") is None: + raise unittest.SkipTest("Install the standalone funasr-onnx distribution first.") + +import soundfile +from funasr_onnx.utils.frontend import WavFrontend +from funasr_onnx.vad_bin import Fsmn_vad + +# frontend_conf of funasr/models/fsmn_vad_streaming/template.yaml +FRONTEND_CONF = { + "fs": 16000, + "window": "hamming", + "n_mels": 80, + "frame_length": 25, + "frame_shift": 10, + "dither": 0.0, + "lfr_m": 5, + "lfr_n": 1, +} + +# 16 kHz test files: length and noise ranges in samples, silence elsewhere. +AUDIO = { + "a": (48000, [(8000, 24000)]), # 3 s, noise at 0.5-1.5 s + "b": (72000, [(16000, 32000), (48000, 56000)]), # 4.5 s, noise at 1-2 s and 3-3.5 s + "silence": (32000, []), # 2 s +} + +# Segments (ms) of each file on its own. +SEGMENTS = {"a": [[220, 1660]], "b": [[720, 2160], [2720, 3660]], "silence": []} + + +def fake_infer(inputs): + # Score loud frames as speech instead of running the ONNX session. + sil = np.where(inputs[0][0].mean(-1) > 5.0, 0.05, 0.95) + scores = np.stack((sil, 1.0 - sil), axis=-1)[None].astype(np.float32) + return scores, inputs[1:] + + +def make_model(batch_size=1): + # Keep __call__, WavFrontend and E2EVadModel real; no model files. + model = Fsmn_vad.__new__(Fsmn_vad) + model.frontend = WavFrontend(cmvn_file=None, **FRONTEND_CONF) + model.batch_size = batch_size + # E2EVadModel's defaults equal the template's model_conf. + model.vad_scorer_config = {} + model.max_end_sil = 800 + model.encoder_conf = {"fsmn_layers": 1, "proj_dim": 2, "lorder": 2} + model.infer = fake_infer + return model + + +class FsmnVadMultipleInputsTest(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.wav = {} + for name, (length, noise) in AUDIO.items(): + rng = np.random.default_rng(0) + audio = np.zeros(length, dtype=np.float32) + for start, end in noise: + audio[start:end] = rng.uniform(-0.3, 0.3, end - start) + self.wav[name] = os.path.join(tmp.name, name + ".wav") + soundfile.write(self.wav[name], audio, 16000) + + def test_single_input(self): + # A path or an array gives a list with one segment list. + model = make_model() + for name in AUDIO: + with self.subTest(name=name): + self.assertEqual(model(self.wav[name]), [SEGMENTS[name]]) + audio, _ = soundfile.read(self.wav[name], dtype="float32") + self.assertEqual(model(audio), [SEGMENTS[name]]) + + def test_list_input(self): + # A list of paths gives one segment list per file, in input order. + model = make_model() + for names in ([], ["a", "b"], ["b", "a"], ["silence", "b"], ["a", "silence", "b"]): + with self.subTest(names=names): + result = model([self.wav[name] for name in names]) + self.assertEqual(result, [SEGMENTS[name] for name in names]) + + def test_batch_size_above_one(self): + # batch_size=2 or 3 gives the batch_size=1 results, for one file and for lists. + for batch_size in (2, 3): + model = make_model(batch_size=batch_size) + for name in AUDIO: + with self.subTest(batch_size=batch_size, name=name): + self.assertEqual(model(self.wav[name]), [SEGMENTS[name]]) + for names in (["a", "b"], ["a", "a"], ["silence", "b"], ["a", "silence", "b"]): + with self.subTest(batch_size=batch_size, names=names): + result = model([self.wav[name] for name in names]) + self.assertEqual(result, [SEGMENTS[name] for name in names]) + + +if __name__ == "__main__": + unittest.main()