fix(onnx): return one segment list per input from Fsmn_vad

.github/workflows/test-funasr-onnx-package.yml

@@ -12,6 +12,7 @@ on:
       - "tests/test_funasr_onnx_paraformer_online_overlap.py"
       - "tests/test_funasr_onnx_paraformer_online_empty_final.py"
       - "tests/test_funasr_onnx_vad_online_short_chunks.py"
+      - "tests/test_funasr_onnx_vad_multiple_inputs.py"
       - "tests/test_rich_transcription_postprocess.py"
       - "tests/test_rich_transcription_event_segments.py"
   push:
@@ -27,6 +28,7 @@ on:
       - "tests/test_funasr_onnx_paraformer_online_overlap.py"
       - "tests/test_funasr_onnx_paraformer_online_empty_final.py"
       - "tests/test_funasr_onnx_vad_online_short_chunks.py"
+      - "tests/test_funasr_onnx_vad_multiple_inputs.py"
       - "tests/test_rich_transcription_postprocess.py"
       - "tests/test_rich_transcription_event_segments.py"
   workflow_dispatch:
@@ -110,6 +112,9 @@ jobs:
       - name: Verify streaming VAD short chunks
         run: python tests/test_funasr_onnx_vad_online_short_chunks.py
 
+      - name: Verify offline VAD results per input
+        run: python tests/test_funasr_onnx_vad_multiple_inputs.py
+
       - name: Verify source distribution error paths
         run: |
           python -m pip install --force-reinstall --no-deps dist/funasr-onnx/*.tar.gz

runtime/python/onnxruntime/funasr_onnx/vad_bin.py

@@ -96,10 +96,11 @@ def __call__(self, audio_in: Union[str, np.ndarray, List[str]], **kwargs) -> Lis
         waveform_list = self.load_data(audio_in, self.frontend.opts.frame_opts.samp_freq)
         waveform_nums = len(waveform_list)
         is_final = kwargs.get("kwargs", False)
-        segments = [[]] * self.batch_size
-        for beg_idx in range(0, waveform_nums, self.batch_size):
+        segments = [[] for _ in range(waveform_nums)]
+        # E2EVadModel and prepare_cache handle one waveform at a time, so batch_size is not used.
+        for beg_idx in range(0, waveform_nums):
             vad_scorer = E2EVadModel(self.vad_scorer_config)
-            end_idx = min(waveform_nums, beg_idx + self.batch_size)
+            end_idx = beg_idx + 1
             waveform = waveform_list[beg_idx:end_idx]
             feats, feats_len = self.extract_feat(waveform)
             waveform = np.array(waveform)
@@ -137,8 +138,7 @@ def __call__(self, audio_in: Union[str, np.ndarray, List[str]], **kwargs) -> Lis
                     # segments = self.vad_scorer(scores, waveform[0][None, :], is_final=is_final, max_end_sil=self.max_end_sil)
 
                     if segments_part:
-                        for batch_num in range(0, self.batch_size):
-                            segments[batch_num] += segments_part[batch_num]
+                        segments[beg_idx] += segments_part[0]
 
             except ONNXRuntimeError:
                 # logging.warning(traceback.format_exc())

tests/test_funasr_onnx_vad_multiple_inputs.py

@@ -0,0 +1,104 @@
+"""Run against an installed funasr-onnx package."""
+
+import importlib.util
+import os
+import tempfile
+import unittest
+
+import numpy as np
+
+if importlib.util.find_spec("funasr_onnx") is None:
+    raise unittest.SkipTest("Install the standalone funasr-onnx distribution first.")
+
+import soundfile
+from funasr_onnx.utils.frontend import WavFrontend
+from funasr_onnx.vad_bin import Fsmn_vad
+
+# frontend_conf of funasr/models/fsmn_vad_streaming/template.yaml
+FRONTEND_CONF = {
+    "fs": 16000,
+    "window": "hamming",
+    "n_mels": 80,
+    "frame_length": 25,
+    "frame_shift": 10,
+    "dither": 0.0,
+    "lfr_m": 5,
+    "lfr_n": 1,
+}
+
+# 16 kHz test files: length and noise ranges in samples, silence elsewhere.
+AUDIO = {
+    "a": (48000, [(8000, 24000)]),  # 3 s, noise at 0.5-1.5 s
+    "b": (72000, [(16000, 32000), (48000, 56000)]),  # 4.5 s, noise at 1-2 s and 3-3.5 s
+    "silence": (32000, []),  # 2 s
+}
+
+# Segments (ms) of each file on its own.
+SEGMENTS = {"a": [[220, 1660]], "b": [[720, 2160], [2720, 3660]], "silence": []}
+
+
+def fake_infer(inputs):
+    # Score loud frames as speech instead of running the ONNX session.
+    sil = np.where(inputs[0][0].mean(-1) > 5.0, 0.05, 0.95)
+    scores = np.stack((sil, 1.0 - sil), axis=-1)[None].astype(np.float32)
+    return scores, inputs[1:]
+
+
+def make_model(batch_size=1):
+    # Keep __call__, WavFrontend and E2EVadModel real; no model files.
+    model = Fsmn_vad.__new__(Fsmn_vad)
+    model.frontend = WavFrontend(cmvn_file=None, **FRONTEND_CONF)
+    model.batch_size = batch_size
+    # E2EVadModel's defaults equal the template's model_conf.
+    model.vad_scorer_config = {}
+    model.max_end_sil = 800
+    model.encoder_conf = {"fsmn_layers": 1, "proj_dim": 2, "lorder": 2}
+    model.infer = fake_infer
+    return model
+
+
+class FsmnVadMultipleInputsTest(unittest.TestCase):
+    def setUp(self):
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.wav = {}
+        for name, (length, noise) in AUDIO.items():
+            rng = np.random.default_rng(0)
+            audio = np.zeros(length, dtype=np.float32)
+            for start, end in noise:
+                audio[start:end] = rng.uniform(-0.3, 0.3, end - start)
+            self.wav[name] = os.path.join(tmp.name, name + ".wav")
+            soundfile.write(self.wav[name], audio, 16000)
+
+    def test_single_input(self):
+        # A path or an array gives a list with one segment list.
+        model = make_model()
+        for name in AUDIO:
+            with self.subTest(name=name):
+                self.assertEqual(model(self.wav[name]), [SEGMENTS[name]])
+                audio, _ = soundfile.read(self.wav[name], dtype="float32")
+                self.assertEqual(model(audio), [SEGMENTS[name]])
+
+    def test_list_input(self):
+        # A list of paths gives one segment list per file, in input order.
+        model = make_model()
+        for names in ([], ["a", "b"], ["b", "a"], ["silence", "b"], ["a", "silence", "b"]):
+            with self.subTest(names=names):
+                result = model([self.wav[name] for name in names])
+                self.assertEqual(result, [SEGMENTS[name] for name in names])
+
+    def test_batch_size_above_one(self):
+        # batch_size=2 or 3 gives the batch_size=1 results, for one file and for lists.
+        for batch_size in (2, 3):
+            model = make_model(batch_size=batch_size)
+            for name in AUDIO:
+                with self.subTest(batch_size=batch_size, name=name):
+                    self.assertEqual(model(self.wav[name]), [SEGMENTS[name]])
+            for names in (["a", "b"], ["a", "a"], ["silence", "b"], ["a", "silence", "b"]):
+                with self.subTest(batch_size=batch_size, names=names):
+                    result = model([self.wav[name] for name in names])
+                    self.assertEqual(result, [SEGMENTS[name] for name in names])
+
+
+if __name__ == "__main__":
+    unittest.main()