fix(postprocess): keep a trailing @@ piece instead of dropping it

funasr/utils/postprocess_utils.py

@@ -269,6 +269,12 @@ def sentence_postprocess(words: List[Any], time_stamp: List[List] = None):
             else:
                 word_lists.append(ch)
 
+    # a trailing "@@" piece (e.g. a streaming chunk ending inside a word) is still a word
+    if word_item:
+        word_lists.append(word_item)
+        if time_stamp is not None:
+            ts_lists.append([begin, end])
+
     if time_stamp is not None:
         word_lists, ts_lists = abbr_dispose(word_lists, ts_lists)
         real_word_lists = []

runtime/python/libtorch/funasr_torch/utils/postprocess_utils.py

@@ -235,6 +235,12 @@ def sentence_postprocess(words: List[Any], time_stamp: List[List] = None):
             else:
                 raise ValueError("invalid character: {}".format(ch))
 
+    # a trailing "@@" piece (e.g. a streaming chunk ending inside a word) is still a word
+    if word_item:
+        word_lists.append(word_item)
+        if time_stamp is not None:
+            ts_lists.append([begin, end])
+
     if time_stamp is not None:
         word_lists, ts_lists = abbr_dispose(word_lists, ts_lists)
         real_word_lists = []

runtime/python/onnxruntime/funasr_onnx/utils/postprocess_utils.py

@@ -237,6 +237,12 @@ def sentence_postprocess(words: List[Any], time_stamp: List[List] = None):
             else:
                 raise ValueError("invalid character: {}".format(ch))
 
+    # a trailing "@@" piece (e.g. a streaming chunk ending inside a word) is still a word
+    if word_item:
+        word_lists.append(word_item)
+        if time_stamp is not None:
+            ts_lists.append([begin, end])
+
     if time_stamp is not None:
         word_lists, ts_lists = abbr_dispose(word_lists, ts_lists)
         real_word_lists = []

tests/test_sentence_postprocess_timestamps.py

@@ -88,6 +88,23 @@ def test_filtered_only_tokens_need_no_timestamps(self):
                     ("", [], []),
                 )
 
+    def test_trailing_bpe_piece_is_kept(self):
+        # a streaming chunk can end inside a word, e.g. ["hello", "wor@@"]
+        for path in POSTPROCESS_UTILS:
+            for tokens, expected in (
+                (["hello", "wor@@"], ("hello wor", ["hello", "wor"])),
+                (["你", "好", "wor@@"], ("你好wor", ["你", "好", "wor"])),
+            ):
+                with self.subTest(path=path, tokens=tokens):
+                    self.assertEqual(_load_sentence_postprocess(path)(tokens), expected)
+
+    def test_trailing_bpe_piece_keeps_its_timestamp(self):
+        self.assert_timestamps(
+            ["hello", "wor@@", "l@@"],
+            ["hello", "worl"],
+            [[0, 100], [100, 300]],
+        )
+
     def test_untimed_transcription_still_accepts_filtered_tokens(self):
         for path in POSTPROCESS_UTILS:
             with self.subTest(path=path):