From 83de4060de043f9ea0a0c318edc9a818c54f28d5 Mon Sep 17 00:00:00 2001
From: LauraGPT <18321252+LauraGPT@users.noreply.github.com>
Date: Sun, 30 Aug 2026 21:21:23 +0000
Subject: [PATCH] fix(auto-model): align sentencepiece word markers

Signed-off-by: LauraGPT <18321252+LauraGPT@users.noreply.github.com>
---
 funasr/auto/auto_model.py     |  2 +-
 tests/test_punc_model_none.py | 50 +++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py
index 1ee47022e..16eacfbe4 100644
--- a/funasr/auto/auto_model.py
+++ b/funasr/auto/auto_model.py
@@ -193,7 +193,7 @@ def _merge_timestamp_units(text, words, timestamps, punc_array, punc_model):
         return None
 
     def normalize(value):
-        return "".join(value.split()).casefold()
+        return "".join(value.split()).replace("▁", "").casefold()
 
     if len(words) != len(timestamps):
         return None
diff --git a/tests/test_punc_model_none.py b/tests/test_punc_model_none.py
index b8dc22256..e0b66f41f 100644
--- a/tests/test_punc_model_none.py
+++ b/tests/test_punc_model_none.py
@@ -390,6 +390,56 @@ def test_sentence_timestamp_splits_one_asr_word_across_punctuation_tokens(
             ],
         )
 
+    @patch(
+        "funasr.auto.auto_model._get_punc_tokens",
+        return_value=["你", "好", "真", "的"],
+    )
+    @patch("funasr.auto.auto_model.slice_padding_audio_samples")
+    @patch("funasr.auto.auto_model.load_audio_text_image_video")
+    @patch("funasr.auto.auto_model.prepare_data_iterator")
+    def test_sentence_timestamp_ignores_sentencepiece_word_boundary_marker(
+        self, mock_prep, mock_load, mock_slice, _mock_get_punc_tokens
+    ):
+        punc_model = MagicMock()
+        punc_model.punc_list = None
+        am = self._make_auto_model(punc_model=punc_model)
+        tag = "<|zh|><|NEUTRAL|><|Speech|><|woitn|>"
+        results_seq = [
+            [{"key": "test_utt", "value": [[0, 2000]]}],
+            [
+                {
+                    "text": f"{tag}你好真的",
+                    "timestamp": [[0, 500], [500, 1000], [1000, 2000]],
+                    "words": ["你", "好", "▁真的"],
+                }
+            ],
+            [{"text": "你好，真的。", "punc_array": [1, 2, 1, 3]}],
+        ]
+        am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0))
+        mock_prep.return_value = (["test_utt"], [np.zeros(32000, dtype=np.float32)])
+        mock_load.return_value = np.zeros(32000, dtype=np.float32)
+        mock_slice.return_value = ([np.zeros(32000, dtype=np.float32)], [32000])
+
+        results = am.inference_with_vad("dummy_input", sentence_timestamp=True)
+
+        self.assertEqual(
+            results[0]["sentence_info"],
+            [
+                {
+                    "text": "你好，",
+                    "start": 0,
+                    "end": 1000,
+                    "timestamp": [[0, 500], [500, 1000]],
+                },
+                {
+                    "text": "真的。",
+                    "start": 1000,
+                    "end": 2000,
+                    "timestamp": [[1000, 1500], [1500, 2000]],
+                },
+            ],
+        )
+
     @patch("funasr.auto.auto_model.distribute_spk")
     @patch("funasr.auto.auto_model.postprocess")
     @patch("funasr.auto.auto_model.sv_chunk")