From 2e97ae57e50b0e9bdf11c66fd5216a7891cc61ca Mon Sep 17 00:00:00 2001 From: DZH166 <248446924+DZH166@users.noreply.github.com> Date: Sun, 15 Nov 2026 06:14:16 +0800 Subject: [PATCH] Python: fix text_chunker's never-true split flag so the separator-tier loop can stop early --- python/semantic_kernel/text/text_chunker.py | 48 ++++++++++------- python/tests/unit/text/test_text_chunker.py | 59 +++++++++++++++++++++ 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/python/semantic_kernel/text/text_chunker.py b/python/semantic_kernel/text/text_chunker.py index f79796399b6d..2ead6514e8f2 100644 --- a/python/semantic_kernel/text/text_chunker.py +++ b/python/semantic_kernel/text/text_chunker.py @@ -197,10 +197,10 @@ def _split_str_lines( text = text.replace("\r\n", "\n") lines: list[str] = [] - was_split = False + all_chunks_fit = False for split_option in separators: if not lines: - lines, was_split = _split_str( + lines, all_chunks_fit = _split_str( text=text, max_tokens=max_tokens, separators=split_option, @@ -208,15 +208,15 @@ def _split_str_lines( token_counter=token_counter, ) else: - lines, was_split = _split_list( + lines, all_chunks_fit = _split_list( text=lines, max_tokens=max_tokens, separators=split_option, trim=trim, token_counter=token_counter, ) - if was_split: - break # pragma: no cover + if all_chunks_fit: + break return lines @@ -228,10 +228,14 @@ def _split_str( trim: bool, token_counter: Callable = _token_counter, ) -> tuple[list[str], bool]: - """Split text into lines.""" - input_was_split = False + """Split text into lines. + + Returns the chunks and whether every returned chunk fits within ``max_tokens``; + a chunk that does not fit needs a weaker separator tier to be split further. + """ + all_chunks_fit = False if not text: - return [], input_was_split # pragma: no cover + return [], all_chunks_fit # pragma: no cover if trim: text = text.strip() @@ -239,7 +243,9 @@ def _split_str( text_as_is = [text] if token_counter(text) <= max_tokens: - return text_as_is, input_was_split + # A chunk trimmed down to "" cannot be finalized here; report it as not + # resolved so the next, weaker tier drops it. + return text_as_is, bool(text) half = len(text) // 2 @@ -260,12 +266,13 @@ def _split_str( # distance is increasing, so we can stop searching break else: - return text_as_is, input_was_split + return text_as_is, False if 0 < cutpoint < len(text): lines = [] + all_chunks_fit = True for text_part in [text[:cutpoint], text[cutpoint:]]: - split, has_split = _split_str( + split, part_fits = _split_str( text=text_part, max_tokens=max_tokens, separators=separators, @@ -273,11 +280,11 @@ def _split_str( token_counter=token_counter, ) lines.extend(split) - input_was_split = input_was_split or has_split + all_chunks_fit = all_chunks_fit and part_fits else: - return text_as_is, input_was_split + return text_as_is, False - return lines, input_was_split + return lines, all_chunks_fit def _split_list( @@ -287,14 +294,17 @@ def _split_list( trim: bool, token_counter: Callable = _token_counter, ) -> tuple[list[str], bool]: - """Split list of string into lines.""" + """Split list of string into lines. + + Returns the chunks and whether every returned chunk fits within ``max_tokens``. + """ if not text: return [], False # pragma: no cover lines = [] - input_was_split = False + all_chunks_fit = True for line in text: - split_str, was_split = _split_str( + split_str, part_fits = _split_str( text=line, max_tokens=max_tokens, separators=separators, @@ -302,6 +312,6 @@ def _split_list( token_counter=token_counter, ) lines.extend(split_str) - input_was_split = input_was_split or was_split + all_chunks_fit = all_chunks_fit and part_fits - return lines, input_was_split + return lines, all_chunks_fit diff --git a/python/tests/unit/text/test_text_chunker.py b/python/tests/unit/text/test_text_chunker.py index 648f84b752c5..50583d6ebb5f 100644 --- a/python/tests/unit/text/test_text_chunker.py +++ b/python/tests/unit/text/test_text_chunker.py @@ -532,3 +532,62 @@ def test_split_md_on_newlines(): max_token_per_line = 15 split = split_markdown_paragraph(test, max_token_per_line) assert expected == split + + +def test_split_str_flags_fully_resolved_input(): + """Test _split_str() reporting whether every returned chunk fits the token budget (issue #14490)""" + from semantic_kernel.text.text_chunker import _split_str + + # Text that fits the budget is reported as fully resolved. + lines, all_fit = _split_str("Short text.", max_tokens=100, separators=["."], trim=True) + assert lines == ["Short text."] + assert all_fit is True + + # A split that resolves every chunk is reported as fully resolved. + lines, all_fit = _split_str("One. Two.", max_tokens=1, separators=["."], trim=True) + assert lines == ["One.", "Two."] + assert all_fit is True + + # A chunk that no separator in the current tier can split is reported as not + # resolved, so the caller keeps trying the weaker tiers. + lines, all_fit = _split_str("OneLongUnbreakableWord", max_tokens=1, separators=["."], trim=True) + assert lines == ["OneLongUnbreakableWord"] + assert all_fit is False + + +def test_split_text_lines_stops_after_fully_resolved_tier(monkeypatch): + """Test that no weaker separator tier runs once one tier resolved every chunk (issue #14490)""" + from semantic_kernel.text import text_chunker as tc + + calls = {"count": 0} + original_split_list = tc._split_list + + def counting_split_list(*args, **kwargs): + calls["count"] += 1 + return original_split_list(*args, **kwargs) + + monkeypatch.setattr(tc, "_split_list", counting_split_list) + + text = "\n".join(f"Line number {i} is short." for i in range(30)) + split = tc.split_plaintext_lines(text, max_token_per_line=50) + + # The newline tier fully resolves this input; before the fix every one of the + # nine weaker tiers re-scanned the finished chunks (9 _split_list calls). + assert calls["count"] == 0 + assert split == [ + "\n".join(f"Line number {i} is short." for i in range(0, 8)), + "\n".join(f"Line number {i} is short." for i in range(8, 15)), + "\n".join(f"Line number {i} is short." for i in range(15, 22)), + "\n".join(f"Line number {i} is short." for i in range(22, 30)), + ] + + +def test_split_text_lines_keeps_splitting_oversized_remainders(): + """Test that a tier leaving an oversized chunk without its separators does not stop the loop (issue #14490)""" + from semantic_kernel.text.text_chunker import _token_counter + + text = ("One. Two. Three. Four. Five. Six. Seven. Eight. Nine. Ten. " * 3) + "A" * 300 + split = split_plaintext_lines(text, max_token_per_line=50) + + assert split + assert all(_token_counter(chunk) <= 50 for chunk in split)