Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 29 additions & 19 deletions python/semantic_kernel/text/text_chunker.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,26 +197,26 @@ def _split_str_lines(

text = text.replace("\r\n", "\n")
lines: list[str] = []
was_split = False
all_chunks_fit = False
for split_option in separators:
if not lines:
lines, was_split = _split_str(
lines, all_chunks_fit = _split_str(
text=text,
max_tokens=max_tokens,
separators=split_option,
trim=trim,
token_counter=token_counter,
)
else:
lines, was_split = _split_list(
lines, all_chunks_fit = _split_list(
text=lines,
max_tokens=max_tokens,
separators=split_option,
trim=trim,
token_counter=token_counter,
)
if was_split:
break # pragma: no cover
if all_chunks_fit:
break

return lines

Expand All @@ -228,18 +228,24 @@ def _split_str(
trim: bool,
token_counter: Callable = _token_counter,
) -> tuple[list[str], bool]:
"""Split text into lines."""
input_was_split = False
"""Split text into lines.

Returns the chunks and whether every returned chunk fits within ``max_tokens``;
a chunk that does not fit needs a weaker separator tier to be split further.
"""
all_chunks_fit = False
if not text:
return [], input_was_split # pragma: no cover
return [], all_chunks_fit # pragma: no cover

if trim:
text = text.strip()

text_as_is = [text]

if token_counter(text) <= max_tokens:
return text_as_is, input_was_split
# A chunk trimmed down to "" cannot be finalized here; report it as not
# resolved so the next, weaker tier drops it.
return text_as_is, bool(text)

half = len(text) // 2

Expand All @@ -260,24 +266,25 @@ def _split_str(
# distance is increasing, so we can stop searching
break
else:
return text_as_is, input_was_split
return text_as_is, False

if 0 < cutpoint < len(text):
lines = []
all_chunks_fit = True
for text_part in [text[:cutpoint], text[cutpoint:]]:
split, has_split = _split_str(
split, part_fits = _split_str(
text=text_part,
max_tokens=max_tokens,
separators=separators,
trim=trim,
token_counter=token_counter,
)
lines.extend(split)
input_was_split = input_was_split or has_split
all_chunks_fit = all_chunks_fit and part_fits
else:
return text_as_is, input_was_split
return text_as_is, False

return lines, input_was_split
return lines, all_chunks_fit


def _split_list(
Expand All @@ -287,21 +294,24 @@ def _split_list(
trim: bool,
token_counter: Callable = _token_counter,
) -> tuple[list[str], bool]:
"""Split list of string into lines."""
"""Split list of string into lines.

Returns the chunks and whether every returned chunk fits within ``max_tokens``.
"""
if not text:
return [], False # pragma: no cover

lines = []
input_was_split = False
all_chunks_fit = True
for line in text:
split_str, was_split = _split_str(
split_str, part_fits = _split_str(
text=line,
max_tokens=max_tokens,
separators=separators,
trim=trim,
token_counter=token_counter,
)
lines.extend(split_str)
input_was_split = input_was_split or was_split
all_chunks_fit = all_chunks_fit and part_fits

return lines, input_was_split
return lines, all_chunks_fit
59 changes: 59 additions & 0 deletions python/tests/unit/text/test_text_chunker.py
Original file line number Diff line number Diff line change
Expand Up @@ -532,3 +532,62 @@ def test_split_md_on_newlines():
max_token_per_line = 15
split = split_markdown_paragraph(test, max_token_per_line)
assert expected == split


def test_split_str_flags_fully_resolved_input():
"""Test _split_str() reporting whether every returned chunk fits the token budget (issue #14490)"""
from semantic_kernel.text.text_chunker import _split_str

# Text that fits the budget is reported as fully resolved.
lines, all_fit = _split_str("Short text.", max_tokens=100, separators=["."], trim=True)
assert lines == ["Short text."]
assert all_fit is True

# A split that resolves every chunk is reported as fully resolved.
lines, all_fit = _split_str("One. Two.", max_tokens=1, separators=["."], trim=True)
assert lines == ["One.", "Two."]
assert all_fit is True

# A chunk that no separator in the current tier can split is reported as not
# resolved, so the caller keeps trying the weaker tiers.
lines, all_fit = _split_str("OneLongUnbreakableWord", max_tokens=1, separators=["."], trim=True)
assert lines == ["OneLongUnbreakableWord"]
assert all_fit is False


def test_split_text_lines_stops_after_fully_resolved_tier(monkeypatch):
"""Test that no weaker separator tier runs once one tier resolved every chunk (issue #14490)"""
from semantic_kernel.text import text_chunker as tc

calls = {"count": 0}
original_split_list = tc._split_list

def counting_split_list(*args, **kwargs):
calls["count"] += 1
return original_split_list(*args, **kwargs)

monkeypatch.setattr(tc, "_split_list", counting_split_list)

text = "\n".join(f"Line number {i} is short." for i in range(30))
split = tc.split_plaintext_lines(text, max_token_per_line=50)

# The newline tier fully resolves this input; before the fix every one of the
# nine weaker tiers re-scanned the finished chunks (9 _split_list calls).
assert calls["count"] == 0
assert split == [
"\n".join(f"Line number {i} is short." for i in range(0, 8)),
"\n".join(f"Line number {i} is short." for i in range(8, 15)),
"\n".join(f"Line number {i} is short." for i in range(15, 22)),
"\n".join(f"Line number {i} is short." for i in range(22, 30)),
]


def test_split_text_lines_keeps_splitting_oversized_remainders():
"""Test that a tier leaving an oversized chunk without its separators does not stop the loop (issue #14490)"""
from semantic_kernel.text.text_chunker import _token_counter

text = ("One. Two. Three. Four. Five. Six. Seven. Eight. Nine. Ten. " * 3) + "A" * 300
split = split_plaintext_lines(text, max_token_per_line=50)

assert split
assert all(_token_counter(chunk) <= 50 for chunk in split)
Loading