From 8549fa819769d1674df5086a93e760203510c77c Mon Sep 17 00:00:00 2001 From: abhay-codes07 Date: Thu, 20 Aug 2026 20:20:54 +0530 Subject: [PATCH] fix(llm): slice every text document type, not just five of them (#2900) _read_files caps each unit at _FILE_CHAR_CAP (20,000 characters) before joining it into the user message. expand_oversized_files exists so an oversized document is split into contiguous FileSlices covering the whole file -- but it only splits suffixes in _SPLITTABLE_TEXT_SUFFIXES, and that list drifted from detect.DOC_EXTENSIONS as the latter grew. reach the semantic pass : .html .md .mdx .pdf .qmd .rst .skill .txt .yaml .yml sliced when oversized : .markdown .md .mdx .rst .txt never sliced : .html .pdf .qmd .skill .yaml .yml Measured on a 38,411-character fixture against the 20,000 cap: .md units=3 chars reaching the LLM=38411 LOST=0 .qmd units=1 chars reaching the LLM=20000 LOST=18411 .html units=1 chars reaching the LLM=20000 LOST=18411 .yaml units=1 chars reaching the LLM=20000 LOST=18411 .skill units=1 chars reaching the LLM=20000 LOST=18411 Everything past 20k was invisible to the semantic pass, so nothing in it could become a node -- with no warning, no _partial_files marker (that machinery tracks truncated RESPONSES, not truncated inputs), and nothing in the report. The graph looks complete. graphify's own .skill files are in the gap. These five are plain text and _file_to_text reads them with a straight read_text, so a character range over the raw file is exactly what the model is shown -- they slice identically to .md, which has worked this way since #1369. .pdf is deliberately NOT added. Its bytes are not what the model sees: _file_to_text routes it through extract_pdf_text while read_slice_text does path.read_text(), so slicing it needs the converter threaded through the slicing path first. Left alone rather than half-done -- a paper over 20k characters is still truncated today. The contract test is the durable half: it asserts every non-binary DOC_EXTENSION is splittable, so adding a document type without deciding how it gets sliced fails loudly instead of quietly losing the tail. --- graphify/file_slice.py | 14 ++- tests/test_oversized_document_slicing.py | 114 +++++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 tests/test_oversized_document_slicing.py diff --git a/graphify/file_slice.py b/graphify/file_slice.py index 30dc49cfbd..670c36231b 100644 --- a/graphify/file_slice.py +++ b/graphify/file_slice.py @@ -26,7 +26,19 @@ # `_file_to_text` is a straight ``read_text`` (so a char range matches the bytes # the model is shown). Deliberately excludes code (.py, .ts, ...) and binary # docs (.pdf) — those are never sliced. -_SPLITTABLE_TEXT_SUFFIXES = frozenset({".md", ".mdx", ".markdown", ".txt", ".rst"}) +# +# This set has to keep pace with ``detect.DOC_EXTENSIONS``: anything classified +# as a document reaches the semantic pass, and anything the pass sees that is +# NOT listed here is silently cut at ``_FILE_CHAR_CAP`` by ``_read_files``. The +# two lists drifted as DOC_EXTENSIONS grew — .qmd, .skill, .html, .yaml and .yml +# were documents that never got sliced, so a 38k-character one reached the model +# as its first 20k with no warning and no partial marker (#2900). +# ``tests/test_oversized_document_slicing.py`` pins the relationship so a future +# addition to DOC_EXTENSIONS fails loudly instead of quietly losing content. +_SPLITTABLE_TEXT_SUFFIXES = frozenset({ + ".md", ".mdx", ".markdown", ".txt", ".rst", + ".qmd", ".skill", ".html", ".yaml", ".yml", +}) # Boundary preferences, strongest first. A Markdown heading (``\n#``) keeps a # section with its title; a blank line keeps a paragraph intact; a bare newline diff --git a/tests/test_oversized_document_slicing.py b/tests/test_oversized_document_slicing.py new file mode 100644 index 0000000000..04e151bbbd --- /dev/null +++ b/tests/test_oversized_document_slicing.py @@ -0,0 +1,114 @@ +"""Every document type that reaches the semantic pass must be sliceable. + +`_read_files` caps each unit at `_FILE_CHAR_CAP` (20,000 characters) before +joining it into the user message. `expand_oversized_files` exists so an +oversized document is split into contiguous `FileSlice`s that together cover the +whole file — but it only splits suffixes listed in `_SPLITTABLE_TEXT_SUFFIXES`. + +That list and `detect.DOC_EXTENSIONS` drifted. `.qmd`, `.skill`, `.html`, +`.yaml` and `.yml` were classified as documents, so they reached the model, and +were not splittable, so a 38,411-character one arrived as its first 20,000 +characters — no warning, no `_partial_files` marker, nothing in the report +(#2900). + +The contract test below is the point of this file: it fails when a new suffix is +added to `DOC_EXTENSIONS` without deciding how it gets sliced, so the two lists +cannot drift apart again silently. +""" +from pathlib import Path + +import pytest + +from graphify.detect import DOC_EXTENSIONS +from graphify.file_slice import ( + _SPLITTABLE_TEXT_SUFFIXES, + expand_oversized_files, + is_splittable_text, + read_slice_text, + slice_boundaries, +) +from graphify.llm import _FILE_CHAR_CAP, _read_files + +# Document suffixes whose bytes are NOT what the model is shown, so a character +# range over the raw file would be meaningless. `_file_to_text` routes these +# through a converter; slicing them needs that converter threaded through +# `read_slice_text` first, which is deliberately out of scope here. +BINARY_DOC_SUFFIXES = frozenset({".pdf"}) + +BIG = "# Heading\n\n" + ("The parser calls the tokenizer. " * 1200) + + +# --------------------------------------------------------------------------- +# The contract +# --------------------------------------------------------------------------- + +def test_every_text_document_type_is_splittable(): + """The guard against re-drift: a plain-text document type that reaches the + semantic pass must be sliceable, or an oversized one loses its tail.""" + missing = sorted( + (DOC_EXTENSIONS - BINARY_DOC_SUFFIXES) - _SPLITTABLE_TEXT_SUFFIXES + ) + assert not missing, ( + f"these document types reach the LLM but are never sliced, so anything " + f"over {_FILE_CHAR_CAP} characters is silently truncated: {missing}" + ) + + +@pytest.mark.parametrize( + "ext", sorted(DOC_EXTENSIONS - BINARY_DOC_SUFFIXES) +) +def test_an_oversized_document_reaches_the_model_whole(tmp_path, ext): + f = tmp_path / f"doc{ext}" + f.write_text(BIG, encoding="utf-8") + units = expand_oversized_files([f], _FILE_CHAR_CAP) + assert len(units) > 1, f"{ext} was not sliced; its tail never reaches the model" + # The slices tile the file exactly — no gap, no overlap, nothing dropped. + assert "".join(read_slice_text(u) for u in units) == BIG + + +# --------------------------------------------------------------------------- +# Slicing itself is unchanged for the types that already worked +# --------------------------------------------------------------------------- + +def test_a_small_document_is_still_passed_through_whole(tmp_path): + f = tmp_path / "small.qmd" + f.write_text("# tiny\n\nnothing to slice\n", encoding="utf-8") + units = expand_oversized_files([f], _FILE_CHAR_CAP) + assert units == [f], "a file under the cap must pass through as a plain Path" + + +def test_binary_documents_are_still_not_sliced(tmp_path): + """A PDF's bytes are not what the model is shown, so slicing the raw file + would be wrong. It must keep passing through untouched.""" + f = tmp_path / "paper.pdf" + f.write_bytes(b"%PDF-1.4\n" + b"x" * 40_000) + assert not is_splittable_text(f) + assert expand_oversized_files([f], _FILE_CHAR_CAP) == [f] + + +def test_code_files_are_still_not_sliced(tmp_path): + f = tmp_path / "mod.py" + f.write_text("def f():\n pass\n" * 4000, encoding="utf-8") + assert not is_splittable_text(f) + assert expand_oversized_files([f], _FILE_CHAR_CAP) == [f] + + +def test_slices_stay_within_the_cap(tmp_path): + for start, end in slice_boundaries(BIG, _FILE_CHAR_CAP): + assert end - start <= _FILE_CHAR_CAP + + +# --------------------------------------------------------------------------- +# End to end through the prompt builder +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("ext", [".qmd", ".html", ".yaml", ".yml", ".skill"]) +def test_the_tail_of_a_big_document_reaches_the_prompt(tmp_path, ext): + """The symptom a user would notice: content past 20k was invisible to the + semantic pass, so nothing in the tail could ever become a node.""" + marker = "UNIQUE_TAIL_MARKER_XYZZY" + f = tmp_path / f"doc{ext}" + f.write_text(BIG + "\n\n" + marker + "\n", encoding="utf-8") + units = expand_oversized_files([f], _FILE_CHAR_CAP) + joined = "".join(_read_files([u], tmp_path) for u in units) + assert marker in joined