diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index ddd9201..049a73e 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -23,6 +23,15 @@ from .runtime import _aborted, _tuning +def decode_source_bytes(raw: bytes) -> str: + """Decode source bytes into Elasticsearch-safe Unicode. + + Valid UTF-8 is preserved exactly. Undecodable input is deterministically replaced with + U+FFFD instead of surrogateescape code points, which Elasticsearch rejects as invalid UTF-8. + """ + return raw.decode("utf-8", errors="replace") + + def file_attributes( path: pathlib.Path, *, binary: bool = False, is_symlink: bool | None = None ) -> list[str]: @@ -344,7 +353,7 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: if abs_path.is_symlink(): # core.symlinks=true: actual filesystem symlink try: - git_target_path: str | None = os.readlink(abs_path) + git_target_path: str | None = decode_source_bytes(os.readlink(os.fsencode(abs_path))) except OSError: git_target_path = None try: @@ -353,9 +362,10 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: git_target_size = None else: # core.symlinks=false: file content IS the target path (no trailing newline) - git_target_path = raw.decode("utf-8", errors="surrogateescape") if raw is not None else None - if git_target_path is not None: - resolved = abs_path.parent / git_target_path + raw_target_path = os.fsdecode(raw) if raw is not None else None + git_target_path = decode_source_bytes(raw) if raw is not None else None + if raw_target_path is not None: + resolved = abs_path.parent / raw_target_path try: raw = resolved.read_bytes() git_target_size = resolved.stat().st_size @@ -384,7 +394,7 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: actions = [{"_index": f_index, "_id": file_id, "_source": file_doc}] if raw is None or binary: return actions - content = raw.decode("utf-8", errors="surrogateescape") + content = decode_source_bytes(raw) ff = file_doc["file"] if incremental: line_docs = iter_incremental_line_docs( diff --git a/tests/test_documents.py b/tests/test_documents.py index 5a4bc56..4cffe51 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -4,6 +4,7 @@ multiprocessing pool.""" # Standard packages +import json import os import pathlib import stat @@ -15,6 +16,7 @@ build_file_actions, build_file_doc, build_incremental_file_doc, + decode_source_bytes, file_attributes, iter_incremental_line_docs, iter_line_docs, @@ -44,6 +46,21 @@ def _set_worker_ctx_incremental(host: str, org: str, repo: str, repo_dir, ) +class TestDecodeSourceBytes: + def test_valid_utf8_is_preserved_exactly(self): + raw = "café\n東京\n🙂".encode("utf-8") + assert decode_source_bytes(raw) == "café\n東京\n🙂" + + def test_invalid_legacy_bytes_and_lone_bytes_are_replaced(self): + # Latin-1/Windows-1252 byte values and a lone byte are not valid UTF-8. Replace each + # invalid byte rather than introducing surrogate code points that JSON cannot encode. + raw = b"caf\xe9\nprice \x80\nlone \xff" + content = decode_source_bytes(raw) + assert content == "caf�\nprice �\nlone �" + assert not any(0xD800 <= ord(char) <= 0xDFFF for char in content) + content.encode("utf-8") + + class TestBuildFileDoc: def test_nested_dir(self, tmp_path): p = tmp_path / "a.txt" @@ -330,6 +347,31 @@ def test_binary_file_yields_only_file_doc(self, tmp_path): assert len(actions) == 1 assert actions[0]["_index"] == files_index("github", "acme", "widgets") + def test_invalid_utf8_text_is_not_binary_and_lines_are_normalized(self, tmp_path): + (tmp_path / "legacy.txt").write_bytes(b"caf\xe9\nprice \x80\nlone \xff\n") + _set_worker_ctx("github", "acme", "widgets", "deadbeef", tmp_path) + actions = _build_one_file_actions("legacy.txt") + + assert "binary" not in actions[0]["_source"]["file"].get("attributes", []) + line_actions = actions[1:] + assert [action["_source"]["line"] for action in line_actions] == [ + {"number": 1, "content": "caf�"}, + {"number": 2, "content": "price �"}, + {"number": 3, "content": "lone �"}, + ] + + def test_invalid_utf8_actions_are_deterministic_and_json_encodable(self, tmp_path): + (tmp_path / "legacy.txt").write_bytes(b"caf\xe9\nlone \xff\n") + _set_worker_ctx("github", "acme", "widgets", "deadbeef", tmp_path) + + first = _build_one_file_actions("legacy.txt") + second = _build_one_file_actions("legacy.txt") + + assert first == second + serialized = json.dumps(first, ensure_ascii=False) + serialized.encode("utf-8") + assert json.loads(serialized) == first + def test_nul_beyond_8kb_window_is_treated_as_text(self, tmp_path): # Binary detection only sniffs the first 8 KB -- a NUL byte after that point should # not trigger binary classification.