fix(index): normalize invalid UTF-8 source bytes

src/sourcerer/commands/index/documents.py

@@ -23,6 +23,15 @@
 from .runtime import _aborted, _tuning
 
 
+def decode_source_bytes(raw: bytes) -> str:
+    """Decode source bytes into Elasticsearch-safe Unicode.
+
+    Valid UTF-8 is preserved exactly. Undecodable input is deterministically replaced with
+    U+FFFD instead of surrogateescape code points, which Elasticsearch rejects as invalid UTF-8.
+    """
+    return raw.decode("utf-8", errors="replace")
+
+
 def file_attributes(
     path: pathlib.Path, *, binary: bool = False, is_symlink: bool | None = None
 ) -> list[str]:
@@ -344,7 +353,7 @@ def _build_one_file_actions(rel_path: str) -> list[dict]:
         if abs_path.is_symlink():
             # core.symlinks=true: actual filesystem symlink
             try:
-                git_target_path: str | None = os.readlink(abs_path)
+                git_target_path: str | None = decode_source_bytes(os.readlink(os.fsencode(abs_path)))
             except OSError:
                 git_target_path = None
             try:
@@ -353,9 +362,10 @@ def _build_one_file_actions(rel_path: str) -> list[dict]:
                 git_target_size = None
         else:
             # core.symlinks=false: file content IS the target path (no trailing newline)
-            git_target_path = raw.decode("utf-8", errors="surrogateescape") if raw is not None else None
-            if git_target_path is not None:
-                resolved = abs_path.parent / git_target_path
+            raw_target_path = os.fsdecode(raw) if raw is not None else None
+            git_target_path = decode_source_bytes(raw) if raw is not None else None
+            if raw_target_path is not None:
+                resolved = abs_path.parent / raw_target_path
                 try:
                     raw = resolved.read_bytes()
                     git_target_size = resolved.stat().st_size
@@ -384,7 +394,7 @@ def _build_one_file_actions(rel_path: str) -> list[dict]:
     actions = [{"_index": f_index, "_id": file_id, "_source": file_doc}]
     if raw is None or binary:
         return actions
-    content = raw.decode("utf-8", errors="surrogateescape")
+    content = decode_source_bytes(raw)
     ff = file_doc["file"]
     if incremental:
         line_docs = iter_incremental_line_docs(

tests/test_documents.py

@@ -4,6 +4,7 @@
 multiprocessing pool."""
 
 # Standard packages
+import json
 import os
 import pathlib
 import stat
@@ -15,6 +16,7 @@
     build_file_actions,
     build_file_doc,
     build_incremental_file_doc,
+    decode_source_bytes,
     file_attributes,
     iter_incremental_line_docs,
     iter_line_docs,
@@ -44,6 +46,21 @@ def _set_worker_ctx_incremental(host: str, org: str, repo: str, repo_dir,
     )
 
 
+class TestDecodeSourceBytes:
+    def test_valid_utf8_is_preserved_exactly(self):
+        raw = "café\n東京\n🙂".encode("utf-8")
+        assert decode_source_bytes(raw) == "café\n東京\n🙂"
+
+    def test_invalid_legacy_bytes_and_lone_bytes_are_replaced(self):
+        # Latin-1/Windows-1252 byte values and a lone byte are not valid UTF-8. Replace each
+        # invalid byte rather than introducing surrogate code points that JSON cannot encode.
+        raw = b"caf\xe9\nprice \x80\nlone \xff"
+        content = decode_source_bytes(raw)
+        assert content == "caf�\nprice �\nlone �"
+        assert not any(0xD800 <= ord(char) <= 0xDFFF for char in content)
+        content.encode("utf-8")
+
+
 class TestBuildFileDoc:
     def test_nested_dir(self, tmp_path):
         p = tmp_path / "a.txt"
@@ -330,6 +347,31 @@ def test_binary_file_yields_only_file_doc(self, tmp_path):
         assert len(actions) == 1
         assert actions[0]["_index"] == files_index("github", "acme", "widgets")
 
+    def test_invalid_utf8_text_is_not_binary_and_lines_are_normalized(self, tmp_path):
+        (tmp_path / "legacy.txt").write_bytes(b"caf\xe9\nprice \x80\nlone \xff\n")
+        _set_worker_ctx("github", "acme", "widgets", "deadbeef", tmp_path)
+        actions = _build_one_file_actions("legacy.txt")
+
+        assert "binary" not in actions[0]["_source"]["file"].get("attributes", [])
+        line_actions = actions[1:]
+        assert [action["_source"]["line"] for action in line_actions] == [
+            {"number": 1, "content": "caf�"},
+            {"number": 2, "content": "price �"},
+            {"number": 3, "content": "lone �"},
+        ]
+
+    def test_invalid_utf8_actions_are_deterministic_and_json_encodable(self, tmp_path):
+        (tmp_path / "legacy.txt").write_bytes(b"caf\xe9\nlone \xff\n")
+        _set_worker_ctx("github", "acme", "widgets", "deadbeef", tmp_path)
+
+        first = _build_one_file_actions("legacy.txt")
+        second = _build_one_file_actions("legacy.txt")
+
+        assert first == second
+        serialized = json.dumps(first, ensure_ascii=False)
+        serialized.encode("utf-8")
+        assert json.loads(serialized) == first
+
     def test_nul_beyond_8kb_window_is_treated_as_text(self, tmp_path):
         # Binary detection only sniffs the first 8 KB -- a NUL byte after that point should
         # not trigger binary classification.