feat: implement and test parsing

ingest/PARSER.md

@@ -0,0 +1,61 @@
+# Parse chunks
+
+This parser takes chunks that have already been created. It does not read a
+repository, split files, generate embeddings, or modify the existing chunker.
+
+```bash
+uv run python -m ingest.parser chunks.jsonl --out parsed_chunks.jsonl
+```
+
+Each input line is a JSON object with `text`, `path`, and `start_line` (a positive,
+one-based original file line number). All other fields are preserved. The existing
+chunker's full records work directly; their metadata header is excluded from syntax
+analysis without changing the stored text or shifting original source positions.
+
+For example, an input chunk can be:
+
+```json
+{"text":"def hello():\n    return 'hi'\n","path":"src/demo.py","start_line":20}
+```
+
+Each output line contains two fields:
+
+- `chunk`: the complete original input record, unchanged. Embeddings can consume
+  `record["chunk"]["text"]`; original IDs, citations and token counts remain here.
+- `parsed`: detected language, parse status, named definitions with original file
+  line ranges, and Markdown headings with levels and original line numbers.
+
+The output is a separate analysis format. The nested `chunk` retains the existing
+chunk contract; the outer wrapper is not a replacement for that contract.
+
+Python usage:
+
+```python
+from ingest.parser import parse_chunk
+
+result = parse_chunk({
+    "text": "def hello():\n    return 'hi'\n",
+    "path": "src/demo.py",
+    "start_line": 20,
+})
+print(result["parsed"]["definitions"])
+```
+
+Code is analyzed using the project's existing Tree-sitter dependency. Grammars may
+download on first use. No tokenizer or model is loaded. Definitions include names
+such as `Greeter.greet` when the enclosing class is present in the same chunk.
+
+Statuses are `ok` (no syntax errors reported), `partial` (syntax errors, possibly
+because the chunk cuts through code), `unavailable` (grammar could not be loaded),
+and `text` (plain text or RST, retained without structural analysis). `ok` does not
+guarantee a fragment is complete. Chunks are never dropped because parsing is partial
+or unavailable. Missing surrounding classes, Markdown fences, or headings cannot be
+recovered from an isolated chunk. Only `#`-style Markdown headings are extracted.
+
+Invalid input reports its JSONL line number and leaves previous output intact.
+Input and output must be different paths. Filtering and secret scanning belong
+upstream; this parser preserves the chunks it receives.
+
+```bash
+uv run pytest tests/test_parser.py
+```

ingest/parser.py

@@ -0,0 +1,150 @@
+"""Parse existing chunks without splitting, merging, or changing their text.
+
+    uv run python -m ingest.parser chunks.jsonl --out parsed_chunks.jsonl
+
+Input: one JSON object per line, with text, path, and start_line. The existing
+ingest.chunker output is accepted directly, including its source header.
+Output: {"chunk": <original record>, "parsed": <analysis>}, one per input chunk.
+All original metadata is preserved. No tokenizer, model, or repository is loaded.
+"""
+
+import argparse
+import json
+import os
+import re
+import tempfile
+from pathlib import Path
+
+import tree_sitter_language_pack as tslp
+
+from ingest.chunker import DEFINITION_RE, DOC_LANGUAGES, HEADING_RE, detect_language
+
+
+def _source_text(chunk: dict) -> str:
+    """Remove only the exact metadata header emitted by the existing chunker."""
+    header = f"# {chunk.get('repo')} {chunk['path']}:{chunk['start_line']}-{chunk.get('end_line')}"
+    if chunk.get("symbol"):
+        header += f" {chunk['symbol']}"
+    first, separator, rest = chunk["text"].partition("\n")
+    return rest if separator and first == header else chunk["text"]
+
+
+def _headings(text: str, start_line: int) -> list[dict]:
+    headings = []
+    fence = None
+    for offset, line in enumerate(text.splitlines()):
+        marker = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line)
+        if marker:
+            run, tail = marker.groups()
+            if fence is None:
+                fence = run
+            elif run[0] == fence[0] and len(run) >= len(fence) and not tail.strip():
+                fence = None
+            continue
+        match = HEADING_RE.match(line) if fence is None else None
+        if match:
+            headings.append({"title": match[2], "level": len(match[1]), "line": start_line + offset})
+    return headings
+
+
+def parse_chunk(chunk: dict) -> dict:
+    """Return an unchanged chunk plus syntax information found within it.
+
+    Positions refer to the original file using the supplied start_line. For
+    fragments, partial means Tree-sitter found a syntax error; it does not prove
+    a chunk is complete when no error is found. No missing context is invented.
+    """
+    if not isinstance(chunk, dict):
+        raise ValueError("each chunk must be a JSON object")
+    for key in ("text", "path"):
+        if not isinstance(chunk.get(key), str) or not chunk[key].strip():
+            raise ValueError(f"{key} must be a nonempty string")
+    start = chunk.get("start_line")
+    if type(start) is not int or start < 1:
+        raise ValueError("start_line must be a positive integer")
+    if "end_line" in chunk and (type(chunk["end_line"]) is not int or chunk["end_line"] < start):
+        raise ValueError("end_line must be an integer at least start_line")
+    if "language" in chunk and (not isinstance(chunk["language"], str) or not chunk["language"]):
+        raise ValueError("language must be a nonempty string")
+
+    fallback = "text" if Path(chunk["path"]).suffix.lower() == ".txt" else None
+    language = chunk.get("language") or fallback or detect_language(chunk["path"])
+    source = _source_text(chunk)
+    parsed = {"language": language, "status": "ok", "definitions": [], "headings": []}
+    result = {"chunk": dict(chunk), "parsed": parsed}
+    if language in DOC_LANGUAGES:
+        if language == "markdown":
+            parsed["headings"] = _headings(source, start)
+        else:
+            parsed["status"] = "text"
+        return result
+
+    try:
+        parser = tslp.get_parser(language)
+    except Exception:
+        # Grammar downloads can fail offline; retain the chunk and report it.
+        parsed["status"] = "unavailable"
+        return result
+    root = parser.parse(source.encode("utf-8")).root_node
+    parsed["status"] = "partial" if root.has_error else "ok"
+
+    stack = [(root, [])]
+    while stack:
+        node, scope = stack.pop()
+        name_node = node.child_by_field_name("name")
+        if name_node is None and node.type == "impl_item":
+            name_node = node.child_by_field_name("type")
+        if DEFINITION_RE.search(node.type) and name_node is not None:
+            name = name_node.text.decode("utf-8")
+            scope = scope + [name]
+            last_row = node.end_point[0] - (node.end_point[1] == 0 and node.end_point[0] > node.start_point[0])
+            parsed["definitions"].append({
+                "name": ".".join(scope),
+                "type": node.type,
+                "start_line": start + node.start_point[0],
+                "end_line": start + last_row,
+            })
+        stack.extend((child, scope) for child in reversed(node.named_children))
+    return result
+
+
+def parse_jsonl(input_path: Path, output_path: Path) -> int:
+    """Stream chunks to a separate output file; preserve prior output on failure."""
+    if input_path.resolve() == output_path.resolve():
+        raise ValueError("input and output must be different files")
+    temporary = None
+    count = 0
+    try:
+        with input_path.open(encoding="utf-8") as source:
+            with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=output_path.parent, delete=False) as target:
+                temporary = Path(target.name)
+                for line_number, line in enumerate(source, 1):
+                    if not line.strip():
+                        continue
+                    try:
+                        record = parse_chunk(json.loads(line))
+                    except ValueError as error:
+                        raise ValueError(f"input line {line_number}: {error}") from error
+                    target.write(json.dumps(record) + "\n")
+                    count += 1
+        os.replace(temporary, output_path)
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)
+    return count
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("chunks", type=Path, help="JSONL from the chunker")
+    parser.add_argument("--out", type=Path, default=Path("parsed_chunks.jsonl"))
+    args = parser.parse_args()
+    try:
+        count = parse_jsonl(args.chunks, args.out)
+    except (ValueError, OSError) as error:
+        parser.exit(1, f"Parser failed: {error}\n")
+    print(f"Parsed {count} chunks into {args.out}")
+
+
+if __name__ == "__main__":
+    main()

tests/test_parser.py

@@ -0,0 +1,93 @@
+import copy
+import json
+
+import pytest
+
+from ingest.chunker import chunk_file
+from ingest.parser import parse_chunk, parse_jsonl
+from tests.test_contracts import validator
+
+
+def chunk(text, path="example.py", start_line=10, **metadata):
+    return {"text": text, "path": path, "start_line": start_line, **metadata}
+
+
+def test_python_definitions_preserve_chunk_and_file_positions():
+    original = chunk("class Greeter:\n    def greet(self):\n        return 'hello'\n", id="unchanged")
+    before = copy.deepcopy(original)
+    result = parse_chunk(original)
+    assert original == before == result["chunk"]
+    assert result["parsed"]["status"] == "ok"
+    assert result["parsed"]["definitions"] == [
+        {"name": "Greeter", "type": "class_definition", "start_line": 10, "end_line": 12},
+        {"name": "Greeter.greet", "type": "function_definition", "start_line": 11, "end_line": 12},
+    ]
+
+
+def test_accepts_existing_chunker_records_without_header_line_offset():
+    source = b"# documentation\ndef hello():\n    return 'hello'\n"
+    records = chunk_file("org/repo", "example.py", source, "a" * 40, lambda s: len(s.split()))
+    for original in records:
+        result = parse_chunk(original)
+        validator("chunk.schema.json").validate(result["chunk"])
+        assert result["chunk"] == original
+        assert result["parsed"]["definitions"][0]["start_line"] == 2
+
+
+def test_markdown_heading_positions_and_fences():
+    original = chunk("# Project\n````python\n```\n# hidden\n````\n## Setup\n", path="README.md")
+    result = parse_chunk(original)
+    assert result["parsed"]["headings"] == [
+        {"title": "Project", "level": 1, "line": 10},
+        {"title": "Setup", "level": 2, "line": 15},
+    ]
+    assert result["chunk"]["text"] == original["text"]
+
+
+def test_partial_code_is_preserved_and_flagged():
+    original = chunk("def unfinished(\n")
+    result = parse_chunk(original)
+    assert result["parsed"]["status"] == "partial"
+    assert result["chunk"] == original
+
+
+def test_grammar_unavailable_does_not_drop_chunk(monkeypatch):
+    from ingest import parser
+
+    def unavailable(language):
+        raise RuntimeError("offline")
+
+    monkeypatch.setattr(parser.tslp, "get_parser", unavailable)
+    original = chunk("some code", language="python")
+    assert parse_chunk(original)["parsed"]["status"] == "unavailable"
+    assert parse_chunk(original)["chunk"] == original
+
+
+def test_plain_text_preserved():
+    original = chunk("Project overview.", path="notes.txt")
+    result = parse_chunk(original)
+    assert result["parsed"]["status"] == "text"
+    assert result["chunk"] == original
+
+
+@pytest.mark.parametrize("bad", [None, [], {}, chunk(""), chunk("x", start_line=0), chunk("x", start_line=True), chunk("x", end_line=3), chunk("x", language=12)])
+def test_rejects_invalid_chunk(bad):
+    with pytest.raises(ValueError):
+        parse_chunk(bad)
+
+
+def test_jsonl_one_output_per_input_and_atomic_failure(tmp_path):
+    source = tmp_path / "chunks.jsonl"
+    target = tmp_path / "parsed.jsonl"
+    chunks = [chunk("# Project\n", path="README.md"), chunk("def run():\n    pass\n")]
+    source.write_text("\n".join(json.dumps(c) for c in chunks) + "\n\n")
+    assert parse_jsonl(source, target) == 2
+    saved = target.read_text()
+    assert [json.loads(line)["chunk"] for line in saved.splitlines()] == chunks
+    source.write_text(source.read_text() + "{broken\n")
+    with pytest.raises(ValueError, match="input line 4"):
+        parse_jsonl(source, target)
+    assert target.read_text() == saved
+    assert sorted(p.name for p in tmp_path.iterdir()) == ["chunks.jsonl", "parsed.jsonl"]
+    with pytest.raises(ValueError, match="different files"):
+        parse_jsonl(source, source)