diff --git a/ingest/PARSER.md b/ingest/PARSER.md new file mode 100644 index 0000000..bb3ebd1 --- /dev/null +++ b/ingest/PARSER.md @@ -0,0 +1,61 @@ +# Parse chunks + +This parser takes chunks that have already been created. It does not read a +repository, split files, generate embeddings, or modify the existing chunker. + +```bash +uv run python -m ingest.parser chunks.jsonl --out parsed_chunks.jsonl +``` + +Each input line is a JSON object with `text`, `path`, and `start_line` (a positive, +one-based original file line number). All other fields are preserved. The existing +chunker's full records work directly; their metadata header is excluded from syntax +analysis without changing the stored text or shifting original source positions. + +For example, an input chunk can be: + +```json +{"text":"def hello():\n return 'hi'\n","path":"src/demo.py","start_line":20} +``` + +Each output line contains two fields: + +- `chunk`: the complete original input record, unchanged. Embeddings can consume + `record["chunk"]["text"]`; original IDs, citations and token counts remain here. +- `parsed`: detected language, parse status, named definitions with original file + line ranges, and Markdown headings with levels and original line numbers. + +The output is a separate analysis format. The nested `chunk` retains the existing +chunk contract; the outer wrapper is not a replacement for that contract. + +Python usage: + +```python +from ingest.parser import parse_chunk + +result = parse_chunk({ + "text": "def hello():\n return 'hi'\n", + "path": "src/demo.py", + "start_line": 20, +}) +print(result["parsed"]["definitions"]) +``` + +Code is analyzed using the project's existing Tree-sitter dependency. Grammars may +download on first use. No tokenizer or model is loaded. Definitions include names +such as `Greeter.greet` when the enclosing class is present in the same chunk. + +Statuses are `ok` (no syntax errors reported), `partial` (syntax errors, possibly +because the chunk cuts through code), `unavailable` (grammar could not be loaded), +and `text` (plain text or RST, retained without structural analysis). `ok` does not +guarantee a fragment is complete. Chunks are never dropped because parsing is partial +or unavailable. Missing surrounding classes, Markdown fences, or headings cannot be +recovered from an isolated chunk. Only `#`-style Markdown headings are extracted. + +Invalid input reports its JSONL line number and leaves previous output intact. +Input and output must be different paths. Filtering and secret scanning belong +upstream; this parser preserves the chunks it receives. + +```bash +uv run pytest tests/test_parser.py +``` diff --git a/ingest/parser.py b/ingest/parser.py new file mode 100644 index 0000000..87b00f4 --- /dev/null +++ b/ingest/parser.py @@ -0,0 +1,150 @@ +"""Parse existing chunks without splitting, merging, or changing their text. + + uv run python -m ingest.parser chunks.jsonl --out parsed_chunks.jsonl + +Input: one JSON object per line, with text, path, and start_line. The existing +ingest.chunker output is accepted directly, including its source header. +Output: {"chunk": , "parsed": }, one per input chunk. +All original metadata is preserved. No tokenizer, model, or repository is loaded. +""" + +import argparse +import json +import os +import re +import tempfile +from pathlib import Path + +import tree_sitter_language_pack as tslp + +from ingest.chunker import DEFINITION_RE, DOC_LANGUAGES, HEADING_RE, detect_language + + +def _source_text(chunk: dict) -> str: + """Remove only the exact metadata header emitted by the existing chunker.""" + header = f"# {chunk.get('repo')} {chunk['path']}:{chunk['start_line']}-{chunk.get('end_line')}" + if chunk.get("symbol"): + header += f" {chunk['symbol']}" + first, separator, rest = chunk["text"].partition("\n") + return rest if separator and first == header else chunk["text"] + + +def _headings(text: str, start_line: int) -> list[dict]: + headings = [] + fence = None + for offset, line in enumerate(text.splitlines()): + marker = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line) + if marker: + run, tail = marker.groups() + if fence is None: + fence = run + elif run[0] == fence[0] and len(run) >= len(fence) and not tail.strip(): + fence = None + continue + match = HEADING_RE.match(line) if fence is None else None + if match: + headings.append({"title": match[2], "level": len(match[1]), "line": start_line + offset}) + return headings + + +def parse_chunk(chunk: dict) -> dict: + """Return an unchanged chunk plus syntax information found within it. + + Positions refer to the original file using the supplied start_line. For + fragments, partial means Tree-sitter found a syntax error; it does not prove + a chunk is complete when no error is found. No missing context is invented. + """ + if not isinstance(chunk, dict): + raise ValueError("each chunk must be a JSON object") + for key in ("text", "path"): + if not isinstance(chunk.get(key), str) or not chunk[key].strip(): + raise ValueError(f"{key} must be a nonempty string") + start = chunk.get("start_line") + if type(start) is not int or start < 1: + raise ValueError("start_line must be a positive integer") + if "end_line" in chunk and (type(chunk["end_line"]) is not int or chunk["end_line"] < start): + raise ValueError("end_line must be an integer at least start_line") + if "language" in chunk and (not isinstance(chunk["language"], str) or not chunk["language"]): + raise ValueError("language must be a nonempty string") + + fallback = "text" if Path(chunk["path"]).suffix.lower() == ".txt" else None + language = chunk.get("language") or fallback or detect_language(chunk["path"]) + source = _source_text(chunk) + parsed = {"language": language, "status": "ok", "definitions": [], "headings": []} + result = {"chunk": dict(chunk), "parsed": parsed} + if language in DOC_LANGUAGES: + if language == "markdown": + parsed["headings"] = _headings(source, start) + else: + parsed["status"] = "text" + return result + + try: + parser = tslp.get_parser(language) + except Exception: + # Grammar downloads can fail offline; retain the chunk and report it. + parsed["status"] = "unavailable" + return result + root = parser.parse(source.encode("utf-8")).root_node + parsed["status"] = "partial" if root.has_error else "ok" + + stack = [(root, [])] + while stack: + node, scope = stack.pop() + name_node = node.child_by_field_name("name") + if name_node is None and node.type == "impl_item": + name_node = node.child_by_field_name("type") + if DEFINITION_RE.search(node.type) and name_node is not None: + name = name_node.text.decode("utf-8") + scope = scope + [name] + last_row = node.end_point[0] - (node.end_point[1] == 0 and node.end_point[0] > node.start_point[0]) + parsed["definitions"].append({ + "name": ".".join(scope), + "type": node.type, + "start_line": start + node.start_point[0], + "end_line": start + last_row, + }) + stack.extend((child, scope) for child in reversed(node.named_children)) + return result + + +def parse_jsonl(input_path: Path, output_path: Path) -> int: + """Stream chunks to a separate output file; preserve prior output on failure.""" + if input_path.resolve() == output_path.resolve(): + raise ValueError("input and output must be different files") + temporary = None + count = 0 + try: + with input_path.open(encoding="utf-8") as source: + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=output_path.parent, delete=False) as target: + temporary = Path(target.name) + for line_number, line in enumerate(source, 1): + if not line.strip(): + continue + try: + record = parse_chunk(json.loads(line)) + except ValueError as error: + raise ValueError(f"input line {line_number}: {error}") from error + target.write(json.dumps(record) + "\n") + count += 1 + os.replace(temporary, output_path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + return count + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("chunks", type=Path, help="JSONL from the chunker") + parser.add_argument("--out", type=Path, default=Path("parsed_chunks.jsonl")) + args = parser.parse_args() + try: + count = parse_jsonl(args.chunks, args.out) + except (ValueError, OSError) as error: + parser.exit(1, f"Parser failed: {error}\n") + print(f"Parsed {count} chunks into {args.out}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..9b4fc1c --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,93 @@ +import copy +import json + +import pytest + +from ingest.chunker import chunk_file +from ingest.parser import parse_chunk, parse_jsonl +from tests.test_contracts import validator + + +def chunk(text, path="example.py", start_line=10, **metadata): + return {"text": text, "path": path, "start_line": start_line, **metadata} + + +def test_python_definitions_preserve_chunk_and_file_positions(): + original = chunk("class Greeter:\n def greet(self):\n return 'hello'\n", id="unchanged") + before = copy.deepcopy(original) + result = parse_chunk(original) + assert original == before == result["chunk"] + assert result["parsed"]["status"] == "ok" + assert result["parsed"]["definitions"] == [ + {"name": "Greeter", "type": "class_definition", "start_line": 10, "end_line": 12}, + {"name": "Greeter.greet", "type": "function_definition", "start_line": 11, "end_line": 12}, + ] + + +def test_accepts_existing_chunker_records_without_header_line_offset(): + source = b"# documentation\ndef hello():\n return 'hello'\n" + records = chunk_file("org/repo", "example.py", source, "a" * 40, lambda s: len(s.split())) + for original in records: + result = parse_chunk(original) + validator("chunk.schema.json").validate(result["chunk"]) + assert result["chunk"] == original + assert result["parsed"]["definitions"][0]["start_line"] == 2 + + +def test_markdown_heading_positions_and_fences(): + original = chunk("# Project\n````python\n```\n# hidden\n````\n## Setup\n", path="README.md") + result = parse_chunk(original) + assert result["parsed"]["headings"] == [ + {"title": "Project", "level": 1, "line": 10}, + {"title": "Setup", "level": 2, "line": 15}, + ] + assert result["chunk"]["text"] == original["text"] + + +def test_partial_code_is_preserved_and_flagged(): + original = chunk("def unfinished(\n") + result = parse_chunk(original) + assert result["parsed"]["status"] == "partial" + assert result["chunk"] == original + + +def test_grammar_unavailable_does_not_drop_chunk(monkeypatch): + from ingest import parser + + def unavailable(language): + raise RuntimeError("offline") + + monkeypatch.setattr(parser.tslp, "get_parser", unavailable) + original = chunk("some code", language="python") + assert parse_chunk(original)["parsed"]["status"] == "unavailable" + assert parse_chunk(original)["chunk"] == original + + +def test_plain_text_preserved(): + original = chunk("Project overview.", path="notes.txt") + result = parse_chunk(original) + assert result["parsed"]["status"] == "text" + assert result["chunk"] == original + + +@pytest.mark.parametrize("bad", [None, [], {}, chunk(""), chunk("x", start_line=0), chunk("x", start_line=True), chunk("x", end_line=3), chunk("x", language=12)]) +def test_rejects_invalid_chunk(bad): + with pytest.raises(ValueError): + parse_chunk(bad) + + +def test_jsonl_one_output_per_input_and_atomic_failure(tmp_path): + source = tmp_path / "chunks.jsonl" + target = tmp_path / "parsed.jsonl" + chunks = [chunk("# Project\n", path="README.md"), chunk("def run():\n pass\n")] + source.write_text("\n".join(json.dumps(c) for c in chunks) + "\n\n") + assert parse_jsonl(source, target) == 2 + saved = target.read_text() + assert [json.loads(line)["chunk"] for line in saved.splitlines()] == chunks + source.write_text(source.read_text() + "{broken\n") + with pytest.raises(ValueError, match="input line 4"): + parse_jsonl(source, target) + assert target.read_text() == saved + assert sorted(p.name for p in tmp_path.iterdir()) == ["chunks.jsonl", "parsed.jsonl"] + with pytest.raises(ValueError, match="different files"): + parse_jsonl(source, source)