From 57edf8936784dd6cddc2432e6738664554c467fe Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Fri, 21 Aug 2026 01:44:15 +0000 Subject: [PATCH] fix(extract): preserve existing semantic layer on --code-only --force (#2923) graphify extract --code-only --force previously rewrote graph.json with only the AST tier, silently dropping every doc/paper/image node plus its connected hyperedges. --force disables incremental mode so the merge path that would otherwise carry the surviving semantic tier forward never ran. A code-only run cannot touch the semantic tier at all (no LLM dispatch), so discarding the existing semantic layer is a destructive side effect with no correctness justification. Re-enable the incremental merge when --force and --code-only are combined and an existing graph.json is present; the AST tier is still fully replaced (full re-scan, semantic cache reads skipped) while doc/paper/image nodes are carried forward via build_merge / merge_raw_extraction. graph_stale_sources still prunes semantic nodes for files deleted from disk between the prior extract and this one, so the merge cannot resurrect nodes for sources that no longer exist. Adds two regression tests in test_extract_code_only_cli.py: - test_code_only_force_preserves_existing_semantic_layer: seeded graph with AST + SEMANTIC nodes; verifies the SEMANTIC tier survives --code-only --force. - test_code_only_force_prunes_removed_semantic_files: deletes NOTES.txt between seed and re-run; verifies its semantic nodes are pruned, not resurrected, by the merge. Ref: #2923 --- graphify/cli.py | 14 ++++ tests/test_extract_code_only_cli.py | 115 ++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/graphify/cli.py b/graphify/cli.py index 02e55b9447..899930aac0 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -3129,6 +3129,20 @@ def _parse_float(name: str, raw: str) -> float: # --force: full scan, not the manifest-gated incremental diff — a warm # unchanged tree would otherwise dispatch zero files (#1894). incremental_mode = incremental_mode and not force + # #2923: --force --code-only must NOT drop the existing semantic layer. + # The AST pass is fully replaced (full re-scan, semantic cache reads + # skipped), but the semantic pass is itself skipped entirely, so + # doc/paper/image nodes from the existing graph carry forward via the + # incremental merge (build_merge / merge_raw_extraction keep them + # because no new semantic-tier sources are dispatched). Without this, + # a single --code-only --force silently erases every doc/paper/image + # node plus its connected hyperedges. + if force and code_only and existing_graph_path.exists(): + incremental_mode = True + print( + "[graphify extract] --force --code-only: full AST re-scan, " + "existing semantic layer preserved (no semantic pass this run)" + ) if force: print("[graphify extract] --force: full re-scan, semantic cache reads skipped") elif incremental_mode and not manifest_path.exists(): diff --git a/tests/test_extract_code_only_cli.py b/tests/test_extract_code_only_cli.py index 93fec48d34..9853bbdd1f 100644 --- a/tests/test_extract_code_only_cli.py +++ b/tests/test_extract_code_only_cli.py @@ -258,3 +258,118 @@ def test_extract_names_skipped_sensitive_files(tmp_path): out = r.stdout + r.stderr assert "skipped as potentially sensitive" in out assert "github_token.txt" in out, "the skipped filename must be surfaced (#2106)" + + +def test_code_only_force_preserves_existing_semantic_layer(tmp_path): + """#2923 regression: --code-only --force must not drop the existing semantic + layer. The AST pass is fully replaced (full re-scan, semantic cache reads + skipped) but the semantic pass is itself skipped, so doc/paper/image nodes + from graph.json must be carried forward. Before the fix this combination + silently rewrote graph.json with only the AST tier, losing every semantic + node and every hyperedge connected to one. + """ + repo = _mixed_repo(tmp_path) + out = repo / "graphify-out" + out.mkdir() + graph = out / "graph.json" + # Seed a graph.json as if a prior full extract with an LLM backend had run: + # 2 AST nodes from app.py + 4 SEMANTIC nodes from README.md/NOTES.txt. + graph.write_text(json.dumps({ + "nodes": [ + {"id": "app_py", "label": "app.py", "type": "file", + "source_file": "app.py", "origin": "AST"}, + {"id": "app_hello", "label": "hello()", "type": "function", + "source_file": "app.py", "origin": "AST"}, + {"id": "readme_md", "label": "readme.md", "type": "file", + "source_file": "README.md", "origin": "SEMANTIC"}, + {"id": "readme_design", "label": "Design", "type": "concept", + "source_file": "README.md", "origin": "SEMANTIC"}, + {"id": "notes_txt", "label": "NOTES.txt", "type": "file", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + {"id": "notes_architecture", "label": "Architecture", "type": "concept", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + ], + "edges": [ + {"id": "e1", "source": "app_py", "target": "app_hello", + "relation": "contains", "source_file": "app.py"}, + {"id": "e2", "source": "readme_md", "target": "readme_design", + "relation": "concept_about", "source_file": "README.md"}, + {"id": "e3", "source": "notes_txt", "target": "notes_architecture", + "relation": "concept_about", "source_file": "NOTES.txt"}, + ], + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + })) + + r = _run(repo, "--code-only", "--force", "--no-cluster") + assert r.returncode == 0, r.stderr + + out_graph = json.loads(graph.read_text()) + semantic_labels = {n["label"] for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC"} + semantic_source_files = { + Path(str(n["source_file"])).name.lower() + for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC" + } + # Every seeded semantic node must survive. The AST pass may add new nodes + # (or relabel existing ones — e.g. hello vs hello()) but it must not + # silently drop the semantic tier. + assert {"readme.md", "notes.txt"}.issubset(semantic_source_files), ( + "code-only --force erased the existing semantic layer (#2923); " + f"semantic nodes remaining: {semantic_labels}" + ) + # Hyperedges are also semantic tier; the seeded graph had none but the + # AST re-extract must not have invented any non-semantic work, and the + # surviving edges list must not have been wholesale replaced. + assert "edges" in out_graph, "graph.json must still have an edges key" + # And the user-visible console line must explain why a semantic-layer- + # preserving branch fired. + assert "existing semantic layer preserved" in r.stdout + r.stderr, ( + "the --force --code-only print must announce the semantic-preserving branch" + ) + + +def test_code_only_force_prunes_removed_semantic_files(tmp_path): + """#2923 follow-up: --code-only --force preserves surviving semantic nodes + but must still prune semantic nodes for files that have been removed from + disk (the doc/paper/image tier cannot outlive the corpus it indexes). + """ + repo = _mixed_repo(tmp_path) + out = repo / "graphify-out" + out.mkdir() + graph = out / "graph.json" + graph.write_text(json.dumps({ + "nodes": [ + {"id": "app_py", "label": "app.py", "type": "file", + "source_file": "app.py", "origin": "AST"}, + {"id": "app_hello", "label": "hello()", "type": "function", + "source_file": "app.py", "origin": "AST"}, + {"id": "notes_txt", "label": "NOTES.txt", "type": "file", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + {"id": "notes_architecture", "label": "Architecture", "type": "concept", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + ], + "edges": [], + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + })) + + # Delete NOTES.txt between seed and re-run. The merge's graph_stale_sources + # path must drop its semantic nodes because the file no longer exists. + (repo / "NOTES.txt").unlink() + + r = _run(repo, "--code-only", "--force", "--no-cluster") + assert r.returncode == 0, r.stderr + out_graph = json.loads(graph.read_text()) + remaining_sources = { + Path(n["source_file"]).name + for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC" + } + assert "NOTES.txt" not in remaining_sources, ( + "NOTES.txt was deleted from disk; its semantic nodes must be pruned " + "(#2923 follow-up)" + )