diff --git a/graphify/cache.py b/graphify/cache.py index 833fdaa86..e254621e3 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -992,6 +992,22 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast", # across chunks without losing the truncated one (it stays partial). if not allow_partial and isinstance(result, dict) and result.get("partial"): return None + # An entry carrying edges but ZERO nodes is a model-omission artifact + # (#2927) that older versions cached keyed by content hash, replaying + # the empty node set forever — the file sat at 0 nodes while every + # edge pointing into it dangled. Treat it as a MISS so the file is + # re-dispatched and a complete extraction overwrites the poisoned key + # (self-healing, same mechanism as the ``partial`` check above). + # Hyperedge-only entries remain valid output (#1920). No + # ``allow_partial`` escape hatch here: a zero-node entry carries + # nothing worth merging onto a slice union. + if ( + kind.startswith("semantic") + and isinstance(result, dict) + and not result.get("nodes") + and result.get("edges") + ): + return None if ( kind.startswith("semantic") and isinstance(result, dict) @@ -1329,6 +1345,13 @@ def save_semantic_cache( so the flag survives even when a caller (e.g. cli.py's final save) does not pass ``partial_source_files``. + A per-file result carrying edges but ZERO nodes is a model-omission + artifact (#2927): caching it keyed by content hash makes every later run a + hit, permanently freezing the file out of the graph while the console + claims "a re-run will retry them". Such groups are NOT written — the file + stays uncached and genuinely re-dispatches next run. Hyperedge-only + results remain valid output and cache normally (#1920). + ``prompt`` is the extraction prompt that produced these results — text, or a Path to the prompt file. It stamps entries into the p{fingerprint}/ namespace so a later run under a different prompt re-extracts rather than @@ -1492,6 +1515,7 @@ def hyperedge_dangles(h: dict) -> bool: saved = 0 skipped_not_file = 0 + edges_only_skips = 0 for fpath, result in by_file.items(): cache_path = source_path(fpath) p = resolved_source_path(fpath) @@ -1528,6 +1552,17 @@ def hyperedge_dangles(h: dict) -> bool: } else: _prev_partial = False + # Edges-but-no-nodes is a model-omission artifact (#2927): writing + # it keyed by content hash would make every later run a cache hit, + # permanently freezing the file out of the graph while the console + # claims "a re-run will retry them". Leave it uncached so the next + # run genuinely re-dispatches the file. Evaluated AFTER the merge + # so a multi-chunk file whose earlier slice produced nodes still + # accumulates (the union carries nodes), and hyperedge-only + # results stay cacheable (#1920). + if not result["nodes"] and result["edges"]: + edges_only_skips += 1 + continue # A file is partial if the caller named it, any of its grouped items # carries the intrinsic ``_partial`` marker, OR the entry it merged # onto was already partial (an empty-parse truncation leaves a @@ -1558,4 +1593,12 @@ def hyperedge_dangles(h: dict) -> bool: RuntimeWarning, stacklevel=2, ) + if edges_only_skips: + warnings.warn( + f"save_semantic_cache: {edges_only_skips} file(s) produced edges but no " + "nodes — the model omitted them from its response (#2927). Nothing was " + "cached for them, so the next run will re-dispatch and retry them.", + RuntimeWarning, + stacklevel=2, + ) return saved diff --git a/graphify/cli.py b/graphify/cli.py index 02e55b944..89e32f5b6 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -97,6 +97,15 @@ def _stamped_manifest_files( have no source_file entry in sem_result — leaving their semantic_hash empty so detect_incremental re-queues them (#933). + A file attributed only EDGES (no nodes, no hyperedges) did not really + produce output — the model omitted the file from its response (#2927). + Counting edge coverage as output stamped such a file up-to-date, so + detect_incremental never re-dispatched it and it stayed frozen at 0 nodes + even after the cache stopped serving the empty entry. Edges therefore do + not contribute to stamping; the file stays unstamped and is re-queued + next run (same #933 mechanism). Hyperedges still count (#1920); the + stamping condition mirrors save_semantic_cache's cache-write rule. + A file in ``partial_source_files`` DID produce output this run, but only a truncated fragment of it, so it is excluded from stamping too — otherwise detect_incremental would see it "done" and never re-dispatch it, leaving the @@ -133,7 +142,10 @@ def _resolve(value: str) -> Path: return p sem_extracted: set[Path] = set() - for coll in ("nodes", "edges", "hyperedges"): + # Edges deliberately excluded (#2927) — see the docstring: edge-only + # attribution is the model-omission shape, and stamping it froze the file + # out of warm incremental runs. Nodes and hyperedges (#1920) count. + for coll in ("nodes", "hyperedges"): for item in sem_result.get(coll, []): sf = item.get("source_file", "") if sf: diff --git a/tests/test_cache.py b/tests/test_cache.py index 920526dc1..2818fcb3f 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1749,3 +1749,123 @@ def test_corrupt_semantic_entry_warns_and_is_a_miss(tmp_path): # The corrupt entry is a miss, so the file is re-dispatched for extraction. assert nodes == [] assert uncached == [str(f)] + + +def test_save_semantic_cache_skips_edges_only_results(tmp_path): + """#2927: a dispatched file whose semantic result carries edges but ZERO + nodes is a model omission. Caching it keyed by content hash made every + later run a hit, permanently freezing the file out of the graph while the + console claimed "a re-run will retry them". The write must be skipped so + the file stays uncached and genuinely re-dispatches on the next run.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "hub.md" + f.write_text("# Hub\n\nMost-referenced doc.\n") + + with pytest.warns(RuntimeWarning, match="edges but no nodes"): + saved = save_semantic_cache( + [], + [{"source": "a", "target": "b", "source_file": "hub.md"}], + root=tmp_path, + ) + + assert saved == 0 + # No entry was written for the content hash... + entry = cache_dir(tmp_path, "semantic") / f"{file_hash(f, tmp_path)}.json" + assert not entry.exists() + # ...so the next run reports the file as uncached (re-dispatch). + nodes, edges, _, uncached = check_semantic_cache([str(f)], root=tmp_path) + assert nodes == [] and edges == [] + assert uncached == [str(f)] + + +def test_save_semantic_cache_edges_only_merges_onto_node_entry(tmp_path): + """#2927 skip must not break multi-chunk accumulation: an edges-only slice + merging onto a prior entry that HAS nodes still saves the union. Only a + zero-node FINAL result is skipped.""" + from graphify.cache import load_cached, save_semantic_cache + + f = tmp_path / "big.md" + f.write_text("# Big\n\n" + "para\n" * 200) + save_semantic_cache([{"id": "n1", "source_file": "big.md"}], [], root=tmp_path) + + saved = save_semantic_cache( + [], + [{"source": "n1", "target": "x", "source_file": "big.md"}], + root=tmp_path, + merge_existing=True, + ) + assert saved == 1 + loaded = load_cached(f, tmp_path, kind="semantic") + assert [n["id"] for n in loaded["nodes"]] == ["n1"] + assert [e["source"] for e in loaded["edges"]] == ["n1"] + + +def test_save_semantic_cache_hyperedge_only_still_caches(tmp_path): + """#1920 must survive the #2927 fix: a hyperedge-only result is valid + output and caches normally.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "hyper.md" + f.write_text("# Hyper\n") + hyper = { + "id": "h1", "label": "L", "nodes": ["a", "b", "c"], + "relation": "participate_in", "source_file": "hyper.md", + } + saved = save_semantic_cache([], [], [hyper], root=tmp_path) + assert saved == 1 + + nodes, _, hyperedges, uncached = check_semantic_cache([str(f)], root=tmp_path) + assert uncached == [] + # Replay absolutizes source_file (#777); identity is the hyperedge id. + assert [h["id"] for h in hyperedges] == ["h1"] + + +def test_edges_only_semantic_entry_is_a_miss_and_self_heals(tmp_path): + """#2927 healing: entries already poisoned by older versions (edges, no + nodes) must read as a MISS so the file re-dispatches and a complete + extraction overwrites the poisoned key. Hyperedge-only entries (#1920) + keep serving, and AST entries are untouched by the rule.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "hub.md" + f.write_text("# Hub\n") + poison = { + "nodes": [], + "edges": [{"source": "a", "target": "b", "source_file": "hub.md"}], + } + save_cached(f, poison, tmp_path, kind="semantic") + assert load_cached(f, tmp_path, kind="semantic") is None + + _, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path) + assert uncached == [str(f)] + + # Self-heal: a complete extraction overwrites the same content-hash key. + save_semantic_cache( + [{"id": "hub", "source_file": "hub.md"}], + [{"source": "a", "target": "hub", "source_file": "hub.md"}], + root=tmp_path, + ) + loaded = load_cached(f, tmp_path, kind="semantic") + assert [n["id"] for n in loaded["nodes"]] == ["hub"] + _, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path) + assert uncached == [] + + # Hyperedge-only entries remain valid output (#1920). + h = tmp_path / "hyper.md" + h.write_text("# Hyper\n") + save_cached( + h, + {"nodes": [], "edges": [], "hyperedges": [ + {"id": "h1", "nodes": ["a", "b", "c"], "source_file": "hyper.md"}, + ]}, + tmp_path, kind="semantic", + ) + served = load_cached(h, tmp_path, kind="semantic") + assert served is not None and served["hyperedges"] + + # AST kind is outside the rule's scope. + a = tmp_path / "mod.py" + a.write_text("x = 1\n") + save_cached(a, {"nodes": [], "edges": []}, tmp_path, kind="ast") + assert load_cached(a, tmp_path, kind="ast") is not None diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index fc98dc6e0..2122b2401 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -429,9 +429,11 @@ def test_stamped_manifest_files_normalizes_both_sides(tmp_path): "document": [str(fresh_doc), str(cached_doc), str(omitted_doc)], } sem_result = { - # fresh extraction: root-relative source_file - "nodes": [{"id": "n1", "source_file": "fresh.md"}], - # cache replay: absolute source_file (edge-only coverage counts too) + # fresh extraction: root-relative source_file; cache replay: absolute + "nodes": [ + {"id": "n1", "source_file": "fresh.md"}, + {"id": "n2", "source_file": str(cached_doc)}, + ], "edges": [{"source": "a", "target": "b", "source_file": str(cached_doc)}], } @@ -440,6 +442,32 @@ def test_stamped_manifest_files_normalizes_both_sides(tmp_path): assert out["document"] == [str(fresh_doc), str(cached_doc)] +def test_stamped_manifest_files_drops_edges_only_docs(tmp_path): + """#2927: a doc attributed only edges (no nodes, no hyperedges) was + model-omitted from its response. Stamping it up-to-date froze it out of + every warm incremental run at 0 nodes; it must stay unstamped so + detect_incremental re-queues it, matching the cache-write skip.""" + from graphify.cli import _stamped_manifest_files + + edges_only_doc = tmp_path / "hub.md"; edges_only_doc.write_text("# hub") + node_doc = tmp_path / "ok.md"; node_doc.write_text("# ok") + + files_by_type = {"document": [str(edges_only_doc), str(node_doc)]} + sem_result = { + "nodes": [{"id": "n1", "source_file": str(node_doc)}], + "edges": [ + {"source": "a", "target": "b", "source_file": str(edges_only_doc)}, + ], + "hyperedges": [], + } + + out = _stamped_manifest_files(files_by_type, sem_result, tmp_path) + assert str(edges_only_doc) not in out["document"], ( + "an edges-only doc must not be stamped (#2927)" + ) + assert str(node_doc) in out["document"] + + def test_stamped_manifest_files_counts_hyperedge_only_docs(tmp_path): """#1920: a doc whose only chunk output is a hyperedge (3+ nodes sharing a concept) is valid output — the semantic cache persists it per source_file —