Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions graphify/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -992,6 +992,22 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast",
# across chunks without losing the truncated one (it stays partial).
if not allow_partial and isinstance(result, dict) and result.get("partial"):
return None
# An entry carrying edges but ZERO nodes is a model-omission artifact
# (#2927) that older versions cached keyed by content hash, replaying
# the empty node set forever — the file sat at 0 nodes while every
# edge pointing into it dangled. Treat it as a MISS so the file is
# re-dispatched and a complete extraction overwrites the poisoned key
# (self-healing, same mechanism as the ``partial`` check above).
# Hyperedge-only entries remain valid output (#1920). No
# ``allow_partial`` escape hatch here: a zero-node entry carries
# nothing worth merging onto a slice union.
if (
kind.startswith("semantic")
and isinstance(result, dict)
and not result.get("nodes")
and result.get("edges")
):
return None
if (
kind.startswith("semantic")
and isinstance(result, dict)
Expand Down Expand Up @@ -1329,6 +1345,13 @@ def save_semantic_cache(
so the flag survives even when a caller (e.g. cli.py's final save) does not
pass ``partial_source_files``.
A per-file result carrying edges but ZERO nodes is a model-omission
artifact (#2927): caching it keyed by content hash makes every later run a
hit, permanently freezing the file out of the graph while the console
claims "a re-run will retry them". Such groups are NOT written — the file
stays uncached and genuinely re-dispatches next run. Hyperedge-only
results remain valid output and cache normally (#1920).
``prompt`` is the extraction prompt that produced these results — text, or
a Path to the prompt file. It stamps entries into the p{fingerprint}/
namespace so a later run under a different prompt re-extracts rather than
Expand Down Expand Up @@ -1492,6 +1515,7 @@ def hyperedge_dangles(h: dict) -> bool:

saved = 0
skipped_not_file = 0
edges_only_skips = 0
for fpath, result in by_file.items():
cache_path = source_path(fpath)
p = resolved_source_path(fpath)
Expand Down Expand Up @@ -1528,6 +1552,17 @@ def hyperedge_dangles(h: dict) -> bool:
}
else:
_prev_partial = False
# Edges-but-no-nodes is a model-omission artifact (#2927): writing
# it keyed by content hash would make every later run a cache hit,
# permanently freezing the file out of the graph while the console
# claims "a re-run will retry them". Leave it uncached so the next
# run genuinely re-dispatches the file. Evaluated AFTER the merge
# so a multi-chunk file whose earlier slice produced nodes still
# accumulates (the union carries nodes), and hyperedge-only
# results stay cacheable (#1920).
if not result["nodes"] and result["edges"]:
edges_only_skips += 1
continue
# A file is partial if the caller named it, any of its grouped items
# carries the intrinsic ``_partial`` marker, OR the entry it merged
# onto was already partial (an empty-parse truncation leaves a
Expand Down Expand Up @@ -1558,4 +1593,12 @@ def hyperedge_dangles(h: dict) -> bool:
RuntimeWarning,
stacklevel=2,
)
if edges_only_skips:
warnings.warn(
f"save_semantic_cache: {edges_only_skips} file(s) produced edges but no "
"nodes — the model omitted them from its response (#2927). Nothing was "
"cached for them, so the next run will re-dispatch and retry them.",
RuntimeWarning,
stacklevel=2,
)
return saved
14 changes: 13 additions & 1 deletion graphify/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,15 @@ def _stamped_manifest_files(
have no source_file entry in sem_result — leaving their semantic_hash
empty so detect_incremental re-queues them (#933).

A file attributed only EDGES (no nodes, no hyperedges) did not really
produce output — the model omitted the file from its response (#2927).
Counting edge coverage as output stamped such a file up-to-date, so
detect_incremental never re-dispatched it and it stayed frozen at 0 nodes
even after the cache stopped serving the empty entry. Edges therefore do
not contribute to stamping; the file stays unstamped and is re-queued
next run (same #933 mechanism). Hyperedges still count (#1920); the
stamping condition mirrors save_semantic_cache's cache-write rule.

A file in ``partial_source_files`` DID produce output this run, but only a
truncated fragment of it, so it is excluded from stamping too — otherwise
detect_incremental would see it "done" and never re-dispatch it, leaving the
Expand Down Expand Up @@ -133,7 +142,10 @@ def _resolve(value: str) -> Path:
return p

sem_extracted: set[Path] = set()
for coll in ("nodes", "edges", "hyperedges"):
# Edges deliberately excluded (#2927) — see the docstring: edge-only
# attribution is the model-omission shape, and stamping it froze the file
# out of warm incremental runs. Nodes and hyperedges (#1920) count.
for coll in ("nodes", "hyperedges"):
for item in sem_result.get(coll, []):
sf = item.get("source_file", "")
if sf:
Expand Down
120 changes: 120 additions & 0 deletions tests/test_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -1749,3 +1749,123 @@ def test_corrupt_semantic_entry_warns_and_is_a_miss(tmp_path):
# The corrupt entry is a miss, so the file is re-dispatched for extraction.
assert nodes == []
assert uncached == [str(f)]


def test_save_semantic_cache_skips_edges_only_results(tmp_path):
"""#2927: a dispatched file whose semantic result carries edges but ZERO
nodes is a model omission. Caching it keyed by content hash made every
later run a hit, permanently freezing the file out of the graph while the
console claimed "a re-run will retry them". The write must be skipped so
the file stays uncached and genuinely re-dispatches on the next run."""
from graphify.cache import check_semantic_cache, save_semantic_cache

f = tmp_path / "hub.md"
f.write_text("# Hub\n\nMost-referenced doc.\n")

with pytest.warns(RuntimeWarning, match="edges but no nodes"):
saved = save_semantic_cache(
[],
[{"source": "a", "target": "b", "source_file": "hub.md"}],
root=tmp_path,
)

assert saved == 0
# No entry was written for the content hash...
entry = cache_dir(tmp_path, "semantic") / f"{file_hash(f, tmp_path)}.json"
assert not entry.exists()
# ...so the next run reports the file as uncached (re-dispatch).
nodes, edges, _, uncached = check_semantic_cache([str(f)], root=tmp_path)
assert nodes == [] and edges == []
assert uncached == [str(f)]


def test_save_semantic_cache_edges_only_merges_onto_node_entry(tmp_path):
"""#2927 skip must not break multi-chunk accumulation: an edges-only slice
merging onto a prior entry that HAS nodes still saves the union. Only a
zero-node FINAL result is skipped."""
from graphify.cache import load_cached, save_semantic_cache

f = tmp_path / "big.md"
f.write_text("# Big\n\n" + "para\n" * 200)
save_semantic_cache([{"id": "n1", "source_file": "big.md"}], [], root=tmp_path)

saved = save_semantic_cache(
[],
[{"source": "n1", "target": "x", "source_file": "big.md"}],
root=tmp_path,
merge_existing=True,
)
assert saved == 1
loaded = load_cached(f, tmp_path, kind="semantic")
assert [n["id"] for n in loaded["nodes"]] == ["n1"]
assert [e["source"] for e in loaded["edges"]] == ["n1"]


def test_save_semantic_cache_hyperedge_only_still_caches(tmp_path):
"""#1920 must survive the #2927 fix: a hyperedge-only result is valid
output and caches normally."""
from graphify.cache import check_semantic_cache, save_semantic_cache

f = tmp_path / "hyper.md"
f.write_text("# Hyper\n")
hyper = {
"id": "h1", "label": "L", "nodes": ["a", "b", "c"],
"relation": "participate_in", "source_file": "hyper.md",
}
saved = save_semantic_cache([], [], [hyper], root=tmp_path)
assert saved == 1

nodes, _, hyperedges, uncached = check_semantic_cache([str(f)], root=tmp_path)
assert uncached == []
# Replay absolutizes source_file (#777); identity is the hyperedge id.
assert [h["id"] for h in hyperedges] == ["h1"]


def test_edges_only_semantic_entry_is_a_miss_and_self_heals(tmp_path):
"""#2927 healing: entries already poisoned by older versions (edges, no
nodes) must read as a MISS so the file re-dispatches and a complete
extraction overwrites the poisoned key. Hyperedge-only entries (#1920)
keep serving, and AST entries are untouched by the rule."""
from graphify.cache import check_semantic_cache, save_semantic_cache

f = tmp_path / "hub.md"
f.write_text("# Hub\n")
poison = {
"nodes": [],
"edges": [{"source": "a", "target": "b", "source_file": "hub.md"}],
}
save_cached(f, poison, tmp_path, kind="semantic")
assert load_cached(f, tmp_path, kind="semantic") is None

_, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path)
assert uncached == [str(f)]

# Self-heal: a complete extraction overwrites the same content-hash key.
save_semantic_cache(
[{"id": "hub", "source_file": "hub.md"}],
[{"source": "a", "target": "hub", "source_file": "hub.md"}],
root=tmp_path,
)
loaded = load_cached(f, tmp_path, kind="semantic")
assert [n["id"] for n in loaded["nodes"]] == ["hub"]
_, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path)
assert uncached == []

# Hyperedge-only entries remain valid output (#1920).
h = tmp_path / "hyper.md"
h.write_text("# Hyper\n")
save_cached(
h,
{"nodes": [], "edges": [], "hyperedges": [
{"id": "h1", "nodes": ["a", "b", "c"], "source_file": "hyper.md"},
]},
tmp_path, kind="semantic",
)
served = load_cached(h, tmp_path, kind="semantic")
assert served is not None and served["hyperedges"]

# AST kind is outside the rule's scope.
a = tmp_path / "mod.py"
a.write_text("x = 1\n")
save_cached(a, {"nodes": [], "edges": []}, tmp_path, kind="ast")
assert load_cached(a, tmp_path, kind="ast") is not None
34 changes: 31 additions & 3 deletions tests/test_extract_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -429,9 +429,11 @@ def test_stamped_manifest_files_normalizes_both_sides(tmp_path):
"document": [str(fresh_doc), str(cached_doc), str(omitted_doc)],
}
sem_result = {
# fresh extraction: root-relative source_file
"nodes": [{"id": "n1", "source_file": "fresh.md"}],
# cache replay: absolute source_file (edge-only coverage counts too)
# fresh extraction: root-relative source_file; cache replay: absolute
"nodes": [
{"id": "n1", "source_file": "fresh.md"},
{"id": "n2", "source_file": str(cached_doc)},
],
"edges": [{"source": "a", "target": "b", "source_file": str(cached_doc)}],
}

Expand All @@ -440,6 +442,32 @@ def test_stamped_manifest_files_normalizes_both_sides(tmp_path):
assert out["document"] == [str(fresh_doc), str(cached_doc)]


def test_stamped_manifest_files_drops_edges_only_docs(tmp_path):
"""#2927: a doc attributed only edges (no nodes, no hyperedges) was
model-omitted from its response. Stamping it up-to-date froze it out of
every warm incremental run at 0 nodes; it must stay unstamped so
detect_incremental re-queues it, matching the cache-write skip."""
from graphify.cli import _stamped_manifest_files

edges_only_doc = tmp_path / "hub.md"; edges_only_doc.write_text("# hub")
node_doc = tmp_path / "ok.md"; node_doc.write_text("# ok")

files_by_type = {"document": [str(edges_only_doc), str(node_doc)]}
sem_result = {
"nodes": [{"id": "n1", "source_file": str(node_doc)}],
"edges": [
{"source": "a", "target": "b", "source_file": str(edges_only_doc)},
],
"hyperedges": [],
}

out = _stamped_manifest_files(files_by_type, sem_result, tmp_path)
assert str(edges_only_doc) not in out["document"], (
"an edges-only doc must not be stamped (#2927)"
)
assert str(node_doc) in out["document"]


def test_stamped_manifest_files_counts_hyperedge_only_docs(tmp_path):
"""#1920: a doc whose only chunk output is a hyperedge (3+ nodes sharing a
concept) is valid output — the semantic cache persists it per source_file —
Expand Down
Loading