diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index 3632fd4126..0d07b30b41 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index cf116869f2..7b127d3052 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -687,6 +687,60 @@ def test_generated_runbooks_pass_root_to_save_manifest(): assert checked >= 4, f"expected save_manifest calls across the runbooks, found {checked}" +def test_update_runbook_stamps_manifest_before_build_merge_mutation(tmp_path): + """#2865: the update runbook must stamp the manifest from raw extraction + BEFORE build_merge() mutates it in place. + + build/build_from_json normalizes legacy extraction aliases (path -> source_file, + source -> source_file). _stamped_manifest_files() reads source_file directly from + the extraction to determine which semantic files produced output. If run after + build_merge(), _stamped_manifest_files() observes the mutated in-place dictionary + rather than the raw extraction. + """ + from graphify.build import build_merge + from graphify.cli import _stamped_manifest_files + + doc = tmp_path / "legacy_doc.md" + doc.write_text("doc content", encoding="utf-8") + files_by_type = {"document": [str(doc)], "code": []} + + # Raw extraction using legacy 'path' alias without 'source_file' + raw_extraction = { + "nodes": [{"id": "doc_node", "name": "Doc Node", "path": str(doc), "file_type": "document"}], + "edges": [], + "hyperedges": [], + } + + # 1. Before build_merge, _stamped_manifest_files() sees raw extraction + # where 'source_file' is absent, so the file is not yet recognized in sem_extracted. + stamped_before = _stamped_manifest_files(files_by_type, raw_extraction, tmp_path) + assert stamped_before["document"] == [], "Raw extraction with only 'path' alias should have no 'source_file'" + + # 2. build_merge mutates raw_extraction in place, converting 'path' to 'source_file'. + graph_path = tmp_path / "graph.json" + build_merge([raw_extraction], graph_path=graph_path, root=tmp_path) + assert "source_file" in raw_extraction["nodes"][0], "build_merge must fold 'path' alias to 'source_file' in place" + + # 3. Running after build_merge observes the mutated dictionary. + stamped_after = _stamped_manifest_files(files_by_type, raw_extraction, tmp_path) + assert stamped_after["document"] == [str(doc)] + assert stamped_before != stamped_after, "Mutation changed the manifest stamping outcome" + + # 4. Verify that all rendered update runbooks compute _stamped_manifest_files before build_merge + targets = sorted((REPO_ROOT / "graphify" / "skills").glob("*/references/update.md")) + assert len(targets) == 14, f"Expected 14 split platform update runbooks, found {len(targets)}" + for path in targets: + lines = path.read_text(encoding="utf-8").splitlines() + stamped_lines = [i for i, ln in enumerate(lines) if "_manifest_files = _stamped_manifest_files" in ln] + build_merge_lines = [i for i, ln in enumerate(lines) if "G = build_merge(" in ln] + assert stamped_lines, f"{path}: missing _stamped_manifest_files call" + assert build_merge_lines, f"{path}: missing build_merge call" + assert stamped_lines[0] < build_merge_lines[0], ( + f"{path}: _stamped_manifest_files (line {stamped_lines[0]}) must appear " + f"BEFORE build_merge (line {build_merge_lines[0]}) (#2865)" + ) + + def test_devin_keeps_its_multi_field_frontmatter(): """devin renders inline, so its 4+-field frontmatter is preserved verbatim.""" platforms = gen.load_platforms() diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index 3632fd4126..0d07b30b41 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -87,12 +87,30 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from graphify.build import build_merge +from graphify.cli import _stamped_manifest_files from graphify.detect import save_manifest # Load new extraction and incremental state new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) + +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (computed from new_extraction BEFORE build_merge() mutates it in place): +# a changed doc whose chunk failed must stay unstamped so the next --update re-queues +# it, otherwise it is marked done and its content is lost forever (#2015, #2865). +# Mirrors the library extract path (cli._stamped_manifest_files + clear_semantic + scan_corpus). +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} + # prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are # handled by build_merge's replace-on-re-extract (#1344): every source_file in # new_chunks is dropped from the base before merge, so old/stale nodes don't survive. @@ -142,24 +160,6 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') "