build: a stopped query or index no longer loses the main graph (0.1.5)

plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py

@@ -452,8 +452,7 @@ def ensure_graph(repo, db):
     if nowait:
         os.makedirs(os.path.join(repo, '.axiomcode'), exist_ok=True)
         log = open(os.path.join(repo, '.axiomcode', 'first-build.log'), 'w')
-        kw = dict(start_new_session=True) if os.name != 'nt' else dict(creationflags=0x00000008 | 0x00000200)   # DETACHED | NEW_GROUP, as ax_fresh.kick
-        p = subprocess.Popen([bash, build, repo], stdin=subprocess.DEVNULL, stdout=log, stderr=log, close_fds=True, env=env, **kw)
+        p = _start_build([bash, build, repo], log, env)
         # until the build holds its lock a second query would see no build and start another, so that much is always waited
         up = time.time() + 10
         while p.poll() is None and not ax_fresh.building(repo) and time.time() < up: time.sleep(0.05)
@@ -469,7 +468,7 @@ def ensure_graph(repo, db):
     os.makedirs(os.path.join(repo, '.axiomcode'), exist_ok=True)
     logp = os.path.join(repo, '.axiomcode', 'first-build.log')
     with open(logp, 'w') as log:
-        p = subprocess.Popen([bash, build, repo], stdin=subprocess.DEVNULL, stdout=log, stderr=log, close_fds=True, env=env)
+        p = _start_build([bash, build, repo], log, env)
     shown = [0]
     def relay():
         try:
@@ -482,6 +481,25 @@ def relay():
     return p.returncode == 0 and os.path.exists(db)
 
 
+def _start_build(argv, log, env):
+    """the build a query starts, in a session (a process group) of its own, as ax_fresh.kick starts the refresher.
+
+    THE BUILD IS NOT THE QUERY'S (#1555). It goes on solving the other languages after the query has its answer, and a
+    query is stopped all the time: a caller's timeout, Ctrl-C, an agent host that ends the command's process group when
+    it returns or times out. Started in the query's group, the build was stopped with it -- on a repository in several
+    languages, whose first build outlasts a two-minute timeout, before the main graph was published, or while it was
+    being indexed, which deleted the solved graph as a failure. No graph.sqlite was left, the next query built again
+    from nothing, was stopped again, and every query rebuilt forever. In its own session the build finishes whatever
+    becomes of the query; a later query finds it running and waits for it rather than starting another."""
+    if os.name != 'nt':
+        return subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=log, stderr=log, close_fds=True, env=env, start_new_session=True)
+    # DETACHED | NEW_GROUP, and out of the caller's job object where the job allows it (see ax_fresh.kick)
+    for flags in (0x00000008 | 0x00000200 | 0x01000000, 0x00000008 | 0x00000200):
+        try: return subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=log, stderr=log, close_fds=True, env=env, creationflags=flags)
+        except OSError: continue
+    return subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=log, stderr=log, close_fds=True, env=env)
+
+
 def _published(repo):
     """the main graph is out while the build goes on solving the repository's other languages: the answer is given now,
     and names on stderr (a `graph refresh:` line, which the MCP server carries into the answer) what it cannot see yet"""

plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py

@@ -378,6 +378,15 @@ def pending(repo):
     old = [l for l in langs if os.path.isfile(os.path.join(repo, '.axiomcode', 'lang', l, 'out', 'graph.sqlite'))]
     return dict(pending=langs, pending_old=old, first=all(len(r) > 1 and r[1] == 'first' for r in rows))
 
+def unbuilt(repo):
+    """the languages a build that was stopped after publishing the main graph left unbuilt (axiomcode-build writes them
+    to out/partial), [] when none"""
+    try: return open(os.path.join(out_dir(repo), 'partial')).read().split()
+    except OSError: return []
+
+def unbuilt_row(repo):
+    return f".axiomcode/out/partial (the {', '.join(unbuilt(repo))} graph was not built: its build was stopped)"
+
 def status(repo):
     if not has_graph(repo): return dict(state='no graph')
     if graph_broken(repo): return dict(state='building' if building(repo) else 'stale', changed=['.axiomcode/out/graph.sqlite (the pointer to the graph is broken)'], added=[], removed=[])
@@ -389,7 +398,8 @@ def status(repo):
     if not (changed or added or removed):
         # a build that already published this tree's main graph and is still solving other languages: the graph a query
         # reads is current, and waiting would be waiting for other languages' compiles
-        return dict(state='building', **pending(repo)) if busy else dict(state='fresh')
+        if busy: return dict(state='building', **pending(repo))
+        return dict(state='stale', changed=[unbuilt_row(repo)], added=[], removed=[]) if unbuilt(repo) else dict(state='fresh')
     d = dict(state='building' if busy else 'stale', changed=changed, added=added, removed=removed)
     if not busy and st.get('failed_table') == change_key(c): d['failed'] = st.get('failed_log', ''); d['failed_reason'] = st.get('failed_reason', '')
     return d
@@ -451,6 +461,7 @@ def worker(repo):
             else:
                 c = changes(repo, t)
                 if graph_broken(repo) and not (c and any(c)): c = [['.axiomcode/out/graph.sqlite (broken pointer)'], [], []]
+                elif unbuilt(repo) and not (c and any(c)): c = [[unbuilt_row(repo)], [], []]
                 if (c is None or not any(c)) and not base_moved(repo):
                     write_state(repo, state='fresh', checked=time.time(), checked_by=os.environ.get('AXIOMCODE_REFRESH_TRIGGER', '')); return 0
                 c = c or [[], [], []]
@@ -718,13 +729,26 @@ def expected_left(repo):
     if st.get('state') == 'building' and st.get('started'): return secs - (time.time() - st['started'])
     return secs + float(os.environ.get('AXIOMCODE_REFRESH_DEBOUNCE') or 2)
 
-def wait_fresh(repo, seconds, say=False):
+def _waits_on(s, files):
+    """the refresh has published the main graph and is still solving languages (#1555) of which one holds an edited file
+    the wait is for (`files`; True: any language at all): the graph that answers for that edit is not out yet"""
+    langs = set(s.get('pending') or ())
+    if not langs or not files: return False
+    if files is True: return True
+    import ax_langs
+    return any(l in langs for f in files for l in ax_langs.BY_EXT.get(os.path.splitext(f)[1].lower(), ()))
+
+def wait_fresh(repo, seconds, say=False, files=()):
     """wait until the graph matches the files again (the refresh swapped in its graph), a rebuild fails, or `seconds`
-    pass. With `say`, a progress line on stderr every 10 s: a wait is never silent. True when the graph is current"""
+    pass. With `say`, a progress line on stderr every 10 s: a wait is never silent. True when the graph is current.
+    THE GRAPH OF THE EDIT, NOT THE MAIN ONE. A refresh publishes the main language's graph first and goes on with the
+    others (#1555); from then on the file table matches, and a wait that ended there answered a question about a Python
+    edit in a TypeScript repository from the previous Python graph: `nothing named` for the function just added. So a
+    wait for `files` goes on while a language that holds one of them is still being solved (files=True: any language)."""
     end = t0 = time.time(); end += seconds; last = t0
     while True:
         s = status(repo)
-        if s['state'] in ('fresh', 'no graph', 'unknown') or not edited(s): return True
+        if (s['state'] in ('fresh', 'no graph', 'unknown') or not edited(s)) and not _waits_on(s, files): return True
         if s.get('failed'): return False
         if s['state'] == 'stale': kick(repo, 'a query')
         now = time.time()
@@ -768,7 +792,7 @@ def with_note(n):                                   # the answer as it is, then
         left = expected_left(repo)
         print(f"waiting for the graph to refresh (--fresh): {len(edited(s))} file(s) changed since the graph was built" +
               (f", the last build took {int(build_seconds(repo)[0])} s" if left is not None else '') + " …", file=sys.stderr, flush=True)
-        if wait_fresh(repo, float(os.environ.get('AXIOMCODE_FRESH_MAX') or 600), say=True): passthrough()
+        if wait_fresh(repo, float(os.environ.get('AXIOMCODE_FRESH_MAX') or 600), say=True, files=True): passthrough()
         s = status(repo)
     stale = Stale(edited(s)); r = run()
     out = r.stdout.decode('utf-8', 'replace')
@@ -782,7 +806,7 @@ def with_note(n):                                   # the answer as it is, then
             print(f"waiting for the graph to refresh: this answer touches {', '.join(stale.files[:3])}" + (' …' if len(stale.files) > 3 else '') +
                   f", edited since the graph was built; up to {int(budget)} s" + (f" (the last build took {int(build_seconds(repo)[0])} s)" if left is not None else '') + " …",
                   file=sys.stderr, flush=True)
-            if wait_fresh(repo, budget, say=True): passthrough()
+            if wait_fresh(repo, budget, say=True, files=stale.files): passthrough()
             s = status(repo); stale = Stale(edited(s))
             if edited(s):
                 r = run(); out = r.stdout.decode('utf-8', 'replace')

plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build

@@ -242,7 +242,7 @@ LANGDIR="$REPO/.axiomcode/lang"      # one graph directory per language other th
 # file the parser reads with its hash; when none differs the graph is current whatever git says: a dirty tree whose
 # edits are already indexed is not rebuilt, and a tree edited and restored, which git calls clean against a graph built
 # from the edit, is. A commit that changed no file moves the stamp, not the graph.
-if [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then
+if [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then
   if [ "$(cat "$OUT/stamp")" != "$STAMP" ]; then echo "$STAMP" > "$OUT/stamp"; INDEXED_TREE="$OLD_INDEXED"; set_base "$OLD_INDEXED"; PREV=""; keep_base_graph; commit_base; fi
   has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO"
   [ -f "$REPO/.axiomcode/engine" ] || printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine" || true
@@ -353,18 +353,34 @@ unpend(){ grep -v "^$1 " "$PENDING" > "$PENDING.new" 2>/dev/null || true; mv -f
 EPID=""
 stop_engine(){ if [ -n "$EPID" ] && kill -0 "$EPID" 2>/dev/null; then kill -TERM -- "-$EPID" 2>/dev/null || kill -TERM "$EPID" 2>/dev/null || true; fi; rm -f "$PROGRESS" "$PENDING" "$PENDING.new"; }
 trap stop_engine EXIT
-trap 'exit 130' INT; trap 'exit 143' TERM HUP
+# A BUILD STOPPED ONCE ITS MAIN LANGUAGE IS SOLVED PUBLISHES THAT GRAPH FIRST. The solved main graph is most of what the
+# build cost; stopped while it was being indexed (a caller's timeout ends the whole process group, the indexer included),
+# the build took the indexer's death for a failure, deleted the solved graph and left no graph.sqlite, so the next query
+# built everything again, was stopped again, and a repository in several languages rebuilt forever. A signal that
+# arrives after the engine names the main language solved, or while it is published, is held until graph.sqlite points
+# at it (and the indexer runs in a process group of its own, which a signal to the build's does not reach); the build
+# then stops, the other languages unpublished. One that arrives earlier stops the build at once, as it always did.
+SIG=""; SIGS=0; PUBLISHING=""; MAIN_DONE=""
+on_signal(){ SIGS=$((SIGS + 1)); SIG="$1"
+  if [ -z "$MAIN_DONE" ] && { [ -n "$PUBLISHING" ] || grep -q "^$LANG_ARG ok" "$PROGRESS" 2>/dev/null; }; then return 0; fi
+  exit "$1"; }
+trap 'on_signal 130' INT; trap 'on_signal 143' TERM HUP
+# run a command in a process group of its own and wait for it; a held signal interrupts `wait`, not the command
+shielded(){ local p rc n; set -m; "$@" & p=$!; set +m
+  while :; do n=$SIGS; rc=0; wait "$p" || rc=$?; [ "$SIGS" != "$n" ] && [ "$rc" -gt 128 ] && kill -0 "$p" 2>/dev/null && continue; break; done
+  return "$rc"; }
 # an engine older than this script has no --progress and refuses an option it does not know: it gets none, and its
 # graphs are published together when it ends, as they always were
 PROG=(); grep -q -- '--progress' "$ENGINE/bin/axiomcode" 2>/dev/null && PROG=(--progress "$PROGRESS")
 set -m
 "$ENGINE/bin/axiomcode" all "$SRC" "$OUT" --language "$SOLVE" ${PROG[@]+"${PROG[@]}"} ${LIB[@]+"${LIB[@]}"} < /dev/null > "$REPO/.axiomcode/build.log" 2>&1 &
 EPID=$!
 set +m
-MAIN_DONE=""; QUEUED=(); READY=(); FAILED=(); DONE=" "; SEEN=0
+QUEUED=(); READY=(); FAILED=(); DONE=" "; SEEN=0
 # THE MAIN LANGUAGE'S GRAPH BECOMES THE GRAPH. Everything a finished build did for it happens here, as soon as it is
 # solved: index, stamp, file table, baseline, pointer.
 publish_main(){
+  PUBLISHING=1
   # the new database: a FILE under $OUT/$LANG_ARG. Matched as `*/$LANG_ARG/*` it was any path with the language's name
   # in it, the repository's own included: a repository at .../python/app matched the graph.sqlite SYMLINK, the link was
   # pointed at itself, the previous graph deleted, and every verb after it reported no graph.
@@ -374,7 +390,7 @@ publish_main(){
   # A QUERY NEVER READS A HALF-BUILT GRAPH. The new database gets its symbols, refs and views BEFORE graph.sqlite points
   # at it; until the pointer moves, every verb (and the hooks) keeps answering from the previous graph. Only then is the
   # old one removed: a reader that opened it keeps the file it opened.
-  AXIOMCODE_INDEX_DB="$DB" python3 "$H/axiomcode-index" "$REPO" || { restore; echo "indexing the new graph failed — the graph that was there is untouched"; exit 1; }
+  shielded env AXIOMCODE_INDEX_DB="$DB" python3 "$H/axiomcode-index" "$REPO" || { restore; echo "indexing the new graph failed — the graph that was there is untouched"; exit 1; }
   stamp_meta "$DB"
   echo "$STAMP" > "$OUT/stamp"
   # the engine that built this graph: the next build, the background refresh above all, starts its search there (see the top)
@@ -396,7 +412,13 @@ publish_main(){
   done
   commit_base
   rm -rf "$PREV"
-  MAIN_DONE=1
+  MAIN_DONE=1; PUBLISHING=""
+  # the languages still unsolved are named in $OUT/partial: the graph is not up to date while it is there, so the next
+  # build (a refresh the next query starts, or an index) builds them rather than calling the file table current
+  if [ -n "$SIG" ]; then
+    [ -s "$PENDING" ] && cut -d' ' -f1 "$PENDING" > "$OUT/partial"
+    echo "$LANG_ARG graph published at $OUT/graph.sqlite; stopped before the other languages were built"; exit "$SIG"
+  fi
   python3 - "$OUT/graph.sqlite" <<'PY'
 import sqlite3, sys
 rows = sqlite3.connect(sys.argv[1]).execute("SELECT tier, count(*) FROM call_edges GROUP BY tier ORDER BY 2 DESC").fetchall()
@@ -445,7 +467,7 @@ while :; do
   running=1; kill -0 "$EPID" 2>/dev/null || running=0
   drain
   [ "$running" = 1 ] || break
-  sleep 1
+  sleep 1 || true          # a held signal ends the sleep, not the build
 done
 ENGINE_RC=0; wait "$EPID" || ENGINE_RC=$?; EPID=""
 drain
@@ -459,7 +481,7 @@ fi
 for l in ${OTHER_LANGS//,/ }; do
   case "$DONE" in *" $l "*) ;; *) publish_other "$l" ok;; esac
 done
-rm -f "$PROGRESS" "$PENDING"
+rm -f "$PROGRESS" "$PENDING" "$OUT/partial"
 if [ ${#FAILED[@]} -gt 0 ]; then
   for l in "${FAILED[@]}"; do
     if [ -f "$LANGDIR/$l/out/graph.sqlite" ]; then echo "⚠️  the $l graph did not build: $l questions are answered from its PREVIOUS graph, which predates this tree; see $REPO/.axiomcode/build.log"

tests/README.md

@@ -61,7 +61,8 @@ One check needs no graph and is its own script:
                                          unelevated Windows user (WinError 1314); indexes a case, so it needs the engine
     python3 tests/multi_language.py      a repository in several languages is indexed in all of them and every query asks
                                          each graph: nothing dropped, a one-graph answer unchanged, an edit reported once,
-                                         refreshed, upgraded from a one-language graph (builds real graphs, needs the engine)
+                                         refreshed, upgraded from a one-language graph; a first query or an index stopped
+                                         mid-build still leaves the main graph published (builds real graphs, needs the engine)
     python3 tests/latency.py             what a query repeats on every call is done once, with the same answer: the non-source
                                          scan cached per graph, a repeated impact solve read back, one tree walk for every language,
                                          the verb and the Python found without starting programs, a first build that reports its