Query latency: do once what every query repeated; first build reports its stage instead of blocking

bin/axiomcode

@@ -57,7 +57,13 @@ if [ -L "$self" ]; then
   fi
   self="$r"
 fi
-ROOT="$(d="$(cd "$(dirname "$self")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")"
+# NO PROGRAM IS STARTED TO FIND IT: the parent of a path is taken by parameter expansion, not `dirname`. Every query goes
+# through here, and under Git Bash on Windows each program started is a fork emulation that costs tens of milliseconds
+# idle and far more on a busy machine; the walk used to start one per level, the verb check below four more.
+case "$self" in */*) d="${self%/*}";; *) d=.;; esac
+d="$(cd "${d:-/}" && pwd)"
+while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="${d%/*}"; d="${d:-/}"; done
+ROOT="$d"
 # No marker found means every path built from ROOT is nonsense; say so here rather than fail
 # later on a path the user cannot interpret.
 [ -d "$ROOT/graph" ] || { echo "axiomcode: cannot locate the package root from $0 (resolved to $self)" >&2; exit 2; }
@@ -76,8 +82,20 @@ QUERY="$ROOT/plugins/axiomcode/skills/axiomcode/scripts/axiomcode"
 # `tests` is the frontend's alias for test-impact; `test` is THIS command's engine suite. One letter
 # apart, entirely different jobs, and putting both on one command is a trap — so the alias is not
 # routed here, and the name is refused below with both readings spelled out.
-query_verbs(){ [ -f "$QUERY" ] && bash "$QUERY" --verbs 2>/dev/null | grep -vx tests || true; }
-is_query_verb(){ query_verbs | grep -qx -- "$1"; }
+# The list is read from the frontend's own dispatch table, the lines `  <verb>[|<alias>]) exec …` that its `--verbs`
+# prints, so the two entry points still cannot drift -- but read here, in this shell, instead of by a second bash, a
+# sed, a tr and two greps on every query.
+QVERBS=()
+load_verbs(){
+  [ ${#QVERBS[@]} -gt 0 ] && return 0; [ -f "$QUERY" ] || return 0
+  local l v x re='^  ([a-z|-]*)\) *exec '
+  while IFS= read -r l || [ -n "$l" ]; do
+    [[ $l =~ $re ]] || continue; v="${BASH_REMATCH[1]}"
+    for x in ${v//|/ }; do [ "$x" = tests ] || QVERBS+=("$x"); done
+  done < "$QUERY"
+}
+query_verbs(){ load_verbs; [ ${#QVERBS[@]} -gt 0 ] && printf '%s\n' "${QVERBS[@]}"; return 0; }
+is_query_verb(){ load_verbs; local x; for x in ${QVERBS[@]+"${QVERBS[@]}"}; do [ "$x" = "$1" ] && return 0; done; return 1; }
 usage(){
   awk 'NR > 2 && /^# ─/ {exit} NR > 2' "$0" | sed 's/^# \{0,1\}//'
   if [ -f "$QUERY" ]; then

plugins/axiomcode/mcp/find-python.js

@@ -14,6 +14,9 @@
 'use strict';
 const { spawnSync } = require('child_process');
 const { which } = require('./which.js');
+const crypto = require('crypto');
+const fs = require('fs');
+const os = require('os');
 const path = require('path');
 
 const SHIM = path.join(__dirname, '..', 'skills', 'axiomcode', 'scripts', 'pyshim');
@@ -24,14 +27,67 @@ function candidates() {
 }
 
 // { cmd, exe } or { error }: cmd is how the candidate was named, exe the interpreter file it runs.
+// THE PROBE IS REMEMBERED. It starts an interpreter, and it ran on every `axiomcode` command and on every hook, which fire
+// on every tool call an agent makes: one Python start each time, before the work, and on a busy Windows machine that is
+// the slowest process there is to start. Its answer depends only on what each candidate name resolves to on PATH, so it is
+// kept in the temp directory under a key of PATH and AXIOMCODE_PYTHON, with the file every candidate it tried resolved to
+// (and that file's mtime). A later call re-resolves those names -- a stat per directory, no process -- and probes again
+// only when one of them resolves elsewhere, or the file changed: a Python installed, removed or upgraded.
+// AXIOMCODE_NO_PYTHON_CACHE=1 probes every time.
+function onPath(name) {
+  if (process.platform === 'win32' || path.isAbsolute(name) || /[\\/]/.test(name)) {
+    const p = process.platform === 'win32' ? which(name) : name;
+    if (!p) return null;
+    try { return [p, fs.statSync(p).mtimeMs]; } catch { return null; }
+  }
+  for (const d of (process.env.PATH || '').split(path.delimiter)) {
+    if (!d || !path.isAbsolute(d)) continue;
+    const p = path.join(d, name);
+    try { const st = fs.statSync(p); if (st.isFile()) return [p, st.mtimeMs]; } catch { /* not here */ }
+  }
+  return null;
+}
+
+function cacheFile() {
+  const key = crypto.createHash('sha1').update([process.platform, process.arch, process.env.PATH || process.env.Path || '',
+    process.env.AXIOMCODE_PYTHON || ''].join('\0')).digest('hex').slice(0, 16);
+  return path.join(os.tmpdir(), `axiomcode-python-${key}.json`);
+}
+
+function cached() {
+  if (process.env.AXIOMCODE_NO_PYTHON_CACHE) return null;
+  try {
+    const c = JSON.parse(fs.readFileSync(cacheFile(), 'utf8'));
+    for (const [name, was] of c.seen) {
+      const now = onPath(name);
+      if (JSON.stringify(now) !== JSON.stringify(was)) return null;
+    }
+    if (!fs.statSync(c.exe).isFile()) return null;
+    return { cmd: c.cmd, exe: c.exe };
+  } catch { return null; }
+}
+
+function remember(found, seen) {
+  if (process.env.AXIOMCODE_NO_PYTHON_CACHE) return;
+  try {
+    const f = cacheFile(), tmp = `${f}.${process.pid}`;
+    fs.writeFileSync(tmp, JSON.stringify({ cmd: found.cmd, exe: found.exe, seen }));
+    fs.renameSync(tmp, f);
+  } catch { /* a read-only temp directory only costs the probe next time */ }
+}
+
 function findPython() {
+  const hit = cached();
+  if (hit) return hit;
+  const seen = [];
   for (const cmd of candidates()) {
+    seen.push([cmd[0], onPath(cmd[0])]);
     const exe0 = which(cmd[0]);                                  // PATH only: never a python.exe in the current directory
     if (!exe0) continue;
     const r = spawnSync(exe0, [...cmd.slice(1), '-c', 'import sys; print(sys.executable)'],
                         { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true, timeout: 15000 });
     const exe = r.status === 0 && String(r.stdout).trim();
-    if (exe) return { cmd, exe };
+    if (exe) { const found = { cmd, exe }; remember(found, seen); return found; }
   }
   return { error: 'axiomcode needs Python 3, and no python3, python' + (process.platform === 'win32' ? ' or py -3' : '') +
     ' on PATH runs.\n   • install it (https://www.python.org/downloads/), or\n' +

plugins/axiomcode/mcp/server.py

@@ -41,11 +41,23 @@ def _timer(interval):
 def run(args, cwd=None, timeout=900):
     for a in args[1:]:
         if os.path.isdir(a) and os.path.isdir(os.path.join(a, '.axiomcode')): SEEN.add(os.path.realpath(a))
+    # A QUERY ON A REPOSITORY WITH NO GRAPH YET does not hold the call for the length of a first build (minutes; past this
+    # server's timeout it came back as a bare "Error executing tool"): the build is started in the background and the
+    # answer says what stage it is at (ax_contract.ensure_graph). `index` is asked for a build and still waits for it.
+    env = dict(os.environ, AXIOMCODE_BUILD_NOWAIT='1') if args and args[0] != 'index' else None
     try:
-        r = subprocess.run([BASH, AX, *args], cwd=cwd or None, capture_output=True, text=True, timeout=timeout)
+        r = subprocess.run([BASH, AX, *args], cwd=cwd or None, capture_output=True, text=True, timeout=timeout, env=env)
     except OSError as e:
         return (f"axiomcode could not start bash ({BASH}): {e}. On Windows it needs the bash that comes with "
                 "Git for Windows; install it, or set AXIOMCODE_BASH to its bin\\bash.exe.")
+    except subprocess.TimeoutExpired:
+        repo = next((a for a in reversed(args[1:]) if os.path.isdir(a)), cwd or os.getcwd())
+        try:
+            sys.path.insert(0, os.path.dirname(AX)); import ax_contract, ax_fresh
+            if ax_fresh.building(os.path.realpath(repo)): return ax_contract.building_note(os.path.realpath(repo))
+        except Exception: pass
+        return (f"axiomcode {args[0] if args else ''} did not answer within {timeout} s. Nothing was changed; "
+                f"see {os.path.join(repo, '.axiomcode', 'build.log')} if a build was running, and ask again.")
     out = (r.stdout or '') + (('\n' + r.stderr.strip()) if r.returncode and r.stderr.strip() else '')
     # an answer given from a graph that predates some edit says so, and names the files (#1305)
     if not r.returncode: out += ''.join('\n' + l for l in (r.stderr or '').splitlines() if l.startswith('graph refresh:'))

plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py

@@ -23,7 +23,7 @@
   5. STATE THE BOUND. Say what the answer cannot see — the relations this graph does not encode — so a
      partial list is not read as a complete one.
 """
-import collections, difflib, os, re, subprocess, sys
+import collections, difflib, os, re, subprocess, sys, time
 
 SPLIT = re.compile(r'[^A-Za-z0-9]+')
 CAMEL = re.compile(r'[A-Z]+(?![a-z])|[A-Z][a-z0-9]*|[a-z0-9]+')
@@ -388,11 +388,32 @@ def ensure_graph(repo, db):
     sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); import ax_fresh
     rr = os.path.realpath(repo); auto = os.environ.get('AXIOMCODE_AUTOBUILD')
     if ax_fresh.building(rr):
-        ax_fresh.wait_build(rr, float(os.environ.get('AXIOMCODE_BUILD_WAIT') or 900) if auto else 0)
+        # a build is running: never a silent wait. Under the MCP server answer at once with the stage it is at; from a
+        # shell wait for it, printing the stage as it moves; either way, use the graph the moment it exists
+        if auto and not os.environ.get('AXIOMCODE_BUILD_NOWAIT'):
+            print(f"a graph build is already running for {repo}; waiting for it …", file=sys.stderr)
+            _follow(rr, lambda: ax_fresh.building(rr))
         if os.path.exists(db): return True
+        if auto and os.environ.get('AXIOMCODE_BUILD_NOWAIT'):
+            _BUILD_NOTE.append(building_note(rr)); return False
     if not auto or os.environ.get('AXIOMCODE_GRAPH'): return False
     build = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'axiomcode-build')
     if not os.path.exists(build): return False
+    import ax_fresh
+    bash = os.environ.get('AXIOMCODE_BASH') or 'bash'
+    # NEVER A SILENT WAIT. A first build takes minutes (a whole-program solve, and on a machine that has never built this
+    # language, a one-time compile of its rules), and it used to run with nothing said after its first line, so a caller
+    # could not tell a build from a hang. Under the MCP server (AXIOMCODE_BUILD_NOWAIT) the build is started in the
+    # background and the call answers within AXIOMCODE_BUILD_WAIT seconds either way: with the graph when it was quick,
+    # else with the stage the build is at, so the agent is told to ask again rather than left waiting past the host's
+    # timeout. From a shell the call waits for it, printing the stage as it moves. A call that finds a build already
+    # running (another query's, or an `axiomcode index`) reports on that one rather than queueing a second behind it.
+    nowait = bool(os.environ.get('AXIOMCODE_BUILD_NOWAIT'))
+    if ax_fresh.building(repo):
+        if nowait: _BUILD_NOTE.append(building_note(repo)); return False
+        print(f"a graph build is already running for {repo}; waiting for it …", file=sys.stderr)
+        _follow(repo, lambda: ax_fresh.building(repo))
+        return os.path.exists(db)
     env = None
     if ax_fresh.has_graph(rr):
         # a graph WAS built here and its pointer is broken: a repair, which keeps the baseline `changed` and test-impact
@@ -402,12 +423,71 @@ def ensure_graph(repo, db):
         env = dict(os.environ, AXIOMCODE_KEEP_BASE='1')
     else:
         print(f"no graph for {repo} yet — building one (this is the only slow call; later ones read it) …", file=sys.stderr)
-    r = subprocess.run([os.environ.get('AXIOMCODE_BASH') or 'bash', build, repo], env=env)
-    return r.returncode == 0 and os.path.exists(db)
+    if nowait:
+        os.makedirs(os.path.join(repo, '.axiomcode'), exist_ok=True)
+        log = open(os.path.join(repo, '.axiomcode', 'first-build.log'), 'w')
+        kw = dict(start_new_session=True) if os.name != 'nt' else dict(creationflags=0x00000008 | 0x00000200)   # DETACHED | NEW_GROUP, as ax_fresh.kick
+        p = subprocess.Popen([bash, build, repo], stdin=subprocess.DEVNULL, stdout=log, stderr=log, close_fds=True, env=env, **kw)
+        # until the build holds its lock a second query would see no build and start another, so that much is always waited
+        up = time.time() + 10
+        while p.poll() is None and not ax_fresh.building(repo) and time.time() < up: time.sleep(0.05)
+        end = time.time() + float(os.environ.get('AXIOMCODE_BUILD_WAIT') or 60)
+        while p.poll() is None and time.time() < end: time.sleep(0.5)
+        if p.poll() is None: _BUILD_NOTE.append(building_note(repo)); return False
+        return p.returncode == 0 and os.path.exists(db)
+    p = subprocess.Popen([bash, build, repo], env=env)
+    _follow(repo, lambda: p.poll() is None)
+    return p.returncode == 0 and os.path.exists(db)
+
+
+_BUILD_NOTE = []
+TICK = 20.0
+
+
+def build_stage(repo):
+    """(stage, done, of): the last `▶` step the running build's log names, and how many of its languages it has
+    started solving, from .axiomcode/build.log"""
+    stage, langs, solving = '', [], 0
+    try: text = open(os.path.join(repo, '.axiomcode', 'build.log'), errors='replace').read()
+    except OSError: text = ''
+    for l in text.splitlines():
+        if not l.startswith('\u25b6 '): continue
+        stage = l[2:].strip()
+        if stage.startswith('languages:'): langs = stage.split(':', 1)[1].split()
+        elif stage.startswith('solving '): solving += 1
+    return stage, solving, len(langs)
+
+
+def building_note(repo):
+    """one line: a graph is being built, what it is doing and for how long, and what to do meanwhile"""
+    try: took = time.time() - os.path.getmtime(os.path.join(repo, '.axiomcode', 'build.lock'))
+    except OSError: took = 0
+    stage, n, of = build_stage(repo)
+    where = (f"language {n} of {of}, " if of and n else '') + (stage or 'starting')
+    if 'compiling souffle program' in stage:
+        where += ' — a one-time compile of this language\'s rules on this machine, a few minutes; later builds reuse it'
+    return (f"the graph for {repo} is being built ({where}; {int(took // 60)}m{int(took % 60):02d}s so far). "
+            f"Nothing else is needed: ask again in a minute or two, or read the code directly meanwhile. "
+            f"Progress: {os.path.join(repo, '.axiomcode', 'build.log')}")
+
+
+def _follow(repo, running):
+    """wait while running(), printing the build's stage to stderr whenever it moves, and every TICK seconds at least"""
+    last, said = None, time.time()
+    while running():
+        time.sleep(0.5)
+        stage, n, of = build_stage(repo)
+        if (stage and stage != last) or time.time() - said >= TICK:
+            try: took = time.time() - os.path.getmtime(os.path.join(repo, '.axiomcode', 'build.lock'))
+            except OSError: took = 0
+            print(f"  … {'language %d of %d, ' % (n, of) if of and n else ''}{stage or 'starting'} ({int(took)}s)", file=sys.stderr, flush=True)
+            last, said = stage, time.time()
 
 
 def no_graph(repo, db):
     """what to say when there is still no graph after ensure_graph has had its turn."""
+    if _BUILD_NOTE:
+        return _BUILD_NOTE[-1]
     if os.environ.get('AXIOMCODE_GRAPH'):
         return f"no graph at {db} (AXIOMCODE_GRAPH is set, so nothing was built into it)"
     if os.environ.get('AXIOMCODE_AUTOBUILD'):

plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py

@@ -103,21 +103,27 @@ def git_ignored_dirs(root):
 def watched(root, lang):
     """every file the parser of `lang` reads under root; `lang` may be a comma list (a repository in several
     languages, each with its own graph), and a file two of them read is yielded once"""
-    if ',' in lang:
-        seen = set()
-        for l in lang.split(','):
-            for p in watched(root, l):
-                if p not in seen: seen.add(p); yield p
-        return
-    exts, names, prune = EXT.get(lang, ()), NAMES.get(lang, ()), PRUNE.get(lang, PRUNE_ALL)
-    ignored = git_ignored_dirs(root); real = os.path.realpath(root)
+    # ONE WALK FOR EVERY LANGUAGE. A repository in five languages was walked five times on every freshness check, that is
+    # before every query; the tree is walked once now, a directory only one language prunes (C#'s obj/bin/packages)
+    # entered for the others and its files kept from that one. The files yielded are the same set.
+    langs = [l for l in lang.split(',') if l] if lang else ['']
+    common = set.intersection(*[set(PRUNE.get(l, PRUNE_ALL)) for l in langs])
+    extra = {l: set(PRUNE.get(l, PRUNE_ALL)) - common for l in langs}
+    spec = [(l, EXT.get(l, ()), NAMES.get(l, ())) for l in langs]
+    hidden = {root: frozenset()}                                   # the languages that pruned each directory walked
+    ignored = git_ignored_dirs(root); real = os.path.realpath(root)   # a .gitignore'd directory is pruned for every language
     for d, subdirs, files in os.walk(root):
+        off = hidden.pop(d, frozenset())
         rd = os.path.join(real, os.path.relpath(d, root)) if ignored else d
-        subdirs[:] = [s for s in subdirs if s not in prune and not (ignored and os.path.normpath(os.path.join(rd, s)) in ignored)]
+        subdirs[:] = [s for s in subdirs if s not in common and not all(l in off or s in extra[l] for l in langs)
+                      and not (ignored and os.path.normpath(os.path.join(rd, s)) in ignored)]
+        for s in subdirs: hidden[os.path.join(d, s)] = off | {l for l in langs if s in extra[l]}
         for f in files:
-            if f.endswith(exts) or f in names or (lang == 'java' and os.path.basename(d) == 'services' and 'META-INF' in d) \
-                    or (lang == 'python' and python_script(os.path.join(d, f), f)):
-                yield os.path.join(d, f)
+            for l, exts, names in spec:
+                if l in off: continue
+                if f.endswith(exts) or f in names or (l == 'java' and os.path.basename(d) == 'services' and 'META-INF' in d) \
+                        or (l == 'python' and python_script(os.path.join(d, f), f)):
+                    yield os.path.join(d, f); break
 
 def digest(p):
     h = hashlib.sha1()