From 8bad51c1499440fc96d7771267ad3f7d8c2c7ba5 Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Sat, 17 Oct 2026 09:13:22 +0330 Subject: [PATCH 1/3] paper: arXiv-ready LaTeX source, competitor table, reviewer risks paper/main.tex (article class, one- or two-column via the Makefile), seven table files, refs.bib (39 entries; arXiv author lists and titles read from the abstract pages), a static checker (check.py: braces, environments, inputs, cite keys, labels, ASCII) and REVIEWER-RISKS.md (the ten hardest objections, our answer, the missing experiment). Integrates session 19: SWE-Gym third holdout, learned-reranker negative result, opt-in dense short-list vote on Lite-274 (func Acc@5 0.460 -> 0.485). Verified dense-vote number is a \todo. Positioned honestly below SweRankEmbed-Small and CodeRankEmbed. Not compiled (no TeX on this machine); check.py passes. Co-Authored-By: Claude Opus 5.5 --- paper/.gitignore | 8 + paper/Makefile | 44 ++++ paper/REVIEWER-RISKS.md | 241 +++++++++++++++++++ paper/check.py | 131 +++++++++++ paper/main.tex | 481 ++++++++++++++++++++++++++++++++++++++ paper/refs.bib | 411 ++++++++++++++++++++++++++++++++ paper/tables/ablation.tex | 20 ++ paper/tables/cost.tex | 28 +++ paper/tables/dense.tex | 38 +++ paper/tables/holdouts.tex | 35 +++ paper/tables/lite274.tex | 52 +++++ paper/tables/negative.tex | 30 +++ paper/tables/plain.tex | 20 ++ 13 files changed, 1539 insertions(+) create mode 100644 paper/.gitignore create mode 100644 paper/Makefile create mode 100644 paper/REVIEWER-RISKS.md create mode 100644 paper/check.py create mode 100644 paper/main.tex create mode 100644 paper/refs.bib create mode 100644 paper/tables/ablation.tex create mode 100644 paper/tables/cost.tex create mode 100644 paper/tables/dense.tex create mode 100644 paper/tables/holdouts.tex create mode 100644 paper/tables/lite274.tex create mode 100644 paper/tables/negative.tex create mode 100644 paper/tables/plain.tex diff --git a/paper/.gitignore b/paper/.gitignore new file mode 100644 index 0000000..8d38e42 --- /dev/null +++ b/paper/.gitignore @@ -0,0 +1,8 @@ +*.aux +*.bbl +*.blg +*.log +*.out +*.toc +*.pdf +arxiv.tar.gz diff --git a/paper/Makefile b/paper/Makefile new file mode 100644 index 0000000..d301873 --- /dev/null +++ b/paper/Makefile @@ -0,0 +1,44 @@ +# Build the paper. Needs a TeX distribution (pdflatex + bibtex), or tectonic (`make tectonic`). +# `make check` runs the static checks only (Python 3, no TeX needed). + +TEX ?= pdflatex +BIBTEX ?= bibtex +PYTHON ?= python3 +FLAGS = -interaction=nonstopmode -halt-on-error +SOURCES = main.tex refs.bib $(wildcard tables/*.tex) + +.PHONY: all twocol tectonic check todo arxiv clean + +all: main.pdf + +main.pdf: $(SOURCES) + $(TEX) $(FLAGS) main.tex + $(BIBTEX) main + $(TEX) $(FLAGS) main.tex + $(TEX) $(FLAGS) main.tex + +# Two-column layout from the same source. +twocol: main-2col.pdf + +main-2col.pdf: $(SOURCES) + $(TEX) $(FLAGS) -jobname=main-2col "\def\twocolumnmode{1}\input{main.tex}" + $(BIBTEX) main-2col + $(TEX) $(FLAGS) -jobname=main-2col "\def\twocolumnmode{1}\input{main.tex}" + $(TEX) $(FLAGS) -jobname=main-2col "\def\twocolumnmode{1}\input{main.tex}" + +tectonic: $(SOURCES) + tectonic main.tex + +check: + $(PYTHON) check.py + +# Fails while any \todo remains (use before submission). +todo: + @! grep -n '\\todo{' main.tex tables/*.tex + +# arXiv wants the .bbl, not the .bib. Builds first, then packs the sources. +arxiv: main.pdf + tar czf arxiv.tar.gz main.tex main.bbl tables/*.tex + +clean: + rm -f *.aux *.bbl *.blg *.log *.out *.toc *.pdf arxiv.tar.gz diff --git a/paper/REVIEWER-RISKS.md b/paper/REVIEWER-RISKS.md new file mode 100644 index 0000000..9faf819 --- /dev/null +++ b/paper/REVIEWER-RISKS.md @@ -0,0 +1,241 @@ +# Reviewer risks: the 10 hardest objections + +These are written the way a hostile but fair reviewer would put them. Each one gives our answer +as the paper stands today and, where the answer is not enough, the experiment that would close +the gap. Numbers come from `docs/research/contributions-log.md` (cited as log §x) and +`docs/research/related-work-2026.md`. They are ordered by how likely each objection is to sink +the paper. + +--- + +## 1. "This is below a 137M retriever on every metric. Why publish a weaker method?" + +SweRankEmbed-Small has file Acc@1 0.664 and function Acc@5 0.631 on the same 274 instances. Ours +are 0.500 and 0.460, or 0.485 with the dense vote. CodeRankEmbed is also ahead (0.526 / 0.518). + +**Our answer.** We do not claim state of the art, and the abstract says so. Our claim is a +different point on the cost/accuracy curve: + +- no trained model at query time; +- no embedding of the whole repository; +- zero tokens; +- 2.9 s per issue on a laptop CPU, cold index included; +- every number from a holdout run once. + +None of the methods in Table 1 sits at this point. + +**What is missing.** We have not shown that this point *matters*. The answer is only convincing if +we show one of these: + +- (a) a setting where the dense retriever cannot run: CPU-only, a repository that changes often, or + a privacy-restricted repository, with its real cost measured (see #2); +- (b) that our engine as a first stage plus a cheap reranker reaches the dense retriever's accuracy + at lower total cost (see #3). + +Experiment: run SweRankEmbed-Small and CodeRankEmbed **ourselves on CPU** over the same 274 +instances. Report the indexing time per repository, the time per query, and the accuracy. Then +report our engine plus SweRankEmbed-Small used only as a short-list reranker over our top 30 +definitions. If that hybrid gets close to 0.63 function Acc@5 at a small fraction of the indexing +cost, the paper has a real result. If not, we should say so plainly. + +## 2. "The CPU cost of dense retrieval is exaggerated." + +Our 2.6 chunks/s comes from one measurement: jina-code-v2, int8, 6 threads, 256-token chunks, on a +machine already loaded with other runs (log §8.1). A reviewer will point out four things: + +- a 137M encoder with batching and ONNX/OpenVINO on an idle CPU is much faster than that; +- the index is built once per repository and updated incrementally; +- spread over many issues on the same repository, the per-issue cost is small; +- SWE-bench has only 12 Lite repositories. + +**Our answer.** The measurement is what it is, and the paper calls it "hours for django-sized +repositories", not a precise figure. Today that is all we have. + +**What is missing.** A clean cost table: + +- idle machine, Linux and Windows; +- batch sizes 1, 8 and 32; fp32 and int8; +- CodeRankEmbed and SweRankEmbed-Small on 3 to 5 repositories of different sizes; +- the incremental re-index cost after a typical commit; +- peak memory. + +Report both the amortised per-issue cost and the cold cost. This could shrink our cost argument, so +run it before a reviewer does. + +## 3. "The paper is about coding agents, but there is no agent and no LLM stage." + +The title and introduction motivate localisation as the first step of an agent. But no experiment +measures an agent, and no experiment measures an LLM reading `where_to_look`. The 3B local-LLM pilot +(15 issues) was negative. The API-model stage was blocked by unfunded keys (log §8.13, §8.9). + +**Our answer.** The evidence that the engine is a good first stage is indirect: + +- 45% of function misses have every gold function in the list but outside the top 5 (log §8.18), so + ordering, not recall, is the bottleneck; +- the dense vote improves exactly that ordering. + +The paper names the missing LLM stage as its main gap (§Threats). + +**What is missing.** This is the most important experiment, in two parts. + +- (a) An Agentless-style single LLM call that picks files and functions from our top 10 / top 30, + using one open model (Qwen2.5-Coder-7B/32B) and one API model. Report Acc@1/5 and tokens per issue + next to Agentless and LocAgent. +- (b) If budget allows, an end-to-end resolve rate. Take one agent (OpenHands or SWE-agent) with and + without `where_to_look` in its first prompt, on a 100-issue Lite subset. Report tokens per resolved + issue. + +## 4. "'Pre-registered holdout' is a self-attested git commit. This is not pre-registration." + +The gold, the split scripts and the dense-vote decision (608acb4) are committed before the runs. +But the repository is ours, and commits can be rewritten. "Run once" is also only our word. + +**Our answer.** The commits were pushed to a public GitHub repository before each run, and GitHub +records push times. All result files, including the rejected runs, are in the repository. The +negative results table shows that we report what failed. + +**What is missing.** + +- Archive the gold and decision files on Zenodo or OSF, which gives each one a DOI and a timestamp + that we cannot change. +- Add a table of commit hashes and push times next to each holdout run. +- For the final paper, consider a real pre-registration on OSF for one new holdout. A good candidate + is SWE-bench-Live, or issues created after our release date. + +Without these steps, call it a "commit-locked holdout", not "pre-registered". + +## 5. "The dense vote went through forking paths, and p = 0.065 is not significant." + +The vote failed the bar on dev. It was then re-measured on a *new* split until it passed, and only +then run on Lite. On Lite, the CI [0.004, 0.051] barely excludes zero, and the sign test (9 issues +gained, 2 lost) gives p = 0.065. + +**Our answer.** The weight and k were not changed between dev and SWE-Gym. The decision was +committed before Lite. The effect is the same direction and size on every split (+0.032 pooled on +dev + gym, +0.026 on Lite). The paper reports the dev failure and states this threat explicitly. It +is opt-in, not the default. + +**What is missing.** + +- The Verified run with the locked decision. It is in progress and marked `\todo` in the paper. +- If Verified is flat, the paper must say so, and the vote stays a "promising opt-in". +- A Holm correction across the three holdouts (Lite, Verified, Gym-by-port) would pre-empt the + multiple-comparisons version of this objection. + +## 6. "Your BM25 is suspiciously weak." + +On the same 274 instances our BM25 has file Acc@1 0.299, against 0.387 for LocAgent's BM25. A +reviewer may suspect a weak baseline that inflates "×2.3 over BM25" and "+0.21 over BM25". + +**Our answer.** Table 1 reports both. Against LocAgent's stronger BM25 the gains are still +0.11 +file Acc@1 and +0.14 function Acc@5. The SWE-Gym ×2.3 figure is relative to our own BM25 only. + +**What is missing.** + +- A third, standard BM25: Pyserini/Lucene with default k1 = 0.9, b = 0.4, over files and over + functions with the same file aggregation. +- A sentence on *why* ours is lower. Possible causes are tokenisation of identifiers, scoring whole + files versus taking the best function, and handling of test files. +- Recompute the SWE-Gym ratio against the standard BM25. + +This is cheap and removes an easy attack. + +## 7. "Python only, one benchmark family. The claims about generality are not supported." + +Every SWE-bench-style set is Python. The plain-language holdouts cover five languages, but with 12 +questions each, one annotator, and gold written by the authors. One question moves R@3 by 0.083. + +**Our answer.** The paper reports plain-language results as "a direction, not a precise effect". +The engine's parsers cover Rust, TypeScript, Go, Java and others, but the SWE-bench-style numbers +apply only to Python. + +**What is missing.** + +- A multilingual issue benchmark: SWE-bench-java, Multi-SWE-bench, SWE-PolyBench, or SweRank+'s + multilingual set, which also gives a head-to-head with SweRankEmbedMulti. +- Plain-language holdouts of 50 or more questions with a second annotator and inter-annotator + agreement. +- MULocBench's non-code issues, because the related work cites it as motivation. + +## 8. "Contamination and leakage: the engine was hand-tuned over many sessions by people who saw the benchmarks." + +The heuristics were written by developers who knew SWE-bench well. That includes hygiene rules for +issue templates, rule-code tokens, and traceback parsing. 24 Lite instances were looked at. Django +and the other Lite repositories are well known. + +**Our answer.** + +- The strict row excludes the 24 looked-at instances, and the 250-instance subset shows they do not + flatter the result. +- Tuning used only dev, whose repositories are disjoint from Lite. +- SWE-Gym's seven repositories were never seen before the v1.4.0 run, and the gain held there + (×2.3). +- The engine has no learned weights, so pre-training contamination does not apply to it. It does + apply to the encoders we compare against. + +**What is missing.** + +- A holdout made of issues *created after* the v1.4.0 release date, for example from SWE-bench-Live + or new issues mined the same way as SWE-Gym. No heuristic could have been fitted to them. +- A list of every rule in the hygiene component, with where it came from (which dev issue). + +## 9. "No research novelty: BM25F + RRF + traceback parsing is engineering." + +**Our answer.** The contribution is the measurement, not the components: + +- the ablation shows which LLM-free component carries the gain (definition-level ranking, −0.10 + Acc@1 when removed); +- the query-length split for cross-encoders; +- the finding that blind graph expansion hurts while LLM-guided expansion helps; +- the finding that a learned reranker over lexical features does not transfer (coefficients change + sign); +- the explanation that dev understates function-level gains because of gold size; +- a dozen negative results with numbers. + +Negative results and protocols are publishable at venues that value them (MSR, ICSE/FSE "new ideas +and emerging results", or the TOSEM/EMSE replication tracks). This paper is probably a better fit +there than at a main ML track. + +**What is missing.** + +- A sharper single insight in the abstract. The best candidate is "ordering, not recall, is the + bottleneck for LLM-free localisers: 45% of misses rank the gold at about 36". +- A per-component failure taxonomy on the holdout (not only dev), so the analysis section stands on + held-out data. + +## 10. "The function-level gold may not be LocAgent's gold, so the head-to-head is not exact." + +We rebuild function gold ourselves (the innermost function containing a removed line or insertion +point, from Python `ast`). The *count* matches LocAgent's 274 exactly, but the *per-instance* gold +sets might differ. For example, LocAgent may count class-level edits, decorators, or added +functions differently. A difference of a few instances moves Acc@5 by 0.01 to 0.02. That is the +same size as the dense-vote effect, and close to the gap to CodeRankEmbed. + +**Our answer.** Matching the 274 count is evidence that the gold extraction is right, but it is not +proof. + +**What is missing.** + +- Score our stored outputs with LocAgent's released evaluation script and gold, if it is public. +- Otherwise, take its released per-instance predictions or gold, report the per-instance agreement + rate of the gold sets, and re-score. + +This is cheap and should be done before submission. + +--- + +## Smaller objections to pre-empt in the text + +- **Timing hygiene.** Timing comes from one Windows laptop that was shared with other jobs. Re-time + on an idle Linux machine and report p50/p90 by repository size (part of #2). +- **CI overlap between versions.** v1.3 and v1.4 file-level results are identical, and the + function-level CIs overlap. Paired tests are reported only for the dense vote. Add paired tests + for v1.3 → v1.4 at function Acc@5 (gained/lost counts are in the stored results). +- **Ablation is on v1.2.0.** The ablation predates two votes. Re-run the ablation on v1.4.0 with the + named-files and traceback votes as rows. +- **Packet size / tokens.** The packet's median of 11.1k tokens is reported, but packet precision + is not. The agreed "next metric" (token_precision) would answer "how much of what you hand the + agent is useful?" +- **Upstream credit.** NeuroMesh is a fork of pinoox/neuromesh. The acknowledgements credit the + upstream project. The paper must make clear which components were added in this work (graph + link-resolution fixes, report mode, definition ranking, named files, traceback vote, dense vote). diff --git a/paper/check.py b/paper/check.py new file mode 100644 index 0000000..b134b69 --- /dev/null +++ b/paper/check.py @@ -0,0 +1,131 @@ +"""Static checks for the paper source, for machines without a TeX installation. + +Checks: balanced braces and \\begin/\\end pairs per file, every \\input file exists, every +\\cite key is in refs.bib, every \\ref has a \\label, no duplicate labels or bib keys, bib entries +brace-balanced, no non-ASCII characters in .tex files. Lists every \\todo. Exit code 1 on any error. + + python paper/check.py # from the repository root or from paper/ +""" + +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parent + + +def strip_comments(text): + # A % starts a comment unless escaped as \%. + return "\n".join(re.sub(r"(? 127: + errors.append(f"{path.name}:{n}: non-ASCII character {ch!r}") + break + text = strip_comments(raw) + body = re.sub(r"\\[{}]", "", text) # escaped braces + depth = 0 + for n, line in enumerate(body.splitlines(), 1): + for ch in line: + depth += ch == "{" + depth -= ch == "}" + if depth < 0: + errors.append(f"{path.name}:{n}: unmatched '}}'") + depth = 0 + if depth: + errors.append(f"{path.name}: {depth} unclosed '{{'") + stack = [] + for m in re.finditer(r"\\(begin|end)\{([^}]+)\}", text): + line = text.count("\n", 0, m.start()) + 1 + if m.group(1) == "begin": + stack.append((m.group(2), line)) + elif not stack or stack[-1][0] != m.group(2): + errors.append(f"{path.name}:{line}: \\end{{{m.group(2)}}} does not close " + f"{stack[-1] if stack else 'anything'}") + else: + stack.pop() + for env, line in stack: + errors.append(f"{path.name}:{line}: \\begin{{{env}}} never closed") + return text + + +def bib_keys(path, errors): + raw = path.read_text(encoding="utf-8") + keys = [] + for m in re.finditer(r"^@(\w+)\{([^,\s]+),", raw, re.M): + keys.append(m.group(2)) + start, depth, i = m.end(), 1, m.end() + while i < len(raw) and depth: + depth += raw[i] == "{" + depth -= raw[i] == "}" + i += 1 + if depth: + errors.append(f"refs.bib: entry {m.group(2)} not brace-balanced") + seen = set() + for k in keys: + if k in seen: + errors.append(f"refs.bib: duplicate key {k}") + seen.add(k) + return seen + + +def main(): + errors = [] + texts = {} + main_tex = ROOT / "main.tex" + queue = [main_tex] + while queue: + path = queue.pop() + if path in texts: + continue + if not path.exists(): + errors.append(f"missing input file {path.relative_to(ROOT)}") + continue + texts[path] = check_tex(path, errors) + for m in re.finditer(r"\\input\{([^}]+)\}", texts[path]): + name = m.group(1) + queue.append(ROOT / (name if name.endswith(".tex") else name + ".tex")) + all_text = "\n".join(texts.values()) + + keys = bib_keys(ROOT / "refs.bib", errors) + cited = set() + for m in re.finditer(r"\\cite\w*\*?(?:\[[^\]]*\])*\{([^}]+)\}", all_text): + cited.update(k.strip() for k in m.group(1).split(",")) + for k in sorted(cited - keys): + errors.append(f"cited but not in refs.bib: {k}") + unused = sorted(keys - cited) + + labels = re.findall(r"\\label\{([^}]+)\}", all_text) + for lab in {l for l in labels if labels.count(l) > 1}: + errors.append(f"duplicate label {lab}") + for ref in sorted(set(re.findall(r"\\ref\{([^}]+)\}", all_text)) - set(labels)): + errors.append(f"\\ref to undefined label {ref}") + + todos = [] + for path, text in texts.items(): + for n, line in enumerate(text.splitlines(), 1): + for m in re.finditer(r"\\todo\{([^}]*)", line): + todos.append(f"{path.name}:{n}: {m.group(1)}") + + print(f"files: {', '.join(sorted(p.name for p in texts))}") + print(f"citations: {len(cited)} keys used, {len(keys)} in refs.bib" + + (f" (unused: {', '.join(unused)})" if unused else "")) + print(f"labels: {len(labels)}") + print(f"TODOs ({len(todos)}):") + for t in todos: + print(" " + t) + if errors: + print(f"ERRORS ({len(errors)}):") + for e in errors: + print(" " + e) + return 1 + print("OK: no syntax errors found") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/paper/main.tex b/paper/main.tex new file mode 100644 index 0000000..d8280c0 --- /dev/null +++ b/paper/main.tex @@ -0,0 +1,481 @@ +% Where to look: LLM-free, CPU-only code localisation for coding agents, measured on holdouts. +% +% Source of every number: docs/research/contributions-log.md (sections cited in comments as "log sec. x"). +% Competitor numbers: docs/research/related-work-2026.md (PR #146). +% Build: `make` (one column) or `make twocol`. Two-column mode is selected by defining +% \twocolumnmode to 1 before this file is read (see the Makefile). +\providecommand{\twocolumnmode}{0} +\if1\twocolumnmode + \documentclass[10pt,twocolumn]{article} + \PassOptionsToPackage{margin=0.75in}{geometry} +\else + \documentclass[11pt]{article} + \PassOptionsToPackage{margin=1in}{geometry} +\fi + +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{lmodern} +\usepackage{geometry} +\usepackage{microtype} +\usepackage{amsmath,amssymb} +\usepackage{booktabs} +\usepackage{graphicx} +\usepackage{xcolor} +\usepackage{url} +\usepackage[numbers,sort&compress]{natbib} +\usepackage[colorlinks=true,linkcolor=blue!50!black,citecolor=blue!50!black,urlcolor=blue!50!black]{hyperref} + +% Unmeasured or unconfirmed content. `make check` lists every occurrence; none may remain at submission. +\newcommand{\todo}[1]{\textcolor{red}{\textbf{[TODO:~#1]}}} +\newcommand{\nm}{NeuroMesh} +\newcommand{\accat}[1]{Acc@#1} + +\title{Where to Look: LLM-Free, CPU-Only Code Localisation\\for Coding Agents, Measured on Holdouts} +\author{Parsa Karkooti\thanks{\todo{affiliation and co-authors}}} +\date{} + +\begin{document} +\maketitle + +% --------------------------------------------------------------------------------------------- +\begin{abstract} +Coding agents spend much of their budget finding the code a task needs. Strong localisers today +run an LLM agent over the repository, or embed every function of it with a trained retriever, +usually on a GPU. We ask how far a local engine gets with neither: a per-project code graph, +field-weighted lexical ranking, a definition-level ranking of long reports, report hygiene, and the +report's own traceback, all on a laptop CPU in a few seconds per issue. Design decisions were +tuned on SWE-bench dev only (the last one also on a SWE-Gym split, after the released engine had +been run on it once), and each released version was run once on held-out data. On 276 held-out SWE-bench Lite issues the engine reaches file-level +\accat{1}/3/5 of 0.507/0.717/0.750 (BM25 0.301/0.507/0.587) at 2.9\,s per issue including a cold +index; on the 407 SWE-bench Verified issues outside Lite 0.435/0.671/0.732 (BM25 +0.194/0.373/0.477); on 280 SWE-Gym issues from seven repositories it never saw, file \accat{1} +0.279/0.271 on the two halves (BM25 0.121/0.121). At function level it lists every edited function +among its first five on 0.460 of LocAgent's 274 Lite instances, and 0.485 with an opt-in vote that +embeds only the engine's top 30 definitions on CPU (paired $+0.026$, 95\% CI [0.004, 0.051]). This +is below code-trained dense retrievers --- SweRankEmbed-Small reaches 0.664 file \accat{1} and +0.631 function \accat{5} on the same instances --- and well below LLM agents and rerankers. Our +contribution is the other corner of the trade-off: no training, no GPU, no whole-repository +embedding index, zero tokens, a holdout protocol that caught two results looked-at data had +suggested, and a dozen negative results with numbers. +\end{abstract} + +% --------------------------------------------------------------------------------------------- +\section{Introduction} +\label{sec:intro} + +A coding agent asked to fix an issue first has to find where the fix goes. On SWE-bench +\citep{jimenez2024swebench} this step, \emph{localisation}, is where agent pipelines spend much of +their budget: Agentless \citep{xia2025agentless} prompts a model with the repository tree, then with +file skeletons, then with code; LocAgent \citep{chen2025locagent}, SWE-agent \citep{yang2024sweagent} +and OpenHands \citep{wang2025openhands} run multi-step LLM searches. The strongest model-free +alternatives embed every function of the repository with a trained encoder +\citep{suresh2025cornstack,reddy2026swerank}. A 137M-parameter encoder is small for a GPU, but on a +laptop CPU we measured 2.6 chunks per second for a comparable model (\S\ref{sec:negative}): hours per +large repository, repeated whenever the code changes. Local-first assistants such as Aider's RepoMap +\citep{aider_repomap} ship structural or lexical retrieval but are not measured on localisation +benchmarks. + +We ask a narrow question: \emph{how far does a local engine get with no LLM and no GPU, if every +design decision is measured on a tuning split and confirmed once on held-out data?} The engine, +\nm{} \citep{neuromesh}, builds a per-project code graph in seconds, ranks files with field-weighted +BM25 \citep{robertson2004bm25f}, ranks \emph{definitions} for long reports, removes issue-template +scaffolding from the query, and reads the report's own pointers --- the files and functions its +traceback runs through. It returns a ranked list of files and of functions (\texttt{where\_to\_look}, +\texttt{functions\_to\_look}) that an agent opens first. + +The answer is \emph{not} state of the art, and we say so up front. On LocAgent's 274 Lite instances +the engine is 0.03 below CodeRankEmbed and 0.16 below SweRankEmbed-Small at file \accat{1}, and +0.27 below LocAgent with Claude-3.5 at function \accat{5} (Table~\ref{tab:lite274}). What it offers +is a measured point no published method occupies: no learned model at query time, no +whole-repository index beyond a code graph, zero tokens, 2.9\,s per issue on a laptop, and numbers +from holdouts run once. + +\paragraph{Contributions.} +\begin{enumerate} + \item \textbf{A holdout protocol for retrieval engines} (\S\ref{sec:protocol}): gold locked by + commit before any run, two tuning splits disjoint in repositories from every evaluation set, a + pre-set acceptance bar, one run per released version on each holdout, and per-set labels for what + has been looked at. It caught two results that looked-at sets suggested (a cross-encoder reranker + and a neighbour vote) and confirmed one on a fresh repository. + \item \textbf{An LLM-free localiser} reaching file \accat{1} 0.507 on 276 held-out Lite issues (BM25 + 0.301), 0.435 on 407 Verified issues outside Lite (BM25 0.194) and $2.2$--$2.3\times$ BM25 on seven unseen + SWE-Gym repositories, at 2.9\,s per issue on a laptop CPU (\S\ref{sec:results}). + \item \textbf{A cheap dense short-list vote}: embedding only the engine's top 30 definitions lifts + function \accat{5} on Lite from 0.460 to 0.485, with the decision committed before the holdout run + (\S\ref{sec:dense}). + \item \textbf{Ablations, costs and negative results with numbers} (\S\ref{sec:ablation}, + \S\ref{sec:negative}): a dozen ideas from the bug-localisation and retrieval literature --- commit + history, query expansion, cross-encoder reranking, graph expansion, a learned reranker, a small + local LLM judge --- that did not survive the protocol, and why. +\end{enumerate} + +% --------------------------------------------------------------------------------------------- +\section{Related work} +\label{sec:related} + +\paragraph{LLM localisers.} +Agentless \citep{xia2025agentless} narrows hierarchically (repository tree, files, class and +function skeletons, lines) with one model call per level plus sampling and voting; with GPT-4o it +covers the gold file in 69.7\% and the gold function in 52.0\% of Lite issues at \$0.70 per issue. +SWE-agent \citep{yang2024sweagent}, OpenHands \citep{wang2025openhands} and MoatlessTools +\citep{moatless} search through shell or retrieval tools inside an agent loop. LocAgent +\citep{chen2025locagent} indexes a heterogeneous code graph (files, classes, functions; +contain/import/invoke/inherit edges) that an LLM agent traverses with entity-level BM25. It also +fixed the reference protocol we use: 274 Lite instances whose patch edits an existing function, and +\accat{k} requiring all gold locations in the top $k$. With Claude-3.5 it reaches file \accat{1}/5 +0.777/0.942 and function \accat{5} 0.734 at \$0.66 per issue. Later systems refine the search +policy: iterative graph search (CoSIL, \citealp{jiang2025cosil}), priority scheduling and context +pruning (OrcaLoca, \citealp{yu2025orcaloca}), repository knowledge graphs (KGCompass, +\citealp{yang2025kgcompass}), repository memory (RepoMem, \citealp{wang2025repomem}) and code graphs as +a plug-in (RepoGraph, \citealp{ouyang2025repograph}). Another line trains small open models for +localisation with reinforcement learning \citep{ma2026tooltrain,sutawika2026codescout,xu2026fusesearch}. +With frontier models the numbers keep rising: SWE-Adept \citep{he2026sweadept} reports function +\accat{5} 0.878 on the 274 instances with Claude-Sonnet-4.5 at 348k tokens per issue, and in the same +paper LocAgent re-run with GPT-5.2 scores 0.646 against the 0.734 originally reported --- agent +numbers move by about 0.1 with the model and the run. SHERLOC \citep{tamoyan2026sherloc} adds +structured diagnosis with a reasoning model. Our engine borrows LocAgent's entity-level content +ranking and Agentless's file-to-function narrowing, without the model. + +\paragraph{Dense retrieval and learned rerankers.} +CodeRankEmbed (137M; \citealp{suresh2025cornstack}) ranks function chunks by cosine with the issue; +on the 274 instances it reaches file \accat{1} 0.526 and function \accat{5} 0.518, and a 7B +listwise reranker lifts the latter to 0.675. SweRank \citep{reddy2026swerank} fine-tunes retrievers +and rerankers on issue-localisation data mined from GitHub: SweRankEmbed-Small (137M) reaches +0.664/0.631 (file \accat{1}/function \accat{5}) with no LLM at query time, and +SweRankEmbed-Large with a 32B reranker 0.832/0.814 at \$0.015 per issue on a GPU; SweRank+ +\citep{reddy2025swerankplus} extends this to several languages (function \accat{5} 0.854). CoRet +\citep{fehr2025coret} adds call-graph context to a 130M retriever, SWE-Fixer \citep{xie2025swefixer} +selects files from a BM25 short list with a fine-tuned 7B model, and BLAZE +\citep{chakraborty2025blaze} fine-tunes CodeSage \citep{zhang2024codesage} for cross-project bug +localisation. These are the strongest non-agent localisers. All require training data close to the +task and an embedding of every function in the repository; none reports CPU indexing cost. We +measure that cost, and a variant that embeds only the engine's short list (\S\ref{sec:dense}). + +\paragraph{Classic bug localisation.} +IR-based bug localisation uses report text, history and structure: BugLocator +\citep{zhou2012buglocator} combines a revised VSM with similar fixed bugs, Locus +\citep{wen2016locus} ranks change hunks, and \citet{rahman2018query} reformulate the query. We test +the history and reformulation (RM3, \citealp{abduljaleel2004rm3}) ideas under our protocol; both are +negative (\S\ref{sec:negative}). MULocBench \citep{zhang2025mulocbench} shows that issues outside +code (configuration, documentation, build) are much harder --- LocAgent with Claude-3.5 reaches file +\accat{5} 0.352 there --- which motivates our plain-language holdouts (\S\ref{sec:plain}). + +\paragraph{Ranking machinery.} +We use BM25F \citep{robertson2004bm25f} and reciprocal rank fusion (RRF; +\citealp{cormack2009rrf}); \citet{bruch2023fusion} analyse convex lexical/dense fusion. + +\paragraph{Positioning.} +No published method in this space is free of learned models at query time, and none reports a +pre-registered holdout. Table~\ref{tab:lite274} places our engine among the methods measured on the +same 274 instances with the same metric. + +% --------------------------------------------------------------------------------------------- +\section{System} +\label{sec:system} + +\paragraph{Index.} +A per-project graph of files, definitions (functions, methods, classes), calls and imports, built by +tree-sitter parsers and cached on disk. Link resolution was the cold-start bottleneck: on a django +checkout (3.5k files) four fixes to repeated path normalisation and per-relation neighbour walks +brought the first packet from 3\,min\,20\,s to 26\,s without changing the graph (log~\S8.2). + +\paragraph{Questions.} +A prompt's anchors (identifiers, paths) seed spreading activation over the graph; the activated +nodes form a token-bounded \emph{packet}. Files are ranked by BM25F over four fields: path, symbol +names, comments and body. + +\paragraph{Reports.} +For prompts of at least 60 words the engine treats the prompt as an issue report. (i)~\emph{Hygiene} +removes issue-template scaffolding (``search before asking'', checklists, links, headings) and keeps +code-like tokens that word tokenisers drop (rule codes such as \texttt{L031}). (ii)~A +\emph{definition-level} BM25 ranks every definition's text, with the report title counted three +times; files inherit the rank of their best definition. (iii)~\emph{Named files}: traceback frames +(deepest first), then paths and unique file names in prose, resolved against the index by longest +unique path suffix. The packet order, the definition ranking and the named files are fused by RRF +(named files at weight~2) into the file list \texttt{where\_to\_look}. + +\paragraph{Short questions.} +For short prompts the file list is the RRF of the packet order and a whole-question ranking (S1); +the packet's activation order alone puts hub files first. + +\paragraph{Functions to look at.} +For every prompt the definition ranking (classes removed) is fused by RRF with each definition's file +rank --- Agentless's file-to-function step without an LLM --- and with the functions a traceback runs +through: each frame is resolved to an indexed file, then to the definition named in the frame (or +the innermost one containing its line), test frames skipped, voting at double weight deepest-first. + +\paragraph{Opt-in dense short-list vote.} +With \texttt{NM\_DENSE=1}, jina-embeddings-v2-base-code \citep{gunther2023jina} (int8 ONNX, CPU) +embeds the issue and the engine's top 30 definitions (text from the base commit, cut at 512 tokens); +their cosine order joins the function-list fusion by RRF ($k=60$, weight $0.5$). The file list is not +changed. It costs about 6\,s per issue on CPU and 31 embeddings (30 definitions and the issue), instead of an embedding of every +function in the repository. + +% --------------------------------------------------------------------------------------------- +\section{Evaluation protocol} +\label{sec:protocol} + +\paragraph{Tuning splits.} +\emph{SWE-bench dev} (225 issues from pvlib, pydicom, sqlfluff, astroid, pyvista and marshmallow --- +no repository shared with Lite or Verified), in halves dev-fast (59, ten per repository) and dev-rest +(166). Because one dev-fast issue is worth 0.017, dev cannot resolve differences of 0.02, and its +function gold is unlike Lite's; we therefore added \emph{SWE-Gym} \citep{pan2025swegym}: 280 issues +from seven repositories disjoint from Lite, Verified and dev (pydantic, hydra, conan, dvc, dask, +moto, mypy), at most 40 per repository, patches that edit only existing non-test Python files, +sampled with a fixed seed by a script committed before any run, in halves gym-a and gym-b. + +\paragraph{Acceptance bar.} +A change is kept only if file \accat{1} or function \accat{5} rises by at least 0.02 in \emph{both} +halves of a tuning split, a bar fixed before each measurement. Every idea is first a Python +prototype over stored engine outputs, then an engine port re-measured on the full split. + +\paragraph{Holdouts.} +SWE-bench Lite test (300) and Verified (500), each run once per released version with the release +binary and never inspected per instance. The 24 Lite instances looked at in an early session are +excluded from the strict row (276). LocAgent's 274-instance subset (patches that edit an existing +function) was rebuilt from its definition and matches exactly; it gives the head-to-head. 407 +Verified issues are outside Lite. The released v1.4.0 had never seen the SWE-Gym repositories, so its +single run there is a third holdout, reported before any tuning on SWE-Gym. + +\paragraph{Metrics.} +File-level \accat{k}: all gold files in the top $k$. Function-level \accat{k}: all edited functions in +the top $k$, where function gold is the innermost function containing a removed line or insertion +point of the reference patch at the base commit; an instance without a function list counts as a +miss. 95\% bootstrap confidence intervals \citep{efron1993bootstrap}; paired comparisons report the +bootstrap CI of the difference, issues gained and lost, and a two-sided sign test. + +\paragraph{Plain-language questions.} +Five holdouts (ripgrep/Rust, click/Python, cobra/Go, axios/JavaScript, jsoup/Java), 12 questions +each, gold written from source and locked by commit before the first run; metric R@3 of the file +list. + +% --------------------------------------------------------------------------------------------- +\section{Results} +\label{sec:results} + +\input{tables/lite274} +\input{tables/holdouts} + +\subsection{Head-to-head on LocAgent's 274 Lite instances} +\label{sec:h2h} + +Table~\ref{tab:lite274} compares the engine with every method reported on the same instances and +metric. Without any learned model it is $+0.11$ above LocAgent's BM25 at file \accat{1} (0.500 vs +0.387) and $+0.14$ at function \accat{5} (0.460 vs 0.318). It is above Jina-Code-v2 at file +\accat{1} (0.434), level with E5-base (0.496), and below CodeRankEmbed (0.526) and SweRankEmbed-Small +(0.664). At file \accat{5} it is below every dense retriever (0.755 vs 0.781--0.909). At function +\accat{5} it is above E5-base, Jina-Code-v2 and CodeSage-large (0.394, 0.423, 0.339), equal to +SWE-agent with GPT-4o (0.460), and below CodeRankEmbed (0.518), SweRankEmbed-Small (0.631) and every +LLM agent on Claude-3.5. Function \accat{1} is 0.223 (not reported by the other papers). + +The 24 instances looked at during development are all among the 274; without them (250 instances) +the v1.2.0 engine scored 0.488/0.716/0.756 and BM25 0.304/0.512/0.592, so the looked-at instances do +not flatter the result (log~\S8.7). + +\subsection{Three holdouts} +\label{sec:holdouts} + +Table~\ref{tab:holdouts} gives every SWE-bench-style set. The gain over BM25 holds across all of +them: file \accat{1} $+0.21$ on Lite strict, $+0.24$ on Verified outside Lite, and $2.2$--$2.3\times$ on +the two SWE-Gym halves --- seven repositories the engine was never developed on. On dev the gain was +$+0.15$; the holdout gains are larger, not smaller, which is the expected sign if the tuning did not +overfit dev. + +Function-level gains from dev transfer \emph{larger} than predicted. The traceback vote and function +lists for short reports (v1.3.0 $\to$ v1.4.0) raised dev function \accat{5} by $+0.034$ +(0.290 $\to$ 0.324) but Lite-274 by $+0.066$ (0.394 $\to$ 0.460) and Verified by $+0.083$ (0.346 +$\to$ 0.429). The inputs do not explain this --- tracebacks appear in 20\%, 20\% and 14\% of dev, +Lite and Verified issues, reports under 60 words in 10\%, 12\% and 14\% --- but the gold does: one +edited function in 45\% of dev issues (mean 3.7) against 86\% of Lite (mean 1.17) and 72\% of +Verified (mean 1.87). When every edited function must be in the top $k$, a correctly placed function +pays off far more often on single-function gold. SWE-Gym's gold (mean 3.4 functions, one function in +45\%) resembles dev, which is why its function numbers are low. + +\subsection{Dense short-list vote} +\label{sec:dense} + +\input{tables/dense} + +The dense vote of \S\ref{sec:system} first came up as a near miss on dev: positive in almost every +cell but below the bar on dev-fast (file \accat{1} $+0.017$, one issue; function \accat{5} $+0.019$ +and $+0.013$). Re-measured unchanged on SWE-Gym, it passed (function \accat{5} $+0.030$ and $+0.059$; +Table~\ref{tab:dense}). Pooled over dev and SWE-Gym (476 issues) function \accat{5} rose from 0.286 +to 0.317, $+0.032$ [0.015, 0.050], 17 issues gained and 2 lost. The weight and short-list size were +committed before the Lite run. On Lite-274, run once, function \accat{1}/5/10 went from +0.223/0.460/0.540 to 0.252/0.485/0.555: $+0.026$ at \accat{5}, CI [0.004, 0.051], 9 gained and 2 +lost. The CI excludes zero; with only 11 discordant issues the sign test does not ($p=0.065$). The +effect is the same direction and size as on the tuning splits. It narrows the gap to CodeRankEmbed +(0.518) from $-0.058$ to $-0.033$ and leaves a gap of $-0.146$ to SweRankEmbed-Small, with 30 +embeddings per issue on CPU instead of every function of the repository. On Verified the run with +the locked decision is \todo{Verified function \accat{1}/5/10 with the dense vote (459 instances), +paired $\Delta$@5 with CI and gained/lost --- being measured}. + +The vote also re-ranks files: the dense order alone puts the gold file first more often than the +engine on dev (0.356 vs 0.307 file \accat{1}) while losing depth, but the file vote did not clear the +bar on gym-b ($+0.000$), so the file list is unchanged. + +\subsection{Ablation} +\label{sec:ablation} + +\input{tables/ablation} + +Table~\ref{tab:ablation} removes one component at a time on the Lite strict holdout (engine v1.2.0, +before the named-file and traceback votes). The definition-level ranking --- LocAgent's +entity-content BM25, made LLM-free --- carries most of the gain over the packet ($+0.10$ \accat{1}, +$+0.14$ \accat{3}). Report hygiene is small on Lite ($-0.019$ \accat{1} when removed); its dev gains +came mostly from sqlfluff's template-heavy issues. The localisation list itself matters most: the +packet order alone scores 0.366 \accat{1}, because a packet holds one to five files and caps +\accat{5}. The named-files vote was added after this ablation; its effect is the v1.2.0 $\to$ v1.3.0 +step on Verified outside Lite, file \accat{1} 0.392 $\to$ 0.437 on the same 403 instances, against +$+0.049$ predicted on dev (log~\S8.13). + +\subsection{Plain-language questions} +\label{sec:plain} + +\input{tables/plain} + +Issue reports are long and name code. Developers' questions to an assistant are often short and do +not (``where are retries configured?''). Table~\ref{tab:plain} reports five such holdouts. S1, the +short-question fusion of \S\ref{sec:system}, was chosen on the first three sets (looked at), was +neutral on axios, and was confirmed on jsoup, a language and repository never run before: +$+0.333$ R@3 over the packet order and $+0.083$ over BM25. Twelve questions per set is small (one +question is 0.083); we report these as a direction, not a precise effect. + +\subsection{Cost} +\label{sec:cost} + +\input{tables/cost} + +Table~\ref{tab:cost} gives per-issue cost. The engine needs no network, no GPU and no tokens; the +cold index is included in the 2.9\,s median. Its packet (the context an agent receives) has a median +of 11.1k tokens. The cheapest published setting that beats it on file \accat{1} at the same protocol +without an LLM, SweRankEmbed-Small, needs a contrastively trained encoder and an embedding of every +function; at our measured CPU rate for a model of that size, embedding django's roughly 40k chunks +would take hours. + +% --------------------------------------------------------------------------------------------- +\section{Negative results and what the holdouts caught} +\label{sec:negative} + +\input{tables/negative} + +Table~\ref{tab:negative} lists the ideas that did not survive. Three findings stand out. + +\emph{Query length decides whether a cross-encoder helps.} jina-reranker-v2 reordering the engine's +top 10 files improved R@3 on two short-question sets (ripgrep $+0.375$, click $+0.208$) and hurt on +long issue reports (\accat{3} 0.492 $\to$ 0.373): its 512-token window drops the report's specifics, +which the first stage already reads in full. On a fresh short-question set (cobra) it lost $-0.084$; +without the holdout we would have shipped it for short questions. + +\emph{Blind graph expansion hurts.} Voting for the graph neighbours of the top three files pulls in +the hubs they call (base classes, utilities) ahead of the edited module (file \accat{1} 0.254 +$\to$ 0.136). LocAgent gets value from the graph through an LLM choosing which edges to follow. + +\emph{A learned reranker fits the split, not the task.} Logistic regression and boosted trees over 25 +features of the top 100 function candidates did not pass cross-half on dev ($+0.000$/$+0.019$); +trained on dev and applied to SWE-Gym they gained nothing, and the largest coefficients change sign +between splits (file rank $-0.61$ on dev, $+1.23$ on SWE-Gym). With a few hundred issues per split, +supervised reranking of lexical features does not generalise. + +Two results from the bug-localisation literature were clearly negative on dev. A commit-history prior +in the spirit of BugLocator and Locus (past commits whose message resembles the issue vote for the +files they touched) lowered file \accat{1} from 0.271 to 0.169 on dev-fast even at weight 0.1. RM3 +expansion of the definition query was worse than plain BM25 on its own and no better as a vote. +A local 3B LLM (Qwen2.5-Coder-3B-Instruct \citep{hui2024qwen25coder}, 4-bit, llama.cpp +\citep{llamacpp}) as an Agentless-style file judge over the engine's top 10 candidates copied the engine's order when shown it and did worse +when the order was shuffled (\accat{1} 0.200 $\to$ 0.067 on 15 issues), at about 105\,s per issue: +on a laptop the engine's ranking already beats a small LLM judge. + +Our own evaluation slips are part of the record: a suffix match without a path-segment boundary that +inflated a baseline, a function-level denominator that silently dropped short reports (244 instead +of 274 instances, \accat{5} 0.443 instead of 0.394), and a prototype baseline taken from an older +run. The denominator slip was caught by re-deriving LocAgent's count of 274; every affected number +was re-measured and is reported corrected. + +% --------------------------------------------------------------------------------------------- +\section{Discussion} +\label{sec:discussion} + +\paragraph{Where the engine stands.} +It is the best measured point we know of with no learned model at query time, and it is clearly +below trained dense retrievers and LLM pipelines. We see it as a first stage: its top 30 definitions +are a short list that a dense model or an LLM can reorder at a small fraction of the cost of +searching the repository, as the dense vote shows. An LLM stage over \texttt{where\_to\_look}, +measured in tokens and \accat{1} against Agentless and LocAgent, is the experiment this paper does +not have (\S\ref{sec:threats}). + +\paragraph{Where function misses come from.} +On dev, 45\% of function-level misses have every gold function in the 300-entry list but not in the +top five (median rank 36), 15\% miss the file, and 8\% lack coverage (log~\S8.18). Ordering, not +recall, is the bottleneck --- which is why a reranker of a short list is the natural next stage. + +\paragraph{Why holdouts.} +Two of the ideas that looked best on looked-at sets (the cross-encoder on short questions and the +neighbour vote) lost on fresh data, one decided on looked-at sets (S1) was confirmed on a fresh one, +and the gains that did transfer (definition ranking, named files, traceback vote) transferred with +the predicted sign. None of the competitor papers we read reports a pre-registered holdout; on +benchmarks where agent numbers move by 0.09 between re-runs, we think they should. + +% --------------------------------------------------------------------------------------------- +\section{Threats to validity} +\label{sec:threats} + +\paragraph{Missing LLM stage.} +We do not measure an LLM on top of the engine. A local 3B model was rejected on dev; an API model run +was blocked by unfunded keys at the time of writing. The claim that the engine is a good first stage +for an LLM is supported only by the dense vote and by the 45\% ordering misses. + +\paragraph{Tuning-split mismatch.} +Dev and SWE-Gym differ from the holdouts in gold size (3.4--3.7 gold functions vs 1.17 on Lite), so +tuning-split numbers are lower and function-level gains transfer larger than predicted. A split that +resembles Lite would make the bar more sensitive. + +\paragraph{Forking paths.} +The dense vote failed the bar on dev and was re-measured on SWE-Gym, a second look at the same idea. +We mitigated this by keeping its weight unchanged, committing the decision before the Lite run, and +reporting the dev result; the Lite effect (CI [0.004, 0.051], sign test $p=0.065$) is modest. + +\paragraph{Comparisons across papers.} +All competitor numbers are copied from their papers; our 274 instances are rebuilt from LocAgent's +definition and match its count exactly, but we did not re-run the competitors. Our own BM25 (0.299 +file \accat{1}) is below LocAgent's (0.387) on the same instances, so ``gain over BM25'' depends on +which BM25; we report both. Agent numbers vary by model and run. + +\paragraph{Small plain-language sets.} +Twelve questions per repository, one annotator, gold written by the authors. MULocBench-style +non-code issues are not covered. + +\paragraph{Engineering.} +Timing is from a single Windows laptop shared with other jobs; harness failures (blobless clones, +path-length limits, worktree lock contention) cost instances in early runs and are reported where +they occurred. Benchmark data may appear in the pre-training of the encoders we compare against; the +engine itself has no learned weights, and the dense vote uses an off-the-shelf encoder not trained on +SWE-bench. + +% --------------------------------------------------------------------------------------------- +\section{Conclusion} +\label{sec:conclusion} + +A model-free engine on a laptop CPU localises SWE-bench issues at file \accat{1} 0.507 (Lite, 276 +held-out issues) and 0.435 (Verified outside Lite), more than doubles BM25 on seven repositories it +never saw, and lists every edited function in its top five on 0.460 of LocAgent's Lite instances --- +0.485 when only its top 30 definitions are embedded, on CPU --- in a few seconds and no tokens. It is below trained dense +retrievers and LLM agents. The holdout protocol mattered: two ideas that looked best on looked-at data +lost on fresh data, and most literature priors we tried did not clear a bar fixed before measuring. +What remains: an LLM stage on top of \texttt{where\_to\_look} measured in tokens and accuracy, and +plain-language holdouts of at least fifty questions with a second annotator. + +\paragraph{Availability.} +Engine, harness, gold files, result files and the scripts that rebuild every table are open source +\citep{neuromesh}. \todo{archive the release and result files (Zenodo DOI) and cite the exact tag} + +\paragraph{Acknowledgements.} +\nm{} is a fork of the open-source NeuroMesh project by Pinoox (\url{https://github.com/pinoox/neuromesh}). +\todo{funding and people}. Development of the engine and of this paper used an LLM coding +assistant; no LLM runs at query time in any reported result. \todo{confirm disclosure wording} + +\bibliographystyle{plainnat} +\bibliography{refs} + +\end{document} diff --git a/paper/refs.bib b/paper/refs.bib new file mode 100644 index 0000000..b30b7f4 --- /dev/null +++ b/paper/refs.bib @@ -0,0 +1,411 @@ +% Bibliography for "Where to look". Author lists and titles of every arXiv entry were +% read from the arXiv abstract page (citation_* meta tags) on 2026-10-17. Venues follow +% docs/research/related-work-2026.md §2 (how each venue was confirmed is noted there). +% Entries without an arXiv id (classic IR / bug localisation) are cited from their venue. + +% ---------------------------------------------------------------- benchmarks +@inproceedings{jimenez2024swebench, + title = {{SWE}-bench: Can Language Models Resolve Real-World {GitHub} Issues?}, + author = {Jimenez, Carlos E. and Yang, John and Wettig, Alexander and Yao, Shunyu and Pei, Kexin and Press, Ofir and Narasimhan, Karthik}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year = {2024}, + eprint = {2310.06770}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + url = {https://arxiv.org/abs/2310.06770} +} + +@inproceedings{pan2025swegym, + title = {Training Software Engineering Agents and Verifiers with {SWE-Gym}}, + author = {Pan, Jiayi and Wang, Xingyao and Neubig, Graham and Jaitly, Navdeep and Ji, Heng and Suhr, Alane and Zhang, Yizhe}, + booktitle = {International Conference on Machine Learning (ICML)}, + year = {2025}, + eprint = {2412.21139}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2412.21139} +} + +@misc{zhang2025mulocbench, + title = {A Benchmark for Localizing Code and Non-Code Issues in Software Projects}, + author = {Zhang, Zejun and Wang, Jian and Yang, Qingyun and Pan, Yifan and Tang, Yi and Li, Yi and Xing, Zhenchang and Zhang, Tian and Li, Xuandong and Zhang, Guoan}, + year = {2025}, + eprint = {2509.25242}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2509.25242} +} + +% ---------------------------------------------------------------- LLM localisers and agents +@inproceedings{chen2025locagent, + title = {{LocAgent}: Graph-Guided {LLM} Agents for Code Localization}, + author = {Chen, Zhaoling and Tang, Xiangru and Deng, Gangda and Wu, Fang and Wu, Jialong and Jiang, Zhiwei and Prasanna, Viktor and Cohan, Arman and Wang, Xingyao}, + booktitle = {Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (ACL)}, + year = {2025}, + eprint = {2503.09089}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://aclanthology.org/2025.acl-long.426} +} + +@inproceedings{xia2025agentless, + title = {Agentless: Demystifying {LLM}-based Software Engineering Agents}, + author = {Xia, Chunqiu Steven and Deng, Yinlin and Dunn, Soren and Zhang, Lingming}, + booktitle = {ACM International Conference on the Foundations of Software Engineering (FSE)}, + year = {2025}, + eprint = {2407.01489}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2407.01489} +} + +@inproceedings{yang2024sweagent, + title = {{SWE-agent}: Agent-Computer Interfaces Enable Automated Software Engineering}, + author = {Yang, John and Jimenez, Carlos E. and Wettig, Alexander and Lieret, Kilian and Yao, Shunyu and Narasimhan, Karthik and Press, Ofir}, + booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, + year = {2024}, + eprint = {2405.15793}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2405.15793} +} + +@inproceedings{wang2025openhands, + title = {{OpenHands}: An Open Platform for {AI} Software Developers as Generalist Agents}, + author = {Wang, Xingyao and Li, Boxuan and Song, Yufan and Xu, Frank F. and Tang, Xiangru and Zhuge, Mingchen and Pan, Jiayi and Song, Yueqi and Li, Bowen and Singh, Jaskirat and Tran, Hoang H. and Li, Fuqiang and Ma, Ren and Zheng, Mingzhang and Qian, Bill and Shao, Yanjun and Muennighoff, Niklas and Zhang, Yizhe and Hui, Binyuan and Lin, Junyang and Brennan, Robert and Peng, Hao and Ji, Heng and Neubig, Graham}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year = {2025}, + eprint = {2407.16741}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2407.16741} +} + +@misc{moatless, + title = {{MoatlessTools}}, + author = {{Aide.dev}}, + year = {2024}, + howpublished = {\url{https://github.com/aorwall/moatless-tools}}, + note = {Numbers as reported in LocAgent, Table~4} +} + +@inproceedings{jiang2025cosil, + title = {Issue Localization via {LLM}-Driven Iterative Code Graph Searching}, + author = {Jiang, Zhonghao and Ren, Xiaoxue and Yan, Meng and Jiang, Wei and Li, Yong and Liu, Zhongxin}, + booktitle = {IEEE/ACM International Conference on Automated Software Engineering (ASE)}, + year = {2025}, + eprint = {2503.22424}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2503.22424} +} + +@inproceedings{yu2025orcaloca, + title = {{OrcaLoca}: An {LLM} Agent Framework for Software Issue Localization}, + author = {Yu, Zhongming and Zhang, Hejia and Zhao, Yujie and Huang, Hanxian and Yao, Matrix and Ding, Ke and Zhao, Jishen}, + booktitle = {International Conference on Machine Learning (ICML)}, + year = {2025}, + eprint = {2502.00350}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2502.00350} +} + +@inproceedings{ouyang2025repograph, + title = {{RepoGraph}: Enhancing {AI} Software Engineering with Repository-level Code Graph}, + author = {Ouyang, Siru and Yu, Wenhao and Ma, Kaixin and Xiao, Zilin and Zhang, Zhihan and Jia, Mengzhao and Han, Jiawei and Zhang, Hongming and Yu, Dong}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year = {2025}, + eprint = {2410.14684}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2410.14684} +} + +@misc{yang2025kgcompass, + title = {Enhancing Repository-Level Software Repair via Repository-Aware Knowledge Graphs}, + author = {Yang, Boyang and Ren, Jiadong and Jin, Shunfu and Liu, Yang and Liu, Feng and Le, Bach and Tian, Haoye}, + year = {2025}, + eprint = {2503.21710}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + note = {KGCompass}, + url = {https://arxiv.org/abs/2503.21710} +} + +@misc{wang2025repomem, + title = {Improving Code Localization with Repository Memory}, + author = {Wang, Boshi and Xu, Weijian and Li, Yunsheng and Gao, Mei and Xie, Yujia and Sun, Huan and Chen, Dongdong}, + year = {2025}, + eprint = {2510.01003}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2510.01003} +} + +@inproceedings{ma2026tooltrain, + title = {Tool-integrated Reinforcement Learning for Repo Deep Search}, + author = {Ma, Zexiong and Peng, Chao and Zeng, Qunhong and Gao, Pengfei and Zou, Yanzhen and Xie, Bing}, + booktitle = {IEEE/ACM International Conference on Software Engineering (ICSE)}, + year = {2026}, + eprint = {2508.03012}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + note = {ToolTrain / RepoSearcher}, + url = {https://arxiv.org/abs/2508.03012} +} + +@misc{sutawika2026codescout, + title = {{CodeScout}: An Effective Recipe for Reinforcement Learning of Code Search Agents}, + author = {Sutawika, Lintang and Soni, Aditya Bharat and R, Bharath Sriraam R and Gandhi, Apurva and Yassine, Taha and Vijayvargiya, Sanidhya and Li, Yuchen and Zhou, Xuhui and Zhang, Yilin and Maben, Leander Melroy and Neubig, Graham}, + year = {2026}, + eprint = {2603.17829}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2603.17829} +} + +@inproceedings{xu2026fusesearch, + title = {Learning Adaptive Parallel Execution for Efficient Code Localization}, + author = {Xu, Ke and Xiao, Siyang and Liang, Ming and Yu, Yichen and Wang, Zhixiang and Xu, Jingxuan and Chen, Dajun and Jiang, Wei and Li, Yong}, + booktitle = {Findings of the Association for Computational Linguistics (ACL)}, + year = {2026}, + eprint = {2601.19568}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + note = {FuseSearch}, + url = {https://arxiv.org/abs/2601.19568} +} + +@misc{he2026sweadept, + title = {{SWE-Adept}: An {LLM}-Based Agentic Framework for Deep Codebase Analysis and Structured Issue Resolution}, + author = {He, Kang and Roy, Kaushik}, + year = {2026}, + eprint = {2603.01327}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2603.01327} +} + +@inproceedings{tamoyan2026sherloc, + title = {{SHERLOC}: Structured Diagnostic Localization for Code Repair Agents}, + author = {Tamoyan, Hovhannes and Narenthiran, Sean and Arakelyan, Erik and Mezini, Mira and Ginsburg, Boris}, + booktitle = {Conference on Empirical Methods in Natural Language Processing (EMNLP)}, + year = {2026}, + eprint = {2606.24820}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2606.24820} +} + +@inproceedings{xie2025swefixer, + title = {{SWE-Fixer}: Training Open-Source {LLMs} for Effective and Efficient {GitHub} Issue Resolution}, + author = {Xie, Chengxing and Li, Bowen and Gao, Chang and Du, He and Lam, Wai and Zou, Difan and Chen, Kai}, + booktitle = {Findings of the Association for Computational Linguistics (ACL)}, + year = {2025}, + eprint = {2501.05040}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://aclanthology.org/2025.findings-acl.62} +} + +% ---------------------------------------------------------------- dense retrieval, rerankers, models +@inproceedings{suresh2025cornstack, + title = {{CoRNStack}: High-Quality Contrastive Data for Better Code Retrieval and Reranking}, + author = {Suresh, Tarun and Reddy, Revanth Gangi and Xu, Yifei and Nussbaum, Zach and Mulyar, Andriy and Duderstadt, Brandon and Ji, Heng}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year = {2025}, + eprint = {2412.01007}, + archivePrefix = {arXiv}, + primaryClass = {cs.IR}, + note = {CodeRankEmbed and CodeRankLLM}, + url = {https://arxiv.org/abs/2412.01007} +} + +@inproceedings{reddy2026swerank, + title = {{SweRank}: Software Issue Localization with Code Ranking}, + author = {Reddy, Revanth Gangi and Suresh, Tarun and Doo, JaeHyeok and Liu, Ye and Nguyen, Xuan Phi and Zhou, Yingbo and Yavuz, Semih and Xiong, Caiming and Ji, Heng and Joty, Shafiq}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year = {2026}, + eprint = {2505.07849}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2505.07849} +} + +@misc{reddy2025swerankplus, + title = {{SweRank+}: Multilingual, Multi-Turn Code Ranking for Software Issue Localization}, + author = {Reddy, Revanth Gangi and Liu, Ye and Zhao, Wenting and Doo, JaeHyeok and Suresh, Tarun and Lee, Daniel and Xiong, Caiming and Zhou, Yingbo and Yavuz, Semih and Joty, Shafiq}, + year = {2025}, + eprint = {2512.20482}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2512.20482} +} + +@inproceedings{fehr2025coret, + title = {{CoRet}: Improved Retriever for Code Editing}, + author = {Fehr, Fabio and Sivaprasad, Prabhu Teja and Franceschi, Luca and Zappella, Giovanni}, + booktitle = {Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (ACL), Short Papers}, + year = {2025}, + eprint = {2505.24715}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2505.24715} +} + +@article{chakraborty2025blaze, + title = {{BLAZE}: Cross-Language and Cross-Project Bug Localization via Dynamic Chunking and Hard Example Learning}, + author = {Chakraborty, Partha and Alfadel, Mahmoud and Nagappan, Meiyappan}, + journal = {IEEE Transactions on Software Engineering}, + year = {2025}, + doi = {10.1109/TSE.2025.3579574}, + eprint = {2407.17631}, + archivePrefix = {arXiv}, + primaryClass = {cs.SE}, + url = {https://arxiv.org/abs/2407.17631} +} + +@misc{gunther2023jina, + title = {Jina Embeddings 2: 8192-Token General-Purpose Text Embeddings for Long Documents}, + author = {G{\"u}nther, Michael and Ong, Jackmin and Mohr, Isabelle and Abdessalem, Alaeddine and Abel, Tanguy and Akram, Mohammad Kalim and Guzman, Susana and Mastrapas, Georgios and Sturua, Saba and Wang, Bo and Werk, Maximilian and Wang, Nan and Xiao, Han}, + year = {2023}, + eprint = {2310.19923}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {The jina-embeddings-v2-base-code model used in this paper belongs to this family}, + url = {https://arxiv.org/abs/2310.19923} +} + +@misc{wang2022e5, + title = {Text Embeddings by Weakly-Supervised Contrastive Pre-training}, + author = {Wang, Liang and Yang, Nan and Huang, Xiaolong and Jiao, Binxing and Yang, Linjun and Jiang, Daxin and Majumder, Rangan and Wei, Furu}, + year = {2022}, + eprint = {2212.03533}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + url = {https://arxiv.org/abs/2212.03533} +} + +@inproceedings{zhang2024codesage, + title = {Code Representation Learning at Scale}, + author = {Zhang, Dejiao and Ahmad, Wasi and Tan, Ming and Ding, Hantian and Nallapati, Ramesh and Roth, Dan and Ma, Xiaofei and Xiang, Bing}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year = {2024}, + eprint = {2402.01935}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {CodeSage}, + url = {https://arxiv.org/abs/2402.01935} +} + +@misc{hui2024qwen25coder, + title = {{Qwen2.5-Coder} Technical Report}, + author = {Hui, Binyuan and Yang, Jian and Cui, Zeyu and Yang, Jiaxi and Liu, Dayiheng and Zhang, Lei and Liu, Tianyu and Zhang, Jiajun and Yu, Bowen and Lu, Keming and others}, + year = {2024}, + eprint = {2409.12186}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + url = {https://arxiv.org/abs/2409.12186} +} + +% ---------------------------------------------------------------- ranking machinery +@inproceedings{robertson2004bm25f, + title = {Simple {BM25} Extension to Multiple Weighted Fields}, + author = {Robertson, Stephen and Zaragoza, Hugo and Taylor, Michael}, + booktitle = {Proceedings of the 13th ACM International Conference on Information and Knowledge Management (CIKM)}, + pages = {42--49}, + year = {2004}, + doi = {10.1145/1031171.1031181} +} + +@inproceedings{cormack2009rrf, + title = {Reciprocal Rank Fusion Outperforms {Condorcet} and Individual Rank Learning Methods}, + author = {Cormack, Gordon V. and Clarke, Charles L. A. and B{\"u}ttcher, Stefan}, + booktitle = {Proceedings of the 32nd International ACM SIGIR Conference on Research and Development in Information Retrieval}, + pages = {758--759}, + year = {2009}, + doi = {10.1145/1571941.1572114} +} + +@article{bruch2023fusion, + title = {An Analysis of Fusion Functions for Hybrid Retrieval}, + author = {Bruch, Sebastian and Gai, Siyu and Ingber, Amir}, + journal = {ACM Transactions on Information Systems}, + volume = {42}, + number = {1}, + year = {2023}, + doi = {10.1145/3596512}, + eprint = {2210.11934}, + archivePrefix = {arXiv}, + primaryClass = {cs.IR}, + url = {https://arxiv.org/abs/2210.11934} +} + +@inproceedings{abduljaleel2004rm3, + title = {{UMass} at {TREC} 2004: Novelty and {HARD}}, + author = {Abdul-Jaleel, Nasreen and Allan, James and Croft, W. Bruce and Diaz, Fernando and Larkey, Leah and Li, Xiaoyan and Smucker, Mark D. and Wade, Courtney}, + booktitle = {Proceedings of the Thirteenth Text REtrieval Conference (TREC)}, + year = {2004}, + note = {Origin of the RM3 relevance model} +} + +@book{efron1993bootstrap, + title = {An Introduction to the Bootstrap}, + author = {Efron, Bradley and Tibshirani, Robert J.}, + publisher = {Chapman \& Hall}, + year = {1993} +} + +% ---------------------------------------------------------------- classic bug localisation +@inproceedings{zhou2012buglocator, + title = {Where Should the Bugs Be Fixed? More Accurate Information Retrieval-Based Bug Localization Based on Bug Reports}, + author = {Zhou, Jian and Zhang, Hongyu and Lo, David}, + booktitle = {Proceedings of the 34th International Conference on Software Engineering (ICSE)}, + pages = {14--24}, + year = {2012}, + doi = {10.1109/ICSE.2012.6227210} +} + +@inproceedings{wen2016locus, + title = {Locus: Locating Bugs from Software Changes}, + author = {Wen, Ming and Wu, Rongxin and Cheung, Shing-Chi}, + booktitle = {Proceedings of the 31st IEEE/ACM International Conference on Automated Software Engineering (ASE)}, + pages = {262--273}, + year = {2016}, + doi = {10.1145/2970276.2970359} +} + +@inproceedings{rahman2018query, + title = {Improving {IR}-Based Bug Localization with Context-Aware Query Reformulation}, + author = {Rahman, Mohammad Masudur and Roy, Chanchal K.}, + booktitle = {Proceedings of the 26th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC/FSE)}, + pages = {621--632}, + year = {2018}, + doi = {10.1145/3236024.3236065} +} + +% ---------------------------------------------------------------- tools +@misc{aider_repomap, + title = {Building a better repository map with tree-sitter}, + author = {Gauthier, Paul}, + year = {2023}, + howpublished = {\url{https://aider.chat/2023/10/22/repomap.html}}, + note = {Aider's RepoMap} +} + +@misc{llamacpp, + title = {llama.cpp}, + author = {Gerganov, Georgi and others}, + year = {2023}, + howpublished = {\url{https://github.com/ggml-org/llama.cpp}}, + note = {Release b11172 used in this paper} +} + +@misc{neuromesh, + author = {{NeuroMesh contributors}}, + title = {{NeuroMesh}: a local code-context engine (release v1.4.0)}, + year = {2026}, + howpublished = {\url{https://github.com/ParsaVictor/code-context-engine}}, + note = {Fork of \url{https://github.com/pinoox/neuromesh}} +} diff --git a/paper/tables/ablation.tex b/paper/tables/ablation.tex new file mode 100644 index 0000000..8b997ea --- /dev/null +++ b/paper/tables/ablation.tex @@ -0,0 +1,20 @@ +% Ablation, v1.2.0 engine, Lite strict holdout (276); contributions-log sec. 8.11. +\begin{table}[t] +\centering +\small +\caption{Ablation on the Lite strict holdout (276 issues), one run per removed component, engine +v1.2.0 (before the named-files and traceback votes). File Acc@$k$.} +\label{tab:ablation} +\begin{tabular}{@{}lcccc@{}} +\toprule +removed & @1 & @3 & @5 & @10\\ +\midrule +nothing (v1.2.0) & 0.486 & 0.710 & 0.750 & 0.804\\ +report hygiene + code tokens & 0.467 & 0.707 & 0.743 & 0.801\\ +definition-level ranking & \textbf{0.388} & \textbf{0.569} & \textbf{0.652} & 0.764\\ +localisation list (packet order only) & 0.366 & 0.536 & 0.558 & --\\ +\midrule +plain BM25, for reference & 0.301 & 0.507 & 0.587 & 0.721\\ +\bottomrule +\end{tabular} +\end{table} diff --git a/paper/tables/cost.tex b/paper/tables/cost.tex new file mode 100644 index 0000000..dc4e725 --- /dev/null +++ b/paper/tables/cost.tex @@ -0,0 +1,28 @@ +% Cost per issue; ours: contributions-log sec. 8.1, sec. 8.2, sec. 8.9, sec. 8.13, sec. 8.18; +% others: related-work-2026.md sec. 1-sec. 2 (SweRank T5, LocAgent T5, Agentless T1). +\begin{table}[t] +\centering +\small +\caption{Cost of localising one issue. Ours measured on a 6-core laptop CPU (Ryzen~5 7530U), cold +index included, no network and no GPU. Others as reported by their authors.} +\label{tab:cost} +\setlength{\tabcolsep}{3pt} +\begin{tabular}{@{}p{0.52\columnwidth}p{0.42\columnwidth}@{}} +\toprule +setting & cost\\ +\midrule +NeuroMesh, Lite & p50 2.9\,s, p90 10.6\,s; 0 tokens\\ +NeuroMesh, Verified & p50 2.5\,s, p90 8.8\,s; 0 tokens\\ +\quad index build, django (3.5k files) & 26\,s cold\\ +\quad + dense short-list vote & +$\approx$6\,s p50 (30 definitions + issue)\\ +Whole-repository embedding, CPU & 2.6 chunks/s (jina-code-v2)\\ +Local 3B LLM judge, CPU & $\approx$105\,s per issue\\ +\midrule +SweRankEmbed-L + SweRankLLM-Small (7B) & A100, 12.5\,s, \$0.011\\ +LocAgent + Qwen2.5-7B (ft) & GPU, \$0.05\\ +LocAgent + Claude-3.5 & \$0.66\\ +Agentless + GPT-4o & \$0.70, 78k tokens\\ +SWE-Adept + Claude-Sonnet-4.5 & 348k tokens\\ +\bottomrule +\end{tabular} +\end{table} diff --git a/paper/tables/dense.tex b/paper/tables/dense.tex new file mode 100644 index 0000000..0954f0d --- /dev/null +++ b/paper/tables/dense.tex @@ -0,0 +1,38 @@ +% Dense short-list vote: dev near miss (sec. 8.16), SWE-Gym pass + pooled test, decision locked in +% 608acb4, then Lite once (sec. 8.18). Verified still being measured. +\begin{table}[t] +\centering +\small +\caption{Opt-in dense short-list vote (jina-code-v2, int8, CPU; the issue and the engine's top-30 +definitions are embedded, their cosine order votes by RRF at $w=0.5$). Function Acc@$k$, engine $\to$ +engine + vote. The weight and $k$ were committed before the Lite and Verified runs. Paired +differences: bootstrap 95\% CI, issues gained/lost at @5, two-sided sign test.} +\label{tab:dense} +\setlength{\tabcolsep}{3pt} +\begin{tabular}{@{}lrccc@{}} +\toprule +set & $n$ & @1 & @5 & @10\\ +\midrule +dev-fast (tuning) & 52 & 0.173\,$\to$\,0.212 & 0.308\,$\to$\,0.327 & --\\ +dev-rest (tuning) & 158 & 0.196\,$\to$\,0.215 & 0.329\,$\to$\,0.342 & --\\ +gym-a (tuning) & 140 & 0.152\,$\to$\,0.167 & 0.258\,$\to$\,0.288 & 0.333\,$\to$\,0.364\\ +gym-b (tuning) & 140 & 0.112\,$\to$\,0.134 & 0.254\,$\to$\,0.313 & 0.366\,$\to$\,0.388\\ +\addlinespace +\multicolumn{5}{@{}l}{dev + gym pooled, $n=476$: @5 0.286\,$\to$\,0.317, $+0.032$ [0.015,\,0.050],}\\ +\multicolumn{5}{@{}l}{\quad 17 gained / 2 lost, $p=0.0007$}\\ +\midrule +\multicolumn{5}{@{}l}{\emph{Holdout, run once with the locked decision}}\\ +Lite, LocAgent subset & 274 & 0.223\,$\to$\,\textbf{0.252} & 0.460\,$\to$\,\textbf{0.485} & 0.540\,$\to$\,\textbf{0.555}\\ +Lite, strict & 250 & 0.228\,$\to$\,0.252 & 0.472\,$\to$\,0.496 & 0.556\,$\to$\,0.568\\ +\addlinespace +\multicolumn{5}{@{}l}{Lite 274 paired: @5 $+0.026$ [0.004,\,0.051], 9 gained / 2 lost, $p=0.065$;}\\ +\multicolumn{5}{@{}l}{\quad @1 $+0.029$ (18 / 10)}\\ +\addlinespace +Verified & 459 & \multicolumn{3}{c}{\todo{locked-decision run, being measured}}\\ +\bottomrule +\end{tabular} +\par\smallskip +{\footnotesize Lite numbers come from the Python prototype applied to the stored v1.4.0 run. The Rust +port (\texttt{NM\_DENSE=1}) reproduces the prototype order up to near-tie swaps; on SWE-Gym it gives +@5 0.295 (gym-a) and 0.299 (gym-b).} +\end{table} diff --git a/paper/tables/holdouts.tex b/paper/tables/holdouts.tex new file mode 100644 index 0000000..daf4b1c --- /dev/null +++ b/paper/tables/holdouts.tex @@ -0,0 +1,35 @@ +% File and function level on every SWE-bench-style set, BM25 vs engine. +% contributions-log sec. 8.13 (file, v1.3.0 = v1.4.0 at file level per sec. 8.17), sec. 8.17 (function, v1.4.0), +% sec. 8.18 (SWE-Gym), paper-draft sec. 5.2 (dev). +\begin{table*}[t] +\centering +\small +\caption{File- and function-level localisation on four holdouts and the tuning split. Engine = +NeuroMesh v1.4.0, one run per set (file lists are identical to v1.3.0 on every instance). Function +columns count only instances with function gold ($n_f$); an instance with no function list counts as a +miss. 95\% bootstrap CI for file Acc@1 in brackets.} +\label{tab:holdouts} +\setlength{\tabcolsep}{4pt} +\resizebox{\textwidth}{!}{% +\begin{tabular}{@{}lr cccc cccc r ccc@{}} +\toprule + & & \multicolumn{4}{c}{BM25, file Acc@$k$} & \multicolumn{4}{c}{engine, file Acc@$k$} & & \multicolumn{3}{c}{engine, function Acc@$k$}\\ +\cmidrule(lr){3-6}\cmidrule(lr){7-10}\cmidrule(l){12-14} +set & $n$ & @1 & @3 & @5 & @10 & @1 & @3 & @5 & @10 & $n_f$ & @1 & @5 & @10\\ +\midrule +\multicolumn{14}{@{}l}{\emph{Holdouts (never tuned on; run once per released version)}}\\ +Lite, strict & 276 & 0.301 & 0.507 & 0.587 & 0.721 & \textbf{0.507} {\scriptsize[0.45,0.57]} & 0.717 & 0.750 & 0.815 & 250 & 0.228 & 0.472 & 0.556\\ +Lite, LocAgent subset & 274 & 0.299 & 0.522 & 0.606 & 0.734 & \textbf{0.500} {\scriptsize[0.44,0.56]} & 0.723 & 0.755 & 0.828 & 274 & 0.223 & 0.460 & 0.540\\ +Verified, all & 500 & 0.216 & 0.392 & 0.490 & 0.642 & \textbf{0.446} {\scriptsize[0.40,0.49]} & 0.680 & 0.736 & 0.814 & 459 & 0.198 & 0.429 & 0.497\\ +Verified, not in Lite & 407 & 0.194 & 0.373 & 0.477 & 0.622 & \textbf{0.435} {\scriptsize[0.38,0.48]} & 0.671 & 0.732 & 0.811 & 375 & 0.189 & 0.397 & 0.464\\ +SWE-Gym, gym-a & 140 & 0.121 & 0.257 & 0.329 & 0.407 & \textbf{0.279} {\scriptsize[0.21,0.36]} & 0.450 & 0.536 & 0.650 & -- & 0.152 & 0.258 & 0.333\\ +SWE-Gym, gym-b & 140 & 0.121 & 0.271 & 0.307 & 0.436 & \textbf{0.271} {\scriptsize[0.20,0.35]} & 0.414 & 0.493 & 0.600 & -- & 0.112 & 0.254 & 0.366\\ +\midrule +\multicolumn{14}{@{}l}{\emph{Tuning split (looked at)}}\\ +SWE-bench dev & 225 & 0.160 & 0.347 & 0.427 & 0.538 & 0.308 & 0.527 & 0.621 & 0.692 & 210 & 0.190 & 0.324 & 0.390\\ +\bottomrule +\end{tabular}} +\par\smallskip +{\footnotesize SWE-Gym: 266 of the 280 issues have function gold; the per-half split of $n_f$ is not +recorded in the log.} +\end{table*} diff --git a/paper/tables/lite274.tex b/paper/tables/lite274.tex new file mode 100644 index 0000000..0385730 --- /dev/null +++ b/paper/tables/lite274.tex @@ -0,0 +1,52 @@ +% Head-to-head on LocAgent's 274-instance SWE-bench Lite subset (same instances, same metric). +% Ours: contributions-log sec. 8.7, sec. 8.13, sec. 8.17, sec. 8.18. Others: related-work-2026.md sec. 1 +% (LocAgent Table 4, SweRank Table 1/5), rounded to three decimals. +\begin{table*}[t] +\centering +\small +\caption{SWE-bench Lite, LocAgent's 274 instances (patch edits an existing function); Acc@$k$ = all +gold locations in the top $k$. Rows marked \dag{} are copied from LocAgent~\citep[Table~4]{chen2025locagent} +and SweRank~\citep[Tables~1 and~5]{reddy2026swerank}. ``Model'' is what runs at query time; ``index'' is +what must be computed per repository before the first query. Our rows are a single run of a released +binary on a holdout (\S\ref{sec:protocol}); 95\% bootstrap CIs in brackets. The dense vote changes only +the function list, so its file columns equal the engine's.} +\label{tab:lite274} +\setlength{\tabcolsep}{4pt} +\resizebox{\textwidth}{!}{% +\begin{tabular}{@{}llcccccl@{}} +\toprule + & & \multicolumn{3}{c}{file Acc@$k$} & \multicolumn{2}{c}{function Acc@$k$} & \\ +\cmidrule(lr){3-5}\cmidrule(lr){6-7} +method & model at query time / index & @1 & @3 & @5 & @5 & @10 & cost per issue \\ +\midrule +\multicolumn{8}{@{}l}{\emph{No learned model}}\\ +BM25\dag & -- / inverted index & 0.387 & 0.518 & 0.617 & 0.318 & 0.369 & -- \\ +BM25 (our implementation) & -- / inverted index & 0.299 & 0.522 & 0.606 & -- & -- & -- \\ +\textbf{NeuroMesh v1.4.0 (ours)} & -- / code graph, 26\,s cold (django) & \textbf{0.500} & \textbf{0.723} & \textbf{0.755} & \textbf{0.460} & \textbf{0.540} & CPU, p50 2.9\,s, 0 tokens \\ + & & {\scriptsize[0.44,0.56]} & & & {\scriptsize[0.40,0.52]} & & \\ +\midrule +\multicolumn{8}{@{}l}{\emph{Short-list embedding (30 definitions per issue, no repository embedding)}}\\ +\textbf{NeuroMesh v1.4.0 + dense vote (ours)} & 161M encoder, CPU / code graph & 0.500 & 0.723 & 0.755 & \textbf{0.485} & \textbf{0.555} & CPU, +$\approx$6\,s p50 \\ + & & & & & {\scriptsize[0.43,0.54]} & & \\ +\midrule +\multicolumn{8}{@{}l}{\emph{Dense retrievers (embed every function of the repository)}}\\ +CodeSage-large-v2 (1.3B)~\citep{zhang2024codesage}\dag & -- / all functions & 0.478 & 0.693 & 0.781 & 0.339 & 0.445 & GPU in practice \\ +Jina-Code-v2 (161M)~\citep{gunther2023jina}\dag & -- / all functions & 0.434 & 0.712 & 0.803 & 0.423 & 0.522 & -- \\ +E5-base-v2 (110M)~\citep{wang2022e5}\dag & -- / all functions & 0.496 & 0.745 & 0.803 & 0.394 & 0.511 & -- \\ +CodeRankEmbed (137M)\dag & -- / all functions & 0.526 & 0.777 & 0.847 & 0.518 & 0.588 & -- \\ +SweRankEmbed-Small (137M)\dag & -- / all functions & 0.664 & 0.865 & 0.909 & 0.631 & 0.745 & -- \\ +SweRankEmbed-Large (7B)\dag & -- / all functions & 0.726 & 0.912 & 0.942 & 0.719 & 0.821 & GPU \\ +\midrule +\multicolumn{8}{@{}l}{\emph{Retriever + LLM reranker}}\\ +SweRankEmbed-L + SweRankLLM-L\dag & 32B reranker, GPU / all functions & 0.832 & 0.949 & 0.960 & 0.814 & 0.887 & \$0.015 \\ +\midrule +\multicolumn{8}{@{}l}{\emph{LLM agents}}\\ +SWE-agent + GPT-4o\dag & API LLM / -- & 0.573 & 0.650 & 0.690 & 0.460 & 0.464 & \$0.56 \\ +OpenHands + GPT-4o\dag & API LLM / -- & 0.610 & 0.719 & 0.737 & 0.496 & 0.504 & \$0.83 \\ +Agentless + GPT-4o\dag & API LLM / -- & 0.672 & 0.745 & 0.745 & 0.555 & 0.555 & -- \\ +Agentless + Claude-3.5\dag & API LLM / -- & 0.726 & 0.792 & 0.796 & 0.588 & 0.588 & -- \\ +LocAgent + Qwen2.5-7B (ft)\dag & 7B LLM, GPU / code graph & 0.708 & 0.847 & 0.883 & 0.642 & 0.715 & \$0.05 \\ +LocAgent + Claude-3.5\dag & API LLM / code graph & 0.777 & 0.920 & 0.942 & 0.734 & 0.774 & \$0.66 \\ +\bottomrule +\end{tabular}} +\end{table*} diff --git a/paper/tables/negative.tex b/paper/tables/negative.tex new file mode 100644 index 0000000..4396ebe --- /dev/null +++ b/paper/tables/negative.tex @@ -0,0 +1,30 @@ +% Negative results; contributions-log sec. 8.1, sec. 8.3, sec. 8.4, sec. 8.6, sec. 8.9, sec. 8.13, sec. 8.14, sec. 8.15, sec. 8.16, sec. 8.18. +\begin{table*}[t] +\centering +\small +\caption{Ideas that did not survive the protocol, with the measurement that rejected them. ``dev'' = +SWE-bench dev (tuning split); the bar is file Acc@1 or function Acc@5 $\geq +0.02$ on both halves.} +\label{tab:negative} +\begin{tabular}{@{}p{0.36\textwidth}p{0.20\textwidth}p{0.38\textwidth}@{}} +\toprule +idea (inspiration) & where measured & effect\\ +\midrule +Whole-repository dense chunk retrieval on CPU (CodeRankEmbed, Jina) & astroid, 3,364 chunks & 2.6 chunks/s (1,314\,s for one mid-sized repository); hours for django-sized ones\\ +Cross-encoder rerank of the top 10 files (jina-reranker-v2) & dev-fast, 59 issues & file Acc@3 0.492 $\to$ 0.373 on long reports; 8.0\,s p50 per issue\\ +Same cross-encoder on short questions & ripgrep, click; then fresh cobra & R@3 $+0.375$ / $+0.208$ on the two looked-at sets, $-0.084$ on the fresh one\\ +Graph-neighbour vote of the top three files (LocAgent-style expansion without an LLM) & dev-fast & file Acc@1 0.254 $\to$ 0.136\\ +Body term frequency in BM25F & dev-fast & file Acc@5 0.544 $\to$ 0.509\\ +Commit-history prior (BugLocator, Locus) & dev & file Acc@1 0.271 $\to$ 0.169 (dev-fast) even at weight 0.1\\ +Identifiers in the issue's code blocks $\to$ their definitions & dev & file Acc@1 drops; function Acc@5 $+0.019$ (below the bar)\\ +RM3 pseudo-relevance feedback on the definition ranking & dev & alone: function Acc@1 0.135 $\to$ 0.038 (dev-fast); as a vote no better than plain BM25\\ +Documentation sections as a bridge from question words to identifiers & four plain-language sets & R@3 down on three of four (ripgrep 0.500 $\to$ 0.375, axios 0.833 $\to$ 0.583)\\ +Local 3B LLM as file judge (Qwen2.5-Coder-3B, Agentless-style, one call) & 15 dev-fast issues & copies the engine order (0.200/0.467/0.600 unchanged); shuffled input 0.067/0.400/0.600; $\approx$105\,s per issue\\ +Learned reranker (logistic regression / boosted trees, 25 features, top 100 functions) & dev halves; dev $\to$ SWE-Gym & function Acc@5 $+0.000$ / $+0.019$ cross-half; $+0.000$ trained on dev, applied to SWE-Gym; the file-rank coefficient flips sign ($-0.61$ vs $+1.23$)\\ +\midrule +\multicolumn{3}{@{}l}{\emph{Evaluation slips we made and caught}}\\ +Gold match by plain suffix (no path-segment boundary) & cobra & BM25 R@3 printed 0.958 instead of 0.875\\ +Function denominator without short reports & Lite 274 & 244 instances and Acc@5 0.443 instead of 274 and 0.394 (v1.3.0)\\ +Prototype baseline taken from an older run & dev & every function-level delta re-measured against the true baseline\\ +\bottomrule +\end{tabular} +\end{table*} diff --git a/paper/tables/plain.tex b/paper/tables/plain.tex new file mode 100644 index 0000000..96e5a37 --- /dev/null +++ b/paper/tables/plain.tex @@ -0,0 +1,20 @@ +% Plain-language holdouts, R@3 of the localisation list; contributions-log sec. 8.6, sec. 8.16. +\begin{table}[t] +\centering +\small +\caption{Plain-language questions (12 per repository, gold written from the source and committed +before the first run), R@3 of the localisation list. S1 = fusing the packet order with the +whole-question ranking for short questions; it was chosen on the first three sets.} +\label{tab:plain} +\begin{tabular}{@{}lccc@{}} +\toprule +set (language) & BM25 & before S1 & S1\\ +\midrule +ripgrep (Rust), looked at & 0.500 & 0.208 & 0.500\\ +click (Python), looked at & 0.750 & 0.750 & 0.917\\ +cobra (Go), looked at & 0.875 & 0.792 & 0.875\\ +axios (JS), fresh at its run & 0.583 & 0.833 & 0.833\\ +\textbf{jsoup (Java), fresh, run once} & 0.667 & 0.417 & \textbf{0.750}\\ +\bottomrule +\end{tabular} +\end{table} From 7d6a0d20f25b561da45508511004b29e6aa76546 Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Sat, 17 Oct 2026 09:26:43 +0330 Subject: [PATCH 2/3] paper: fill Verified dense-vote number (459: func Acc@5 0.429 -> 0.458, +0.028 [0.009,0.048], p=0.007) Source: contributions-log.md sec. 8.18 at ce0dceb (PR #147). Abstract, contributions, sec. 5.3, threats and reviewer risk #5 updated to match. Co-Authored-By: Claude Opus 5.5 --- paper/REVIEWER-RISKS.md | 15 +++++++++------ paper/main.tex | 19 +++++++++++++------ paper/tables/dense.tex | 8 ++++++-- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/paper/REVIEWER-RISKS.md b/paper/REVIEWER-RISKS.md index 9faf819..94a9f12 100644 --- a/paper/REVIEWER-RISKS.md +++ b/paper/REVIEWER-RISKS.md @@ -104,21 +104,24 @@ negative results table shows that we report what failed. Without these steps, call it a "commit-locked holdout", not "pre-registered". -## 5. "The dense vote went through forking paths, and p = 0.065 is not significant." +## 5. "The dense vote went through forking paths, and on Lite p = 0.065 is not significant." The vote failed the bar on dev. It was then re-measured on a *new* split until it passed, and only then run on Lite. On Lite, the CI [0.004, 0.051] barely excludes zero, and the sign test (9 issues gained, 2 lost) gives p = 0.065. **Our answer.** The weight and k were not changed between dev and SWE-Gym. The decision was -committed before Lite. The effect is the same direction and size on every split (+0.032 pooled on -dev + gym, +0.026 on Lite). The paper reports the dev failure and states this threat explicitly. It -is opt-in, not the default. +committed before Lite and Verified. The effect is the same direction and size on every split +(+0.032 pooled on dev + gym, +0.026 on Lite, +0.028 on Verified). Verified, run once with the locked +decision, is significant on its own: function Acc@5 0.429 → 0.458, CI [0.009, 0.048], 17 issues +gained and 4 lost, sign test p = 0.007; at @1, +0.039, p = 0.005. The paper reports the dev failure +and states this threat explicitly. It is opt-in, not the default. **What is missing.** -- The Verified run with the locked decision. It is in progress and marked `\todo` in the paper. -- If Verified is flat, the paper must say so, and the vote stays a "promising opt-in". +- The Lite and Verified numbers come from the Python prototype applied to the stored v1.4.0 runs. + A run of the Rust port (`NM_DENSE=1`) on both would remove "prototype vs shipped" as a follow-up + objection. - A Holm correction across the three holdouts (Lite, Verified, Gym-by-port) would pre-empt the multiple-comparisons version of this objection. diff --git a/paper/main.tex b/paper/main.tex index d8280c0..32d51bc 100644 --- a/paper/main.tex +++ b/paper/main.tex @@ -52,7 +52,8 @@ 0.194/0.373/0.477); on 280 SWE-Gym issues from seven repositories it never saw, file \accat{1} 0.279/0.271 on the two halves (BM25 0.121/0.121). At function level it lists every edited function among its first five on 0.460 of LocAgent's 274 Lite instances, and 0.485 with an opt-in vote that -embeds only the engine's top 30 definitions on CPU (paired $+0.026$, 95\% CI [0.004, 0.051]). This +embeds only the engine's top 30 definitions on CPU (paired $+0.026$, 95\% CI [0.004, 0.051]); on +Verified the same vote lifts function \accat{5} from 0.429 to 0.458 ($+0.028$ [0.009, 0.048]). This is below code-trained dense retrievers --- SweRankEmbed-Small reaches 0.664 file \accat{1} and 0.631 function \accat{5} on the same instances --- and well below LLM agents and rerankers. Our contribution is the other corner of the trade-off: no training, no GPU, no whole-repository @@ -102,7 +103,8 @@ \section{Introduction} 0.301), 0.435 on 407 Verified issues outside Lite (BM25 0.194) and $2.2$--$2.3\times$ BM25 on seven unseen SWE-Gym repositories, at 2.9\,s per issue on a laptop CPU (\S\ref{sec:results}). \item \textbf{A cheap dense short-list vote}: embedding only the engine's top 30 definitions lifts - function \accat{5} on Lite from 0.460 to 0.485, with the decision committed before the holdout run + function \accat{5} on Lite from 0.460 to 0.485 and on Verified from 0.429 to 0.458, with the + decision committed before both holdout runs (\S\ref{sec:dense}). \item \textbf{Ablations, costs and negative results with numbers} (\S\ref{sec:ablation}, \S\ref{sec:negative}): a dozen ideas from the bug-localisation and retrieval literature --- commit @@ -307,9 +309,13 @@ \subsection{Dense short-list vote} lost. The CI excludes zero; with only 11 discordant issues the sign test does not ($p=0.065$). The effect is the same direction and size as on the tuning splits. It narrows the gap to CodeRankEmbed (0.518) from $-0.058$ to $-0.033$ and leaves a gap of $-0.146$ to SweRankEmbed-Small, with 30 -embeddings per issue on CPU instead of every function of the repository. On Verified the run with -the locked decision is \todo{Verified function \accat{1}/5/10 with the dense vote (459 instances), -paired $\Delta$@5 with CI and gained/lost --- being measured}. +embeddings per issue on CPU instead of every function of the repository. On Verified (459 issues +with function gold), also run once with the locked decision, function \accat{1}/5/10 went from +0.198/0.429/0.497 to 0.237/0.458/0.514: $+0.028$ at \accat{5}, CI [0.009, 0.048], 17 gained and 4 +lost ($p=0.007$), and $+0.039$ at \accat{1} (28/10, $p=0.005$); on the 375 outside Lite \accat{5} +rose from 0.397 to 0.435. Every split --- dev, both SWE-Gym halves, Lite and Verified --- moves the +same way by $+0.02$ to $+0.06$ at function \accat{5}, and the effect predicted on tuning data +($+0.032$) is what both holdouts show. The vote also re-ranks files: the dense order alone puts the gold file first more often than the engine on dev (0.356 vs 0.307 file \accat{1}) while losing depth, but the file vote did not clear the @@ -434,7 +440,8 @@ \section{Threats to validity} \paragraph{Forking paths.} The dense vote failed the bar on dev and was re-measured on SWE-Gym, a second look at the same idea. We mitigated this by keeping its weight unchanged, committing the decision before the Lite run, and -reporting the dev result; the Lite effect (CI [0.004, 0.051], sign test $p=0.065$) is modest. +reporting the dev result. The effect is modest on each holdout (Lite CI [0.004, 0.051], sign test +$p=0.065$; Verified CI [0.009, 0.048], $p=0.007$), but it has the predicted sign and size on both. \paragraph{Comparisons across papers.} All competitor numbers are copied from their papers; our 274 instances are rebuilt from LocAgent's diff --git a/paper/tables/dense.tex b/paper/tables/dense.tex index 0954f0d..4b0bbb6 100644 --- a/paper/tables/dense.tex +++ b/paper/tables/dense.tex @@ -1,5 +1,5 @@ % Dense short-list vote: dev near miss (sec. 8.16), SWE-Gym pass + pooled test, decision locked in -% 608acb4, then Lite once (sec. 8.18). Verified still being measured. +% 608acb4, then Lite and Verified once (sec. 8.18, log at ce0dceb). \begin{table}[t] \centering \small @@ -28,7 +28,11 @@ \multicolumn{5}{@{}l}{Lite 274 paired: @5 $+0.026$ [0.004,\,0.051], 9 gained / 2 lost, $p=0.065$;}\\ \multicolumn{5}{@{}l}{\quad @1 $+0.029$ (18 / 10)}\\ \addlinespace -Verified & 459 & \multicolumn{3}{c}{\todo{locked-decision run, being measured}}\\ +Verified & 459 & 0.198\,$\to$\,\textbf{0.237} & 0.429\,$\to$\,\textbf{0.458} & 0.497\,$\to$\,\textbf{0.514}\\ +Verified, not in Lite & 375 & 0.189\,$\to$\,0.208 & 0.397\,$\to$\,0.435 & 0.464\,$\to$\,0.483\\ +\addlinespace +\multicolumn{5}{@{}l}{Verified 459 paired: @5 $+0.028$ [0.009,\,0.048], 17 / 4, $p=0.007$;}\\ +\multicolumn{5}{@{}l}{\quad @1 $+0.039$ [0.013,\,0.065], 28 / 10, $p=0.005$; @10 $+0.017$, 8 / 0, $p=0.008$}\\ \bottomrule \end{tabular} \par\smallskip From cb623d3b8b7222984f953b0844a1ac76df117c9c Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Sat, 17 Oct 2026 09:26:57 +0330 Subject: [PATCH 3/3] paper: dense table footnote covers Verified too Co-Authored-By: Claude Opus 5.5 --- paper/tables/dense.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paper/tables/dense.tex b/paper/tables/dense.tex index 4b0bbb6..78e22d6 100644 --- a/paper/tables/dense.tex +++ b/paper/tables/dense.tex @@ -36,7 +36,7 @@ \bottomrule \end{tabular} \par\smallskip -{\footnotesize Lite numbers come from the Python prototype applied to the stored v1.4.0 run. The Rust +{\footnotesize Lite and Verified numbers come from the Python prototype applied to the stored v1.4.0 runs. The Rust port (\texttt{NM\_DENSE=1}) reproduces the prototype order up to near-tie swaps; on SWE-Gym it gives @5 0.295 (gym-a) and 0.299 (gym-b).} \end{table}