diff --git a/crates/neuromesh-cli/src/commands/packet.rs b/crates/neuromesh-cli/src/commands/packet.rs index 6115740..6b31036 100644 --- a/crates/neuromesh-cli/src/commands/packet.rs +++ b/crates/neuromesh-cli/src/commands/packet.rs @@ -113,6 +113,7 @@ pub fn execute(args: &[String]) -> Result<()> { // scored as misses without one). let definitions = neuromesh_context::gold::definition_order(&graph, &prompt, &localization, def_depth()); + let definitions = dense_vote(definitions, &prompt, ¤t_dir); let mut files: Vec = packet_file_names(&view).into_iter().collect(); files.sort(); let reduction = if workspace_tokens > 0 { @@ -531,3 +532,125 @@ mod tests { assert_eq!(parsed.query.as_deref(), Some("Where is HTTPAdapter.send?")); } } + +/// Opt-in dense vote on the function list (`NM_DENSE=1`; session 19, §8.18): +/// jina-code-v2 embeds the issue and the top `DENSE_K` definitions (path, name, +/// body), reorders them by cosine, and that order votes into the list by RRF at +/// weight 0.5. Decided on SWE-bench dev + a SWE-Gym split (func Acc@5 +0.032, +/// p = 0.0007); ~6 s per prompt on CPU, needs `neuromesh install embed jina`. +/// Any failure (model missing, unreadable file) returns the list unchanged. +fn dense_vote( + defs: Vec, + prompt: &str, + root: &std::path::Path, +) -> Vec { + if std::env::var("NM_DENSE").ok().as_deref() != Some("1") || defs.len() < 2 { + return defs; + } + #[cfg(feature = "embeddings")] + { + match dense_order(&defs, prompt, root) { + Ok(order) => return rrf_vote(defs, &order, 0.5), + Err(e) => eprintln!("neuromesh: dense vote skipped ({e})"), + } + } + #[cfg(not(feature = "embeddings"))] + let _ = (prompt, root); + defs +} + +const DENSE_K: usize = 30; + +#[cfg(feature = "embeddings")] +fn dense_order( + defs: &[neuromesh_graph::RankedDefinition], + prompt: &str, + root: &std::path::Path, +) -> std::result::Result, String> { + use neuromesh_core::{EmbeddingConfig, EmbeddingModelId}; + let k = defs.len().min(DENSE_K); + let mut files: std::collections::HashMap<&str, Vec> = Default::default(); + let mut texts = Vec::with_capacity(k + 1); + texts.push(prompt.chars().take(4000).collect::()); + for d in &defs[..k] { + let lines = files.entry(d.path.as_str()).or_insert_with(|| { + std::fs::read_to_string(root.join(&d.path)) + .map(|t| t.lines().map(str::to_string).collect()) + .unwrap_or_default() + }); + let (a, b) = d.lines; + let lo = a.saturating_sub(1).min(lines.len()); + let hi = b.min(lines.len()).max(lo); + let body = lines[lo..hi].join( + " +", + ); + let text = format!( + "{} +{} +{}", + d.path, d.name, body + ); + texts.push(text.chars().take(1500).collect()); + } + let config = EmbeddingConfig { + model: EmbeddingModelId::JinaCodeV2, + matryoshka_dim: EmbeddingModelId::JinaCodeV2.default_matryoshka_dim(), + intra_threads: None, + ..EmbeddingConfig::default() + }; + let mut emb = neuromesh_embed::Embedder::try_new(config).map_err(|e| e.to_string())?; + let vecs = emb.embed_documents(&texts).map_err(|e| e.to_string())?; + let q = &vecs[0]; + let sim = |v: &Vec| v.iter().zip(q).map(|(a, b)| a * b).sum::(); + let mut order: Vec = (0..k).collect(); + order.sort_by(|&i, &j| sim(&vecs[j + 1]).total_cmp(&sim(&vecs[i + 1]))); + Ok(order) +} + +/// Reciprocal-rank fusion (k = 60) of the list's own order (weight 1) and a +/// reordering of its head (`order` indexes into `defs`, weight `w`). +fn rrf_vote( + defs: Vec, + order: &[usize], + w: f32, +) -> Vec { + let mut score: Vec = (0..defs.len()).map(|i| 1.0 / (61.0 + i as f32)).collect(); + for (r, &i) in order.iter().enumerate() { + score[i] += w / (61.0 + r as f32); + } + let mut idx: Vec = (0..defs.len()).collect(); + idx.sort_by(|&a, &b| score[b].total_cmp(&score[a]).then(a.cmp(&b))); + let mut slots: Vec> = + defs.into_iter().map(Some).collect(); + idx.into_iter().filter_map(|i| slots[i].take()).collect() +} + +#[cfg(test)] +mod dense_tests { + use super::*; + + fn d(n: &str) -> neuromesh_graph::RankedDefinition { + neuromesh_graph::RankedDefinition { + path: "a.py".into(), + name: n.into(), + lines: (1, 2), + score: 0.0, + } + } + + #[test] + fn rrf_vote_promotes_dense_head_but_keeps_tail() { + let defs = vec![d("x"), d("y"), d("z"), d("t")]; + let out = rrf_vote(defs, &[2, 0, 1], 0.5); + let names: Vec<_> = out.iter().map(|d| d.name.as_str()).collect(); + assert_eq!(names, ["x", "z", "y", "t"]); + } + + #[test] + fn dense_vote_is_off_by_default() { + std::env::remove_var("NM_DENSE"); + let out = dense_vote(vec![d("x"), d("y")], "q", std::path::Path::new(".")); + assert_eq!(out[0].name, "x"); + } +} diff --git a/docs/research/contributions-log.md b/docs/research/contributions-log.md index b379970..4aac404 100644 --- a/docs/research/contributions-log.md +++ b/docs/research/contributions-log.md @@ -586,3 +586,76 @@ function-level effects on Lite-like data. Against published function-level numbe GPU-scale dense retriever halved (it was −0.12/−0.11 with v1.3.0). Installed as the dogfood binary (v1.3.0 kept as `neuromesh-v1.3.0-backup.exe`); MCP banner 1.4.0, exits on stdin EOF (probe with `NEUROMESH_NO_BROWSER=1`). + +### 8.18 Second tuning split from SWE-Gym; learned reranker rejected; dense short-list vote accepted (session 19, 2026-10-17) + +**Why a second split.** Dev (225) cannot resolve +0.02 (one dev-fast issue = 0.017) and its function +gold is unlike Lite's. `scripts/research/gym_split.py` (committed in d1a0432 *before* any run on it) +samples SWE-Gym (Pan et al. 2024): 7 repositories disjoint from Lite/Verified/dev (pydantic, hydra, +conan, dvc, dask, moto, mypy), patches editing only existing non-test `.py` files, ≤40 per repository, +`random.Random(19)`, halves gym-a / gym-b (140 each) alternating within each repository. Gold: 266 of +280 have function gold, mean 3.4 functions (one function in 45%) — like dev, not like Lite. + +**v1.4.0 on SWE-Gym (release binary, run once, never tuned on these repositories — a third holdout):** + +| | file Acc@1 | @3 | @5 | @10 | func Acc@1 | @5 | @10 | +|---|---|---|---|---|---|---|---| +| BM25 gym-a / gym-b | 0.121 / 0.121 | 0.257 / 0.271 | 0.329 / 0.307 | 0.407 / 0.436 | – | – | – | +| engine gym-a | 0.279 [0.21,0.36] | 0.450 | 0.536 | 0.650 | 0.152 | 0.258 | 0.333 | +| engine gym-b | 0.271 [0.20,0.35] | 0.414 | 0.493 | 0.600 | 0.112 | 0.254 | 0.366 | + +The file-level gain over BM25 (×2.3 at Acc@1) holds on seven repositories the engine never saw. + +**Where function misses come from** (`func_miss.py`, dev): ordering 45% (every gold function in the +300-entry list, not in the top 5; median rank 36), hit 32%, file miss 15%, coverage 8%. + +**Learned reranker (rejected).** `func_ltr.py`: logistic regression / gradient-boosted trees over 25 +features of the top-100 candidates (engine rank/score, file rank, rank in file, name/class/file +mentions, traceback frames, token overlap/IDF, length, kind). Cross-half on dev: func@5 +0.000 / ++0.019 (logreg), fail. On gym logreg passes (+0.030/+0.030) but @1 drops (−0.038/−0.015), GBT fails +(+0.068/+0.007); trained on all dev and applied to all gym: func@5 +0.000. The largest coefficients +change sign between dev and gym (file rank −0.61 vs +1.23) — with a few hundred issues a learned +reranker fits the split, not the task. Negative result. + +**Dense short-list vote (accepted).** The dev near miss of §8.16 (jina-code-v2 int8 on the engine's +top-30 definitions, RRF vote w=0.5) re-measured on SWE-Gym with the same weight: + +| | file Acc@1 a / b | func Acc@1 a / b | func Acc@5 a / b | func Acc@10 a / b | +|---|---|---|---|---| +| engine | 0.279 / 0.271 | 0.152 / 0.112 | 0.258 / 0.254 | 0.333 / 0.366 | +| + dense vote w=0.5 | 0.307 / 0.271 | 0.167 / 0.134 | **0.288 / 0.313** | 0.364 / 0.388 | + +Bar (func@5 ≥ +0.02 on both halves): **passes** (+0.030 / +0.059). Pooled with dev (n=476, `paired.py`): +func@5 0.286 → 0.317, +0.032 [0.015, 0.050], 17 issues gained / 2 lost, sign test p = 0.0007; +func@1 +0.021 (p = 0.08); func@10 +0.021 (p = 0.006). Decision (w=0.5, k=30) committed in 608acb4 +before the run on Lite / Verified. Cost ~6 s per issue on CPU (p50; embedding 30 definitions + issue). + +**Engine port (`NM_DENSE=1`, `packet` function list).** Rust re-implementation (fastembed jina-code-v2, +same texts, 512-token cut, RRF k=60 w=0.5 over the top 30); order matches the prototype up to +near-tie swaps. SWE-Gym run with the port: func Acc@5 0.258 → **0.295** (gym-a), 0.254 → **0.299** +(gym-b); file lists unchanged (the vote touches only the function list — the file vote did not clear +the bar on gym-b, +0.000). + +**Lite, run once with the locked decision** (prototype over the stored v1.4.0 run, `dense-sl-lite.jsonl`): + +| | n | func Acc@1 | Acc@5 | Acc@10 | +|---|---|---|---|---| +| LocAgent subset, v1.4.0 | 274 | 0.223 | 0.460 [0.40,0.52] | 0.540 | +| LocAgent subset, + dense vote | 274 | **0.252** | **0.485** [0.43,0.54] | **0.555** | +| Lite strict, v1.4.0 → + dense | 250 | 0.228 → 0.252 | 0.472 → 0.496 | 0.556 → 0.568 | + +Paired (274): @5 +0.026 [0.004, 0.051], 9 gained / 2 lost, sign test p = 0.065; @1 +0.029 (18/10). +Same direction and size as dev+gym (+0.032). Gap to CodeRankEmbed (0.518 func@5, GPU, whole-repo +index) −0.058 → −0.033, with 30 embeddings per issue on CPU instead of every function of the repository. + +**Verified, run once with the locked decision** (`dense-sl-verified.jsonl` over the stored v1.4.0 run): + +| | n | func Acc@1 | Acc@5 | Acc@10 | +|---|---|---|---|---| +| Verified with function gold, v1.4.0 | 459 | 0.198 | 0.429 [0.38,0.48] | 0.497 | +| + dense vote | 459 | **0.237** | **0.458** [0.41,0.50] | **0.514** | +| not in Lite, v1.4.0 → + dense | 375 | 0.189 → 0.208 | 0.397 → 0.435 | 0.464 → 0.483 | + +Paired (459): @1 +0.039 [0.013, 0.065], 28/10, p = 0.005; @5 +0.028 [0.009, 0.048], 17/4, p = 0.007; +@10 +0.017, 8/0, p = 0.008. Every split — dev, gym-a, gym-b, Lite, Verified — moves the same way by ++0.02 to +0.06 at func Acc@5; the effect predicted on tuning data (+0.032) is what both holdouts show. diff --git a/docs/research/paper-draft.md b/docs/research/paper-draft.md index c7b6a7f..8448289 100644 --- a/docs/research/paper-draft.md +++ b/docs/research/paper-draft.md @@ -12,13 +12,13 @@ retriever (CodeRankEmbed). We ask how far a local engine gets with neither: a pe graph, field-weighted lexical ranking, a definition-level ranking of long reports, and explicit report hygiene, all on a laptop CPU. On SWE-bench Lite, run once after tuning only on the SWE-bench dev split, the engine reaches file-level Acc@1/3/5 of 0.507/0.717/0.750 on 276 held-out issues -(plain BM25 0.301/0.507/0.587) in 2.9 s per issue including a cold index — above a code embedding -model at Acc@1 and level with Agentless+GPT-4o at Acc@5; on SWE-bench Verified's 407 issues outside +(plain BM25 0.301/0.507/0.587) in 2.9 s per issue including a cold index — above Jina-Code-v2 +embeddings at Acc@1/3, within 0.03 of CodeRankEmbed at Acc@1, and level with Agentless+GPT-4o at Acc@5; on SWE-bench Verified's 407 issues outside Lite, 0.435/0.671/0.732 (BM25 0.194/0.373/0.477). We report every component's effect, negative results (dense retrieval on CPU, cross-encoder reranking, blind graph expansion, commit history, pseudo-relevance feedback, documentation bridges), and a holdout protocol that caught two results that looked-at sets had suggested and confirmed a third on a fresh repository. At function -level the engine lists the edited function among its first five on 0.394 of LocAgent's 274 Lite +level the engine lists every edited function among its first five on 0.460 of LocAgent's 274 Lite instances (BM25 0.318; CodeRankEmbed 0.518) with no model at all, 0.429 on Verified. ## 1. Introduction @@ -118,6 +118,7 @@ skipped, voting at double weight deepest-first. | engine v1.2.0 | – | 0.474 | 0.708 | 0.752 | – | – | | engine v1.3.0 | – | 0.500 [0.44,0.56] | 0.723 | 0.755 | 0.394 | 0.482 | | **engine v1.4.0** | – | **0.500** [0.44,0.56] | **0.723** | **0.755** | **0.460** [0.40,0.52] | **0.540** | +| **engine v1.4.0 + dense short-list vote (opt-in)** | – | 0.500 | 0.723 | 0.755 | **0.485** [0.43,0.54] | **0.555** | | Jina-Code-v2 † | – | 0.434 | 0.712 | 0.803 | – | – | | CodeRankEmbed † | – | 0.526 | 0.777 | 0.847 | 0.518 | 0.588 | | Agentless + GPT-4o † | ✓ | 0.672 | 0.745 | 0.745 | – | – | @@ -146,6 +147,36 @@ Function level (all edited functions in top k), v1.3.0 → v1.4.0: Verified 459 larger (one edited function in 45% of dev issues vs 86% of Lite), so dev understates function-level effects on Lite-like data. +### 5.2b A third holdout: SWE-Gym (280 issues, 7 unseen repositories) + +A second tuning split, pre-registered by commit before any run (§4): SWE-Gym issues from pydantic, +hydra, conan, dvc, dask, moto and mypy (≤40 per repository, seed 19, patches that edit only existing +non-test Python files), halves gym-a / gym-b. The release v1.4.0 had never seen these repositories, +so its one run is also a holdout: + +| | file Acc@1 | @3 | @5 | @10 | func Acc@5 | +|---|---|---|---|---|---| +| BM25, gym-a / gym-b | 0.121 / 0.121 | 0.257 / 0.271 | 0.329 / 0.307 | 0.407 / 0.436 | – | +| engine v1.4.0, gym-a | 0.279 | 0.450 | 0.536 | 0.650 | 0.258 | +| engine v1.4.0, gym-b | 0.271 | 0.414 | 0.493 | 0.600 | 0.254 | + +The engine's advantage over BM25 (×2.3 at Acc@1) carries to repositories outside every set it +was developed on. + +### 5.2c Dense vote on the engine's short list (decided on dev + SWE-Gym, run once on holdouts) + +jina-code-v2 (int8, CPU) embeds the issue and only the engine's top 30 functions; its order votes +into the function list by RRF (w = 0.5). Decided on the tuning splits (pooled n = 476: func Acc@5 ++0.032, p = 0.0007) and locked by commit before the holdouts: + +| set | n | func Acc@1 | func Acc@5 | func Acc@10 | paired @5 | +|---|---|---|---|---|---| +| Lite, LocAgent subset | 274 | 0.223 → 0.252 | 0.460 → 0.485 | 0.540 → 0.555 | +0.026 [0.004,0.051], 9/2 | +| Verified | 459 | 0.198 → 0.237 | 0.429 → 0.458 | 0.497 → 0.514 | +0.028 [0.009,0.048], 17/4, p=0.007 | + +Thirty embeddings per issue replace a whole-repository index; the gap to CodeRankEmbed at func +Acc@5 narrows from −0.058 to −0.033. + ### 5.3 Ablation (Lite holdout, 276) | removed | Acc@1 | Acc@3 | Acc@5 | @@ -189,6 +220,7 @@ p50 2.9 s / p90 10.6 s per Lite issue (Verified 2.5 s / 8.8 s) on a 6-core lapto | identifiers in the issue's code blocks → their definitions | file Acc@1 drops; func Acc@5 +0.019 (below the 0.02 bar) | | RM3 pseudo-relevance feedback on the definition ranking | alone worse at @1 (0.135 → 0.038, dev-fast); as a vote no better than plain BM25 | | doc sections as a bridge from question words to identifiers (plain-language) | R@3 down on 3 of 4 sets (ripgrep 0.500 → 0.375, axios 0.833 → 0.583) | +| learned reranker (logistic / boosted trees, 25 features, top-100 functions) | dev func@5 +0.000/+0.019; trained on dev, applied to SWE-Gym +0.000; coefficients flip sign between splits | | evaluation: function-level denominator without short reports | 244 instead of 274 instances, Acc@5 0.443 instead of 0.394 | ## 7. Threats to validity diff --git a/scripts/research/func_ltr.py b/scripts/research/func_ltr.py new file mode 100644 index 0000000..f5a34c7 --- /dev/null +++ b/scripts/research/func_ltr.py @@ -0,0 +1,227 @@ +"""Learned (LLM-free) reranking of the engine's function list — session 19 prototype. + +Ordering is the largest function-level failure on dev (func_miss.py: 45% of +instances have every gold function in the 300-entry list but not in the top 5; +median rank 36). This reranks the top-N candidates with a small linear model over +features read from the stored engine output and the issue text only (no checkout): +engine rank and score, file rank, rank within the file, mentions of the name / +class / file in the issue, traceback frames, name-token overlap, length, kind. + +Protocol (fixed before the first measurement): train on one dev half, score the +other, both directions; kept only if func Acc@5 (or file Acc@1, untouched here) +rises by >= 0.02 on BOTH halves. `--apply` trains on all of dev and writes a +reranked results file for a once-per-version holdout run. + + py -3 func_ltr.py --results dev-s18a-all.jsonl --repos repos --gold-cache func-gold-dev.json \ + --fast devset/dev-fast.json --rest devset/dev-rest.json [--top 100] [--model logreg|gbt] +""" +import argparse +import json +import math +import os +import re +import sys +from collections import Counter, defaultdict + +import numpy as np + +sys.path.insert(0, os.path.dirname(__file__)) +from func_eval import boot_ci, load_gold, same # noqa: E402 + +WORD = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +FRAME = re.compile(r'File "([^"]+)", line (\d+), in ([A-Za-z_][A-Za-z0-9_<>]*)') + + +def split_ident(s): + parts = [] + for w in re.split(r"[_\W]+", s): + parts += re.findall(r"[A-Z]+(?![a-z])|[A-Z]?[a-z]+|\d+", w) + return [p.lower() for p in parts if len(p) > 1] + + +def issue_view(text): + words = WORD.findall(text) + wset = set(words) + lower = Counter(w.lower() for w in words) + toks = Counter(t for w in words for t in split_ident(w)) + frames = FRAME.findall(text) + frame_funcs = [f[2] for f in frames] + frame_files = [f[0].replace("\\", "/") for f in frames] + dotted = set(re.findall(r"[A-Za-z_][\w]*(?:\.[A-Za-z_]\w*)+", text)) + return dict(words=wset, lower=lower, toks=toks, frame_funcs=frame_funcs, + frame_files=frame_files, dotted=dotted, text=text, ntok=sum(toks.values()) or 1) + + +def path_in(p, frame_files): + return any(f.endswith("/" + p) or f.endswith(p) or p.endswith(f.split("site-packages/")[-1]) + for f in frame_files) + + +FEATS = ["lrank", "rrank", "score_rel", "frank", "ftop1", "fabsent", "inrank", "lininrank", + "name_exact", "name_lower", "cls_exact", "qual_dotted", "file_base", "frame_func", + "frame_file", "tok_overlap", "tok_idf", "llen", "is_test", "is_private", "is_dunder", + "is_init", "is_method", "file_share20", "nm_frame_innermost"] + + +def candidates(r, iv, top, df): + defs = r.get("definitions", [])[:top] + files = r.get("ours_files", []) + if not defs: + return [], [] + smax = max(d.get("score", 0) for d in defs) or 1.0 + per_file = Counter() + share20 = Counter(d["path"] for d in defs[:20]) + rows, keys = [], [] + innermost = iv["frame_funcs"][-1] if iv["frame_funcs"] else None + for i, d in enumerate(defs): + p, n = d["path"], d["name"] + last = n.split(".")[-1] + cls = n.split(".")[0] if "." in n else "" + fr = next((j for j, f in enumerate(files) if f == p or f.endswith("/" + p) or p.endswith("/" + f)), None) + within = per_file[p] + per_file[p] += 1 + base = os.path.splitext(os.path.basename(p))[0] + ntoks = split_ident(n) + hit = [t for t in ntoks if t in iv["toks"]] + idf = sum(math.log(1 + 300 / (1 + df.get(t, 0))) for t in hit) / (1 + len(ntoks)) + lines = d.get("lines") or [0, 0] + f = dict( + lrank=math.log1p(i), rrank=1.0 / (1 + i), score_rel=d.get("score", 0) / smax, + frank=math.log1p(fr) if fr is not None else math.log1p(30), ftop1=float(fr == 0), + fabsent=float(fr is None), inrank=1.0 / (1 + within), lininrank=math.log1p(within), + name_exact=float(last in iv["words"]), name_lower=math.log1p(iv["lower"].get(last.lower(), 0)), + cls_exact=float(bool(cls) and cls in iv["words"]), + qual_dotted=float(any(x.endswith(n) or x.endswith("." + last) for x in iv["dotted"])), + file_base=float(base in iv["words"] and base != "__init__"), + frame_func=float(last in iv["frame_funcs"]), frame_file=float(path_in(p, iv["frame_files"])), + tok_overlap=len(hit) / (1 + len(ntoks)), tok_idf=idf, + llen=math.log1p(max(0, lines[1] - lines[0])), + is_test=float("test" in p.lower()), is_private=float(last.startswith("_") and not last.startswith("__")), + is_dunder=float(last.startswith("__")), is_init=float(last == "__init__"), + is_method=float("." in n), file_share20=share20[p] / 20.0, + nm_frame_innermost=float(innermost == last), + ) + rows.append([f[k] for k in FEATS]) + keys.append((p, n)) + return rows, keys + + +def label(keys, gold): + return [int(any(same(k, g) for g in gold)) for k in keys] + + +def acc_at(order_keys, gold, k): + top = order_keys[:k] + return int(all(any(same(p, g) for p in top) for g in gold)) + + +def build(res, issues, golds, top, df): + data = [] + for r in res: + gold = golds.get(r["instance_id"]) + if not gold: + continue + iv = issue_view(issues[r["instance_id"]]) + X, keys = candidates(r, iv, top, df) + data.append(dict(iid=r["instance_id"], X=np.array(X) if X else np.zeros((0, len(FEATS))), + keys=keys, y=label(keys, gold) if keys else [], gold=gold, + full=[(d["path"], d["name"]) for d in r.get("definitions", [])])) + return data + + +def fit(train, kind, seed=0): + X = np.vstack([d["X"] for d in train if len(d["keys"])]) + y = np.concatenate([d["y"] for d in train if len(d["keys"])]) + mu, sd = X.mean(0), X.std(0) + 1e-9 + if kind == "gbt": + from sklearn.ensemble import HistGradientBoostingClassifier + m = HistGradientBoostingClassifier(max_iter=200, learning_rate=0.05, max_leaf_nodes=15, + l2_regularization=1.0, random_state=seed) + m.fit(X, y) + return lambda Z: m.predict_proba(Z)[:, 1] + from sklearn.linear_model import LogisticRegression + m = LogisticRegression(C=0.3, class_weight="balanced", max_iter=2000) + m.fit((X - mu) / sd, y) + fit.coef = dict(zip(FEATS, m.coef_[0])) + return lambda Z: m.decision_function((Z - mu) / sd) + + +def rerank(d, scorer): + if not d["keys"]: + return d["full"] + s = scorer(d["X"]) + order = sorted(range(len(d["keys"])), key=lambda i: (-s[i], i)) + head = [d["keys"][i] for i in order] + return head + d["full"][len(head):] + + +def report(name, data, lists): + out = {} + for k in (1, 5, 10): + v = [acc_at(l, d["gold"], k) for d, l in zip(data, lists)] + out[k] = sum(v) / max(len(v), 1) + print(f" {name:28s} n={len(data):3d} func@1 {out[1]:.3f} @5 {out[5]:.3f} @10 {out[10]:.3f}") + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--results", required=True) + ap.add_argument("--repos", required=True) + ap.add_argument("--gold-cache", default="") + ap.add_argument("--fast", required=True) + ap.add_argument("--rest", required=True) + ap.add_argument("--top", type=int, default=100) + ap.add_argument("--model", default="logreg") + ap.add_argument("--apply", default="", help="holdout results jsonl to rerank (train on all dev)") + ap.add_argument("--apply-data", default="") + ap.add_argument("--out", default="") + args = ap.parse_args() + halves = {} + rows = {} + for nm, path in (("fast", args.fast), ("rest", args.rest)): + rr = json.load(open(path, encoding="utf-8")) + halves[nm] = {r["instance_id"] for r in rr} + rows.update({r["instance_id"]: r for r in rr}) + issues = {i: r["problem_statement"] for i, r in rows.items()} + res = [json.loads(l) for l in open(args.results, encoding="utf-8") if l.strip()] + res = [r for r in res if "error" not in r and r["instance_id"] in rows] + golds = load_gold(rows, [r["instance_id"] for r in res], args.repos, args.gold_cache) + df = Counter() + for t in issues.values(): + df.update(set(x for w in WORD.findall(t) for x in split_ident(w))) + data = build(res, issues, golds, args.top, df) + parts = {h: [d for d in data if d["iid"] in halves[h]] for h in halves} + print(f"model={args.model} top={args.top}") + deltas = {} + for test, train in (("fast", "rest"), ("rest", "fast")): + print(f"[test {test}, train {train}]") + base = report("engine", parts[test], [d["full"] for d in parts[test]]) + sc = fit(parts[train], args.model) + new = report("reranked", parts[test], [rerank(d, sc) for d in parts[test]]) + deltas[test] = {k: new[k] - base[k] for k in new} + for h, dd in deltas.items(): + print(f" delta {h}: " + " ".join(f"@{k} {v:+.3f}" for k, v in dd.items())) + ok = all(dd[5] >= 0.02 for dd in deltas.values()) + print("BAR (func@5 >= +0.02 both halves):", "PASS" if ok else "fail") + sc = fit(data, args.model) + if args.model == "logreg": + print("coef (all dev): " + ", ".join(f"{k} {v:+.2f}" for k, v in sorted( + fit.coef.items(), key=lambda kv: -abs(kv[1])))) + if args.apply: + arows = {r["instance_id"]: r for r in json.load(open(args.apply_data, encoding="utf-8"))} + ares = [json.loads(l) for l in open(args.apply, encoding="utf-8") if l.strip()] + with open(args.out, "w", encoding="utf-8") as fo: + for r in ares: + if "error" not in r and r.get("definitions") and r["instance_id"] in arows: + iv = issue_view(arows[r["instance_id"]]["problem_statement"]) + X, keys = candidates(r, iv, args.top, df) + d = dict(X=np.array(X), keys=keys, full=[(x["path"], x["name"]) for x in r["definitions"]]) + by = {(x["path"], x["name"]): x for x in r["definitions"]} + r["definitions_ltr"] = [by[k] for k in rerank(d, sc)] + fo.write(json.dumps(r) + "\n") + print("wrote", args.out) + + +if __name__ == "__main__": + main() diff --git a/scripts/research/func_miss.py b/scripts/research/func_miss.py new file mode 100644 index 0000000..a34bf6a --- /dev/null +++ b/scripts/research/func_miss.py @@ -0,0 +1,78 @@ +"""Where do function-level misses come from? (session 19, analysis only — dev split) + +For every instance with function gold: rank of each gold function in the engine's +`definitions` list (or absent), rank of its file in `ours_files`, list length. +Buckets: hit@5 / gold present but ranked >5 (ordering) / gold file ranked but +function absent from the list (coverage) / file not ranked (file miss). + + py -3 func_miss.py --results dev-s18a-all.jsonl --data devset/swe-dev.json --repos repos \ + --gold-cache func-gold-dev.json +""" +import argparse +import json +import os +import sys +from collections import Counter + +sys.path.insert(0, os.path.dirname(__file__)) +from func_eval import load_gold, same # noqa: E402 + + +def file_rank(files, gp): + for i, f in enumerate(files): + if f == gp or f.endswith("/" + gp) or gp.endswith("/" + f): + return i + return None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--results", required=True) + ap.add_argument("--data", required=True) + ap.add_argument("--repos", required=True) + ap.add_argument("--gold-cache", default="") + args = ap.parse_args() + rows = {r["instance_id"]: r for r in json.load(open(args.data, encoding="utf-8"))} + res = [json.loads(l) for l in open(args.results, encoding="utf-8") if l.strip()] + res = [r for r in res if "error" not in r] + golds = load_gold(rows, [r["instance_id"] for r in res], args.repos, args.gold_cache) + buckets, lens, ranks, ngold = Counter(), [], [], Counter() + for r in res: + gold = golds[r["instance_id"]] + if not gold: + continue + ngold[min(len(gold), 3)] += 1 + preds = [(d["path"], d["name"]) for d in r.get("definitions", [])] + lens.append(len(preds)) + worst = "hit@5" + for g in gold: + rk = next((i for i, p in enumerate(preds) if same(p, g)), None) + fr = file_rank(r.get("ours_files", []), g[0]) + if rk is not None and rk < 5: + b = "hit@5" + elif rk is not None: + b = "ordering" + ranks.append(rk) + elif fr is not None: + b = "coverage(file ranked %s)" % ("top1" if fr == 0 else "top5" if fr < 5 else ">5") + else: + b = "file-miss" + order = ["hit@5", "ordering", "coverage(file ranked top1)", "coverage(file ranked top5)", + "coverage(file ranked >5)", "file-miss"] + if order.index(b) > order.index(worst): + worst = b + buckets[worst] += 1 + n = sum(buckets.values()) + print(f"instances {n}; gold size 1/2/3+: {dict(sorted(ngold.items()))}") + for b, c in buckets.most_common(): + print(f" {b:32s} {c:4d} {c / n:.3f}") + lens.sort() + print(f"list length median {lens[len(lens) // 2]} min {lens[0]} max {lens[-1]}") + if ranks: + ranks.sort() + print(f"ordering misses: rank median {ranks[len(ranks) // 2]}, <=10 {sum(x < 10 for x in ranks)}, " + f"<=20 {sum(x < 20 for x in ranks)}, n {len(ranks)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/research/gym_split.py b/scripts/research/gym_split.py new file mode 100644 index 0000000..aa517a2 --- /dev/null +++ b/scripts/research/gym_split.py @@ -0,0 +1,60 @@ +"""Second tuning split from SWE-Gym (session 19, phase B) — pre-registered. + +SWE-bench dev (225 issues, 6 repositories) is too small to separate +0.02 effects +(one issue in dev-fast is 0.017), and its function gold is unlike Lite's (one +edited function in 45% of dev issues vs 86% of Lite). SWE-Gym (Pan et al. 2024, +2438 issues) uses 11 repositories disjoint from SWE-bench Lite/Verified/dev. + +Selection, fixed before any engine run on it: + repositories pydantic, hydra, conan, dvc, dask, moto, mypy (the four largest + checkouts — pandas, MONAI, modin, bokeh — left out for disk/IO) + instances patch edits only existing non-test .py files (no new files) + sample at most 40 per repository, random.Random(19) shuffle of sorted ids + halves gym-a / gym-b alternate within each repository's sample +Decision bar (same as dev): file Acc@1 or func Acc@5 >= +0.02 on BOTH halves; +an idea that was a near miss on dev is kept only if it also clears it here. + + py -3 gym_split.py --parquet gym/gym.parquet --out-dir devset +""" +import argparse +import json +import os +import random +import re + +REPOS = ["pydantic/pydantic", "facebookresearch/hydra", "conan-io/conan", "iterative/dvc", + "dask/dask", "getmoto/moto", "python/mypy"] +CAP = 40 +SEED = 19 + + +def eligible(patch): + files = re.findall(r"^diff --git a/(\S+) b/(\S+)", patch, re.M) + if not files or "\nnew file mode" in patch or "\ndeleted file mode" in patch: + return False + return all(a.endswith(".py") and "test" not in a.lower() for a, _ in files) + + +def main(): + import pandas as pd + ap = argparse.ArgumentParser() + ap.add_argument("--parquet", required=True) + ap.add_argument("--out-dir", required=True) + args = ap.parse_args() + df = pd.read_parquet(args.parquet) + a, b = [], [] + for repo in REPOS: + rows = [r for r in df[df.repo == repo].to_dict("records") if eligible(r["patch"])] + rows.sort(key=lambda r: r["instance_id"]) + random.Random(SEED).shuffle(rows) + for i, r in enumerate(rows[:CAP]): + r = {k: (v.tolist() if hasattr(v, "tolist") else v) for k, v in r.items()} + (a if i % 2 == 0 else b).append(r) + print(f"{repo:28s} eligible {len(rows):4d} taken {min(CAP, len(rows))}") + for name, part in (("gym-a", a), ("gym-b", b), ("gym-all", a + b)): + json.dump(part, open(os.path.join(args.out_dir, name + ".json"), "w", encoding="utf-8")) + print(name, len(part)) + + +if __name__ == "__main__": + main() diff --git a/scripts/research/paired.py b/scripts/research/paired.py new file mode 100644 index 0000000..a0a51f5 --- /dev/null +++ b/scripts/research/paired.py @@ -0,0 +1,68 @@ +"""Paired comparison of two function lists per instance (exact sign test on +discordant pairs + paired bootstrap CI of the difference), pooling several +(results, ext, gold) sets. Session 19: dense short-list vote vs engine. + + py -3 paired.py --set dev-s18a-all.jsonl:dense-sl-dev.jsonl:func-gold-dev.json \ + --set gym-v140-all.jsonl:dense-sl-gym.jsonl:func-gold-gym.json [--w 0.5] [--k 5] +""" +import argparse +import json +import math +import os +import random +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +from func_eval import same # noqa: E402 + + +def rrf(lists, weights, k=60): + s = {} + for lst, w in zip(lists, weights): + for i, f in enumerate(lst): + s[f] = s.get(f, 0.0) + w / (k + i + 1) + return [f for f, _ in sorted(s.items(), key=lambda kv: -kv[1])] + + +def hit(preds, gold, k): + return int(all(any(same(p, g) for p in preds[:k]) for g in gold)) + + +def load(p): + return {r["instance_id"]: r for r in map(json.loads, filter(str.strip, open(p, encoding="utf-8"))) + if "error" not in r} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--set", action="append", required=True) + ap.add_argument("--w", type=float, default=0.5) + ap.add_argument("--k", type=int, default=5) + a = ap.parse_args() + pairs = [] + for spec in a.set: + rp, ep, gp = spec.split(":") + res, ext, gold = load(rp), load(ep), json.load(open(gp, encoding="utf-8")) + for iid, r in res.items(): + g = {tuple(x) for x in gold.get(iid, [])} + if not g: + continue + base = [(d["path"], d["name"]) for d in r.get("definitions", [])] + dense = [tuple(d) for d in ext.get(iid, {}).get("dense_defs", [])] + new = rrf([base, dense], [1.0, a.w]) if dense else base + pairs.append((hit(base, g, a.k), hit(new, g, a.k))) + n = len(pairs) + b = sum(x for x, _ in pairs) / n + c = sum(y for _, y in pairs) / n + up = sum(1 for x, y in pairs if y > x) + down = sum(1 for x, y in pairs if x > y) + m = up + down + p = min(1.0, 2 * sum(math.comb(m, i) for i in range(min(up, down) + 1)) / 2 ** m) if m else 1.0 + rnd = random.Random(0) + diffs = sorted(sum(y - x for x, y in (rnd.choice(pairs) for _ in range(n))) / n for _ in range(4000)) + print(f"n={n} func@{a.k} {b:.3f} -> {c:.3f} (+{c - b:.3f}, 95% CI [{diffs[100]:+.3f},{diffs[3899]:+.3f}])" + f" gained {up} lost {down} sign-test p={p:.4f}") + + +if __name__ == "__main__": + main()