Session 19: SWE-Gym split, opt-in dense function vote (Lite func@5 0.460 -> 0.485)

crates/neuromesh-cli/src/commands/packet.rs

@@ -113,6 +113,7 @@ pub fn execute(args: &[String]) -> Result<()> {
     // scored as misses without one).
     let definitions =
         neuromesh_context::gold::definition_order(&graph, &prompt, &localization, def_depth());
+    let definitions = dense_vote(definitions, &prompt, &current_dir);
     let mut files: Vec<String> = packet_file_names(&view).into_iter().collect();
     files.sort();
     let reduction = if workspace_tokens > 0 {
@@ -531,3 +532,125 @@ mod tests {
         assert_eq!(parsed.query.as_deref(), Some("Where is HTTPAdapter.send?"));
     }
 }
+
+/// Opt-in dense vote on the function list (`NM_DENSE=1`; session 19, §8.18):
+/// jina-code-v2 embeds the issue and the top `DENSE_K` definitions (path, name,
+/// body), reorders them by cosine, and that order votes into the list by RRF at
+/// weight 0.5. Decided on SWE-bench dev + a SWE-Gym split (func Acc@5 +0.032,
+/// p = 0.0007); ~6 s per prompt on CPU, needs `neuromesh install embed jina`.
+/// Any failure (model missing, unreadable file) returns the list unchanged.
+fn dense_vote(
+    defs: Vec<neuromesh_graph::RankedDefinition>,
+    prompt: &str,
+    root: &std::path::Path,
+) -> Vec<neuromesh_graph::RankedDefinition> {
+    if std::env::var("NM_DENSE").ok().as_deref() != Some("1") || defs.len() < 2 {
+        return defs;
+    }
+    #[cfg(feature = "embeddings")]
+    {
+        match dense_order(&defs, prompt, root) {
+            Ok(order) => return rrf_vote(defs, &order, 0.5),
+            Err(e) => eprintln!("neuromesh: dense vote skipped ({e})"),
+        }
+    }
+    #[cfg(not(feature = "embeddings"))]
+    let _ = (prompt, root);
+    defs
+}
+
+const DENSE_K: usize = 30;
+
+#[cfg(feature = "embeddings")]
+fn dense_order(
+    defs: &[neuromesh_graph::RankedDefinition],
+    prompt: &str,
+    root: &std::path::Path,
+) -> std::result::Result<Vec<usize>, String> {
+    use neuromesh_core::{EmbeddingConfig, EmbeddingModelId};
+    let k = defs.len().min(DENSE_K);
+    let mut files: std::collections::HashMap<&str, Vec<String>> = Default::default();
+    let mut texts = Vec::with_capacity(k + 1);
+    texts.push(prompt.chars().take(4000).collect::<String>());
+    for d in &defs[..k] {
+        let lines = files.entry(d.path.as_str()).or_insert_with(|| {
+            std::fs::read_to_string(root.join(&d.path))
+                .map(|t| t.lines().map(str::to_string).collect())
+                .unwrap_or_default()
+        });
+        let (a, b) = d.lines;
+        let lo = a.saturating_sub(1).min(lines.len());
+        let hi = b.min(lines.len()).max(lo);
+        let body = lines[lo..hi].join(
+            "
+",
+        );
+        let text = format!(
+            "{}
+{}
+{}",
+            d.path, d.name, body
+        );
+        texts.push(text.chars().take(1500).collect());
+    }
+    let config = EmbeddingConfig {
+        model: EmbeddingModelId::JinaCodeV2,
+        matryoshka_dim: EmbeddingModelId::JinaCodeV2.default_matryoshka_dim(),
+        intra_threads: None,
+        ..EmbeddingConfig::default()
+    };
+    let mut emb = neuromesh_embed::Embedder::try_new(config).map_err(|e| e.to_string())?;
+    let vecs = emb.embed_documents(&texts).map_err(|e| e.to_string())?;
+    let q = &vecs[0];
+    let sim = |v: &Vec<f32>| v.iter().zip(q).map(|(a, b)| a * b).sum::<f32>();
+    let mut order: Vec<usize> = (0..k).collect();
+    order.sort_by(|&i, &j| sim(&vecs[j + 1]).total_cmp(&sim(&vecs[i + 1])));
+    Ok(order)
+}
+
+/// Reciprocal-rank fusion (k = 60) of the list's own order (weight 1) and a
+/// reordering of its head (`order` indexes into `defs`, weight `w`).
+fn rrf_vote(
+    defs: Vec<neuromesh_graph::RankedDefinition>,
+    order: &[usize],
+    w: f32,
+) -> Vec<neuromesh_graph::RankedDefinition> {
+    let mut score: Vec<f32> = (0..defs.len()).map(|i| 1.0 / (61.0 + i as f32)).collect();
+    for (r, &i) in order.iter().enumerate() {
+        score[i] += w / (61.0 + r as f32);
+    }
+    let mut idx: Vec<usize> = (0..defs.len()).collect();
+    idx.sort_by(|&a, &b| score[b].total_cmp(&score[a]).then(a.cmp(&b)));
+    let mut slots: Vec<Option<neuromesh_graph::RankedDefinition>> =
+        defs.into_iter().map(Some).collect();
+    idx.into_iter().filter_map(|i| slots[i].take()).collect()
+}
+
+#[cfg(test)]
+mod dense_tests {
+    use super::*;
+
+    fn d(n: &str) -> neuromesh_graph::RankedDefinition {
+        neuromesh_graph::RankedDefinition {
+            path: "a.py".into(),
+            name: n.into(),
+            lines: (1, 2),
+            score: 0.0,
+        }
+    }
+
+    #[test]
+    fn rrf_vote_promotes_dense_head_but_keeps_tail() {
+        let defs = vec![d("x"), d("y"), d("z"), d("t")];
+        let out = rrf_vote(defs, &[2, 0, 1], 0.5);
+        let names: Vec<_> = out.iter().map(|d| d.name.as_str()).collect();
+        assert_eq!(names, ["x", "z", "y", "t"]);
+    }
+
+    #[test]
+    fn dense_vote_is_off_by_default() {
+        std::env::remove_var("NM_DENSE");
+        let out = dense_vote(vec![d("x"), d("y")], "q", std::path::Path::new("."));
+        assert_eq!(out[0].name, "x");
+    }
+}

docs/research/contributions-log.md

@@ -586,3 +586,76 @@ function-level effects on Lite-like data. Against published function-level numbe
 GPU-scale dense retriever halved (it was −0.12/−0.11 with v1.3.0).
 Installed as the dogfood binary (v1.3.0 kept as `neuromesh-v1.3.0-backup.exe`); MCP banner 1.4.0,
 exits on stdin EOF (probe with `NEUROMESH_NO_BROWSER=1`).
+
+### 8.18 Second tuning split from SWE-Gym; learned reranker rejected; dense short-list vote accepted (session 19, 2026-10-17)
+
+**Why a second split.** Dev (225) cannot resolve +0.02 (one dev-fast issue = 0.017) and its function
+gold is unlike Lite's. `scripts/research/gym_split.py` (committed in d1a0432 *before* any run on it)
+samples SWE-Gym (Pan et al. 2024): 7 repositories disjoint from Lite/Verified/dev (pydantic, hydra,
+conan, dvc, dask, moto, mypy), patches editing only existing non-test `.py` files, ≤40 per repository,
+`random.Random(19)`, halves gym-a / gym-b (140 each) alternating within each repository. Gold: 266 of
+280 have function gold, mean 3.4 functions (one function in 45%) — like dev, not like Lite.
+
+**v1.4.0 on SWE-Gym (release binary, run once, never tuned on these repositories — a third holdout):**
+
+| | file Acc@1 | @3 | @5 | @10 | func Acc@1 | @5 | @10 |
+|---|---|---|---|---|---|---|---|
+| BM25 gym-a / gym-b | 0.121 / 0.121 | 0.257 / 0.271 | 0.329 / 0.307 | 0.407 / 0.436 | – | – | – |
+| engine gym-a | 0.279 [0.21,0.36] | 0.450 | 0.536 | 0.650 | 0.152 | 0.258 | 0.333 |
+| engine gym-b | 0.271 [0.20,0.35] | 0.414 | 0.493 | 0.600 | 0.112 | 0.254 | 0.366 |
+
+The file-level gain over BM25 (×2.3 at Acc@1) holds on seven repositories the engine never saw.
+
+**Where function misses come from** (`func_miss.py`, dev): ordering 45% (every gold function in the
+300-entry list, not in the top 5; median rank 36), hit 32%, file miss 15%, coverage 8%.
+
+**Learned reranker (rejected).** `func_ltr.py`: logistic regression / gradient-boosted trees over 25
+features of the top-100 candidates (engine rank/score, file rank, rank in file, name/class/file
+mentions, traceback frames, token overlap/IDF, length, kind). Cross-half on dev: func@5 +0.000 /
++0.019 (logreg), fail. On gym logreg passes (+0.030/+0.030) but @1 drops (−0.038/−0.015), GBT fails
+(+0.068/+0.007); trained on all dev and applied to all gym: func@5 +0.000. The largest coefficients
+change sign between dev and gym (file rank −0.61 vs +1.23) — with a few hundred issues a learned
+reranker fits the split, not the task. Negative result.
+
+**Dense short-list vote (accepted).** The dev near miss of §8.16 (jina-code-v2 int8 on the engine's
+top-30 definitions, RRF vote w=0.5) re-measured on SWE-Gym with the same weight:
+
+| | file Acc@1 a / b | func Acc@1 a / b | func Acc@5 a / b | func Acc@10 a / b |
+|---|---|---|---|---|
+| engine | 0.279 / 0.271 | 0.152 / 0.112 | 0.258 / 0.254 | 0.333 / 0.366 |
+| + dense vote w=0.5 | 0.307 / 0.271 | 0.167 / 0.134 | **0.288 / 0.313** | 0.364 / 0.388 |
+
+Bar (func@5 ≥ +0.02 on both halves): **passes** (+0.030 / +0.059). Pooled with dev (n=476, `paired.py`):
+func@5 0.286 → 0.317, +0.032 [0.015, 0.050], 17 issues gained / 2 lost, sign test p = 0.0007;
+func@1 +0.021 (p = 0.08); func@10 +0.021 (p = 0.006). Decision (w=0.5, k=30) committed in 608acb4
+before the run on Lite / Verified. Cost ~6 s per issue on CPU (p50; embedding 30 definitions + issue).
+
+**Engine port (`NM_DENSE=1`, `packet` function list).** Rust re-implementation (fastembed jina-code-v2,
+same texts, 512-token cut, RRF k=60 w=0.5 over the top 30); order matches the prototype up to
+near-tie swaps. SWE-Gym run with the port: func Acc@5 0.258 → **0.295** (gym-a), 0.254 → **0.299**
+(gym-b); file lists unchanged (the vote touches only the function list — the file vote did not clear
+the bar on gym-b, +0.000).
+
+**Lite, run once with the locked decision** (prototype over the stored v1.4.0 run, `dense-sl-lite.jsonl`):
+
+| | n | func Acc@1 | Acc@5 | Acc@10 |
+|---|---|---|---|---|
+| LocAgent subset, v1.4.0 | 274 | 0.223 | 0.460 [0.40,0.52] | 0.540 |
+| LocAgent subset, + dense vote | 274 | **0.252** | **0.485** [0.43,0.54] | **0.555** |
+| Lite strict, v1.4.0 → + dense | 250 | 0.228 → 0.252 | 0.472 → 0.496 | 0.556 → 0.568 |
+
+Paired (274): @5 +0.026 [0.004, 0.051], 9 gained / 2 lost, sign test p = 0.065; @1 +0.029 (18/10).
+Same direction and size as dev+gym (+0.032). Gap to CodeRankEmbed (0.518 func@5, GPU, whole-repo
+index) −0.058 → −0.033, with 30 embeddings per issue on CPU instead of every function of the repository.
+
+**Verified, run once with the locked decision** (`dense-sl-verified.jsonl` over the stored v1.4.0 run):
+
+| | n | func Acc@1 | Acc@5 | Acc@10 |
+|---|---|---|---|---|
+| Verified with function gold, v1.4.0 | 459 | 0.198 | 0.429 [0.38,0.48] | 0.497 |
+| + dense vote | 459 | **0.237** | **0.458** [0.41,0.50] | **0.514** |
+| not in Lite, v1.4.0 → + dense | 375 | 0.189 → 0.208 | 0.397 → 0.435 | 0.464 → 0.483 |
+
+Paired (459): @1 +0.039 [0.013, 0.065], 28/10, p = 0.005; @5 +0.028 [0.009, 0.048], 17/4, p = 0.007;
+@10 +0.017, 8/0, p = 0.008. Every split — dev, gym-a, gym-b, Lite, Verified — moves the same way by
++0.02 to +0.06 at func Acc@5; the effect predicted on tuning data (+0.032) is what both holdouts show.

docs/research/paper-draft.md

@@ -12,13 +12,13 @@ retriever (CodeRankEmbed). We ask how far a local engine gets with neither: a pe
 graph, field-weighted lexical ranking, a definition-level ranking of long reports, and explicit
 report hygiene, all on a laptop CPU. On SWE-bench Lite, run once after tuning only on the SWE-bench
 dev split, the engine reaches file-level Acc@1/3/5 of 0.507/0.717/0.750 on 276 held-out issues
-(plain BM25 0.301/0.507/0.587) in 2.9 s per issue including a cold index — above a code embedding
-model at Acc@1 and level with Agentless+GPT-4o at Acc@5; on SWE-bench Verified's 407 issues outside
+(plain BM25 0.301/0.507/0.587) in 2.9 s per issue including a cold index — above Jina-Code-v2
+embeddings at Acc@1/3, within 0.03 of CodeRankEmbed at Acc@1, and level with Agentless+GPT-4o at Acc@5; on SWE-bench Verified's 407 issues outside
 Lite, 0.435/0.671/0.732 (BM25 0.194/0.373/0.477). We report every component's effect,
 negative results (dense retrieval on CPU, cross-encoder reranking, blind graph expansion, commit
 history, pseudo-relevance feedback, documentation bridges), and a holdout protocol that caught two
 results that looked-at sets had suggested and confirmed a third on a fresh repository. At function
-level the engine lists the edited function among its first five on 0.394 of LocAgent's 274 Lite
+level the engine lists every edited function among its first five on 0.460 of LocAgent's 274 Lite
 instances (BM25 0.318; CodeRankEmbed 0.518) with no model at all, 0.429 on Verified.
 
 ## 1. Introduction
@@ -118,6 +118,7 @@ skipped, voting at double weight deepest-first.
 | engine v1.2.0 | – | 0.474 | 0.708 | 0.752 | – | – |
 | engine v1.3.0 | – | 0.500 [0.44,0.56] | 0.723 | 0.755 | 0.394 | 0.482 |
 | **engine v1.4.0** | – | **0.500** [0.44,0.56] | **0.723** | **0.755** | **0.460** [0.40,0.52] | **0.540** |
+| **engine v1.4.0 + dense short-list vote (opt-in)** | – | 0.500 | 0.723 | 0.755 | **0.485** [0.43,0.54] | **0.555** |
 | Jina-Code-v2 † | – | 0.434 | 0.712 | 0.803 | – | – |
 | CodeRankEmbed † | – | 0.526 | 0.777 | 0.847 | 0.518 | 0.588 |
 | Agentless + GPT-4o † | ✓ | 0.672 | 0.745 | 0.745 | – | – |
@@ -146,6 +147,36 @@ Function level (all edited functions in top k), v1.3.0 → v1.4.0: Verified 459
 larger (one edited function in 45% of dev issues vs 86% of Lite), so dev understates
 function-level effects on Lite-like data.
 
+### 5.2b A third holdout: SWE-Gym (280 issues, 7 unseen repositories)
+
+A second tuning split, pre-registered by commit before any run (§4): SWE-Gym issues from pydantic,
+hydra, conan, dvc, dask, moto and mypy (≤40 per repository, seed 19, patches that edit only existing
+non-test Python files), halves gym-a / gym-b. The release v1.4.0 had never seen these repositories,
+so its one run is also a holdout:
+
+| | file Acc@1 | @3 | @5 | @10 | func Acc@5 |
+|---|---|---|---|---|---|
+| BM25, gym-a / gym-b | 0.121 / 0.121 | 0.257 / 0.271 | 0.329 / 0.307 | 0.407 / 0.436 | – |
+| engine v1.4.0, gym-a | 0.279 | 0.450 | 0.536 | 0.650 | 0.258 |
+| engine v1.4.0, gym-b | 0.271 | 0.414 | 0.493 | 0.600 | 0.254 |
+
+The engine's advantage over BM25 (×2.3 at Acc@1) carries to repositories outside every set it
+was developed on.
+
+### 5.2c Dense vote on the engine's short list (decided on dev + SWE-Gym, run once on holdouts)
+
+jina-code-v2 (int8, CPU) embeds the issue and only the engine's top 30 functions; its order votes
+into the function list by RRF (w = 0.5). Decided on the tuning splits (pooled n = 476: func Acc@5
++0.032, p = 0.0007) and locked by commit before the holdouts:
+
+| set | n | func Acc@1 | func Acc@5 | func Acc@10 | paired @5 |
+|---|---|---|---|---|---|
+| Lite, LocAgent subset | 274 | 0.223 → 0.252 | 0.460 → 0.485 | 0.540 → 0.555 | +0.026 [0.004,0.051], 9/2 |
+| Verified | 459 | 0.198 → 0.237 | 0.429 → 0.458 | 0.497 → 0.514 | +0.028 [0.009,0.048], 17/4, p=0.007 |
+
+Thirty embeddings per issue replace a whole-repository index; the gap to CodeRankEmbed at func
+Acc@5 narrows from −0.058 to −0.033.
+
 ### 5.3 Ablation (Lite holdout, 276)
 
 | removed | Acc@1 | Acc@3 | Acc@5 |
@@ -189,6 +220,7 @@ p50 2.9 s / p90 10.6 s per Lite issue (Verified 2.5 s / 8.8 s) on a 6-core lapto
 | identifiers in the issue's code blocks → their definitions | file Acc@1 drops; func Acc@5 +0.019 (below the 0.02 bar) |
 | RM3 pseudo-relevance feedback on the definition ranking | alone worse at @1 (0.135 → 0.038, dev-fast); as a vote no better than plain BM25 |
 | doc sections as a bridge from question words to identifiers (plain-language) | R@3 down on 3 of 4 sets (ripgrep 0.500 → 0.375, axios 0.833 → 0.583) |
+| learned reranker (logistic / boosted trees, 25 features, top-100 functions) | dev func@5 +0.000/+0.019; trained on dev, applied to SWE-Gym +0.000; coefficients flip sign between splits |
 | evaluation: function-level denominator without short reports | 244 instead of 274 instances, Acc@5 0.443 instead of 0.394 |
 
 ## 7. Threats to validity