From 79b257d3d54870beccbda72aafdbf52891009c0b Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Sat, 17 Oct 2026 09:04:54 +0330 Subject: [PATCH 1/7] research: pre-register SweRankEmbed-Small as dense short-list embedder (bar: func@5 >= +0.02 on dev-fast, dev-rest, gym-a, gym-b; w=0.5, k=30) Co-Authored-By: Claude Opus 5.5 --- docs/research/prereg-s19-swerank.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/research/prereg-s19-swerank.md diff --git a/docs/research/prereg-s19-swerank.md b/docs/research/prereg-s19-swerank.md new file mode 100644 index 0000000..463b42c --- /dev/null +++ b/docs/research/prereg-s19-swerank.md @@ -0,0 +1,24 @@ +# Pre-registration: SweRankEmbed-Small as the dense short-list embedder (session 19, 2026-10-17) + +Committed before any SweRankEmbed-Small number is computed on dev or SWE-Gym. + +**Question.** Does Salesforce SweRankEmbed-Small (137M NomicBERT, CLS pooling, query prefix +"Represent this query for searching relevant code: ", 512-token cut; run in fp32 torch on CPU by +`scripts/research/swerank_embed.py`) beat the engine as the embedder of the dense short-list vote, +in place of jina-code-v2 (§8.18)? + +**Fixed setup (identical to §8.18 except the embedder).** Engine runs `dev-s18a-all.jsonl` (dev, 225) +and `gym-v140-all.jsonl` (SWE-Gym, 280); top k=30 definitions; same definition texts +(`path\nname\nbody`, 1500 chars) and issue text (4000 chars); RRF k=60, vote weight w=0.5; +`scripts/research/dense_shortlist.py --model swerank-embed-small --threads 6`, own cache file. + +**Primary bar.** func Acc@5 of engine + SweRank vote (w=0.5) ≥ engine + 0.02 on **each** of the four +halves: dev-fast, dev-rest, gym-a, gym-b (`priors.py --prior ext`). Pass on all four → SweRank is a +candidate replacement for jina-code; any half below +0.02 → rejected as the vote embedder. + +**Secondary (reported, not decisive).** w=1; dense order alone (SweRank cosine order of the top 30, +no fusion); pooled dev+gym paired comparison vs engine and vs the jina vote (`paired.py`, sign test, +bootstrap CI); seconds per issue (p50) on CPU. + +**Out of scope here.** Lite and Verified are holdouts: no run on them in this branch; the main session +decides whether a Lite/Verified run follows. From d47346defa0b0c01c491beb29c7ad37ef231ccc3 Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Sat, 17 Oct 2026 09:09:07 +0330 Subject: [PATCH 2/7] research: dense_shortlist --model swerank-embed-small (torch NomicBERT forward, CLS pooling, query prefix; jina default unchanged) Co-Authored-By: Claude Opus 5.5 --- scripts/research/dense_shortlist.py | 14 ++++- scripts/research/swerank_embed.py | 95 +++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 scripts/research/swerank_embed.py diff --git a/scripts/research/dense_shortlist.py b/scripts/research/dense_shortlist.py index 9269497..211927a 100644 --- a/scripts/research/dense_shortlist.py +++ b/scripts/research/dense_shortlist.py @@ -7,9 +7,12 @@ ONNX), reorders them by cosine, and writes `dense_defs` / `dense_files` for `priors.py --prior ext`. Source text comes from the clone at the base commit (`git show`), so no checkout is needed. Embeddings are cached by text hash. +`--model swerank-embed-small` swaps in SweRankEmbed-Small (torch, CLS pooling, +query instruction prefix; `swerank_embed.py`) — use its own `--cache` file. py -3 scripts/research/dense_shortlist.py --results dev-s18a-all.jsonl --data devset/swe-dev.json --repos repos --out dense-sl-dev.jsonl [--k 30] [--cache emb-cache/shortlist.npz] + [--model jina-code-v2|swerank-embed-small] """ import argparse import json @@ -32,6 +35,7 @@ def main(): ap.add_argument("--cache", default="") ap.add_argument("--threads", type=int, default=4) ap.add_argument("--subset", default="") + ap.add_argument("--model", default="jina-code-v2", choices=["jina-code-v2", "swerank-embed-small"]) args = ap.parse_args() rows = {r["instance_id"]: r for r in json.load(open(args.data, encoding="utf-8"))} res = [json.loads(l) for l in open(args.results, encoding="utf-8") if l.strip()] @@ -42,7 +46,13 @@ def main(): done = set() if os.path.exists(args.out): done = {json.loads(l)["instance_id"] for l in open(args.out, encoding="utf-8") if l.strip()} - emb = Embedder("jina-code-v2", 512, threads=args.threads) + prefix = "" + if args.model == "jina-code-v2": + emb = Embedder("jina-code-v2", 512, threads=args.threads) + else: + from swerank_embed import QUERY_PREFIX, SweRankEmbedder + emb = SweRankEmbedder(args.model, 512, threads=args.threads) + prefix = QUERY_PREFIX cache = Cache(args.cache) if args.cache else None with open(args.out, "a", encoding="utf-8") as out: for n, r in enumerate(res, 1): @@ -59,7 +69,7 @@ def main(): a, b = d["lines"] body = "\n".join(lines[max(a - 1, 0):b]) texts.append(f"{d['path']}\n{d['name']}\n{body}"[:1500]) - query = row["problem_statement"][:4000] + query = prefix + row["problem_statement"][:4000] t0 = time.time() todo = [t for t in [query] + texts if cache is None or key(t) not in cache.vec] if todo: diff --git a/scripts/research/swerank_embed.py b/scripts/research/swerank_embed.py new file mode 100644 index 0000000..c209907 --- /dev/null +++ b/scripts/research/swerank_embed.py @@ -0,0 +1,95 @@ +"""SweRankEmbed-Small (Salesforce, 137M, NomicBERT) on CPU with plain torch. + +The published checkpoint needs `trust_remote_code` (NomicBERT modelling code +fetched from another repository); this re-implements its forward pass instead +(post-norm encoder, rotary attention base 1000 on the full head, SwiGLU MLP, no +biases in attention/MLP, token-type embeddings, CLS pooling, L2 norm), so only +the weights and tokenizer are read from +`%LOCALAPPDATA%/neuromesh/models/swerank-embed-small`. Queries carry the model +card's instruction prefix (`QUERY_PREFIX`); documents carry none. +""" +import json +import os + +import numpy as np + +MODELS = os.path.join(os.environ.get("LOCALAPPDATA", ""), "neuromesh", "models") +QUERY_PREFIX = "Represent this query for searching relevant code: " + + +class SweRankEmbedder: + def __init__(self, model="swerank-embed-small", max_tokens=512, threads=0): + import torch + from safetensors.torch import load_file + from tokenizers import Tokenizer + + self.torch = torch + if threads: + torch.set_num_threads(threads) + d = os.path.join(MODELS, model) + cfg = json.load(open(os.path.join(d, "config.json"), encoding="utf-8")) + self.n_layer, self.n_head, self.dim = cfg["n_layer"], cfg["n_head"], cfg["n_embd"] + self.eps = cfg["layer_norm_epsilon"] + self.base = float(cfg["rotary_emb_base"]) + sd = load_file(os.path.join(d, "model.safetensors")) + self.w = {k.removeprefix("bert."): v.float() for k, v in sd.items()} + self.tok = Tokenizer.from_file(os.path.join(d, "tokenizer.json")) + self.tok.no_padding() + self.tok.enable_truncation(max_tokens) + self.max_tokens = max_tokens + hd = self.dim // self.n_head + self.inv_freq = 1.0 / (self.base ** (torch.arange(0, hd, 2, dtype=torch.float32) / hd)) + + def _ln(self, x, name): + F = self.torch.nn.functional + return F.layer_norm(x, (self.dim,), self.w[name + ".weight"], self.w[name + ".bias"], self.eps) + + def _lin(self, x, name): + b = self.w.get(name + ".bias") + y = x @ self.w[name + ".weight"].T + return y + b if b is not None else y + + def _rotary(self, x, cos, sin): + x1, x2 = x.chunk(2, dim=-1) + return x * cos + self.torch.cat((-x2, x1), dim=-1) * sin + + def forward(self, ids, mask): + torch = self.torch + F = torch.nn.functional + w = self.w + b, s = ids.shape + h = w["embeddings.word_embeddings.weight"][ids] + w["embeddings.token_type_embeddings.weight"][0] + h = self._ln(h, "emb_ln") + freqs = torch.outer(torch.arange(s, dtype=torch.float32), self.inv_freq) + cos = torch.cat((freqs.cos(), freqs.cos()), -1)[None, None] + sin = torch.cat((freqs.sin(), freqs.sin()), -1)[None, None] + bias = (1.0 - mask[:, None, None, :].float()) * torch.finfo(torch.float32).min + hd = self.dim // self.n_head + for i in range(self.n_layer): + p = f"encoder.layers.{i}." + qkv = self._lin(h, p + "attn.Wqkv").view(b, s, 3, self.n_head, hd).permute(2, 0, 3, 1, 4) + q, k, v = self._rotary(qkv[0], cos, sin), self._rotary(qkv[1], cos, sin), qkv[2] + a = F.scaled_dot_product_attention(q, k, v, attn_mask=bias) + a = self._lin(a.transpose(1, 2).reshape(b, s, self.dim), p + "attn.out_proj") + h = self._ln(h + a, p + "norm1") + m = self._lin(h, p + "mlp.fc11") * F.silu(self._lin(h, p + "mlp.fc12")) + h = self._ln(h + self._lin(m, p + "mlp.fc2"), p + "norm2") + return h[:, 0] + + def embed(self, texts, max_tokens=None, batch=16): + torch = self.torch + enc = [self.tok.encode(t).ids for t in texts] + order = sorted(range(len(texts)), key=lambda i: len(enc[i])) + out = np.zeros((len(texts), self.dim), dtype=np.float32) + with torch.inference_mode(): + for st in range(0, len(order), batch): + idx = order[st:st + batch] + n = max(len(enc[i]) for i in idx) + arr = torch.zeros((len(idx), n), dtype=torch.long) + mask = torch.zeros((len(idx), n), dtype=torch.long) + for j, i in enumerate(idx): + arr[j, :len(enc[i])] = torch.tensor(enc[i]) + mask[j, :len(enc[i])] = 1 + v = torch.nn.functional.normalize(self.forward(arr, mask), dim=-1) + out[idx] = v.numpy() + return out From 51f589bbb405fa700da4eb495e48c35743bbd445 Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Sat, 17 Oct 2026 11:43:39 +0330 Subject: [PATCH 3/7] research: dense_compare.py (engine / vote w / dense alone per half, secs) and paired.py vote-vs-vote mode Co-Authored-By: Claude Opus 5.5 --- scripts/research/dense_compare.py | 60 +++++++++++++++++++++++++++++++ scripts/research/paired.py | 9 ++++- 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 scripts/research/dense_compare.py diff --git a/scripts/research/dense_compare.py b/scripts/research/dense_compare.py new file mode 100644 index 0000000..1e1680b --- /dev/null +++ b/scripts/research/dense_compare.py @@ -0,0 +1,60 @@ +"""Function Acc@k per half for several dense short-list runs side by side +(engine, engine + vote at w, dense order alone), and seconds per issue. +Session 19: jina-code-v2 vs SweRankEmbed-Small. + + py -3 dense_compare.py --results dev-s18a-all.jsonl --gold func-gold-dev.json + --halves devset/dev-fast.json,devset/dev-rest.json + --run jina=dense-sl-dev.jsonl --run swerank=dense-sw-dev.jsonl [--w 0.5,1] +""" +import argparse +import json +import os +import statistics +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +from paired import hit, load, rrf # noqa: E402 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--results", required=True) + ap.add_argument("--gold", required=True) + ap.add_argument("--halves", required=True) + ap.add_argument("--run", action="append", required=True) + ap.add_argument("--w", default="0.5,1") + a = ap.parse_args() + res = load(a.results) + gold = {i: {tuple(x) for x in g} for i, g in json.load(open(a.gold, encoding="utf-8")).items() if g} + runs = {name: load(p) for name, p in (s.split("=", 1) for s in a.run)} + ws = [float(w) for w in a.w.split(",")] + lists = {"engine": {}} + for iid, r in res.items(): + lists["engine"][iid] = [(d["path"], d["name"]) for d in r.get("definitions", [])] + for name, ext in runs.items(): + for w in ws: + lists[f"{name} w={w:g}"] = {} + lists[f"{name} alone"] = {} + for iid, base in lists["engine"].items(): + dense = [tuple(d) for d in ext.get(iid, {}).get("dense_defs", [])] + for w in ws: + lists[f"{name} w={w:g}"][iid] = rrf([base, dense], [1.0, w]) if dense else base + lists[f"{name} alone"][iid] = dense or base + halves = [(os.path.basename(h).replace(".json", ""), {r["instance_id"] for r in json.load(open(h, encoding="utf-8"))}) + for h in a.halves.split(",")] + halves.append(("all", set(res))) + for hname, ids in halves: + ids = sorted(i for i in ids if i in res and i in gold) + print(f"-- {hname}: {len(ids)} issues with function gold") + base5 = sum(hit(lists["engine"][i], gold[i], 5) for i in ids) / len(ids) + for v, lst in lists.items(): + acc = {k: sum(hit(lst[i], gold[i], k) for i in ids) / len(ids) for k in (1, 5, 10)} + print(f" {v:16s} func @1 {acc[1]:.3f} @5 {acc[5]:.3f} @10 {acc[10]:.3f} d@5 {acc[5] - base5:+.3f}") + for name, ext in runs.items(): + secs = [r["secs"] for r in ext.values() if r["instance_id"] in res] + print(f"secs/issue {name}: n={len(secs)} p50 {statistics.median(secs):.1f} mean {statistics.mean(secs):.1f}" + f" p90 {sorted(secs)[int(0.9 * len(secs))]:.1f}") + + +if __name__ == "__main__": + main() diff --git a/scripts/research/paired.py b/scripts/research/paired.py index a0a51f5..1685df9 100644 --- a/scripts/research/paired.py +++ b/scripts/research/paired.py @@ -4,6 +4,9 @@ py -3 paired.py --set dev-s18a-all.jsonl:dense-sl-dev.jsonl:func-gold-dev.json \ --set gym-v140-all.jsonl:dense-sl-gym.jsonl:func-gold-gym.json [--w 0.5] [--k 5] + +A fourth field (`results:ext:gold:ext0`) compares against the engine fused with +`ext0` at the same weight instead of the bare engine (vote vs vote). """ import argparse import json @@ -41,8 +44,9 @@ def main(): a = ap.parse_args() pairs = [] for spec in a.set: - rp, ep, gp = spec.split(":") + rp, ep, gp, *e0 = spec.split(":") res, ext, gold = load(rp), load(ep), json.load(open(gp, encoding="utf-8")) + ext0 = load(e0[0]) if e0 else None for iid, r in res.items(): g = {tuple(x) for x in gold.get(iid, [])} if not g: @@ -50,6 +54,9 @@ def main(): base = [(d["path"], d["name"]) for d in r.get("definitions", [])] dense = [tuple(d) for d in ext.get(iid, {}).get("dense_defs", [])] new = rrf([base, dense], [1.0, a.w]) if dense else base + if ext0 is not None: + d0 = [tuple(d) for d in ext0.get(iid, {}).get("dense_defs", [])] + base = rrf([base, d0], [1.0, a.w]) if d0 else base pairs.append((hit(base, g, a.k), hit(new, g, a.k))) n = len(pairs) b = sum(x for x, _ in pairs) / n From 326724a81820d6a2051b97656f4a247a4c5c8ba9 Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Sat, 17 Oct 2026 11:43:53 +0330 Subject: [PATCH 4/7] research: pre-register k=100 SweRank short-list test (same bar, w=0.5) before running it Co-Authored-By: Claude Opus 5.5 --- docs/research/prereg-s19-swerank.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/research/prereg-s19-swerank.md b/docs/research/prereg-s19-swerank.md index 463b42c..d5f22dd 100644 --- a/docs/research/prereg-s19-swerank.md +++ b/docs/research/prereg-s19-swerank.md @@ -22,3 +22,15 @@ bootstrap CI); seconds per issue (p50) on CPU. **Out of scope here.** Lite and Verified are holdouts: no run on them in this branch; the main session decides whether a Lite/Verified run follows. + +## Addendum (k=100), committed after the k=30 result and before any k=100 number + +k=30 result: the SweRank vote cleared the bar on all four halves (§8.19). The engine's top 30 holds +every gold function for only 0.50 / 0.47 of dev / gym issues; the top 100 holds them for 0.63 / 0.62. +Second test, same embedder, texts, query, RRF k=60 and w=0.5, only the short list widened to the +engine's top **k=100** definitions (`--k 100`, same cache file, `--threads 3`). + +**Bar (unchanged):** func Acc@5 of engine + SweRank vote over the top 100 (w=0.5) ≥ engine + 0.02 on +each of dev-fast, dev-rest, gym-a, gym-b. **Secondary:** SweRank order alone over the top 100; w=1; +paired vs the k=30 SweRank vote (pooled); seconds per issue with the text cache warm from k=30. +The k=30 decision is not revisited by this test. From 582fb83cb2c0e5be6ff393d3122c6d9d5089dfed Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Sat, 17 Oct 2026 11:44:24 +0330 Subject: [PATCH 5/7] =?UTF-8?q?log:=20=C2=A78.19=20SweRankEmbed-Small=20sh?= =?UTF-8?q?ort-list=20vote=20clears=20the=20pre-registered=20bar=20on=20de?= =?UTF-8?q?v=20and=20SWE-Gym=20halves;=20+0.015=20over=20jina,=20not=20sig?= =?UTF-8?q?nificant?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- docs/research/contributions-log.md | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/research/contributions-log.md b/docs/research/contributions-log.md index d053886..f7445ca 100644 --- a/docs/research/contributions-log.md +++ b/docs/research/contributions-log.md @@ -660,6 +660,37 @@ Paired (459): @1 +0.039 [0.013, 0.065], 28/10, p = 0.005; @5 +0.028 [0.009, 0.04 @10 +0.017, 8/0, p = 0.008. Every split — dev, gym-a, gym-b, Lite, Verified — moves the same way by +0.02 to +0.06 at func Acc@5; the effect predicted on tuning data (+0.032) is what both holdouts show. +### 8.19 SweRankEmbed-Small as the short-list embedder (session 19, 2026-10-17) + +**Question.** SweRankEmbed-Small (Salesforce, 137M NomicBERT, reports func Acc@5 0.631 on Lite-274 as a +whole-repository retriever) in place of jina-code-v2 for the dense vote of §8.18, everything else fixed +(top k=30 engine definitions, same texts, RRF k=60, w=0.5). Bar pre-registered in fca506a +(`docs/research/prereg-s19-swerank.md`) before any number: func Acc@5 ≥ engine + 0.02 on each of +dev-fast, dev-rest, gym-a, gym-b. No ONNX export exists; `scripts/research/swerank_embed.py` is a +plain-torch re-implementation of the NomicBERT forward (no `trust_remote_code`), fp32 on CPU, CLS +pooling, the model card's query prefix, 512-token cut. + +| func Acc@5 | dev-fast (52) | dev-rest (158) | gym-a (132) | gym-b (134) | +|---|---|---|---|---| +| engine | 0.308 | 0.329 | 0.258 | 0.254 | +| + jina vote w=0.5 (§8.18) | 0.327 (+0.019) | 0.342 (+0.013) | 0.288 (+0.030) | 0.313 (+0.060) | +| + SweRank vote w=0.5 | **0.346 (+0.038)** | **0.361 (+0.032)** | **0.318 (+0.061)** | **0.306 (+0.052)** | +| + SweRank vote w=1 | 0.346 | 0.354 | 0.318 | 0.321 | +| SweRank order alone (top 30) | 0.288 | 0.361 | 0.280 | 0.313 | +| jina order alone (top 30) | 0.269 | 0.278 | 0.258 | 0.269 | + +Bar: **passes on all four halves** (jina did not on dev: +0.019 / +0.013). Pooled dev+gym (n=476, +`paired.py`) vs engine: func@5 0.286 → 0.332, +0.046 [0.027, 0.067], 24 gained / 2 lost, p < 0.0001; +func@10 +0.027 (13/0, p = 0.0002); func@1 +0.013 (12/6, p = 0.24). **Head-to-head vs the jina vote** +(same w): func@5 0.317 → 0.332, +0.015 [−0.002, 0.032], 11/4, p = 0.12; func@1 −0.008 (9/13); func@10 ++0.006 — better, not significantly. SweRank's own cosine order is far better than jina's (dev all +0.343 vs 0.276), so the model ranks the short list well on its own, not only as a tie-breaker. + +Cost (CPU, 31 texts per issue, fp32 torch, 6 threads): dev p50 10.8 s / issue (jina int8 ONNX 10.3 s); +gym p50 17.9 s (run while another CPU-heavy job held the machine; jina 6.2 s). The model is 2× jina's +size in fp32; an int8 ONNX export would be the fair speed comparison. Licence cc-by-nc-4.0 +(non-commercial) — the engine cannot ship it as a default, unlike jina-code (Apache-2.0). + ### 8.20 Rerank ceiling: the engine already finds the code (session 19, 2026-10-17) Share of issues whose every gold function is in the engine's first k (v1.4.0, `NM_DEFINITIONS=300`; From 480e92ff777edd6e52afbd7a63ad72508bc4bed8 Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Sat, 17 Oct 2026 11:47:17 +0330 Subject: [PATCH 6/7] research: fix holdout selection rule (k=100 only if it passes all four halves and pooled func@5 >= k=30) before any k=100 number Co-Authored-By: Claude Opus 5.5 --- docs/research/prereg-s19-swerank.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/research/prereg-s19-swerank.md b/docs/research/prereg-s19-swerank.md index d5f22dd..ccff1a0 100644 --- a/docs/research/prereg-s19-swerank.md +++ b/docs/research/prereg-s19-swerank.md @@ -34,3 +34,10 @@ engine's top **k=100** definitions (`--k 100`, same cache file, `--threads 3`). each of dev-fast, dev-rest, gym-a, gym-b. **Secondary:** SweRank order alone over the top 100; w=1; paired vs the k=30 SweRank vote (pooled); seconds per issue with the text cache warm from k=30. The k=30 decision is not revisited by this test. + +## Holdout selection rule, committed before any k=100 number was looked at + +Lite/Verified are run once with exactly one variant: SweRank vote w=0.5 at k=100 if k=100 passes the +bar on all four halves AND its pooled func@5 is >= the k=30 pooled func@5; otherwise k=30. No other +variant is run on the holdouts. (The main session runs the holdout, over the regenerated 300-entry +lists `lite-v140d300-all.jsonl` / `verified-v140d300-all.jsonl`.) From 0630bd5abd8f8e9a6c6107448eeb2fcbbb4fb06d Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Sat, 17 Oct 2026 18:45:24 +0330 Subject: [PATCH 7/7] =?UTF-8?q?log:=20=C2=A78.19=20k=3D100=20SweRank=20vot?= =?UTF-8?q?e=20fails=20the=20bar=20on=20dev-rest=20(+0.006);=20holdout=20v?= =?UTF-8?q?ariant=20by=20the=20fixed=20rule=20=3D=20k=3D30,=20w=3D0.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- docs/research/contributions-log.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/research/contributions-log.md b/docs/research/contributions-log.md index f7445ca..5cb7902 100644 --- a/docs/research/contributions-log.md +++ b/docs/research/contributions-log.md @@ -691,6 +691,26 @@ gym p50 17.9 s (run while another CPU-heavy job held the machine; jina 6.2 s). T size in fp32; an int8 ONNX export would be the fair speed comparison. Licence cc-by-nc-4.0 (non-commercial) — the engine cannot ship it as a default, unlike jina-code (Apache-2.0). +**k=100 (pre-registered in the addendum before any number; same bar, w=0.5).** The engine's top 30 +holds every gold function for 0.50 / 0.47 of dev / gym issues, the top 100 for 0.63 / 0.62. + +| func Acc@5 | dev-fast | dev-rest | gym-a | gym-b | pooled (476) | +|---|---|---|---|---|---| +| engine | 0.308 | 0.329 | 0.258 | 0.254 | 0.286 | +| SweRank vote k=30, w=0.5 | 0.346 | 0.361 | 0.318 | 0.306 | 0.332 | +| SweRank vote k=100, w=0.5 | 0.346 (+0.038) | **0.335 (+0.006)** | 0.326 (+0.068) | 0.328 (+0.075) | 0.332 | +| SweRank vote k=100, w=1 | 0.346 | 0.367 | 0.318 | 0.328 | – | +| SweRank order alone, top 100 | 0.288 | 0.380 | 0.288 | 0.321 | – | + +Bar: **fails on dev-rest** (+0.006). Pooled vs engine +0.046 [0.021, 0.071], 30/8 (k=30: 24/2); vs the +k=30 vote +0.000 (6/6) at @5, +0.002 at @1, +0.004 at @10. Widening the list adds as many losses as +gains at w=0.5: 70 more candidates also bring 70 more chances to push a gold function out of the top 5. +w=1 and dense-alone over the top 100 look better on dev-rest, but they are secondary, not tested +variants. **Holdout choice by the rule fixed before the k=100 numbers: k=30, w=0.5.** Cost at k=100: +p50 14.6 s (dev) / 20.0 s (gym) per issue with the k=30 texts cached, 6 threads, while a second copy of +the run competed for the CPU (an orphaned process; its duplicate rows matched and were dropped) — +upper bounds. + ### 8.20 Rerank ceiling: the engine already finds the code (session 19, 2026-10-17) Share of issues whose every gold function is in the engine's first k (v1.4.0, `NM_DEFINITIONS=300`;