Preserve fallback duplicate matches while indexing MinHash candidates

project/TICKETS.md

@@ -24,4 +24,8 @@ New identities are reserved exclusively by project/new-ticket.sh.
 | **ticket-018** | [`README.md`](./ticket-018/README.md) | - | - | - | - | - |
 | **ticket-019** | [`README.md`](./ticket-019/README.md) | - | - | - | - | - |
 | **ticket-020** | [`README.md`](./ticket-020/README.md) | - | - | - | - | - |
+| **ticket-021** | [`README.md`](./ticket-021/README.md) | - | - | - | - | - |
+| **ticket-024** | [`README.md`](./ticket-024/README.md) | - | - | - | - | - |
+| **ticket-025** | [`README.md`](./ticket-025/README.md) | - | - | - | - | - |
+| **ticket-026** | [`README.md`](./ticket-026/README.md) | - | - | - | - | - |
 <!-- AUTO:TICKET_INDEX:END -->

project/ticket-027/README.md

@@ -0,0 +1,27 @@
+# Ticket 027: Index fallback MinHash candidates without losing duplicate matches
+
+- **ID**: ticket-027
+- **Owner**: tom (session request); implementation agent: codex
+- **Status**: IN_PROGRESS
+- **Workflow state**: PUBLICATION
+- **Created**: 2026-11-06
+
+## Goal and scope
+
+SESSION_EXECUTION_AUTHORIZATION: user requested compact, complete project analysis and repeatedly instructed “kontynuuj, scalaj”. This ticket implements the upstream reDUP performance repair tracked by PLF-19981 under coordination PLF-19979.
+
+Without datasketch, every query currently compares all signatures, causing quadratic work across workspace inventories. Index disjoint signature bands to retain every result accepted by the existing floating-point threshold comparison. Keep original scoring, equal-score occurrence ordering and duplicate grouping. Rebuild after additions or a threshold change; preserve exhaustive behavior for unsupported thresholds or native scoring. The source scope is exactly lsh_matcher.py and its existing regression tests; dependency/backend selection is unchanged.
+
+## Acceptance criteria
+
+- [x] AC-01: Record existing user execution/publication authorization and exact native allocation.
+- [x] AC-02: Match exhaustive fallback results, scores, ordering and groups, including floating-point boundaries and index additions.
+- [x] AC-03: Demonstrate bounded candidate comparison reduction on an unrelated-signature inventory without capping results.
+- [x] AC-04: Pass native affected tests, supported-version tests and governance checks.
+- [ ] AC-05: Independent protected Validator approval and merge of the exact frozen PR head.
+
+## Delivery evidence
+
+Host evidence: tellmesh/.planfile/evidence/project-analysis-remaining-20261106. Native lease ticket-027--fallback-minhash-index, revision 1, fencing token 1. Preserve all other leases, branches and worktrees.
+
+Validation: 294 passed / 14 optional-backend skips on each Python 3.10 and 3.13 (locked dev dependencies); 63 passed / 10 skips in affected tests with the existing native extension environment. Ruff and the native governance gate pass. The new comparison-budget regression failed before implementation (1000 scores versus 1 required) and passes afterward. The bounded 1000-vector / 1000-query benchmark retained every ordered result: 1,000,000 to 1,000 Jaccard calls, 15.346 s to 0.463 s on this host. Dense candidate sets may still require exhaustive scoring. Independent approval/merge remains pending.

project/ticket-027/intent.json

@@ -0,0 +1,88 @@
+{
+  "schema": "new-project.intent/v3",
+  "ticket": "ticket-027",
+  "summary": "Index fallback MinHash candidates without losing duplicate matches",
+  "workstream": "application",
+  "classification": {
+    "kind": "BUG",
+    "priority": "P2",
+    "origin": "regression"
+  },
+  "allowedPaths": [
+    "project/ticket-027/**",
+    "TODO.md",
+    "project/TICKETS.md",
+    "src/redup/core/lsh_matcher.py",
+    "tests/test_lsh_matcher.py"
+  ],
+  "forbiddenPaths": [
+    "project/ticket-*/user-*.md"
+  ],
+  "stacks": [],
+  "dependsOn": [],
+  "conflictsWith": [],
+  "integrationTicket": null,
+  "delivery": {
+    "acceptedBaseSha": "adba8c1a82468267e2d1902b8b68ff1fcf35378b",
+    "targetBranch": "main",
+    "outcome": "Fallback signature candidate indexing retains exhaustive duplicate results while avoiding unrelated pair scoring.",
+    "nonGoals": [
+      "Changing native or datasketch similarity semantics",
+      "Dependency changes or source/result caps"
+    ],
+    "complexity": "S",
+    "estimatedMinutes": 90,
+    "budgets": {
+      "maxImplementationFiles": 2,
+      "maxAffectedComponents": 1,
+      "maxPublicInterfaceChanges": 0,
+      "maxRuntimeDependencies": 0
+    },
+    "architecture": {
+      "status": "accepted",
+      "decision": "Add a cached component-local d+1 disjoint-band index; score candidates with the original method in original order.",
+      "components": [
+        {
+          "name": "fallback-lsh",
+          "paths": [
+            "src/redup/core/lsh_matcher.py",
+            "tests/test_lsh_matcher.py"
+          ]
+        }
+      ],
+      "responsibilityChanges": false,
+      "interfaceChanges": [],
+      "dataChanges": [
+        {
+          "kind": "component-local-state",
+          "component": "fallback-lsh",
+          "description": "Ephemeral signature band tables invalidated on additions or threshold changes."
+        }
+      ],
+      "ui": {
+        "impact": "none",
+        "states": [],
+        "evidence": []
+      },
+      "rollback": "Revert the source/test implementation commit through independently reviewed publication."
+    },
+    "runtimeDependencies": [],
+    "validation": [
+      {
+        "criterion": "AC-02",
+        "commands": [
+          "python -m pytest tests/test_lsh_matcher.py -q"
+        ],
+        "evidence": "Native exact-parity regression log in external continuation evidence."
+      },
+      {
+        "criterion": "AC-04",
+        "commands": [
+          "python -m pytest -q",
+          "bash project/governance-check.sh"
+        ],
+        "evidence": "Native supported-version and governance logs; protected exact-head checks."
+      }
+    ]
+  }
+}

src/redup/core/lsh_matcher.py

@@ -2,9 +2,12 @@
 
 from __future__ import annotations
 
+import math
 from collections.abc import Iterable
 from typing import Any
 
+from redup.core import tokens_hasher
+
 try:
     from datasketch import MinHash, MinHashLSH
 except ImportError:
@@ -56,11 +59,7 @@ def _create_minhash(text: str, num_perm: int = 128) -> MinHash | None:
     tokens = normalized.split()
 
     # Native signature: one Rust call (min over a set is order/duplicate invariant).
-    if (
-        _native is not None
-        and _np is not None
-        and hasattr(_native, "compute_minhash_signature")
-    ):
+    if _native is not None and _np is not None and hasattr(_native, "compute_minhash_signature"):
         try:
             signature = _native.compute_minhash_signature(list(dict.fromkeys(tokens)), num_perm)
             mh = _make_minhash_from_signature(signature, num_perm)
@@ -86,12 +85,14 @@ def __init__(self, threshold: float = 0.8, num_perm: int = 128):
         self.blocks: list[CodeBlock] = []
         self.minhashes: list[Any] = []
         self.lsh = None
+        self._simple_signature_index: tuple[Any, ...] | None = None
 
         if MinHashLSH is not None:
             self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
 
     def add(self, block: CodeBlock) -> None:
         """Add a code block to the LSH index."""
+        self._simple_signature_index = None
         self.blocks.append(block)
 
         # Create MinHash
@@ -118,11 +119,10 @@ def add_blocks(self, blocks: list[CodeBlock]) -> None:
         ):
             try:
                 feature_lists = [
-                    list(dict.fromkeys(_normalize_text(b.text).split()))
-                    for b in blocks
+                    list(dict.fromkeys(_normalize_text(b.text).split())) for b in blocks
                 ]
                 signatures = _native.batch_minhash_signature(feature_lists, self.num_perm)
-                for block, sig in zip(blocks, signatures):
+                for block, sig in zip(blocks, signatures, strict=False):
                     self.blocks.append(block)
                     minhash = _make_minhash_from_signature(sig, self.num_perm)
                     if minhash and self.lsh:
@@ -169,12 +169,57 @@ def _compare_minhashes(
                         result.append((self.blocks[idx], similarity))
         return sorted(result, key=lambda x: x[1], reverse=True)
 
+    def _simple_candidate_indices(self, query: _SimpleMinHash) -> Iterable[int]:
+        """Retain every equality-count threshold match using disjoint bands.
+
+        With at most d unequal positions, d+1 disjoint bands contain an equal
+        band. Candidate scoring still uses the original Jaccard implementation.
+        Native scoring and unusual thresholds retain exhaustive comparisons.
+        """
+        threshold = self.threshold
+        if (
+            not isinstance(threshold, (int, float))
+            or not math.isfinite(threshold)
+            or not 0 < threshold <= 1
+            or tokens_hasher.redup_fast_hash is not None
+        ):
+            return range(len(self.minhashes))
+        width = len(query.hash_values)
+        if not width:
+            return ()
+        required = math.ceil(threshold * width)
+        # Match the original floating-point ratio comparison, including 7/25.
+        while required > 0 and (required - 1) / width >= threshold:
+            required -= 1
+        while required / width < threshold:
+            required += 1
+        bands = width - required + 1
+        key = (id(self.minhashes), len(self.minhashes), threshold, width)
+        cached = self._simple_signature_index
+        if cached is None or cached[0] != key:
+            spans = [(i * width // bands, (i + 1) * width // bands) for i in range(bands)]
+            tables: list[dict[tuple[int, ...], list[int]]] = [{} for _ in spans]
+            for number, stored in enumerate(self.minhashes):
+                if not isinstance(stored, _SimpleMinHash) or len(stored.hash_values) != width:
+                    continue
+                for table, (start, stop) in zip(tables, spans, strict=True):
+                    band = tuple(stored.hash_values[start:stop])
+                    table.setdefault(band, []).append(number)
+            cached = (key, spans, tables)
+            self._simple_signature_index = cached
+        _, spans, tables = cached
+        candidates = set()
+        for table, (start, stop) in zip(tables, spans, strict=True):
+            candidates.update(table.get(tuple(query.hash_values[start:stop]), ()))
+        return sorted(candidates)
+
     def _find_near_duplicates_simple(self, block: CodeBlock) -> list[tuple[CodeBlock, float]]:
-        """Fallback near-duplicate detection without LSH."""
+        """Fallback detection with exact signature candidate pruning."""
         query_minhash = _create_simple_minhash(block.text, self.num_perm)
         result = []
 
-        for i, stored_minhash in enumerate(self.minhashes):
+        for i in self._simple_candidate_indices(query_minhash):
+            stored_minhash = self.minhashes[i]
             if isinstance(stored_minhash, _SimpleMinHash):
                 similarity = query_minhash.jaccard(stored_minhash)
                 if similarity >= self.threshold:

tests/test_lsh_matcher.py

@@ -1,8 +1,10 @@
 """Regression coverage for real LSH candidates and mutable scanner blocks."""
 
+import random
+
 import pytest
 
-from redup.core import lsh_matcher
+from redup.core import lsh_matcher, tokens_hasher
 from redup.core.models import DuplicateType, ScanConfig
 from redup.core.pipeline.duplicate_finder import find_near_duplicate_groups
 from redup.core.scanner import CodeBlock
@@ -126,3 +128,113 @@ def test_index_add_blocks_batch(backend):
     assert len(index.blocks) == 3
     matches = index.find_near_duplicates(block("query.py", "alpha beta gamma delta epsilon"))
     assert len(matches) == 2
+
+
+@pytest.fixture
+def simple_backend(monkeypatch):
+    monkeypatch.setattr(lsh_matcher, "MinHash", None)
+    monkeypatch.setattr(lsh_matcher, "MinHashLSH", None)
+    monkeypatch.setattr(tokens_hasher, "redup_fast_hash", None)
+
+
+def signature(values):
+    result = object.__new__(lsh_matcher._SimpleMinHash)
+    result.num_perm = len(values)
+    result.hash_values = values
+    return result
+
+
+def exhaustive_matches(index, query):
+    matches = []
+    for candidate, stored in zip(index.blocks, index.minhashes, strict=True):
+        if isinstance(stored, lsh_matcher._SimpleMinHash):
+            score = query.jaccard(stored)
+            if score >= index.threshold:
+                matches.append((candidate, score))
+    return sorted(matches, key=lambda item: item[1], reverse=True)
+
+
+@pytest.mark.parametrize("width", [7, 25, 128])
+@pytest.mark.parametrize("threshold", [0, 0.1, 7 / 25, 0.5, 0.85, 1, 1.1])
+def test_fallback_matches_exhaustive_scores_and_stable_order(
+    simple_backend, monkeypatch, width, threshold
+):
+    rng = random.Random(731)
+    anchor = [rng.getrandbits(128) for _ in range(width)]
+    vectors = [anchor, list(anchor)]
+    for equal in range(width + 1):
+        vector = [rng.getrandbits(128) for _ in range(width)]
+        for position in rng.sample(range(width), equal):
+            vector[position] = anchor[position]
+        vectors.append(vector)
+    signatures = {str(i): signature(vector) for i, vector in enumerate(vectors)}
+    monkeypatch.setattr(lsh_matcher, "_create_simple_minhash", lambda text, _: signatures[text])
+    index = lsh_matcher.LSHIndex(threshold=threshold, num_perm=width)
+    index.add_blocks([block(str(i), str(i)) for i in range(len(vectors))])
+    for query in index.blocks[:6]:
+        assert index.find_near_duplicates(query) == exhaustive_matches(
+            index, signatures[query.text]
+        )
+
+
+def test_fallback_prunes_comparisons_and_refreshes_after_add(simple_backend, monkeypatch):
+    signatures = {str(i): signature([i * 1000 + j for j in range(128)]) for i in range(1000)}
+    signatures["duplicate"] = signatures["0"]
+    monkeypatch.setattr(lsh_matcher, "_create_simple_minhash", lambda text, _: signatures[text])
+    index = lsh_matcher.LSHIndex(threshold=0.85)
+    index.add_blocks([block(str(i), str(i)) for i in range(1000)])
+    calls = []
+    original = lsh_matcher._SimpleMinHash.jaccard
+
+    def counted(left, right):
+        calls.append(1)
+        return original(left, right)
+
+    monkeypatch.setattr(lsh_matcher._SimpleMinHash, "jaccard", counted)
+    assert index.find_near_duplicates(index.blocks[0]) == [(index.blocks[0], 1.0)]
+    assert len(calls) == 1
+    calls.clear()
+    duplicate = block("duplicate.py", "duplicate")
+    index.add(duplicate)
+    assert index.find_near_duplicates(index.blocks[0]) == [(index.blocks[0], 1.0), (duplicate, 1.0)]
+    assert len(calls) == 2
+
+
+def test_fallback_refreshes_after_threshold_change(simple_backend, monkeypatch):
+    signatures = {
+        "query": signature(list(range(25))),
+        "partial": signature(list(range(7)) + [-1] * 18),
+    }
+    monkeypatch.setattr(lsh_matcher, "_create_simple_minhash", lambda text, _: signatures[text])
+    index = lsh_matcher.LSHIndex(threshold=0.85, num_perm=25)
+    query, partial = block(text="query"), block("partial.py", "partial")
+    index.add_blocks([query, partial])
+    assert index.find_near_duplicates(query) == [(query, 1.0)]
+    index.threshold = 7 / 25
+    assert index.find_near_duplicates(query) == [(query, 1.0), (partial, 7 / 25)]
+
+
+def test_fallback_preserves_native_scorer_semantics(simple_backend, monkeypatch):
+    from types import SimpleNamespace
+
+    index = lsh_matcher.LSHIndex(threshold=0.85)
+    left, other = block(), block("other.py", "unrelated tokens entirely")
+    index.add_blocks([left, other])
+    # A native scorer is an independent backend; do not assume equality-count semantics.
+    monkeypatch.setattr(
+        tokens_hasher, "redup_fast_hash", SimpleNamespace(minhash_jaccard=lambda a, b: 1.0)
+    )
+    assert index.find_near_duplicates(left) == [(left, 1.0), (other, 1.0)]
+
+
+def test_fallback_groups_match_exhaustive_grouping(simple_backend, monkeypatch):
+    blocks = [block(f"{i}.py", f"alpha beta gamma {i % 3} epsilon") for i in range(30)]
+    indexed = lsh_matcher.find_near_duplicates(blocks, threshold=0.7)
+
+    def exhaustive(self, query):
+        return exhaustive_matches(
+            self, lsh_matcher._create_simple_minhash(query.text, self.num_perm)
+        )
+
+    monkeypatch.setattr(lsh_matcher.LSHIndex, "_find_near_duplicates_simple", exhaustive)
+    assert indexed == lsh_matcher.find_near_duplicates(blocks, threshold=0.7)