From 7bef05ca07a391b278f7baecf8724c0e4c268718 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Fri, 6 Nov 2026 12:37:40 +0100 Subject: [PATCH] perf: index fallback MinHash candidates without losing matches --- project/TICKETS.md | 4 ++ project/ticket-027/README.md | 27 ++++++++ project/ticket-027/intent.json | 88 +++++++++++++++++++++++++ src/redup/core/lsh_matcher.py | 65 ++++++++++++++++--- tests/test_lsh_matcher.py | 114 ++++++++++++++++++++++++++++++++- 5 files changed, 287 insertions(+), 11 deletions(-) create mode 100644 project/ticket-027/README.md create mode 100644 project/ticket-027/intent.json diff --git a/project/TICKETS.md b/project/TICKETS.md index 91aed50..330050b 100644 --- a/project/TICKETS.md +++ b/project/TICKETS.md @@ -24,4 +24,8 @@ New identities are reserved exclusively by project/new-ticket.sh. | **ticket-018** | [`README.md`](./ticket-018/README.md) | - | - | - | - | - | | **ticket-019** | [`README.md`](./ticket-019/README.md) | - | - | - | - | - | | **ticket-020** | [`README.md`](./ticket-020/README.md) | - | - | - | - | - | +| **ticket-021** | [`README.md`](./ticket-021/README.md) | - | - | - | - | - | +| **ticket-024** | [`README.md`](./ticket-024/README.md) | - | - | - | - | - | +| **ticket-025** | [`README.md`](./ticket-025/README.md) | - | - | - | - | - | +| **ticket-026** | [`README.md`](./ticket-026/README.md) | - | - | - | - | - | diff --git a/project/ticket-027/README.md b/project/ticket-027/README.md new file mode 100644 index 0000000..cedee60 --- /dev/null +++ b/project/ticket-027/README.md @@ -0,0 +1,27 @@ +# Ticket 027: Index fallback MinHash candidates without losing duplicate matches + +- **ID**: ticket-027 +- **Owner**: tom (session request); implementation agent: codex +- **Status**: IN_PROGRESS +- **Workflow state**: PUBLICATION +- **Created**: 2026-11-06 + +## Goal and scope + +SESSION_EXECUTION_AUTHORIZATION: user requested compact, complete project analysis and repeatedly instructed “kontynuuj, scalaj”. This ticket implements the upstream reDUP performance repair tracked by PLF-19981 under coordination PLF-19979. + +Without datasketch, every query currently compares all signatures, causing quadratic work across workspace inventories. Index disjoint signature bands to retain every result accepted by the existing floating-point threshold comparison. Keep original scoring, equal-score occurrence ordering and duplicate grouping. Rebuild after additions or a threshold change; preserve exhaustive behavior for unsupported thresholds or native scoring. The source scope is exactly lsh_matcher.py and its existing regression tests; dependency/backend selection is unchanged. + +## Acceptance criteria + +- [x] AC-01: Record existing user execution/publication authorization and exact native allocation. +- [x] AC-02: Match exhaustive fallback results, scores, ordering and groups, including floating-point boundaries and index additions. +- [x] AC-03: Demonstrate bounded candidate comparison reduction on an unrelated-signature inventory without capping results. +- [x] AC-04: Pass native affected tests, supported-version tests and governance checks. +- [ ] AC-05: Independent protected Validator approval and merge of the exact frozen PR head. + +## Delivery evidence + +Host evidence: tellmesh/.planfile/evidence/project-analysis-remaining-20261106. Native lease ticket-027--fallback-minhash-index, revision 1, fencing token 1. Preserve all other leases, branches and worktrees. + +Validation: 294 passed / 14 optional-backend skips on each Python 3.10 and 3.13 (locked dev dependencies); 63 passed / 10 skips in affected tests with the existing native extension environment. Ruff and the native governance gate pass. The new comparison-budget regression failed before implementation (1000 scores versus 1 required) and passes afterward. The bounded 1000-vector / 1000-query benchmark retained every ordered result: 1,000,000 to 1,000 Jaccard calls, 15.346 s to 0.463 s on this host. Dense candidate sets may still require exhaustive scoring. Independent approval/merge remains pending. diff --git a/project/ticket-027/intent.json b/project/ticket-027/intent.json new file mode 100644 index 0000000..996e10b --- /dev/null +++ b/project/ticket-027/intent.json @@ -0,0 +1,88 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-027", + "summary": "Index fallback MinHash candidates without losing duplicate matches", + "workstream": "application", + "classification": { + "kind": "BUG", + "priority": "P2", + "origin": "regression" + }, + "allowedPaths": [ + "project/ticket-027/**", + "TODO.md", + "project/TICKETS.md", + "src/redup/core/lsh_matcher.py", + "tests/test_lsh_matcher.py" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null, + "delivery": { + "acceptedBaseSha": "adba8c1a82468267e2d1902b8b68ff1fcf35378b", + "targetBranch": "main", + "outcome": "Fallback signature candidate indexing retains exhaustive duplicate results while avoiding unrelated pair scoring.", + "nonGoals": [ + "Changing native or datasketch similarity semantics", + "Dependency changes or source/result caps" + ], + "complexity": "S", + "estimatedMinutes": 90, + "budgets": { + "maxImplementationFiles": 2, + "maxAffectedComponents": 1, + "maxPublicInterfaceChanges": 0, + "maxRuntimeDependencies": 0 + }, + "architecture": { + "status": "accepted", + "decision": "Add a cached component-local d+1 disjoint-band index; score candidates with the original method in original order.", + "components": [ + { + "name": "fallback-lsh", + "paths": [ + "src/redup/core/lsh_matcher.py", + "tests/test_lsh_matcher.py" + ] + } + ], + "responsibilityChanges": false, + "interfaceChanges": [], + "dataChanges": [ + { + "kind": "component-local-state", + "component": "fallback-lsh", + "description": "Ephemeral signature band tables invalidated on additions or threshold changes." + } + ], + "ui": { + "impact": "none", + "states": [], + "evidence": [] + }, + "rollback": "Revert the source/test implementation commit through independently reviewed publication." + }, + "runtimeDependencies": [], + "validation": [ + { + "criterion": "AC-02", + "commands": [ + "python -m pytest tests/test_lsh_matcher.py -q" + ], + "evidence": "Native exact-parity regression log in external continuation evidence." + }, + { + "criterion": "AC-04", + "commands": [ + "python -m pytest -q", + "bash project/governance-check.sh" + ], + "evidence": "Native supported-version and governance logs; protected exact-head checks." + } + ] + } +} diff --git a/src/redup/core/lsh_matcher.py b/src/redup/core/lsh_matcher.py index 5a3137e..6f2d997 100644 --- a/src/redup/core/lsh_matcher.py +++ b/src/redup/core/lsh_matcher.py @@ -2,9 +2,12 @@ from __future__ import annotations +import math from collections.abc import Iterable from typing import Any +from redup.core import tokens_hasher + try: from datasketch import MinHash, MinHashLSH except ImportError: @@ -56,11 +59,7 @@ def _create_minhash(text: str, num_perm: int = 128) -> MinHash | None: tokens = normalized.split() # Native signature: one Rust call (min over a set is order/duplicate invariant). - if ( - _native is not None - and _np is not None - and hasattr(_native, "compute_minhash_signature") - ): + if _native is not None and _np is not None and hasattr(_native, "compute_minhash_signature"): try: signature = _native.compute_minhash_signature(list(dict.fromkeys(tokens)), num_perm) mh = _make_minhash_from_signature(signature, num_perm) @@ -86,12 +85,14 @@ def __init__(self, threshold: float = 0.8, num_perm: int = 128): self.blocks: list[CodeBlock] = [] self.minhashes: list[Any] = [] self.lsh = None + self._simple_signature_index: tuple[Any, ...] | None = None if MinHashLSH is not None: self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm) def add(self, block: CodeBlock) -> None: """Add a code block to the LSH index.""" + self._simple_signature_index = None self.blocks.append(block) # Create MinHash @@ -118,11 +119,10 @@ def add_blocks(self, blocks: list[CodeBlock]) -> None: ): try: feature_lists = [ - list(dict.fromkeys(_normalize_text(b.text).split())) - for b in blocks + list(dict.fromkeys(_normalize_text(b.text).split())) for b in blocks ] signatures = _native.batch_minhash_signature(feature_lists, self.num_perm) - for block, sig in zip(blocks, signatures): + for block, sig in zip(blocks, signatures, strict=False): self.blocks.append(block) minhash = _make_minhash_from_signature(sig, self.num_perm) if minhash and self.lsh: @@ -169,12 +169,57 @@ def _compare_minhashes( result.append((self.blocks[idx], similarity)) return sorted(result, key=lambda x: x[1], reverse=True) + def _simple_candidate_indices(self, query: _SimpleMinHash) -> Iterable[int]: + """Retain every equality-count threshold match using disjoint bands. + + With at most d unequal positions, d+1 disjoint bands contain an equal + band. Candidate scoring still uses the original Jaccard implementation. + Native scoring and unusual thresholds retain exhaustive comparisons. + """ + threshold = self.threshold + if ( + not isinstance(threshold, (int, float)) + or not math.isfinite(threshold) + or not 0 < threshold <= 1 + or tokens_hasher.redup_fast_hash is not None + ): + return range(len(self.minhashes)) + width = len(query.hash_values) + if not width: + return () + required = math.ceil(threshold * width) + # Match the original floating-point ratio comparison, including 7/25. + while required > 0 and (required - 1) / width >= threshold: + required -= 1 + while required / width < threshold: + required += 1 + bands = width - required + 1 + key = (id(self.minhashes), len(self.minhashes), threshold, width) + cached = self._simple_signature_index + if cached is None or cached[0] != key: + spans = [(i * width // bands, (i + 1) * width // bands) for i in range(bands)] + tables: list[dict[tuple[int, ...], list[int]]] = [{} for _ in spans] + for number, stored in enumerate(self.minhashes): + if not isinstance(stored, _SimpleMinHash) or len(stored.hash_values) != width: + continue + for table, (start, stop) in zip(tables, spans, strict=True): + band = tuple(stored.hash_values[start:stop]) + table.setdefault(band, []).append(number) + cached = (key, spans, tables) + self._simple_signature_index = cached + _, spans, tables = cached + candidates = set() + for table, (start, stop) in zip(tables, spans, strict=True): + candidates.update(table.get(tuple(query.hash_values[start:stop]), ())) + return sorted(candidates) + def _find_near_duplicates_simple(self, block: CodeBlock) -> list[tuple[CodeBlock, float]]: - """Fallback near-duplicate detection without LSH.""" + """Fallback detection with exact signature candidate pruning.""" query_minhash = _create_simple_minhash(block.text, self.num_perm) result = [] - for i, stored_minhash in enumerate(self.minhashes): + for i in self._simple_candidate_indices(query_minhash): + stored_minhash = self.minhashes[i] if isinstance(stored_minhash, _SimpleMinHash): similarity = query_minhash.jaccard(stored_minhash) if similarity >= self.threshold: diff --git a/tests/test_lsh_matcher.py b/tests/test_lsh_matcher.py index 10515a4..67f9fe5 100644 --- a/tests/test_lsh_matcher.py +++ b/tests/test_lsh_matcher.py @@ -1,8 +1,10 @@ """Regression coverage for real LSH candidates and mutable scanner blocks.""" +import random + import pytest -from redup.core import lsh_matcher +from redup.core import lsh_matcher, tokens_hasher from redup.core.models import DuplicateType, ScanConfig from redup.core.pipeline.duplicate_finder import find_near_duplicate_groups from redup.core.scanner import CodeBlock @@ -126,3 +128,113 @@ def test_index_add_blocks_batch(backend): assert len(index.blocks) == 3 matches = index.find_near_duplicates(block("query.py", "alpha beta gamma delta epsilon")) assert len(matches) == 2 + + +@pytest.fixture +def simple_backend(monkeypatch): + monkeypatch.setattr(lsh_matcher, "MinHash", None) + monkeypatch.setattr(lsh_matcher, "MinHashLSH", None) + monkeypatch.setattr(tokens_hasher, "redup_fast_hash", None) + + +def signature(values): + result = object.__new__(lsh_matcher._SimpleMinHash) + result.num_perm = len(values) + result.hash_values = values + return result + + +def exhaustive_matches(index, query): + matches = [] + for candidate, stored in zip(index.blocks, index.minhashes, strict=True): + if isinstance(stored, lsh_matcher._SimpleMinHash): + score = query.jaccard(stored) + if score >= index.threshold: + matches.append((candidate, score)) + return sorted(matches, key=lambda item: item[1], reverse=True) + + +@pytest.mark.parametrize("width", [7, 25, 128]) +@pytest.mark.parametrize("threshold", [0, 0.1, 7 / 25, 0.5, 0.85, 1, 1.1]) +def test_fallback_matches_exhaustive_scores_and_stable_order( + simple_backend, monkeypatch, width, threshold +): + rng = random.Random(731) + anchor = [rng.getrandbits(128) for _ in range(width)] + vectors = [anchor, list(anchor)] + for equal in range(width + 1): + vector = [rng.getrandbits(128) for _ in range(width)] + for position in rng.sample(range(width), equal): + vector[position] = anchor[position] + vectors.append(vector) + signatures = {str(i): signature(vector) for i, vector in enumerate(vectors)} + monkeypatch.setattr(lsh_matcher, "_create_simple_minhash", lambda text, _: signatures[text]) + index = lsh_matcher.LSHIndex(threshold=threshold, num_perm=width) + index.add_blocks([block(str(i), str(i)) for i in range(len(vectors))]) + for query in index.blocks[:6]: + assert index.find_near_duplicates(query) == exhaustive_matches( + index, signatures[query.text] + ) + + +def test_fallback_prunes_comparisons_and_refreshes_after_add(simple_backend, monkeypatch): + signatures = {str(i): signature([i * 1000 + j for j in range(128)]) for i in range(1000)} + signatures["duplicate"] = signatures["0"] + monkeypatch.setattr(lsh_matcher, "_create_simple_minhash", lambda text, _: signatures[text]) + index = lsh_matcher.LSHIndex(threshold=0.85) + index.add_blocks([block(str(i), str(i)) for i in range(1000)]) + calls = [] + original = lsh_matcher._SimpleMinHash.jaccard + + def counted(left, right): + calls.append(1) + return original(left, right) + + monkeypatch.setattr(lsh_matcher._SimpleMinHash, "jaccard", counted) + assert index.find_near_duplicates(index.blocks[0]) == [(index.blocks[0], 1.0)] + assert len(calls) == 1 + calls.clear() + duplicate = block("duplicate.py", "duplicate") + index.add(duplicate) + assert index.find_near_duplicates(index.blocks[0]) == [(index.blocks[0], 1.0), (duplicate, 1.0)] + assert len(calls) == 2 + + +def test_fallback_refreshes_after_threshold_change(simple_backend, monkeypatch): + signatures = { + "query": signature(list(range(25))), + "partial": signature(list(range(7)) + [-1] * 18), + } + monkeypatch.setattr(lsh_matcher, "_create_simple_minhash", lambda text, _: signatures[text]) + index = lsh_matcher.LSHIndex(threshold=0.85, num_perm=25) + query, partial = block(text="query"), block("partial.py", "partial") + index.add_blocks([query, partial]) + assert index.find_near_duplicates(query) == [(query, 1.0)] + index.threshold = 7 / 25 + assert index.find_near_duplicates(query) == [(query, 1.0), (partial, 7 / 25)] + + +def test_fallback_preserves_native_scorer_semantics(simple_backend, monkeypatch): + from types import SimpleNamespace + + index = lsh_matcher.LSHIndex(threshold=0.85) + left, other = block(), block("other.py", "unrelated tokens entirely") + index.add_blocks([left, other]) + # A native scorer is an independent backend; do not assume equality-count semantics. + monkeypatch.setattr( + tokens_hasher, "redup_fast_hash", SimpleNamespace(minhash_jaccard=lambda a, b: 1.0) + ) + assert index.find_near_duplicates(left) == [(left, 1.0), (other, 1.0)] + + +def test_fallback_groups_match_exhaustive_grouping(simple_backend, monkeypatch): + blocks = [block(f"{i}.py", f"alpha beta gamma {i % 3} epsilon") for i in range(30)] + indexed = lsh_matcher.find_near_duplicates(blocks, threshold=0.7) + + def exhaustive(self, query): + return exhaustive_matches( + self, lsh_matcher._create_simple_minhash(query.text, self.num_perm) + ) + + monkeypatch.setattr(lsh_matcher.LSHIndex, "_find_near_duplicates_simple", exhaustive) + assert indexed == lsh_matcher.find_near_duplicates(blocks, threshold=0.7)