From 8294d907f1e631248cfbd04218f2aed991f0e6b8 Mon Sep 17 00:00:00 2001 From: MSCodeBase Agent Date: Thu, 12 Nov 2026 20:51:49 +0300 Subject: [PATCH] fix(index): build the vector schema from the provider's width (G4) The dense build failed with "Cannot cast to FixedSizeList(384): value at index 0 has length 768" on a table declared 384-d with zero rows. The cast was a symptom. Root cause: `LanceDBManager._open_or_create_table` read the stored width from a sample ROW, and the migration helpers used `table = table or self.table`. An empty LanceDB table is falsy, so on an empty table `self.table` (not yet assigned) raised AttributeError, the broad handler treated that as "table not found", and the dimension check never ran. Contract (new `src/core/indexing/dim_contract.py`): - a provider declares its output width (`embedding_dim` / `provider_name`); - the schema is built from that width; - an empty table with another width is rebuilt (no rows to lose); - a non-empty table with another width refuses loudly: provider name and both numbers (`EmbeddingDimMismatch`), never a silent 768->384 resize; - `LanceDBWriter` no longer pads or truncates a vector. The schema width is read from the SCHEMA, not from a data row. Retries: `bounded_attempts` replaces the open-ended "embed error -> idle-unload -> restart" loop on the llama branch: 3 batch attempts, one revive, two single attempts, then `EmbedderUnavailable` with provider and last error (BLOCKED). Red-first test: `test_embed_batch_raises_blocked_instead_of_restart_loop`. External endpoints: `MSCODEBASE_EMBEDDER_PROVIDER` pins the provider. With it set, the background scanner and the :8080 dimension probe are disabled, so LM Studio / llama-server only answer when named. `auto` keeps the old order. Smoke: the query vector comes from the active provider and its width is checked against the table schema before search; an empty vector table falls back to the text index (shipped default stays text-only). No `--dense` flag was added. Measured (night 12->13.11.2026, temp data root): provider onnx:multilingual-e5-small-int8, dim 384 7307 chunks, fixed_size_list[384], 2210.8 s, 73.3 MB smoke_e2e --project . with MSCODEBASE_DENSE_OFF=false: PASSED (4 checks), step 4 = vector, provider=onnx, dim=384, results=3 Gates: pytest -m "not integration and not benchmark" 2570 passed / 10 skipped (was 2560; +10 new, none lost), ruff OK, text-only smoke PASSED. --- AGENT_DIARY.md | 32 +++ docs/en/CONFIGURATION.md | 1 + docs/ru/CONFIGURATION.md | 1 + docs/zh/CONFIGURATION.md | 1 + scripts/smoke_e2e.py | 121 +++++++--- src/config/settings.py | 10 + src/core/indexing/db_manager.py | 77 ++++--- src/core/indexing/db_writer.py | 28 ++- src/core/indexing/dim_contract.py | 163 ++++++++++++++ src/providers/embedder/remote_embedder.py | 233 ++++++++++++------- tests/test_embedding_dim_contract.py | 263 ++++++++++++++++++++++ 11 files changed, 795 insertions(+), 135 deletions(-) create mode 100644 src/core/indexing/dim_contract.py create mode 100644 tests/test_embedding_dim_contract.py diff --git a/AGENT_DIARY.md b/AGENT_DIARY.md index 04cfc0ab..17bd18d1 100644 --- a/AGENT_DIARY.md +++ b/AGENT_DIARY.md @@ -107,6 +107,38 @@ - *nit 4–6 исправлены в том же проходе:* `None`/отрицательные счётчики теперь fail-open («неизвестно» ≠ «отсутствует»), мёртвая ветка убрана. - P5 снят и записан до мержа: requests 10→12/25 при планке ≥10. +- **G4-retry — dense-smoke на ONNX e5-small-int8, 384 (12→13.11.2026): DONE** + - *Провайдер в прогоне:* `onnx:multilingual-e5-small-int8`, выход 384-d + (закреплён `MSCODEBASE_EMBEDDER_PROVIDER=onnx`; :8080 и LM Studio в сборке + не участвовали). Сборка: **7307 чанков**, `fixed_size_list[384]`, + **2210.8 s**, артефакт **73.3 MB** (temp data root). + - *Smoke на этом индексе:* `✅ SMOKE E2E: PASSED (4 проверки)`, шаг 4 = + `режим=vector, provider=onnx, dim=384, таблица=codebase_chunks, results=3`. + - *Корневая причина G4 (не Cast error):* в `_open_or_create_table` ширина + читалась из строки-сэмпла, а `_migrate_*` делали `table = table or + self.table`. Пустая LanceDB-таблица **falsy** → `AttributeError + 'LanceDBManager' object has no attribute 'table'` → ловился как «таблица не + найдена» → проверка размерности не выполнялась, и 384-схема доживала до + записи 768-векторов. + - *Контракт размерности:* `src/core/indexing/dim_contract.py` — провайдер + объявляет ширину, схема строится от неё, перед записью/поиском + `require_dim_match(provider, provider_dim, schema_dim)`; несовпадение — + `EmbeddingDimMismatch` с именем провайдера и двумя числами. Пустая таблица + пересоздаётся (терять нечего), непустая — отказ (без молчаливого resize). + `db_writer` больше не паддит/обрезает вектор. + - *Ретраи:* `bounded_attempts` — 3 batch + 1 revive + 2 single, затем + `EmbedderUnavailable` с провайдером и последней ошибкой (BLOCKED), без + цикла idle-unload/restart. Покрыто red-first тестом. + - *Внешние эндпоинты:* при закреплённом провайдере сканер и probe :8080 + отключены; `auto` сохраняет прежнее поведение. Тумблер — + `MSCODEBASE_EMBEDDER_PROVIDER` (документирован в CONFIGURATION en/ru/zh). + - *Проверка:* `pytest tests/ -m "not integration and not benchmark"` → + **2570 passed / 10 skipped** (было 2560; +10 новых), rc=0; `ruff_gate` OK; + smoke text-only PASSED. Найдено и закрыто 6 регрессий-мока (`provider_dim` + больше не считает `MagicMock` шириной 1). + - *Осталось:* сборка dense на CPU ~37 мин (не оптимизировалась); `auto` + по-прежнему может переключиться на внешний провайдер — пинится только + явным конфигом. - **E73 (ложный отказ):** `_abstains` решал «субъекта нет» по ПЕРВОМУ идентификатору; на прозе коммитов это `_orm.registry=0` при `Declarative=785`, и 13 из 40 коммитов sqlalchemy отвечали пустотой. Отказ теперь только когда не diff --git a/docs/en/CONFIGURATION.md b/docs/en/CONFIGURATION.md index e2783941..68e24444 100644 --- a/docs/en/CONFIGURATION.md +++ b/docs/en/CONFIGURATION.md @@ -90,6 +90,7 @@ path and LSP. | `LM_STUDIO_HOST` / `LM_STUDIO_PORT` | `127.0.0.1` / `1234` | LM Studio provider | | `OLLAMA_HOST` / `OLLAMA_PORT` | `127.0.0.1` / `11434` | Ollama provider | | `EMBEDDING_MODEL` / `MODEL_NAME` / `EMBEDDING_DIMENSION` | `qwen3-embedding` / `multilingual-e5-small-int8` / `384` | Provider model selection | +| `MSCODEBASE_EMBEDDER_PROVIDER` | `auto` | Pin the embedder (`onnx`, `llama_cpp`, `lm_studio`, `ollama`); external endpoints are only used when named | | `MSCODEBASE_ONNX_PROVIDER` | — | Force a specific ONNX execution provider | | `MSCODEBASE_REMOTE_TOKEN` / `_REMOTE_RATE_LIMIT_RPS` | — | Remote provider auth and rate limit | | `MSCODEBASE_LANGUAGE_PACK` | `false` | Enable tree-sitter-language-pack (downloads parsers on first use) | diff --git a/docs/ru/CONFIGURATION.md b/docs/ru/CONFIGURATION.md index bfab46e8..4a909e1c 100644 --- a/docs/ru/CONFIGURATION.md +++ b/docs/ru/CONFIGURATION.md @@ -93,6 +93,7 @@ dense-пути и LSP. | `LM_STUDIO_HOST` / `LM_STUDIO_PORT` | `127.0.0.1` / `1234` | Провайдер LM Studio | | `OLLAMA_HOST` / `OLLAMA_PORT` | `127.0.0.1` / `11434` | Провайдер Ollama | | `EMBEDDING_MODEL` / `MODEL_NAME` / `EMBEDDING_DIMENSION` | `qwen3-embedding` / `multilingual-e5-small-int8` / `384` | Выбор модели провайдера | +| `MSCODEBASE_EMBEDDER_PROVIDER` | `auto` | Закрепить эмбеддер (`onnx`, `llama_cpp`, `lm_studio`, `ollama`); внешние эндпоинты используются только по имени | | `MSCODEBASE_ONNX_PROVIDER` | — | Принудительно задать конкретный execution provider ONNX | | `MSCODEBASE_REMOTE_TOKEN` / `_REMOTE_RATE_LIMIT_RPS` | — | Аутентификация и лимит частоты для удалённого провайдера | | `MSCODEBASE_LANGUAGE_PACK` | `false` | Включить tree-sitter-language-pack (скачивает парсеры при первом использовании) | diff --git a/docs/zh/CONFIGURATION.md b/docs/zh/CONFIGURATION.md index 3b2a4f7c..d64c325c 100644 --- a/docs/zh/CONFIGURATION.md +++ b/docs/zh/CONFIGURATION.md @@ -88,6 +88,7 @@ | `LM_STUDIO_HOST` / `LM_STUDIO_PORT` | `127.0.0.1` / `1234` | LM Studio provider | | `OLLAMA_HOST` / `OLLAMA_PORT` | `127.0.0.1` / `11434` | Ollama provider | | `EMBEDDING_MODEL` / `MODEL_NAME` / `EMBEDDING_DIMENSION` | `qwen3-embedding` / `multilingual-e5-small-int8` / `384` | provider 模型选择 | +| `MSCODEBASE_EMBEDDER_PROVIDER` | `auto` | 固定 embedder（`onnx`、`llama_cpp`、`lm_studio`、`ollama`）；外部端点仅在显式指定时使用 | | `MSCODEBASE_ONNX_PROVIDER` | — | 强制指定 ONNX 执行 provider | | `MSCODEBASE_REMOTE_TOKEN` / `_REMOTE_RATE_LIMIT_RPS` | — | 远程 provider 认证与速率限制 | | `MSCODEBASE_LANGUAGE_PACK` | `false` | 启用 tree-sitter-language-pack（首次使用时下载 parser） | diff --git a/scripts/smoke_e2e.py b/scripts/smoke_e2e.py index d5671553..3b335fbc 100644 --- a/scripts/smoke_e2e.py +++ b/scripts/smoke_e2e.py @@ -27,6 +27,7 @@ """ import argparse +import os import sqlite3 import sys import time @@ -116,20 +117,53 @@ def check_services() -> dict: return report +def _active_embedder() -> tuple[str, int, str]: + """Which embedder the smoke run must ask, from explicit configuration. + + The script has no `--dense` flag on purpose: `MSCODEBASE_DENSE_OFF` picks + the tier, and `MSCODEBASE_EMBEDDER_PROVIDER` names the provider so a run + can never accidentally validate a different backend's width. + """ + provider = os.getenv("MSCODEBASE_EMBEDDER_PROVIDER", "auto").strip().lower() + if provider == "onnx": + return "onnx", 384, "multilingual-e5-small-int8" + if provider in ("lm_studio", "ollama"): + return ( + provider, + int(os.getenv("EMBEDDING_DIMENSION", "384")), + os.getenv("MODEL_NAME", "multilingual-e5-small-int8"), + ) + # auto / llama_cpp — исторический путь smoke: llama-server на :8080. + return "llama_cpp", int(os.getenv("EMBEDDING_DIMENSION", "768")), "embeddinggemma-2" + + +def _embed_text(text: str) -> tuple[list[float], str, int]: + provider, expected_dim, model = _active_embedder() + if provider == "onnx": + from src.core.embedder.onnx_client import get_onnx_client + + client = get_onnx_client(port=9876, model_name=model) + return client.embed(text), provider, expected_dim + url = ( + "http://127.0.0.1:8080/v1/embeddings" + if provider == "llama_cpp" + else "http://127.0.0.1:1234/v1/embeddings" + ) + r = httpx.post(url, json={"input": [text], "model": model}, timeout=30.0) + r.raise_for_status() + return r.json()["data"][0]["embedding"], provider, expected_dim + + def check_embed() -> tuple[bool, str]: - """Реальный embed через llama.cpp REST API. Проверяет размерность.""" + """Реальный embed через активный провайдер. Проверяет размерность.""" try: - r = httpx.post( - "http://127.0.0.1:8080/v1/embeddings", - json={"input": ["Проверка реального эмбеддинга smoke_e2e"]}, - timeout=15.0, - ) - r.raise_for_status() - data = r.json() - vec = data["data"][0]["embedding"] + vec, provider, expected = _embed_text("Проверка реального эмбеддинга smoke_e2e") dim = len(vec) - ok = dim == 768 - return ok, f"dim={dim} (embeddinggemma-2: 768) {'✅' if ok else '❌'}" + ok = dim == expected + return ok, ( + f"provider={provider}, dim={dim} (expected {expected}) " + f"{'✅' if ok else '❌'}" + ) except Exception as e: # noqa: BLE001 — диагностика return False, f"embed error: {e}" @@ -207,13 +241,18 @@ def check_search(project: Path) -> tuple[bool, str]: Открываем LanceDB напрямую (lancedb.connect) — чтение не требует нашего PID-lock (он защищает запись; MCP другого окна держит lock — создание Indexer через DI упало бы «PID lock held», что и случилось в первом - прогоне). Вектор запроса — реальный embed через llama.cpp (8080). + прогоне). Вектор запроса — real embed активного провайдера + (`MSCODEBASE_EMBEDDER_PROVIDER`), ширина сверяется со схемой таблицы. Если векторного индекса нет (дефолт `MSCODEBASE_DENSE_OFF=true`), шаг проверяет текстовый индекс SQLite: падать на отсутствии вектора значило бы требовать от дефолтной конфигурации чужой режим. """ from src.core.artifact_paths import get_db_path + from src.core.indexing.dim_contract import ( + EmbeddingDimMismatch, + schema_vector_dim, + ) vector_db = Path(str(get_db_path(project))) if not vector_db.exists(): @@ -237,18 +276,33 @@ def check_search(project: Path) -> tuple[bool, str]: ok, detail = _text_index_search(project) return ok, f"в векторной БД нет таблиц; {detail}" table = db.open_table(names[0]) - # Реальный embed запроса через llama.cpp - r = httpx.post( - "http://127.0.0.1:8080/v1/embeddings", - json={"input": ["def __init__ class initialization"]}, - timeout=15.0, - ) - r.raise_for_status() - vec = r.json()["data"][0]["embedding"] + if table.count_rows() == 0: + # An empty vector table is not a dense index; the shipped default is + # text-only, so validate the text index instead of a width mismatch. + ok, detail = _text_index_search(project) + return ok, f"векторная таблица пуста; {detail}" + # Width contract first: a query must match the table's vector width. + # A mismatch is a named failure, not a silent text fallback (G4). + schema_dim = schema_vector_dim(table.schema) + vec, provider, _expected = _embed_text("def __init__ class initialization") + if schema_dim is None: + return False, "vector table has no fixed-size vector column" + if len(vec) != schema_dim: + raise EmbeddingDimMismatch( + f"provider '{provider}' returns {len(vec)}-d vectors, but " + f"the index is {schema_dim}-d; refusing to search a mixed index" + ) df = table.search(vec).limit(3).to_pandas() n = len(df) ok = n > 0 - return ok, f"режим=vector, таблица={names[0]}, results={n} {'✅' if ok else '❌'}" + return ok, ( + f"режим=vector, provider={provider}, dim={len(vec)}, " + f"таблица={names[0]}, results={n} {'✅' if ok else '❌'}" + ) + except EmbeddingDimMismatch as e: + # A vector index exists and its width is the contract: do not hide the + # failure behind a text fallback (that was G4's silent path). + return False, f"dimension contract: {e}" except Exception as e: # noqa: BLE001 — диагностика ok, detail = _text_index_search(project) if ok: @@ -265,8 +319,19 @@ def main() -> int: _setup_env() revision = get_revision() + provider, expected_dim, _model = _active_embedder() + if provider == "onnx": + # A pinned local provider is part of the run, not a precondition: bring + # it up before health so an idle-unloaded server is not a false FAIL. + try: + from src.core.embedder.onnx_client import get_onnx_client + + get_onnx_client(port=9876, model_name=_model).ensure_server_running() + except Exception as exc: # noqa: BLE001 — the embed step reports detail + print(f" ⚠️ ONNX start failed: {exc}") print("🧪 Smoke E2E — реальные сервисы (без моков)") print(f"revision: {revision}") + print(f"embedder provider: {provider} (expected dim {expected_dim})") print("━" * 50) failures = 0 @@ -275,18 +340,18 @@ def main() -> int: svc = check_services() for name, status in svc.items(): print(f" {status} {name}") - # Обязательные live-сервисы (embedder=llama.cpp 8080, reranker 8081) — - # только их отсутствие валит smoke. onnx (9876) — альтернативный embedder-бэкенд, - # не обязателен, когда активен llama.cpp (информационно). - required = [n for n in ("embedder", "reranker") if svc.get(n) == "🔴"] + # The required health entry is the ACTIVE provider, not a fixed llama port: + # an ONNX dense run must not fail because :8080 is intentionally absent. + active_health = "onnx" if provider == "onnx" else "embedder" + required = [n for n in (active_health, "reranker") if svc.get(n) == "🔴"] if required: print(f" ⚠️ Критичные сервисы не отвечают ({', '.join(required)}) — embed/rerank проверки могут упасть") failures += 1 - elif svc.get("onnx") == "🔴": - print(" ℹ️ onnx — альтернативный embedder не запущен (активен llama.cpp); необязательный — пропускаю") + elif provider != "onnx" and svc.get("onnx") == "🔴": + print(" ℹ️ onnx — альтернативный embedder не запущен; необязательный — пропускаю") # 2. Embed - print("2. Реальный embed (llama.cpp):") + print(f"2. Реальный embed ({provider}, expected {expected_dim}-d):") ok, msg = check_embed() print(f" {'✅' if ok else '❌'} {msg}") failures += 0 if ok else 1 diff --git a/src/config/settings.py b/src/config/settings.py index 014a335b..fe833981 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -62,6 +62,16 @@ class EmbeddingConfig: embedding_dimension: int = field( default_factory=lambda: int(os.getenv("EMBEDDING_DIMENSION", "384")) ) + # Which embedder this process may use. "auto" keeps the historical + # behaviour: the local ONNX model first, then local services that answer. + # Any explicit value pins the provider and disables the background scanner, + # so external endpoints (LM Studio :1234, llama-server :8080) are never + # picked up by accident. Dense builds use this to name their provider. + provider: str = field( + default_factory=lambda: os.getenv( + "MSCODEBASE_EMBEDDER_PROVIDER", "auto" + ).strip().lower() + ) # Тумблер авто-запуска llama.cpp (embedder + reranker) при старте MCP. # По умолчанию авто-определение: если llama-server уже запущен на порту 8080, # включаем его автоматически. Иначе — выключаем. diff --git a/src/core/indexing/db_manager.py b/src/core/indexing/db_manager.py index 8c5ce791..68f0cc1f 100644 --- a/src/core/indexing/db_manager.py +++ b/src/core/indexing/db_manager.py @@ -26,6 +26,12 @@ import pyarrow as pa from src.core.indexing.database_lock import DatabaseLock +from src.core.indexing.dim_contract import ( + EmbeddingDimMismatch, + provider_dim, + provider_label, + schema_vector_dim, +) from src.core.indexing.index_guard import IndexGuard from src.core.platform_utils import to_win_long_path @@ -54,6 +60,17 @@ def __init__( self.embedder = embedder self.project_path = project_path self.embedding_dim = embedding_dim + self.provider_name = provider_label(embedder) + + # The schema is built from the provider's declared width; a caller that + # passes a different number is a bug, not a migration opportunity. + declared = provider_dim(embedder) + if declared is not None and declared != embedding_dim: + raise EmbeddingDimMismatch( + f"LanceDBManager: provider '{self.provider_name}' declares " + f"{declared}-d vectors, but the schema was requested at " + f"{embedding_dim}-d. Build the schema from the provider." + ) # Async LanceDB (ленивая инициализация) self._async_db: Optional[Any] = None @@ -200,38 +217,36 @@ def _open_or_create_table(self, schema: pa.Schema): logger.info(f"Opened table: {self.table_name}") - # Dimension mismatch → recreate - _vec_field = next((f for f in table.schema if f.name == "vector"), None) - if _vec_field is not None: - stored_dim = 0 - try: - _t = _vec_field.type - if hasattr(_t, 'value_type'): - _vt = _t.value_type - if hasattr(_vt, 'get_field'): - stored_dim = _vt.get_field("item").type.list_size - except Exception as _dim_err: - logger.debug(f"Schema extraction failed: {_dim_err}") - - # Fallback: read a sample row to get actual vector length - if not stored_dim: - try: - sample = table.limit(1).to_list() - if sample and "vector" in sample[0]: - stored_dim = len(sample[0]["vector"]) - except Exception: - pass - - if stored_dim and stored_dim != self.embedding_dim: + # Dimension mismatch → the schema must follow the provider. + # Read the width from the SCHEMA, not from a sample row: an empty + # table has no rows, and that is how G4's 384-vs-768 mismatch + # reached LanceDB as a cast error instead of a named refusal. + stored_dim = schema_vector_dim(table.schema) + if stored_dim and stored_dim != self.embedding_dim: + rows = int(table.count_rows()) + if rows == 0: logger.warning( - f"Dimension mismatch: index={stored_dim}, " - f"embedder={self.embedding_dim}. Recreating..." + "Dimension mismatch on an EMPTY table: schema=%sd, " + "provider '%s'=%sd. Recreating the schema (no rows to lose).", + stored_dim, + self.provider_name, + self.embedding_dim, ) self.db.drop_table(self.table_name) table = self.db.create_table(self.table_name, schema=schema) - logger.info(f"Table recreated for {self.embedding_dim}dim") self._needs_full_reindex = True + else: + raise EmbeddingDimMismatch( + f"vector schema is {stored_dim}-d with {rows} row(s), " + f"but provider '{self.provider_name}' returns " + f"{self.embedding_dim}-d vectors. Refusing to resize " + "silently. Rebuild the index at the provider's width " + f"(intel_reset_index) or start a {stored_dim}-d provider." + ) + except EmbeddingDimMismatch: + # A width mismatch is a named refusal, not a missing table. + raise except Exception as open_err: logger.debug(f"Table not found: {open_err}. Creating new.") try: @@ -663,7 +678,10 @@ def _migrate_text_full_inplace(self, table=None): table: открытая таблица (в _open_or_create_table self.table ещё не назначен — таблица живёт в локальной переменной). """ - table = table or self.table + if table is None: + table = getattr(self, "table", None) + if table is None: + raise RuntimeError("_migrate_text_full_inplace: no table handle") if "text_full" in [f.name for f in table.schema]: return try: @@ -682,7 +700,10 @@ def _migrate_add_metadata_columns(self, existing_fields, table=None): table: открытая таблица (см. _migrate_text_full_inplace). """ - table = table or self.table + if table is None: + table = getattr(self, "table", None) + if table is None: + raise RuntimeError("_migrate_add_metadata_columns: no table handle") target = {f.name: f for f in self.schema} missing = [name for name in target if name not in existing_fields] if not missing: diff --git a/src/core/indexing/db_writer.py b/src/core/indexing/db_writer.py index f4ce18a6..4c715fd1 100644 --- a/src/core/indexing/db_writer.py +++ b/src/core/indexing/db_writer.py @@ -11,6 +11,13 @@ from datetime import datetime from typing import Any, Dict, List +from src.core.indexing.dim_contract import ( + provider_dim, + provider_label, + require_dim_match, + schema_vector_dim, +) + __all__ = [ "LanceDBWriter", ] @@ -61,10 +68,27 @@ def write_records( ) return [] - _target_dim = self.embedder.embedding_dim or 768 + _target_dim = schema_vector_dim(getattr(self.table, "schema", None)) + if _target_dim is None: + _target_dim = getattr(self.embedder, "embedding_dim", None) + _declared_dim = provider_dim(self.embedder) or _target_dim + require_dim_match( + provider=provider_label(self.embedder), + provider_dim=_declared_dim, + schema_dim=_target_dim, + where=f"write_records({rel_path_str})", + ) + _target_dim = int(_target_dim) for i, vec in enumerate(embeddings): if len(vec) != _target_dim: - embeddings[i] = vec[:_target_dim] + [0.0] * (_target_dim - len(vec)) + # Was: pad/truncate to the schema width. That hides a provider + # swap and stores vectors under the wrong promise (G4). + require_dim_match( + provider=provider_label(self.embedder), + provider_dim=len(vec), + schema_dim=_target_dim, + where=f"write_records({rel_path_str}, chunk {i})", + ) # Guard: проверка нормы вектора (если norm=0, эмбеддер не работал) _norm_sq = sum(v*v for v in embeddings[i]) if _norm_sq < 1e-9: diff --git a/src/core/indexing/dim_contract.py b/src/core/indexing/dim_contract.py new file mode 100644 index 00000000..4692821d --- /dev/null +++ b/src/core/indexing/dim_contract.py @@ -0,0 +1,163 @@ +"""Embedding dimension contract: provider declares, schema follows, mismatch is loud. + +Why (G4, night 12->13.11.2026): this project's LanceDB table was declared as +`vector fixed_size_list[384]` with **zero rows**, while the active +embedder returned 768. The old detection read the width from a data row, found +none, and let the write fail deep inside LanceDB: + + ValueError: lance error: Cannot cast to FixedSizeList(384): + value at index 0 has length 768 + +Silent padding/truncation is worse than the crash: it would store 768-d answers +under a 384-d promise and make similarity meaningless. The rule here is one +line: before any write, provider width == schema width. A mismatch is a fast +refusal that names the provider and both numbers. +""" + +from __future__ import annotations + +import time +from typing import Any, Callable, Optional, TypeVar + +T = TypeVar("T") + + +class EmbeddingDimMismatch(RuntimeError): + """Provider and vector schema disagree about the width.""" + + +class EmbedderUnavailable(RuntimeError): + """The embedder did not answer after the bounded attempts.""" + + +def provider_label(embedder: Any) -> str: + """Human name of the active provider, for error messages and the diary. + + Looks at an explicit `provider_name` first (RemoteEmbedder exposes one), + then falls back to `mode` + model, then to the class name. Never returns an + empty string: an unnamed provider in a mismatch message is useless. + """ + if embedder is None: + return "no-embedder" + + name = getattr(embedder, "provider_name", None) + if callable(name): + try: + name = name() + except Exception: # noqa: BLE001 — a broken label must not hide the error + name = None + if not isinstance(name, str) or not name: + mode = getattr(embedder, "mode", None) or getattr(embedder, "provider", None) + model = getattr(embedder, "model_name", None) or getattr( + embedder, "_model_name", None + ) + name = ":".join( + str(part) + for part in (mode, model) + if isinstance(part, str) and part + ) + if not name: + name = type(embedder).__name__ + return str(name) + + +def provider_dim(embedder: Any) -> Optional[int]: + """Width the provider promises to return, or None when it does not say.""" + if embedder is None: + return None + dim = getattr(embedder, "embedding_dim", None) + if dim is None: + getter = getattr(embedder, "get_embedding_dim", None) + if callable(getter): + try: + dim = getter() + except Exception: # noqa: BLE001 — unknown is not an error here + dim = None + # Only a real number is a declaration; a Mock/attribute of another type is + # "unknown", not a width of 1. + if isinstance(dim, bool) or not isinstance(dim, (int, float)): + return None + return int(dim) if dim else None + + +def schema_vector_dim(schema: Any) -> Optional[int]: + """Width of the `vector` field of an existing LanceDB/pyarrow schema. + + Reads the schema, not a sample row: an empty table has no rows to sample, + which is exactly how G4's 384-vs-768 mismatch slipped through. + """ + if schema is None: + return None + fields = getattr(schema, "names", None) + if fields is None: + return None + if "vector" not in fields: + return None + try: + field = schema.field("vector") + except Exception: # noqa: BLE001 — old pyarrow without field() + field = next((f for f in schema if getattr(f, "name", "") == "vector"), None) + if field is None: + return None + value_type = getattr(field, "type", None) + size = getattr(value_type, "list_size", None) + if size: + return int(size) + value_type = getattr(value_type, "value_type", None) + size = getattr(value_type, "list_size", None) + return int(size) if size else None + + +def require_dim_match( + *, + provider: str, + provider_dim: Optional[int], + schema_dim: Optional[int], + where: str, +) -> None: + """Raise `EmbeddingDimMismatch` unless both widths are known and equal.""" + if provider_dim is None or schema_dim is None: + raise EmbeddingDimMismatch( + f"{where}: cannot verify the vector width — provider '{provider}' " + f"declares {provider_dim}, schema declares {schema_dim}. " + "Refusing to write an unverified vector." + ) + if provider_dim != schema_dim: + raise EmbeddingDimMismatch( + f"{where}: provider '{provider}' returns {provider_dim}-d vectors, " + f"but the vector schema is {schema_dim}-d. Refusing to resize " + "silently. Rebuild the index at the provider's width, or start a " + f"{schema_dim}-d provider." + ) + + +def bounded_attempts( + step: Callable[[int], T], + *, + attempts: int, + provider: str, + label: str, + backoff: tuple[float, ...] = (), + sleep: Callable[[float], None] = time.sleep, +) -> T: + """Run `step` at most `attempts` times, then raise `EmbedderUnavailable`. + + The night incident was an open-ended loop: embed error -> idle-unload -> + restart -> embed error. A retry policy must end in a named BLOCKED answer, + not in a spinner. `step` receives the zero-based attempt number. + """ + if attempts < 1: + raise ValueError("attempts must be >= 1") + last: Optional[BaseException] = None + for attempt in range(attempts): + try: + return step(attempt) + except Exception as exc: # noqa: BLE001 — re-raised below with context + last = exc + if attempt + 1 < attempts: + delay = backoff[min(attempt, len(backoff) - 1)] if backoff else 0.0 + if delay: + sleep(delay) + raise EmbedderUnavailable( + f"{label}: provider '{provider}' failed after {attempts} attempt(s): {last}" + ) from last diff --git a/src/providers/embedder/remote_embedder.py b/src/providers/embedder/remote_embedder.py index bbf1b54f..5d7c45bf 100644 --- a/src/providers/embedder/remote_embedder.py +++ b/src/providers/embedder/remote_embedder.py @@ -13,6 +13,7 @@ import httpx from src.config.settings import get_config +from src.core.indexing.dim_contract import EmbedderUnavailable, bounded_attempts from src.core.interfaces import IEmbedder from src.core.platform_utils import get_extension_dir from src.providers.reranker.llama_install import LLAMA_EMBED_MAX_TOKENS @@ -130,6 +131,10 @@ def __init__( self.model_name = config.embedding.model_name self._breaker = breaker + # Explicit provider selection. "auto" keeps the legacy probing order; + # anything else pins the provider so a dense build cannot silently end + # up on an external endpoint with a different width. + self.provider_preference = getattr(config.embedding, "provider", "auto") or "auto" # The schema and the query vector must agree; the config default is not # the model's dimension (measured 2026-11-08: 384 vs 768). self.embedding_dim = model_dimension( @@ -162,9 +167,12 @@ def __init__( # server runs embeddinggemma-2): the only reliable answer is the server. # Measured 2026-11-08: config said 384, the model returned 768, and dense # search raised "query dim(768) doesn't match the column vector dim(384)". - probed = self._probe_server_dimension() - if probed and not os.getenv("EMBEDDING_DIMENSION", "").strip(): - self.embedding_dim = probed + # Only probe an external llama endpoint when it is allowed to answer: + # with a pinned provider, a stray :8080 must not rewrite the width. + if self.provider_preference in ("auto", "llama_cpp"): + probed = self._probe_server_dimension() + if probed and not os.getenv("EMBEDDING_DIMENSION", "").strip(): + self.embedding_dim = probed # llama.cpp idle timeout self._llama_idle_timeout = 300 @@ -194,12 +202,13 @@ def __init__( # with a different width (measured 2026-11-08: file 384, server 768). # Env still wins, and a probe that needs a moment does not block startup. if not os.getenv("EMBEDDING_DIMENSION", "").strip(): - live_dim = self._probe_server_dimension() - if live_dim and live_dim != self.embedding_dim: - logger.info( - f"embedding_dim {self.embedding_dim} -> {live_dim} (live server)" - ) - self.embedding_dim = live_dim + if self.provider_preference in ("auto", "llama_cpp"): + live_dim = self._probe_server_dimension() + if live_dim and live_dim != self.embedding_dim: + logger.info( + f"embedding_dim {self.embedding_dim} -> {live_dim} (live server)" + ) + self.embedding_dim = live_dim # Shadow Canary: эталонные пары запрос→чанк self._canary_pairs: List[dict] = [] @@ -537,16 +546,22 @@ def _detect_model_dir(self): self._init_thread.start() # Запуск фонового сканера доступности провайдера (LM Studio/Ollama). - # Сканер работает ВСЕГДА: он либо подтверждает LM Studio - # (если _init_provider_async его нашёл), либо ищет его, если - # текущий режим != "lm_studio". + # Только в auto: явно выбранный провайдер не должен подменяться + # ответившим внешним эндпоинтом (у него может быть другая ширина). self._scanner_stop = threading.Event() - self._scanner_thread = threading.Thread( - target=self._provider_scanner_loop, - name="mscodebase-provider-scanner", - daemon=True, - ) - self._scanner_thread.start() + if self.provider_preference == "auto": + self._scanner_thread = threading.Thread( + target=self._provider_scanner_loop, + name="mscodebase-provider-scanner", + daemon=True, + ) + self._scanner_thread.start() + else: + self._scanner_thread = None + logger.info( + "Embedding provider pinned to '%s' — background scanner disabled", + self.provider_preference, + ) # Фоновая предзагрузка ONNX через 15 сек после старта. # К этому моменту MCP сервер уже инициализирован, а модель @@ -603,9 +618,22 @@ def _preload_onnx_delayed(self): """Фоновая предзагрузка ONNX модели через 15 сек после старта MCP. Отключает себя, если llama.cpp работает — ONNX не нужен in-process. + При явном выборе провайдера подмены нет: pinned onnx → грузим ONNX, + pinned external → предзагрузка не нужна вовсе. """ import time as _time + _pref = getattr(self, "provider_preference", "auto") + if _pref not in ("auto", "onnx"): + return + if _pref == "onnx": + self._init_onnx() + if self._onnx_client is not None: + with self._mode_lock: + self.mode = "onnx" + logger.info("✅ ONNX предзагружен (провайдер закреплён: onnx)") + return + # Ждём до 60 секунд пока llama.cpp запустится (он стартует через _warmup_embedder) # Проверяем каждые 5 секунд — если появился, отменяем ONNX загрузку with self._mode_lock: @@ -711,6 +739,19 @@ def _find_reranker_dir(self) -> Optional[str]: return str(candidate) return None + @property + def provider_name(self) -> str: + """Name of the provider that answers: mode + model, e.g. `onnx:...`. + + Used by the dimension contract so a mismatch names the real backend + instead of "embedder". + """ + mode = getattr(self, "mode", "unknown") + model = getattr(self, "_model_name", None) or getattr( + self, "model_name", "" + ) + return f"{mode}:{model}" if model else str(mode) + def get_model_info(self) -> dict: """Возвращает информацию о текущей модели эмбеддера.""" # Если ONNX загружен — показываем реальную модель @@ -728,6 +769,47 @@ def get_model_info(self) -> dict: def _init_provider_async(self): """Фоновая инициализация режима провайдера.""" try: + pref = getattr(self, "provider_preference", "auto") + + # ═══ Explicit provider: no probing of anything else ═══ + if pref == "onnx": + self._init_onnx() + with self._mode_lock: + if self._onnx_client is not None: + self.mode = "onnx" + self._preferred_mode = "onnx" + logger.info("✅ ONNX Client запущен (provider=onnx)!") + else: + self.mode = "fallback" + logger.error( + "❌ provider='onnx' запрошен, но ONNX-сервер недоступен. " + "Внешние эндпоинты не подставляются без явного конфига." + ) + return + if pref in ("llama_cpp", "lm_studio", "ollama"): + checker = { + "llama_cpp": self._check_llama_cpp, + "lm_studio": self._check_lm_studio_raw, + "ollama": self._check_ollama, + }[pref] + if checker(): + with self._mode_lock: + self.mode = pref + logger.info("✅ Провайдер '%s' доступен (pinned).", pref) + else: + with self._mode_lock: + self.mode = "fallback" + logger.error( + "❌ provider='%s' запрошен, но эндпоинт не отвечает. " + "Другие провайдеры не подставляются без явного конфига.", + pref, + ) + return + if pref not in ("auto", ""): + logger.warning( + "Unknown MSCODEBASE_EMBEDDER_PROVIDER='%s' — using auto", pref + ) + # ═══ E5-base ONNX Runtime/OpenVINO (in-process) ═══ _provider = os.getenv("ONNX_PROVIDERS", "cpu").lower() @@ -1043,8 +1125,9 @@ def embed_batch( ) -> List[List[float]]: """Пакетное получение векторов через активный провайдер. - НИКОГДА не меняет mode. Если провайдер недоступен — - возвращает нулевые векторы (режим не сбрасывается). + НИКОГДА не меняет mode. Если провайдер недоступен после ограниченных + ретраев — поднимает `EmbedderUnavailable` с именем провайдера и + последней ошибкой (BLOCKED), а не крутит restart-цикл. """ if not texts: return [] @@ -1057,58 +1140,57 @@ def embed_batch( # Truncate inputs to 512 tokens for llama.cpp (model's max context) texts = self._truncate_for_llama(texts) import time as _retry_time - # Level 1: batch retry (3 attempts with backoff) - for _attempt in range(3): + + provider = f"llama_cpp:{getattr(self, 'model_name', '')}" + + def _touch_llama() -> None: + self._llama_last_used = time.time() try: - payload = {"input": texts} - r = self._http_client.post(self.llama_cpp_url, json=payload) - if r.status_code == 200: - data = r.json().get("data", []) - if data: - data = sorted(data, key=lambda x: x.get("index", 0)) - self._llama_last_used = time.time() - try: - from src.providers.reranker.llama_runner import get_global_runner - get_global_runner()._last_embedder_use = time.time() - except Exception: - pass - return [item["embedding"] for item in data] - else: - logger.warning("llama.cpp: 200 OK but empty data") - elif r.status_code == 500 and _attempt < 2: - _wait = 1 + _attempt * 2 - logger.warning(f"llama.cpp: HTTP 500, retry {_attempt+1}/2, wait {_wait}s") - _retry_time.sleep(_wait) - continue - else: - logger.warning(f"llama.cpp: HTTP {r.status_code}") - except Exception as _exc: - if _attempt < 2: - _retry_time.sleep(1) - continue - logger.warning(f"llama.cpp embed error: {_exc}") - # ─── Level 1.5: embedder мог быть выгружен по idle-timeout ─── - # llama_runner сам убивает llama-server через EMBEDDER_IDLE_TIMEOUT - # (free RAM), поднимать обратно обязан клиент. Иначе :8080 мёртв → - # WinError 10061 → поштучные ретраи → search_code таймаутит - # (инцидент 2026-09-22: max 27s, attempts 2/2). + from src.providers.reranker.llama_runner import get_global_runner + get_global_runner()._last_embedder_use = time.time() + except Exception: + pass + + def _one_batch(_attempt: int) -> List[List[float]]: + r = self._http_client.post(self.llama_cpp_url, json={"input": texts}) + if r.status_code != 200: + raise EmbedderUnavailable(f"llama.cpp HTTP {r.status_code}") + data = r.json().get("data", []) + if not data: + raise EmbedderUnavailable("llama.cpp 200 OK but empty data") + data = sorted(data, key=lambda x: x.get("index", 0)) + _touch_llama() + return [item["embedding"] for item in data] + + # Level 1: a fixed number of batch attempts, then one revive, then + # a bounded per-item pass. The old loop could keep bouncing + # "embed error -> idle-unload -> restart" (G4 night run); a retry + # policy must end in a named BLOCKED answer with the provider. + batch_error: Exception | None = None + try: + return bounded_attempts( + _one_batch, + attempts=3, + provider=provider, + label="embed_batch", + backoff=(1.0, 2.0), + sleep=_retry_time.sleep, + ) + except EmbedderUnavailable as _batch_err: + batch_error = _batch_err + logger.warning( + "llama.cpp embed failed after 3 bounded attempts: %s", _batch_err + ) + + # Level 1.5: the embedder may have been unloaded by idle-timeout. + # One revive, one retry — not a restart loop. if self._revive_llama_cpp(): try: - r = self._http_client.post(self.llama_cpp_url, json={"input": texts}) - if r.status_code == 200: - data = r.json().get("data", []) - if data: - data = sorted(data, key=lambda x: x.get("index", 0)) - self._llama_last_used = time.time() - try: - from src.providers.reranker.llama_runner import get_global_runner - get_global_runner()._last_embedder_use = time.time() - except Exception: - pass - return [item["embedding"] for item in data] + return _one_batch(0) except Exception as _revive_exc: logger.warning(f"llama.cpp retry after revive failed: {_revive_exc}") - # Level 2: single-item retry for each failed text + + # Level 2: single-item retry for each failed text. logger.warning(f"Batch failed, retrying {len(texts)} items individually") results = [None] * len(texts) for idx, text in enumerate(texts): @@ -1120,21 +1202,18 @@ def embed_batch( data = r.json().get("data", []) if data: results[idx] = data[0]["embedding"] - self._llama_last_used = time.time() - try: - from src.providers.reranker.llama_runner import ( - get_global_runner, - ) - get_global_runner()._last_embedder_use = time.time() - except Exception: - pass + _touch_llama() break _retry_time.sleep(1) except Exception: _retry_time.sleep(1) if results[idx] is None: - logger.warning(f"Chunk {idx} failed all retries, zero vector") - raise RuntimeError(f"Embedding failed for chunk {idx} after all retries") + raise EmbedderUnavailable( + f"embed_batch: provider '{provider}' failed for chunk {idx} " + f"after 3 batch attempts + 1 revive + 2 single attempts " + f"(last error: {batch_error}). BLOCKED, not restarted forever." + ) from batch_error + return results return results # ═══ LM Studio ═══ diff --git a/tests/test_embedding_dim_contract.py b/tests/test_embedding_dim_contract.py new file mode 100644 index 00000000..58c3e866 --- /dev/null +++ b/tests/test_embedding_dim_contract.py @@ -0,0 +1,263 @@ +"""Embedding width contract between provider, LanceDB schema and writer (G4). + +The night failure this pins down (12->13.11.2026): the project's vector table +was `fixed_size_list[384]` with zero rows, the active embedder returned +768-d vectors, and the mismatch surfaced as a LanceDB cast error only after the +build had started. Silent padding/truncation would have stored 768-d answers +under a 384-d promise. + +Rules under test: +1. the schema follows the provider's declared width; +2. an empty table with the wrong width is rebuilt (no rows to lose); +3. a non-empty table with the wrong width refuses loudly, naming the provider + and both numbers; +4. the writer never pads or truncates a vector; +5. retries are bounded and end in `EmbedderUnavailable`, not a restart loop. +""" + +from __future__ import annotations + +import threading +from types import SimpleNamespace + +import pytest + +from src.core.indexing.db_manager import LanceDBManager +from src.core.indexing.db_writer import LanceDBWriter +from src.core.indexing.dim_contract import ( + EmbedderUnavailable, + EmbeddingDimMismatch, + bounded_attempts, + provider_dim, + provider_label, + require_dim_match, + schema_vector_dim, +) +from src.providers.embedder.remote_embedder import RemoteEmbedder + + +def _onnx(dim: int = 384) -> SimpleNamespace: + return SimpleNamespace( + embedding_dim=dim, + provider_name="onnx:multilingual-e5-small-int8", + ) + + +def _manager(tmp_path, dim: int, embedder=None) -> LanceDBManager: + project = tmp_path / "project" + project.mkdir(exist_ok=True) + return LanceDBManager( + db_path=tmp_path / "index.lancedb", + embedder=embedder, + project_path=project, + embedding_dim=dim, + ) + + +def _release(mgr: LanceDBManager) -> None: + mgr.close_for_maintenance() + if mgr._db_lock is not None: + mgr._db_lock.release() + + +# ── 1. helpers ────────────────────────────────────────────────────────────── + + +def test_provider_dim_and_label(): + emb = _onnx(384) + assert provider_dim(emb) == 384 + assert provider_label(emb) == "onnx:multilingual-e5-small-int8" + assert provider_dim(None) is None + assert provider_label(None) == "no-embedder" + + +def test_require_dim_match_names_both_numbers(): + with pytest.raises(EmbeddingDimMismatch) as exc: + require_dim_match( + provider="llama_cpp:embeddinggemma-2", + provider_dim=768, + schema_dim=384, + where="write_records(a.py)", + ) + text = str(exc.value) + assert "llama_cpp:embeddinggemma-2" in text + assert "768" in text and "384" in text + + +def test_require_dim_match_refuses_unknown_width(): + with pytest.raises(EmbeddingDimMismatch, match="cannot verify"): + require_dim_match( + provider="mystery", provider_dim=None, schema_dim=384, where="x" + ) + + +# ── 2. schema follows the provider ────────────────────────────────────────── + + +def test_manager_refuses_schema_width_that_contradicts_the_provider(tmp_path): + with pytest.raises(EmbeddingDimMismatch, match="container|declares"): + _manager(tmp_path, dim=768, embedder=_onnx(384)) + + +def test_empty_table_with_other_width_is_rebuilt_for_the_provider(tmp_path): + """G4 repro: 384-d schema, 0 rows, provider returns 768.""" + old = _manager(tmp_path, dim=384, embedder=_onnx(384)) + assert schema_vector_dim(old.table.schema) == 384 + assert old.table.count_rows() == 0 + _release(old) + + new = _manager(tmp_path, dim=768, embedder=_onnx(768)) + assert schema_vector_dim(new.table.schema) == 768, ( + "empty table kept the old width; the schema must follow the provider" + ) + assert new.table.count_rows() == 0 + + +def test_non_empty_table_with_other_width_refuses_and_keeps_rows(tmp_path): + old = _manager(tmp_path, dim=384, embedder=_onnx(384)) + old.table.add( + [ + { + "id": "chunk_0", + "vector": [0.1] * 384, + "text": "def f(): pass", + "text_full": "def f(): pass", + "file_path": "a.py", + "file_hash": "h", + "chunk_index": 0, + "source": "test", + "indexed_at": "2026-11-12T00:00:00", + "summary": "", + "layer": "core", + "module_name": "a", + "hierarchy_level": "function", + "is_public": True, + "symbol_type": "function", + "parent_id": "", + "callees": "", + "health_score": 0.0, + "health_band": "", + "chunk_hash": "c", + "start_line": 1, + "end_line": 2, + "file_mtime_ns": 0, + "file_size": 0, + } + ] + ) + assert old.table.count_rows() == 1 + _release(old) + + with pytest.raises(EmbeddingDimMismatch) as exc: + _manager(tmp_path, dim=768, embedder=_onnx(768)) + text = str(exc.value) + assert "onnx:multilingual-e5-small-int8" in text + assert "384" in text and "768" in text + + +# ── 3. the writer never resizes silently ──────────────────────────────────── + + +def test_writer_refuses_short_vector_instead_of_padding(tmp_path): + mgr = _manager(tmp_path, dim=768, embedder=_onnx(768)) + writer = LanceDBWriter( + table=mgr.table, + table_write_lock=threading.RLock(), + index_lock=None, + embedder=_onnx(768), + db_manager=mgr, + ) + parsed = { + "rel_path": "a.py", + "current_hash": "h2", + "existing_hash": None, + "escaped_path": "a.py", + "chunk_texts": ["def f(): pass"], + "chunk_hashes": ["c2"], + "chunk_texts_full": ["def f(): pass"], + "chunk_metadatas": [{}], + "health": {}, + } + + with pytest.raises(EmbeddingDimMismatch) as exc: + writer.write_records(parsed=parsed, embeddings=[[0.1] * 384]) + + text = str(exc.value) + assert "onnx:multilingual-e5-small-int8" in text + assert "384" in text and "768" in text + assert mgr.table.count_rows() == 0, "a refused write must not reach the table" + + +# ── 4. bounded retries end in a named BLOCKED ─────────────────────────────── + + +def test_bounded_attempts_stops_and_names_the_provider(): + calls = [] + + def _fail(attempt): + calls.append(attempt) + raise RuntimeError("WinError 10061") + + with pytest.raises(EmbedderUnavailable) as exc: + bounded_attempts( + _fail, + attempts=3, + provider="llama_cpp:embeddinggemma-2", + label="embed_batch", + ) + assert calls == [0, 1, 2], "retries must be exactly bounded" + text = str(exc.value) + assert "llama_cpp:embeddinggemma-2" in text + assert "3 attempt(s)" in text + + +def test_bounded_attempts_returns_after_transient_failures(): + calls = [] + + def _flaky(attempt): + calls.append(attempt) + if attempt < 2: + raise RuntimeError("temporary") + return attempt + + assert bounded_attempts( + _flaky, attempts=3, provider="onnx:x", label="embed_batch" + ) == 2 + assert calls == [0, 1, 2] + + +def test_embed_batch_raises_blocked_instead_of_restart_loop(monkeypatch): + """Red-first: old code raised RuntimeError after an unbounded per-item storm.""" + + class _DeadClient: + def __init__(self): + self.posts = 0 + + def post(self, *args, **kwargs): + self.posts += 1 + return SimpleNamespace(status_code=500, json=lambda: {}) + + emb = RemoteEmbedder.__new__(RemoteEmbedder) + emb.mode = "llama_cpp" + emb.model_name = "embeddinggemma-2" + emb.embedding_dim = 768 + emb._mode_lock = threading.Lock() + emb._http_client = _DeadClient() + emb.llama_cpp_url = "http://127.0.0.1:8080/v1/embeddings" + emb._llama_last_used = 0.0 + emb._truncate_for_llama = lambda texts: texts + emb._revive_llama_cpp = lambda: False + + import time as time_mod + + monkeypatch.setattr(time_mod, "sleep", lambda _s: None) + + with pytest.raises(EmbedderUnavailable) as exc: + emb.embed_batch(["a", "b"]) + + text = str(exc.value) + assert "llama_cpp:embeddinggemma-2" in text + assert "BLOCKED" in text + # 3 batch attempts + 2 single-item attempts for the first failing text; + # the loop raises on that text instead of trying the rest. + assert emb._http_client.posts == 3 + 2