\n\n![\"DocuQuery](\"docs/screenshots/hero-dark.png\")
\n\n
\n\n# DocuQuery\n\n### Turn a folder of PDFs, DOCX, and text files into a source you can ask questions.\n\nUpload your documents. DocuQuery reads them, indexes them, and answers questions with the\nexact page and passage the answer came from — no more skimming a 40-page report for one number.\n\n[**Live demo**](#-see-it-work) · [**Quick start**](#-quick-start) · [**Features**](#-features) · [**API reference**](#-api-reference) · [**Roadmap**](#-roadmap)\n\n\n\n\n\n\n\n\n\n
\n\n---\n\n## What DocuQuery actually does\n\n\n---\n\n## What DocuQuery actually does\n\nMost \"chat with your docs\" demos are a thin wrapper around one LLM call. DocuQuery is the\nplumbing that makes that call trustworthy: it parses your files properly, chunks and embeds\nthem, stores the vectors in Postgres, retrieves the passages that actually answer the question,\nand hands the model only that — then shows you exactly where the answer came from.\n- 📂 **Drop in a file** — PDF, DOCX, TXT, or Markdown, up to 50MB\n- ⚡ **It gets indexed automatically** — parsed, cleaned, chunked, embedded, and stored in seconds\n- 💬 **Ask in plain language** — no query syntax, no filters to configure\n- 📎 **Get an answer with receipts** — every claim links back to the filename, page, and passage\n- 🔀 **Pick how strict you want it** — answer strictly from your documents, from general knowledge, or both\n\n---\n\n## 👀 See it work\n\n\n\n\n\n# DocuQuery\n\n### Turn a folder of PDFs, DOCX, and text files into a source you can ask questions.\n\nUpload your documents. DocuQuery reads them, indexes them, and answers questions with the\nexact page and passage the answer came from — no more skimming a 40-page report for one number.\n\n[**Live demo**](#-see-it-work) · [**Quick start**](#-quick-start) · [**Features**](#-features) · [**API reference**](#-api-reference) · [**Roadmap**](#-roadmap)\n\n\n\n\n\n\n\n\n\n
\n![\"A](\"docs/screenshots/demo.gif\")
, HNSW index)"
+ },
+ "metrics": {
+ "document": {
+ "queries": 2,
+ "recall_at_1": 1.0,
+ "precision_at_1": 1.0,
+ "ndcg_at_1": 1.0,
+ "recall_at_3": 1.0,
+ "precision_at_3": 1.0,
+ "ndcg_at_3": 1.0,
+ "recall_at_5": 1.0,
+ "precision_at_5": 0.9,
+ "ndcg_at_5": 1.0,
+ "recall_at_10": 1.0,
+ "precision_at_10": 0.8,
+ "ndcg_at_10": 0.987,
+ "hit_rate_at_5": 1.0,
+ "hit_rate_at_5_note": "identical to recall_at_5: ground truth names one relevant page per query",
+ "mrr": 1.0
+ },
+ "page": {
+ "queries": 2,
+ "recall_at_1": 1.0,
+ "precision_at_1": 1.0,
+ "ndcg_at_1": 1.0,
+ "recall_at_3": 1.0,
+ "precision_at_3": 1.0,
+ "ndcg_at_3": 1.0,
+ "recall_at_5": 1.0,
+ "precision_at_5": 0.9,
+ "ndcg_at_5": 1.0,
+ "recall_at_10": 1.0,
+ "precision_at_10": 0.8,
+ "ndcg_at_10": 0.987,
+ "hit_rate_at_5": 1.0,
+ "hit_rate_at_5_note": "identical to recall_at_5: ground truth names one relevant page per query",
+ "mrr": 1.0
+ },
+ "answer": {
+ "queries": 2,
+ "recall_at_1": 1.0,
+ "precision_at_1": 1.0,
+ "ndcg_at_1": 1.0,
+ "recall_at_3": 1.0,
+ "precision_at_3": 0.3333,
+ "ndcg_at_3": 1.0,
+ "recall_at_5": 1.0,
+ "precision_at_5": 0.2,
+ "ndcg_at_5": 1.0,
+ "recall_at_10": 1.0,
+ "precision_at_10": 0.1,
+ "ndcg_at_10": 1.0,
+ "hit_rate_at_5": 1.0,
+ "hit_rate_at_5_note": "identical to recall_at_5: ground truth names one relevant page per query",
+ "mrr": 1.0,
+ "scorable_queries": 2,
+ "note": "scored only on queries whose generated answer span is present verbatim in the ground-truth passage"
+ }
+ },
+ "queries_with_no_results": 0,
+ "retrieval_latency_ms": {
+ "p50": 22.07,
+ "p95": 488.567,
+ "p99": 488.567,
+ "min": 22.07,
+ "max": 488.567,
+ "mean": 255.318,
+ "n": 2
+ },
+ "per_query": [
+ {
+ "query_id": "q001",
+ "query": "What is the maximum file size supported for uploads in DocuQuery?",
+ "gt_filename": "README.md",
+ "gt_page": 1,
+ "answer_span_scorable": true,
+ "retrieved": [
+ {
+ "rank": 1,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 3,
+ "similarity": 0.8436,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": true
+ },
+ {
+ "rank": 2,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 0,
+ "similarity": 0.8396,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 3,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 16,
+ "similarity": 0.8379,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 4,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 5,
+ "similarity": 0.8316,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 5,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 2,
+ "similarity": 0.8298,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 6,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 12,
+ "similarity": 0.816,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 7,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 13,
+ "similarity": 0.8143,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 8,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 4,
+ "similarity": 0.8138,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 9,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 7,
+ "similarity": 0.8077,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 10,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 18,
+ "similarity": 0.8073,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ }
+ ],
+ "first_relevant_rank": {
+ "document": 1,
+ "page": 1,
+ "answer": 1
+ },
+ "latency_ms": 488.57,
+ "n_retrieved": 10
+ },
+ {
+ "query_id": "q002",
+ "query": "Which package is required for FastAPI to parse multipart/form-data uploads but is never imported directly by the source code?",
+ "gt_filename": "DEPENDENCIES_VERIFIED.md",
+ "gt_page": 1,
+ "answer_span_scorable": true,
+ "retrieved": [
+ {
+ "rank": 1,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 4,
+ "similarity": 0.8855,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": true
+ },
+ {
+ "rank": 2,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 1,
+ "similarity": 0.8843,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 3,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 6,
+ "similarity": 0.8786,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 4,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 3,
+ "similarity": 0.8635,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 5,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 10,
+ "similarity": 0.8532,
+ "relevant_document": false,
+ "relevant_page": false,
+ "relevant_answer": false
+ },
+ {
+ "rank": 6,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 2,
+ "similarity": 0.8456,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 7,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 12,
+ "similarity": 0.8401,
+ "relevant_document": false,
+ "relevant_page": false,
+ "relevant_answer": false
+ },
+ {
+ "rank": 8,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 16,
+ "similarity": 0.8399,
+ "relevant_document": false,
+ "relevant_page": false,
+ "relevant_answer": false
+ },
+ {
+ "rank": 9,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 0,
+ "similarity": 0.8327,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 10,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 11,
+ "similarity": 0.8276,
+ "relevant_document": false,
+ "relevant_page": false,
+ "relevant_answer": false
+ }
+ ],
+ "first_relevant_rank": {
+ "document": 1,
+ "page": 1,
+ "answer": 1
+ },
+ "latency_ms": 22.07,
+ "n_retrieved": 10
+ }
+ ]
+}
\ No newline at end of file
diff --git a/backend/evaluation/run_config_sweep.py b/backend/evaluation/run_config_sweep.py
new file mode 100644
index 0000000..5473486
--- /dev/null
+++ b/backend/evaluation/run_config_sweep.py
@@ -0,0 +1,231 @@
+"""
+Compare retrieval configurations on one fixed query set.
+
+Two kinds of comparison are run:
+
+ index-side chunk size and overlap. Each configuration re-ingests the whole
+ corpus under its own synthetic user, so the index really is
+ rebuilt -- nothing is simulated by re-slicing an existing index.
+
+ query-side similarity threshold and metadata filtering. These reuse the
+ baseline index, because neither of them changes what is stored.
+
+The query set is generated once from the baseline index and reused verbatim by
+every configuration. Ground truth is anchored to (document, page), which does
+not move when chunk size changes, so the comparison is like-for-like.
+
+Usage:
+ python evaluation/run_config_sweep.py --corpus ../corpus --baseline-user-id eval-xxxx
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+from _common import RESULTS_DIR, config_id, load, save
+
+BACKEND_ROOT = Path(__file__).resolve().parent.parent
+
+# (label, chunk_size, chunk_overlap). The first entry must be the shipped
+# default so every improvement is stated relative to what the product does now.
+INDEX_CONFIGS = [
+ ("current-800-120", 800, 120),
+ ("small-400-60", 400, 60),
+ ("large-1200-180", 1200, 180),
+ ("no-overlap-800-0", 800, 0),
+]
+
+# (label, extra flags for run_retrieval_eval.py)
+QUERY_CONFIGS = [
+ ("threshold-0.30-default", []),
+ ("threshold-0.00", ["--no-threshold"]),
+ ("metadata-filter-document", ["--filter-document"]),
+]
+
+
+def run(cmd: list[str]) -> None:
+ print(f"\n$ {' '.join(cmd)}")
+ # Run from backend/ so pydantic-settings finds .env, same as the app does.
+ result = subprocess.run(cmd, cwd=BACKEND_ROOT)
+ if result.returncode != 0:
+ raise SystemExit(f"command failed with exit code {result.returncode}")
+
+
+def headline(metrics: dict) -> dict:
+ """The comparable figures, pulled out of a full result file."""
+ out = {}
+ for level in ("document", "page", "answer"):
+ m = metrics.get(level, {})
+ out[level] = {
+ key: m.get(key)
+ for key in (
+ "queries",
+ "recall_at_1",
+ "recall_at_3",
+ "recall_at_5",
+ "precision_at_5",
+ "ndcg_at_5",
+ "mrr",
+ )
+ }
+ return out
+
+
+def delta(baseline: float | None, candidate: float | None) -> dict:
+ if baseline is None or candidate is None:
+ return {"absolute_pp": None, "relative_pct": None}
+ absolute = round((candidate - baseline) * 100, 2)
+ relative = round((candidate - baseline) / baseline * 100, 2) if baseline else None
+ return {"absolute_pp": absolute, "relative_pct": relative}
+
+
+def main() -> None:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--corpus", required=True)
+ ap.add_argument("--eval-set", default="eval_set.json")
+ ap.add_argument("--top-k", type=int, default=10)
+ ap.add_argument(
+ "--skip-ingest",
+ action="store_true",
+ help="assume every index configuration is already ingested",
+ )
+ ap.add_argument("--out", default="config_sweep.json")
+ args = ap.parse_args()
+
+ eval_set = load(args.eval_set)
+ embedding_model = eval_set.get("generator_model")
+ print(f"[sweep] query set: {args.eval_set} ({len(eval_set['queries'])} queries)")
+
+ from app.core.config import get_settings
+
+ model_label = get_settings().embedding_model
+
+ runs: dict[str, dict] = {}
+
+ for label, chunk_size, overlap in INDEX_CONFIGS:
+ user_id = config_id(
+ chunk_size=chunk_size,
+ chunk_overlap=overlap,
+ embedding_model=model_label,
+ label="",
+ )
+ if not args.skip_ingest:
+ run(
+ [
+ sys.executable,
+ "evaluation/ingest_corpus.py",
+ "--corpus",
+ str(Path(args.corpus).resolve()),
+ "--chunk-size",
+ str(chunk_size),
+ "--chunk-overlap",
+ str(overlap),
+ "--reset",
+ ]
+ )
+ run(
+ [
+ sys.executable,
+ "evaluation/run_retrieval_eval.py",
+ "--user-id",
+ user_id,
+ "--eval-set",
+ args.eval_set,
+ "--top-k",
+ str(args.top_k),
+ "--label",
+ label,
+ ]
+ )
+ result = load(f"retrieval_eval_{label}.json")
+ ingest = load(f"ingest_{user_id}.json")
+ runs[label] = {
+ "kind": "index",
+ "chunk_size": chunk_size,
+ "chunk_overlap": overlap,
+ "user_id": user_id,
+ "chunks_indexed": ingest["total_chunks"],
+ "documents_indexed": ingest["documents_indexed"],
+ "mean_chunk_chars": round(
+ sum(
+ d.get("chunk_chars_mean", 0) * d.get("chunks", 0)
+ for d in ingest["documents"]
+ if d["status"] == "indexed"
+ )
+ / max(1, ingest["total_chunks"]),
+ 1,
+ ),
+ "retrieval_latency_ms": result["retrieval_latency_ms"],
+ "queries_with_no_results": result["queries_with_no_results"],
+ "metrics": headline(result["metrics"]),
+ }
+
+ baseline_label = INDEX_CONFIGS[0][0]
+ baseline_user_id = runs[baseline_label]["user_id"]
+
+ for label, flags in QUERY_CONFIGS:
+ run(
+ [
+ sys.executable,
+ "evaluation/run_retrieval_eval.py",
+ "--user-id",
+ baseline_user_id,
+ "--eval-set",
+ args.eval_set,
+ "--top-k",
+ str(args.top_k),
+ "--label",
+ label,
+ *flags,
+ ]
+ )
+ result = load(f"retrieval_eval_{label}.json")
+ runs[label] = {
+ "kind": "query",
+ "flags": flags,
+ "user_id": baseline_user_id,
+ "retrieval_latency_ms": result["retrieval_latency_ms"],
+ "queries_with_no_results": result["queries_with_no_results"],
+ "metrics": headline(result["metrics"]),
+ }
+
+ baseline_metrics = runs[baseline_label]["metrics"]
+ comparison = {}
+ for label, entry in runs.items():
+ if label == baseline_label:
+ continue
+ comparison[label] = {
+ level: {
+ metric: delta(
+ baseline_metrics[level][metric], entry["metrics"][level][metric]
+ )
+ for metric in ("recall_at_1", "recall_at_3", "recall_at_5", "mrr", "ndcg_at_5")
+ }
+ for level in ("document", "page", "answer")
+ }
+
+ save(
+ args.out,
+ {
+ "eval_set": args.eval_set,
+ "eval_set_size": len(eval_set["queries"]),
+ "top_k": args.top_k,
+ "baseline": baseline_label,
+ "runs": runs,
+ "vs_baseline": comparison,
+ },
+ )
+
+ print("\n=== page-level recall@5 by configuration ===")
+ for label, entry in runs.items():
+ value = entry["metrics"]["page"]["recall_at_5"]
+ marker = " (baseline)" if label == baseline_label else ""
+ print(f"{label:28s} {value}{marker}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/backend/evaluation/run_perf_bench.py b/backend/evaluation/run_perf_bench.py
new file mode 100644
index 0000000..708145f
--- /dev/null
+++ b/backend/evaluation/run_perf_bench.py
@@ -0,0 +1,296 @@
+"""
+Performance benchmark: embedding throughput, retrieval latency, LLM latency.
+
+Ingestion-stage timings (parse / clean / chunk / embed / store) are produced by
+ingest_corpus.py and are not repeated here; this script covers the query side
+plus a standalone embedding throughput measurement.
+
+Every latency figure is reported as a distribution (min / P50 / P95 / P99 / max)
+over a stated number of runs, after a discarded warm-up, so no number here is a
+single lucky run.
+
+The --llm stage sends retrieved passages from the corpus to the configured LLM
+provider over the network and consumes API quota. It is off by default.
+
+Usage:
+ python evaluation/run_perf_bench.py --user-id eval-xxxx
+ python evaluation/run_perf_bench.py --user-id eval-xxxx --llm --llm-queries 20
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import os
+import statistics
+import time
+
+from _common import eval_database_url, load, percentiles, save
+
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+from app.core.config import get_settings
+from app.ingestion.embeddings import get_embedding_service
+from app.llm import get_response_generator
+from app.retrieval.embedding_query import embed_query
+from app.retrieval.exceptions import NoResultsError
+from app.retrieval.retrieval_pipeline import run_retrieval_pipeline
+
+
+def proc_sample() -> dict:
+ try:
+ import psutil
+ except ImportError:
+ return {}
+ p = psutil.Process(os.getpid())
+ return {
+ "rss_mb": round(p.memory_info().rss / 1024**2, 1),
+ "cpu_percent": p.cpu_percent(interval=None),
+ }
+
+
+def bench_embedding(texts: list[str], batch_sizes: tuple[int, ...]) -> dict:
+ """
+ Embedding throughput at several batch sizes.
+
+ The same text list is embedded each time so the only variable is batching.
+ """
+ settings = get_settings()
+ service = get_embedding_service()
+
+ t = time.perf_counter()
+ service.warm_up()
+ cold_load_ms = round((time.perf_counter() - t) * 1000, 2)
+
+ service.embed(texts[:8]) # warm-up, result discarded
+
+ results = {}
+ original = settings.embedding_batch_size
+ for batch_size in batch_sizes:
+ settings.embedding_batch_size = batch_size
+ t = time.perf_counter()
+ vectors = service.embed(texts)
+ elapsed = time.perf_counter() - t
+ results[f"batch_{batch_size}"] = {
+ "texts": len(texts),
+ "seconds": round(elapsed, 3),
+ "embeddings_per_second": round(len(texts) / elapsed, 1),
+ "ms_per_embedding": round(elapsed * 1000 / len(texts), 3),
+ "dim": len(vectors[0]),
+ }
+ print(f"[embed] batch={batch_size}: {results[f'batch_{batch_size}']}")
+ settings.embedding_batch_size = original
+
+ return {
+ "cold_load_ms": cold_load_ms,
+ "model": settings.embedding_model,
+ "by_batch_size": results,
+ "process_after": proc_sample(),
+ }
+
+
+async def bench_query_embedding(queries: list[str], repeats: int) -> dict:
+ samples = []
+ for _ in range(repeats):
+ for query in queries:
+ t = time.perf_counter()
+ await embed_query(query)
+ samples.append((time.perf_counter() - t) * 1000)
+ return percentiles(samples)
+
+
+async def bench_retrieval(db, user_id: str, queries: list[str], top_k: int, repeats: int) -> dict:
+ samples = []
+ empty = 0
+ chunk_counts = []
+ context_chars = []
+ for _ in range(repeats):
+ for query in queries:
+ t = time.perf_counter()
+ try:
+ result = await run_retrieval_pipeline(
+ query=query, db=db, user_id=user_id, top_k=top_k
+ )
+ chunk_counts.append(len(result.chunks))
+ context_chars.append(len(result.context))
+ except NoResultsError:
+ empty += 1
+ samples.append((time.perf_counter() - t) * 1000)
+ return {
+ "latency_ms": percentiles(samples),
+ "queries_per_second_serial": round(1000 / statistics.median(samples), 2) if samples else None,
+ "empty_results": empty,
+ "chunks_returned_mean": round(statistics.mean(chunk_counts), 2) if chunk_counts else None,
+ "context_chars_mean": round(statistics.mean(context_chars), 1) if context_chars else None,
+ "context_chars_max": max(context_chars) if context_chars else None,
+ }
+
+
+async def bench_concurrent_retrieval(
+ engine, user_id: str, queries: list[str], top_k: int, concurrency: int
+) -> dict:
+ """Wall-clock throughput with N retrievals in flight, each on its own session."""
+ Session = async_sessionmaker(engine, expire_on_commit=False)
+
+ async def one(query: str) -> float:
+ async with Session() as db:
+ t = time.perf_counter()
+ try:
+ await run_retrieval_pipeline(query=query, db=db, user_id=user_id, top_k=top_k)
+ except NoResultsError:
+ pass
+ return (time.perf_counter() - t) * 1000
+
+ semaphore = asyncio.Semaphore(concurrency)
+
+ async def guarded(query: str) -> float:
+ async with semaphore:
+ return await one(query)
+
+ t0 = time.perf_counter()
+ samples = await asyncio.gather(*(guarded(q) for q in queries))
+ wall = time.perf_counter() - t0
+
+ return {
+ "concurrency": concurrency,
+ "queries": len(queries),
+ "wall_seconds": round(wall, 3),
+ "queries_per_second": round(len(queries) / wall, 2),
+ "latency_ms": percentiles(list(samples)),
+ }
+
+
+async def bench_llm(db, user_id: str, queries: list[str], top_k: int) -> dict:
+ """
+ Time to first streamed token and full generation latency, over real calls.
+
+ Generation goes through ResponseGenerator.stream -- the same call POST /chat
+ makes -- so context truncation, the versioned system prompt and SSE framing
+ are all in the measured path. Retrieval is timed separately, so
+ time_to_first_token_ms excludes it and end_to_end_ms is what a user waits
+ for from question to last token.
+ """
+ settings = get_settings()
+ generator = get_response_generator()
+
+ ttft, generation, end_to_end, retrieval, chars, tokens, errors = [], [], [], [], [], [], []
+
+ for i, query in enumerate(queries, 1):
+ t_start = time.perf_counter()
+ try:
+ result = await run_retrieval_pipeline(
+ query=query, db=db, user_id=user_id, top_k=top_k
+ )
+ except NoResultsError:
+ continue
+ t_retrieved = time.perf_counter()
+
+ first_token_at = None
+ token_events = 0
+ answer_chars = 0
+ try:
+ async for event in generator.stream(query, result, mode="docuquery"):
+ if not event.startswith("event: token"):
+ continue
+ if first_token_at is None:
+ first_token_at = time.perf_counter()
+ token_events += 1
+ answer_chars += len(event) - len("event: token\ndata: \n\n")
+ except Exception as exc:
+ errors.append(f"{type(exc).__name__}: {exc}")
+ print(f"[{i}] llm error: {type(exc).__name__}: {exc}")
+ continue
+
+ t_end = time.perf_counter()
+ if first_token_at is None:
+ errors.append("stream produced no tokens")
+ continue
+
+ retrieval.append((t_retrieved - t_start) * 1000)
+ ttft.append((first_token_at - t_retrieved) * 1000)
+ generation.append((t_end - t_retrieved) * 1000)
+ end_to_end.append((t_end - t_start) * 1000)
+ chars.append(float(answer_chars))
+ tokens.append(float(token_events))
+ print(
+ f"[{i}/{len(queries)}] ttft={ttft[-1]:.0f}ms "
+ f"total={end_to_end[-1]:.0f}ms tokens={token_events}"
+ )
+
+ return {
+ "provider": settings.llm_provider,
+ "model": settings.llm_model,
+ "temperature": settings.llm_temperature,
+ "max_tokens": settings.llm_max_tokens,
+ "max_context_tokens": settings.llm_max_context_tokens,
+ "completed": len(ttft),
+ "errors": errors,
+ "retrieval_ms": percentiles(retrieval),
+ "time_to_first_token_ms": percentiles(ttft),
+ "generation_ms": percentiles(generation),
+ "end_to_end_ms": percentiles(end_to_end),
+ "streamed_token_events": percentiles(tokens),
+ "answer_chars": percentiles(chars),
+ "note": (
+ "streamed_token_events counts provider stream deltas, not tokeniser "
+ "tokens; answer_chars is JSON-escaped length and is an upper bound"
+ ),
+ }
+
+
+async def main() -> None:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--user-id", required=True)
+ ap.add_argument("--eval-set", default="eval_set.json")
+ ap.add_argument("--top-k", type=int, default=5)
+ ap.add_argument("--repeats", type=int, default=3)
+ ap.add_argument("--embed-texts", type=int, default=256)
+ ap.add_argument("--concurrency", type=int, default=8)
+ ap.add_argument("--llm", action="store_true", help="run real LLM calls (uses quota)")
+ ap.add_argument("--llm-queries", type=int, default=20)
+ ap.add_argument("--out", default="perf_bench.json")
+ args = ap.parse_args()
+
+ eval_set = load(args.eval_set)
+ queries = [q["query"] for q in eval_set["queries"]]
+ passages = [q["gt_passage"] for q in eval_set["queries"]]
+ print(f"[bench] {len(queries)} queries available")
+
+ payload: dict = {"config": vars(args), "process_start": proc_sample()}
+
+ texts = (passages * ((args.embed_texts // max(1, len(passages))) + 1))[: args.embed_texts]
+ payload["embedding"] = bench_embedding(texts, batch_sizes=(1, 8, 32, 64))
+
+ engine = create_async_engine(eval_database_url(), pool_pre_ping=True)
+ Session = async_sessionmaker(engine, expire_on_commit=False)
+
+ async with Session() as db:
+ payload["query_embedding_ms"] = await bench_query_embedding(queries[:20], args.repeats)
+ print(f"[query-embed] {payload['query_embedding_ms']}")
+
+ payload["retrieval"] = await bench_retrieval(
+ db, args.user_id, queries, args.top_k, args.repeats
+ )
+ print(f"[retrieval] {payload['retrieval']['latency_ms']}")
+
+ payload["retrieval_concurrent"] = await bench_concurrent_retrieval(
+ engine, args.user_id, queries, args.top_k, args.concurrency
+ )
+ print(f"[concurrent] {payload['retrieval_concurrent']}")
+
+ if args.llm:
+ async with Session() as db:
+ payload["llm"] = await bench_llm(
+ db, args.user_id, queries[: args.llm_queries], args.top_k
+ )
+ else:
+ payload["llm"] = {"skipped": "run with --llm to measure generation latency"}
+
+ await engine.dispose()
+
+ payload["process_end"] = proc_sample()
+ save(args.out, payload)
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/backend/evaluation/run_retrieval_eval.py b/backend/evaluation/run_retrieval_eval.py
new file mode 100644
index 0000000..2120273
--- /dev/null
+++ b/backend/evaluation/run_retrieval_eval.py
@@ -0,0 +1,264 @@
+"""
+Score the retrieval pipeline against a fixed ground-truth query set.
+
+Every query goes through app.retrieval.run_retrieval_pipeline -- the same
+function POST /retrieve and POST /chat call -- so what is measured is the
+deployed path: query embedding, the per-user SQL filter, the pgvector HNSW
+search, the similarity threshold, deduplication and ranking.
+
+Three relevance levels are scored, from loosest to strictest:
+
+ document the retrieved chunk came from the document the query was written from
+ page ... and from the same page -> page-level citation accuracy
+ answer the retrieved chunk text contains the answer span the generator
+ quoted -> answer-bearing chunk accuracy
+
+Ground truth names exactly one relevant page per query, so Recall@k and
+Hit Rate@k are the same quantity here: the share of queries with at least one
+relevant chunk in the top k. Both names are reported with that caveat attached
+rather than presented as two independent results.
+
+Usage:
+ python evaluation/run_retrieval_eval.py --user-id eval-xxxx
+ python evaluation/run_retrieval_eval.py --user-id eval-xxxx --no-threshold
+ python evaluation/run_retrieval_eval.py --user-id eval-xxxx --filter-document
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import math
+import re
+import time
+
+from _common import eval_database_url, load, percentiles, save
+
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+from app.core.config import get_settings
+from app.retrieval.exceptions import NoResultsError
+from app.retrieval.filters import RetrievalFilter
+from app.retrieval.retrieval_pipeline import run_retrieval_pipeline
+
+K_VALUES = (1, 3, 5, 10)
+LEVELS = ("document", "page", "answer")
+
+
+def normalize(text: str) -> str:
+ return re.sub(r"\s+", " ", text).strip().lower()
+
+
+def relevance(chunk, query: dict, level: str) -> bool:
+ if chunk.document_id != query["gt_document_id"]:
+ return False
+ if level == "document":
+ return True
+ if chunk.page != query["gt_page"]:
+ return False
+ if level == "page":
+ return True
+ span = normalize(query.get("answer_span", ""))
+ return bool(span) and span in normalize(chunk.text)
+
+
+def ndcg_at_k(relevances: list[bool], k: int) -> float:
+ """Binary-gain NDCG. Ideal ranking puts every relevant hit first."""
+ top = relevances[:k]
+ dcg = sum(1.0 / math.log2(i + 2) for i, rel in enumerate(top) if rel)
+ ideal_hits = min(sum(top), k)
+ if ideal_hits == 0:
+ return 0.0
+ idcg = sum(1.0 / math.log2(i + 2) for i in range(ideal_hits))
+ return dcg / idcg
+
+
+def score_level(per_query: list[list[bool]]) -> dict:
+ """Aggregate one relevance level across all queries."""
+ n = len(per_query)
+ if n == 0:
+ return {}
+
+ out: dict = {"queries": n}
+ for k in K_VALUES:
+ hits = sum(1 for rels in per_query if any(rels[:k]))
+ out[f"recall_at_{k}"] = round(hits / n, 4)
+ out[f"precision_at_{k}"] = round(
+ sum(sum(rels[:k]) / k for rels in per_query) / n, 4
+ )
+ out[f"ndcg_at_{k}"] = round(sum(ndcg_at_k(rels, k) for rels in per_query) / n, 4)
+
+ out["hit_rate_at_5"] = out["recall_at_5"]
+ out["hit_rate_at_5_note"] = (
+ "identical to recall_at_5: ground truth names one relevant page per query"
+ )
+
+ reciprocal = []
+ for rels in per_query:
+ rank = next((i + 1 for i, rel in enumerate(rels) if rel), None)
+ reciprocal.append(1.0 / rank if rank else 0.0)
+ out["mrr"] = round(sum(reciprocal) / n, 4)
+
+ return out
+
+
+async def main() -> None:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--user-id", required=True)
+ ap.add_argument("--eval-set", default="eval_set.json")
+ ap.add_argument("--top-k", type=int, default=10)
+ ap.add_argument(
+ "--no-threshold",
+ action="store_true",
+ help="set the similarity threshold to 0 for this run",
+ )
+ ap.add_argument(
+ "--filter-document",
+ action="store_true",
+ help="pass the ground-truth document_id as a metadata filter",
+ )
+ ap.add_argument("--label", default="")
+ ap.add_argument("--out", default=None)
+ args = ap.parse_args()
+
+ settings = get_settings()
+ original_threshold = settings.retrieval_similarity_threshold
+ if args.no_threshold:
+ settings.retrieval_similarity_threshold = 0.0
+ # The pipeline caps results at min(top_k, retrieval_max_context_chunks);
+ # raise the cap so a top_k of 10 is actually returned.
+ settings.retrieval_max_context_chunks = max(
+ settings.retrieval_max_context_chunks, args.top_k
+ )
+
+ eval_set = load(args.eval_set)
+ queries = eval_set["queries"]
+ print(f"[eval-set] {len(queries)} queries from {args.eval_set}")
+
+ engine = create_async_engine(eval_database_url(), pool_pre_ping=True)
+ Session = async_sessionmaker(engine, expire_on_commit=False)
+
+ rows: list[dict] = []
+ latencies: list[float] = []
+ per_level: dict[str, list[list[bool]]] = {level: [] for level in LEVELS}
+ answer_span_scorable = 0
+ empty_results = 0
+
+ async with Session() as db:
+ for i, query in enumerate(queries, 1):
+ filters = (
+ RetrievalFilter(document_id=query["gt_document_id"])
+ if args.filter_document
+ else None
+ )
+
+ t0 = time.perf_counter()
+ try:
+ result = await run_retrieval_pipeline(
+ query=query["query"],
+ db=db,
+ user_id=args.user_id,
+ top_k=args.top_k,
+ filters=filters,
+ )
+ chunks = result.chunks
+ except NoResultsError:
+ chunks = []
+ empty_results += 1
+ latency_ms = (time.perf_counter() - t0) * 1000
+ latencies.append(latency_ms)
+
+ rels = {level: [relevance(c, query, level) for c in chunks] for level in LEVELS}
+
+ # A query can only be scored at answer-span level if the generator
+ # actually quoted a span that appears in the passage it was given.
+ span = normalize(query.get("answer_span", ""))
+ span_scorable = bool(span) and span in normalize(query["gt_passage"])
+ if span_scorable:
+ answer_span_scorable += 1
+
+ for level in LEVELS:
+ if level == "answer" and not span_scorable:
+ continue
+ per_level[level].append(rels[level])
+
+ first_rank = {
+ level: next((j + 1 for j, r in enumerate(rels[level]) if r), None)
+ for level in LEVELS
+ }
+
+ rows.append(
+ {
+ "query_id": query["query_id"],
+ "query": query["query"],
+ "gt_filename": query["gt_filename"],
+ "gt_page": query["gt_page"],
+ "answer_span_scorable": span_scorable,
+ "retrieved": [
+ {
+ "rank": j + 1,
+ "filename": c.filename,
+ "page": c.page,
+ "chunk_index": c.chunk_index,
+ "similarity": round(c.similarity, 4),
+ "relevant_document": rels["document"][j],
+ "relevant_page": rels["page"][j],
+ "relevant_answer": rels["answer"][j],
+ }
+ for j, c in enumerate(chunks)
+ ],
+ "first_relevant_rank": first_rank,
+ "latency_ms": round(latency_ms, 2),
+ "n_retrieved": len(chunks),
+ }
+ )
+
+ if i % 10 == 0:
+ print(f"[{i}/{len(queries)}] scored")
+
+ await engine.dispose()
+
+ metrics = {level: score_level(per_level[level]) for level in LEVELS}
+ metrics["answer"]["scorable_queries"] = answer_span_scorable
+ metrics["answer"]["note"] = (
+ "scored only on queries whose generated answer span is present verbatim "
+ "in the ground-truth passage"
+ )
+
+ payload = {
+ "run": {
+ "label": args.label,
+ "user_id": args.user_id,
+ "eval_set": args.eval_set,
+ "eval_set_size": len(queries),
+ "top_k": args.top_k,
+ "similarity_threshold": settings.retrieval_similarity_threshold,
+ "similarity_threshold_default": original_threshold,
+ "metadata_filter": "document_id" if args.filter_document else "none",
+ "embedding_model": settings.embedding_model,
+ "vector_distance": "cosine (pgvector <=>, HNSW index)",
+ },
+ "metrics": metrics,
+ "queries_with_no_results": empty_results,
+ "retrieval_latency_ms": percentiles(latencies),
+ "per_query": rows,
+ }
+
+ name = args.out or f"retrieval_eval_{args.label or args.user_id}.json"
+ save(name, payload)
+
+ print("\n=== retrieval metrics ===")
+ for level in LEVELS:
+ m = metrics[level]
+ print(
+ f"{level:9s} n={m['queries']:3d} "
+ f"R@1={m['recall_at_1']:.3f} R@3={m['recall_at_3']:.3f} "
+ f"R@5={m['recall_at_5']:.3f} MRR={m['mrr']:.3f} "
+ f"NDCG@5={m['ndcg_at_5']:.3f} P@5={m['precision_at_5']:.3f}"
+ )
+ print(f"\nlatency ms: {payload['retrieval_latency_ms']}")
+ print(f"queries returning nothing: {empty_results}")
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/backend/evaluation/sets/manual.json b/backend/evaluation/sets/manual.json
new file mode 100644
index 0000000..dccf495
--- /dev/null
+++ b/backend/evaluation/sets/manual.json
@@ -0,0 +1,440 @@
+{
+ "status": "DRAFT - not reviewed, not used to produce any reported number.",
+ "description": "Hand-written evaluation slices covering what the LLM-generated set cannot: questions with no answer in the corpus, questions needing two documents, and two-turn exchanges whose second turn is not a standalone query.",
+ "corpus": "corpus/manifest.json",
+ "ground_truth_convention": "Documents are referenced by their manifest `id`, never by database document_id, because the latter is regenerated on every ingest.",
+ "caveats": [
+ "Written from document titles and scope, not by reading every page. Each item needs a human to confirm the answer really is where it is claimed to be, or really is absent.",
+ "The unanswerable slice was the fragile one and this proved correct: 4 of 15 draft items asked about frameworks the NIST and Federal Reserve documents genuinely discuss. Those were replaced and the slice re-verified; see `verification`.",
+ "Multi-hop items assert that two documents are needed. A single document may in practice answer some of them, which would make those items easy rather than wrong."
+ ],
+ "verification": {
+ "status": "verified",
+ "method": "Full text of all 27 corpus PDFs extracted with PyMuPDF and searched for each item's distinguishing terms, case-insensitively and on word boundaries. Word boundaries matter: a substring search reported 903 hits for 'Rust', every one of them inside 'trust'.",
+ "result": "4 of the original 15 were contaminated and were replaced (see each item's why_absent): PCI DSS, HIPAA, Basel III and NBER are all genuinely discussed in the corpus. The remaining 11 verify absent.",
+ "not_verified": "The multi_hop and followup slices are NOT mechanically verified. Their ground-truth document assignments come from document scope and still need a human to confirm the answer is on the page claimed."
+ },
+ "slices": {
+ "unanswerable": {
+ "description": "The corpus does not contain the answer. Measures whether retrieval abstains or returns confident-looking but irrelevant chunks above the similarity threshold.",
+ "expected": "no relevant chunk; ideally nothing above threshold",
+ "items": [
+ {
+ "id": "una-01",
+ "query": "What is the maximum response time an organisation has under GDPR Article 12 to answer a data subject access request?",
+ "why_absent": "EU data-protection law; the corpus is U.S. federal guidance and contains no GDPR article text."
+ },
+ {
+ "id": "una-02",
+ "query": "Under the EU's Digital Operational Resilience Act, how often must a financial entity carry out threat-led penetration testing?",
+ "why_absent": "EU financial-sector regulation. Replaced the original PCI DSS question: 'PCI DSS' and 'Payment Card Industry' occur 18 times across SP 800-92, 800-146, 800-209 and 800-218 as a compliance driver, which made it a poor probe for absence."
+ },
+ {
+ "id": "una-03",
+ "query": "What is the minimum observation window for a SOC 2 Type II examination?",
+ "why_absent": "AICPA attestation standard; absent from this corpus."
+ },
+ {
+ "id": "una-04",
+ "query": "Which risk tier does the EU AI Act assign to remote biometric identification in publicly accessible spaces?",
+ "why_absent": "EU legislation; the AI RMF here is a voluntary U.S. framework and predates it."
+ },
+ {
+ "id": "una-05",
+ "query": "What are the five technical controls required for UK Cyber Essentials certification?",
+ "why_absent": "UK NCSC scheme. 'Cyber Essentials' does not occur in the corpus. Note that the acronym NCSC does appear three times, but it refers to the U.S. ODNI National Counterintelligence and Security Center, not the UK National Cyber Security Centre."
+ },
+ {
+ "id": "una-06",
+ "query": "What maturity levels are defined for the Australian Essential Eight mitigation strategies?",
+ "why_absent": "Australian Signals Directorate guidance; absent from this corpus."
+ },
+ {
+ "id": "una-07",
+ "query": "What is the maximum fine for a data-protection breach under Brazil's LGPD?",
+ "why_absent": "Brazilian data-protection law. Replaced the original HIPAA question: 'HIPAA' occurs 33 times across the corpus as a compliance context, notably in IR 8259 and SP 800-207."
+ },
+ {
+ "id": "una-08",
+ "query": "What reserve requirement ratio does the People's Bank of China apply to large commercial banks?",
+ "why_absent": "Chinese monetary policy. Replaced the original Basel III question: 'Basel III' and 'liquidity coverage ratio' both appear in the 2022 and 2023 Federal Reserve annual reports."
+ },
+ {
+ "id": "una-09",
+ "query": "Which price index does the Bank of Japan use to measure its 2 percent inflation target?",
+ "why_absent": "Japanese monetary policy. Replaced the original NBER question: 'National Bureau of Economic Research' appears in both Federal Reserve annual reports, defining the recession shading on their charts."
+ },
+ {
+ "id": "una-10",
+ "query": "What is the recommended cold-water steeping time for a first-flush Darjeeling?",
+ "why_absent": "Entirely outside the corpus domain; a control for obviously irrelevant queries."
+ },
+ {
+ "id": "una-11",
+ "query": "How does the Rust borrow checker's lifetime elision algorithm decide when a reference outlives its owner?",
+ "why_absent": "Programming-language internals; no document here discusses Rust."
+ },
+ {
+ "id": "una-12",
+ "query": "What is the maximum transmit power permitted for a Wi-Fi 6E access point in the 6 GHz band under FCC rules?",
+ "why_absent": "FCC radio regulation; outside the scope of every document here."
+ },
+ {
+ "id": "una-13",
+ "query": "How many players are on the field per team during a standard rugby union match?",
+ "why_absent": "Entirely outside the corpus domain; a second control query."
+ },
+ {
+ "id": "una-14",
+ "query": "What is the current statutory corporation tax rate for large companies in the Republic of Ireland?",
+ "why_absent": "Foreign tax law; absent from the U.S. federal documents in this corpus."
+ },
+ {
+ "id": "una-15",
+ "query": "Which antibiotic class is first-line treatment for uncomplicated community-acquired pneumonia in adults?",
+ "why_absent": "Clinical medicine; entirely outside the corpus domain."
+ }
+ ]
+ },
+ "multi_hop": {
+ "description": "Answering requires combining two documents. Measures whether top-k retrieval surfaces both, rather than only the more lexically similar one.",
+ "items": [
+ {
+ "id": "mh-01",
+ "query": "How should an organisation verify third-party components in a container image before deploying it?",
+ "gt_document_ids": [
+ "nist-sp-800-190",
+ "nist-sp-800-218"
+ ],
+ "hop_note": "Image-level guidance is in the container security guide; verifying acquired third-party components is an SSDF practice."
+ },
+ {
+ "id": "mh-02",
+ "query": "How does a zero trust policy engine's access decision relate to the authorisation step of the Risk Management Framework?",
+ "gt_document_ids": [
+ "nist-sp-800-207",
+ "nist-sp-800-37r2"
+ ],
+ "hop_note": "Policy engine is defined in the ZTA document; authorise is an RMF step."
+ },
+ {
+ "id": "mh-03",
+ "query": "After a ransomware event, what should an organisation do about media that cannot be verified clean?",
+ "gt_document_ids": [
+ "nist-sp-1800-25",
+ "nist-sp-800-88r1"
+ ],
+ "hop_note": "Recovery context from the data-integrity practice guide; sanitisation decisions from the media sanitization guidelines."
+ },
+ {
+ "id": "mh-04",
+ "query": "Which cloud service model leaves the customer most responsible for securing the operating system, and what does that imply for patching?",
+ "gt_document_ids": [
+ "nist-sp-800-145",
+ "nist-sp-800-146"
+ ],
+ "hop_note": "Service-model definitions in the cloud definition; security implications in the synopsis."
+ },
+ {
+ "id": "mh-05",
+ "query": "Is personally identifiable information also controlled unclassified information, and do the two require different handling?",
+ "gt_document_ids": [
+ "nist-sp-800-122",
+ "nist-sp-800-171r2"
+ ],
+ "hop_note": "PII definition and handling in one document, CUI requirements in the other."
+ },
+ {
+ "id": "mh-06",
+ "query": "Which log sources should be retained to support the detection and analysis phase of incident response?",
+ "gt_document_ids": [
+ "nist-sp-800-92",
+ "nist-sp-800-61r2"
+ ],
+ "hop_note": "Log source guidance in the log management guide; incident response phases in the handling guide."
+ },
+ {
+ "id": "mh-07",
+ "query": "How do the governance activities in the AI risk framework map onto the Identify function of the Cybersecurity Framework?",
+ "gt_document_ids": [
+ "nist-ai-100-1",
+ "nist-cswp-cyberframework-1-1"
+ ],
+ "hop_note": "Govern is an AI RMF function; Identify is a CSF core function."
+ },
+ {
+ "id": "mh-08",
+ "query": "What should a federal agency expect an IoT device manufacturer to have already done before the agency procures the device?",
+ "gt_document_ids": [
+ "nist-ir-8259",
+ "nist-sp-800-213"
+ ],
+ "hop_note": "Manufacturer-side activities in the IR; the federal acquirer's expectations in the SP."
+ },
+ {
+ "id": "mh-09",
+ "query": "How do secure software development practices feed into supply chain risk management for acquired software?",
+ "gt_document_ids": [
+ "nist-sp-800-218",
+ "nist-sp-800-161r1"
+ ],
+ "hop_note": "SSDF practices in one, C-SCRM controls in the other."
+ },
+ {
+ "id": "mh-10",
+ "query": "What inputs does a risk assessment provide to the categorise and select steps of the Risk Management Framework?",
+ "gt_document_ids": [
+ "nist-sp-800-30r1",
+ "nist-sp-800-37r2"
+ ],
+ "hop_note": "Assessment process in the risk assessment guide; the steps in the RMF document."
+ },
+ {
+ "id": "mh-11",
+ "query": "How should backup integrity be validated as part of contingency planning for a destructive-malware event?",
+ "gt_document_ids": [
+ "nist-sp-800-34r1",
+ "nist-sp-1800-25"
+ ],
+ "hop_note": "Contingency planning and backup strategy in one; integrity validation against ransomware in the other."
+ },
+ {
+ "id": "mh-12",
+ "query": "What should happen to storage media at end of life in a storage area network environment?",
+ "gt_document_ids": [
+ "nist-sp-800-209",
+ "nist-sp-800-88r1"
+ ],
+ "hop_note": "Storage infrastructure context in one; the sanitisation decision matrix in the other."
+ },
+ {
+ "id": "mh-13",
+ "query": "How does technical security testing produce evidence for a control assessment during authorisation?",
+ "gt_document_ids": [
+ "nist-sp-800-115",
+ "nist-sp-800-37r2"
+ ],
+ "hop_note": "Testing techniques in one; assessment and authorisation in the other."
+ },
+ {
+ "id": "mh-14",
+ "query": "Which authenticator assurance level is appropriate for resources accessed through a zero trust architecture?",
+ "gt_document_ids": [
+ "nist-sp-800-63-3",
+ "nist-sp-800-207"
+ ],
+ "hop_note": "AAL definitions in the identity guidelines; ZTA access requirements in the other."
+ },
+ {
+ "id": "mh-15",
+ "query": "How did the Federal Reserve's reported balance sheet position change between its 2022 and 2023 annual reports?",
+ "gt_document_ids": [
+ "frb-annual-report-2022",
+ "frb-annual-report-2023"
+ ],
+ "hop_note": "Requires reading the same figure from two different annual reports."
+ }
+ ]
+ },
+ "followup": {
+ "description": "Two-turn exchanges where turn 2 is elliptical and meaningless alone. Measures the value of query condensing: turn-2 retrieval with the raw text vs. with a standalone rewrite.",
+ "items": [
+ {
+ "id": "fu-01",
+ "turn_1": {
+ "query": "What is the purpose of a policy enforcement point in a zero trust architecture?",
+ "gt_document_id": "nist-sp-800-207"
+ },
+ "turn_2": {
+ "query": "How does it differ from the policy engine?",
+ "gt_document_id": "nist-sp-800-207",
+ "depends_on": "'it' refers to the policy enforcement point"
+ },
+ "reference_rewrite": "How does a policy enforcement point differ from a policy engine in a zero trust architecture?"
+ },
+ {
+ "id": "fu-02",
+ "turn_1": {
+ "query": "What are the three cloud service models defined by NIST?",
+ "gt_document_id": "nist-sp-800-145"
+ },
+ "turn_2": {
+ "query": "Which of them gives the customer the least control?",
+ "gt_document_id": "nist-sp-800-145",
+ "depends_on": "'them' refers to the three cloud service models"
+ },
+ "reference_rewrite": "Which NIST cloud service model gives the customer the least control?"
+ },
+ {
+ "id": "fu-03",
+ "turn_1": {
+ "query": "What are the phases of the incident response life cycle?",
+ "gt_document_id": "nist-sp-800-61r2"
+ },
+ "turn_2": {
+ "query": "What happens in the third one?",
+ "gt_document_id": "nist-sp-800-61r2",
+ "depends_on": "'the third one' refers to the third incident response phase"
+ },
+ "reference_rewrite": "What happens during the containment, eradication and recovery phase of incident response?"
+ },
+ {
+ "id": "fu-04",
+ "turn_1": {
+ "query": "What is the difference between clearing and purging media?",
+ "gt_document_id": "nist-sp-800-88r1"
+ },
+ "turn_2": {
+ "query": "When is destruction required instead?",
+ "gt_document_id": "nist-sp-800-88r1",
+ "depends_on": "'instead' contrasts with clearing and purging"
+ },
+ "reference_rewrite": "When is physical destruction of media required instead of clearing or purging?"
+ },
+ {
+ "id": "fu-05",
+ "turn_1": {
+ "query": "What are the core functions of the Cybersecurity Framework?",
+ "gt_document_id": "nist-cswp-cyberframework-1-1"
+ },
+ "turn_2": {
+ "query": "Which one covers incident response?",
+ "gt_document_id": "nist-cswp-cyberframework-1-1",
+ "depends_on": "'one' refers to a core function"
+ },
+ "reference_rewrite": "Which Cybersecurity Framework core function covers incident response?"
+ },
+ {
+ "id": "fu-06",
+ "turn_1": {
+ "query": "What are the four functions of the AI Risk Management Framework?",
+ "gt_document_id": "nist-ai-100-1"
+ },
+ "turn_2": {
+ "query": "What does the first one involve?",
+ "gt_document_id": "nist-ai-100-1",
+ "depends_on": "'the first one' refers to the first AI RMF function"
+ },
+ "reference_rewrite": "What does the Govern function of the AI Risk Management Framework involve?"
+ },
+ {
+ "id": "fu-07",
+ "turn_1": {
+ "query": "What is a container image?",
+ "gt_document_id": "nist-sp-800-190"
+ },
+ "turn_2": {
+ "query": "What are the main risks associated with them?",
+ "gt_document_id": "nist-sp-800-190",
+ "depends_on": "'them' refers to container images"
+ },
+ "reference_rewrite": "What are the main security risks associated with container images?"
+ },
+ {
+ "id": "fu-08",
+ "turn_1": {
+ "query": "What are the steps of the Risk Management Framework?",
+ "gt_document_id": "nist-sp-800-37r2"
+ },
+ "turn_2": {
+ "query": "Which step comes immediately after categorisation?",
+ "gt_document_id": "nist-sp-800-37r2",
+ "depends_on": "depends on the RMF step sequence given in turn 1"
+ },
+ "reference_rewrite": "Which Risk Management Framework step comes immediately after the categorise step?"
+ },
+ {
+ "id": "fu-09",
+ "turn_1": {
+ "query": "What is controlled unclassified information?",
+ "gt_document_id": "nist-sp-800-171r2"
+ },
+ "turn_2": {
+ "query": "How many security requirements protect it?",
+ "gt_document_id": "nist-sp-800-171r2",
+ "depends_on": "'it' refers to controlled unclassified information"
+ },
+ "reference_rewrite": "How many security requirements does NIST specify for protecting controlled unclassified information?"
+ },
+ {
+ "id": "fu-10",
+ "turn_1": {
+ "query": "What is the definition of personally identifiable information?",
+ "gt_document_id": "nist-sp-800-122"
+ },
+ "turn_2": {
+ "query": "What are the confidentiality impact levels for it?",
+ "gt_document_id": "nist-sp-800-122",
+ "depends_on": "'it' refers to personally identifiable information"
+ },
+ "reference_rewrite": "What are the confidentiality impact levels for personally identifiable information?"
+ },
+ {
+ "id": "fu-11",
+ "turn_1": {
+ "query": "What are the practices of the Secure Software Development Framework?",
+ "gt_document_id": "nist-sp-800-218"
+ },
+ "turn_2": {
+ "query": "Which group of them deals with the organisation rather than the code?",
+ "gt_document_id": "nist-sp-800-218",
+ "depends_on": "'them' refers to the SSDF practice groups"
+ },
+ "reference_rewrite": "Which Secure Software Development Framework practice group deals with preparing the organisation rather than the code?"
+ },
+ {
+ "id": "fu-12",
+ "turn_1": {
+ "query": "What is the purpose of a business impact analysis?",
+ "gt_document_id": "nist-sp-800-34r1"
+ },
+ "turn_2": {
+ "query": "What outputs does it produce?",
+ "gt_document_id": "nist-sp-800-34r1",
+ "depends_on": "'it' refers to the business impact analysis"
+ },
+ "reference_rewrite": "What outputs does a business impact analysis produce?"
+ },
+ {
+ "id": "fu-13",
+ "turn_1": {
+ "query": "What are the authenticator assurance levels?",
+ "gt_document_id": "nist-sp-800-63-3"
+ },
+ "turn_2": {
+ "query": "What is required at the highest one?",
+ "gt_document_id": "nist-sp-800-63-3",
+ "depends_on": "'the highest one' refers to the highest AAL"
+ },
+ "reference_rewrite": "What is required at authenticator assurance level 3?"
+ },
+ {
+ "id": "fu-14",
+ "turn_1": {
+ "query": "What is log management?",
+ "gt_document_id": "nist-sp-800-92"
+ },
+ "turn_2": {
+ "query": "What are the main challenges in doing it?",
+ "gt_document_id": "nist-sp-800-92",
+ "depends_on": "'it' refers to log management"
+ },
+ "reference_rewrite": "What are the main challenges in log management?"
+ },
+ {
+ "id": "fu-15",
+ "turn_1": {
+ "query": "What does the Survey of Consumer Finances measure?",
+ "gt_document_id": "frb-scf-2023"
+ },
+ "turn_2": {
+ "query": "How did median family net worth change in the most recent period it covers?",
+ "gt_document_id": "frb-scf-2023",
+ "depends_on": "'it' refers to the Survey of Consumer Finances"
+ },
+ "reference_rewrite": "How did median family net worth change between 2019 and 2022 according to the Survey of Consumer Finances?"
+ }
+ ]
+ }
+ }
+}
diff --git a/backend/migrations/versions/d5e2c8a7f1b3_fail_documents_with_null_embeddings.py b/backend/migrations/versions/d5e2c8a7f1b3_fail_documents_with_null_embeddings.py
new file mode 100644
index 0000000..5a5fefe
--- /dev/null
+++ b/backend/migrations/versions/d5e2c8a7f1b3_fail_documents_with_null_embeddings.py
@@ -0,0 +1,81 @@
+"""mark documents unsearchable when their chunks lost embeddings
+
+Revision ID: d5e2c8a7f1b3
+Revises: a9f3d6c1b5e8
+Create Date: 2026-09-21 00:00:00.000000
+
+Migration a9f3d6c1b5e8 switched the embedding model from bge-base (768 dims) to
+bge-small (384 dims). Existing vectors were incompatible, so it ran
+
+ UPDATE document_chunks SET embedding = NULL
+
+but left `documents.status` alone. Every affected document therefore still reads
+`indexed` while none of its chunks can be matched by a vector search: the UI
+reports the file as ready, retrieval silently never returns it, and the user sees
+an assistant that cannot find something it was told it had ingested.
+
+a9f3d6c1b5e8 has already run in production, so it is not edited. This migration
+corrects the state it left behind, which is the only safe direction.
+
+Repairing the data is deliberately NOT part of this migration -- re-embedding
+requires loading the ONNX model and re-reading the source files, which is not
+work a migration should do while the application waits to boot. `status='failed'`
+is the honest description of these rows, and backend/scripts/reindex.py rebuilds
+them out of band.
+"""
+
+from alembic import op
+
+revision = "d5e2c8a7f1b3"
+down_revision = "a9f3d6c1b5e8"
+branch_labels = None
+depends_on = None
+
+
+# A document is unsearchable if it claims to be indexed but holds at least one
+# chunk with no embedding. Documents with no chunk rows at all are a different
+# defect and are intentionally left alone here.
+_AFFECTED = """
+ id IN (
+ SELECT DISTINCT document_id
+ FROM document_chunks
+ WHERE embedding IS NULL
+ )
+"""
+
+
+def upgrade() -> None:
+ op.execute(
+ f"""
+ UPDATE documents
+ SET status = 'failed'
+ WHERE status = 'indexed'
+ AND {_AFFECTED}
+ """
+ )
+
+
+def downgrade() -> None:
+ """
+ Put the affected documents back to 'indexed'.
+
+ This restores the state a9f3d6c1b5e8 left, which is the state this migration
+ found -- not a healthy state. It is an honest inverse rather than a good one:
+ the rows go back to claiming they are indexed while still holding NULL
+ embeddings.
+
+ The predicate is the same one `upgrade` selected on, so a document that failed
+ for an unrelated reason (a parse error, say) keeps its status, because its
+ chunks have embeddings or it has no chunks. The one case this cannot
+ distinguish is a document that was genuinely 'failed' with NULL-embedding
+ chunks before this migration ran; such a row is promoted to 'indexed' on
+ downgrade. Re-running `upgrade` returns it to 'failed'.
+ """
+ op.execute(
+ f"""
+ UPDATE documents
+ SET status = 'indexed'
+ WHERE status = 'failed'
+ AND {_AFFECTED}
+ """
+ )
diff --git a/backend/pytest.ini b/backend/pytest.ini
index 78c5011..4b7983b 100644
--- a/backend/pytest.ini
+++ b/backend/pytest.ini
@@ -1,3 +1,7 @@
[pytest]
asyncio_mode = auto
testpaths = tests
+markers =
+ integration: runs against a real PostgreSQL database with pgvector. Requires
+ TEST_DATABASE_URL; skipped automatically when it is unset. Select with
+ `-m integration`, exclude with `-m "not integration"`.
diff --git a/backend/requirements.txt b/backend/requirements.txt
index db1067f..c6762d1 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -47,3 +47,4 @@ google-generativeai==0.8.3
# --- Testing ---
pytest==9.1.1
pytest-asyncio==1.4.0
+pytest-cov==6.0.0
diff --git a/backend/scripts/reindex.py b/backend/scripts/reindex.py
new file mode 100644
index 0000000..c9d0737
--- /dev/null
+++ b/backend/scripts/reindex.py
@@ -0,0 +1,171 @@
+"""
+Re-run ingestion for documents whose chunks are no longer searchable.
+
+Migration a9f3d6c1b5e8 cleared every embedding when the model changed dimensions,
+and d5e2c8a7f1b3 marks the documents it stranded as `failed`. This script is the
+repair step: it rebuilds those documents' chunks and embeddings from the files
+still on disk.
+
+ # one document
+ python -m scripts.reindex --document-id 6da1be77-be1f-4d76-9b7d-1a4d19218b9c
+
+ # every failed document whose source file still exists
+ python -m scripts.reindex --all-failed
+
+ # see what would happen and touch nothing
+ python -m scripts.reindex --all-failed --dry-run
+
+Run it from the `backend/` directory with the same environment the API uses, so
+that DATABASE_URL and the embedding model directory resolve identically.
+
+Documents are processed one at a time on purpose. Embedding is CPU-bound and the
+Railway instance that runs this is the same one serving traffic; a parallel
+rebuild is how the earlier OOM crash happened.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+import sys
+from pathlib import Path
+
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.logging import configure_logging
+from app.database.models import Document, DocumentChunk, ProcessingStatus
+from app.database.session import AsyncSessionLocal
+from app.ingestion.pipeline import run_ingestion_pipeline
+
+logger = logging.getLogger("reindex")
+
+
+async def _select_documents(
+ db: AsyncSession, document_id: str | None, all_failed: bool
+) -> list[Document]:
+ if document_id:
+ doc = await db.scalar(select(Document).where(Document.id == document_id))
+ return [doc] if doc else []
+
+ if all_failed:
+ result = await db.execute(
+ select(Document)
+ .where(Document.status == ProcessingStatus.failed)
+ .order_by(Document.upload_time)
+ )
+ return list(result.scalars().all())
+
+ return []
+
+
+async def _reindex_one(db: AsyncSession, doc: Document, dry_run: bool) -> bool:
+ """Rebuild one document's chunks. Returns True when it ends up indexed."""
+ source = Path(doc.storage_path)
+ if not source.is_file():
+ # The row survives a lost file, but nothing can be rebuilt from it. Leaving
+ # the status at `failed` is accurate; the operator has to re-upload.
+ logger.warning(
+ "SKIP %s %r — file is gone (%s)", doc.id, doc.original_filename, doc.storage_path
+ )
+ return False
+
+ if dry_run:
+ logger.info(
+ "WOULD REINDEX %s %r status=%s chunks=%d",
+ doc.id, doc.original_filename, doc.status.value, doc.total_chunks,
+ )
+ return False
+
+ # store_chunks appends rather than replacing, so stale rows -- including the
+ # NULL-embedding ones this script exists to clear -- must go first or the
+ # document ends up with two generations of chunks and doubled retrieval hits.
+ deleted = await db.execute(
+ delete(DocumentChunk).where(DocumentChunk.document_id == doc.id)
+ )
+ doc.total_chunks = 0
+ await db.commit()
+ logger.info(
+ "REINDEX %s %r — cleared %d stale chunk(s)",
+ doc.id, doc.original_filename, deleted.rowcount or 0,
+ )
+
+ # The pipeline owns its own status transitions and swallows exceptions, so the
+ # outcome is read back from the row rather than caught here.
+ await run_ingestion_pipeline(
+ document_id=doc.id,
+ storage_path=doc.storage_path,
+ filename=doc.original_filename,
+ db=db,
+ )
+
+ await db.refresh(doc)
+ if doc.status == ProcessingStatus.indexed:
+ logger.info("OK %s %r — %d chunk(s)", doc.id, doc.original_filename, doc.total_chunks)
+ return True
+
+ logger.error(
+ "FAILED %s %r — status=%s after reindex; see the pipeline log above",
+ doc.id, doc.original_filename, doc.status.value,
+ )
+ return False
+
+
+async def reindex(document_id: str | None, all_failed: bool, dry_run: bool) -> int:
+ async with AsyncSessionLocal() as db:
+ documents = await _select_documents(db, document_id, all_failed)
+
+ if not documents:
+ if document_id:
+ logger.error("No document with id %s", document_id)
+ return 1
+ logger.info("Nothing to do — no documents with status='failed'.")
+ return 0
+
+ logger.info(
+ "%s %d document(s)%s",
+ "Would reindex" if dry_run else "Reindexing",
+ len(documents),
+ " (dry run)" if dry_run else "",
+ )
+
+ succeeded = 0
+ for doc in documents:
+ if await _reindex_one(db, doc, dry_run):
+ succeeded += 1
+
+ if dry_run:
+ return 0
+
+ failed = len(documents) - succeeded
+ logger.info("Done — %d succeeded, %d still failed.", succeeded, failed)
+ # A non-zero exit lets this be chained in a deploy script without the
+ # caller having to parse the log.
+ return 1 if failed else 0
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+ description="Rebuild chunks and embeddings for documents that are not searchable."
+ )
+ target = parser.add_mutually_exclusive_group(required=True)
+ target.add_argument("--document-id", help="Reindex exactly this document.")
+ target.add_argument(
+ "--all-failed",
+ action="store_true",
+ help="Reindex every document with status='failed' whose file still exists.",
+ )
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="List what would be reindexed without modifying anything.",
+ )
+ args = parser.parse_args()
+
+ configure_logging()
+ return asyncio.run(reindex(args.document_id, args.all_failed, args.dry_run))
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
new file mode 100644
index 0000000..56c086b
--- /dev/null
+++ b/backend/tests/conftest.py
@@ -0,0 +1,227 @@
+"""
+Shared pytest fixtures.
+
+The bulk of this suite runs against mocks and needs no services. The fixtures below
+add a second tier: tests marked `@pytest.mark.integration` run against a real
+PostgreSQL database with the pgvector extension, so that SQL, constraints, cascades
+and the ownership filters in the query layer are actually exercised.
+
+Integration tests are opt-in via TEST_DATABASE_URL. When it is unset every
+integration test is skipped, so `pytest -q` on a laptop with no database behaves
+exactly as it did before. CI sets it to a pgvector/pgvector:pg16 service container.
+
+ # local
+ docker run -d --name docuquery-test-db -p 5433:5432 \
+ -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=docuquery_test \
+ pgvector/pgvector:pg16
+ export TEST_DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5433/docuquery_test
+ pytest -q -m integration
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import AsyncGenerator
+
+import pytest
+import pytest_asyncio
+from httpx import ASGITransport, AsyncClient
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+
+BACKEND_DIR = Path(__file__).resolve().parent.parent
+
+# Tables emptied between integration tests. Ordering is irrelevant because the
+# TRUNCATE is a single statement with CASCADE, but document_chunks is listed first
+# to make the dependency direction obvious to a reader.
+_TRUNCATE_TABLES = ("document_chunks", "documents", "users")
+
+# Alembic's env.py reads the app Settings, which refuse to build without AUTH_SECRET
+# whenever DEBUG is false. The migration subprocess therefore needs one of its own;
+# it never signs a cookie that leaves the test process.
+_SUBPROCESS_AUTH_SECRET = "integration-test-secret-not-used-for-any-real-session"
+
+
+def integration_database_url() -> str | None:
+ """The database integration tests run against, or None when they are disabled."""
+ return os.environ.get("TEST_DATABASE_URL") or None
+
+
+def run_alembic(*args: str, database_url: str) -> subprocess.CompletedProcess[str]:
+ """
+ Invoke the Alembic CLI against `database_url` in a subprocess.
+
+ A subprocess rather than an in-process `alembic.command` call for two reasons:
+ migrations/env.py calls `asyncio.run()`, which raises when it is reached from
+ inside pytest-asyncio's already-running event loop; and it resolves the URL from
+ a `functools.lru_cache`d Settings object, so pointing it at the test database
+ in-process would leak that override into every later test in the session.
+ """
+ env = {
+ **os.environ,
+ "DATABASE_URL": database_url,
+ "AUTH_SECRET": os.environ.get("AUTH_SECRET") or _SUBPROCESS_AUTH_SECRET,
+ }
+ return subprocess.run(
+ [sys.executable, "-m", "alembic", *args],
+ cwd=BACKEND_DIR,
+ env=env,
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+
+
+@pytest.fixture(scope="session")
+def database_url() -> str:
+ """Skip the whole integration tier when no test database is configured."""
+ url = integration_database_url()
+ if not url:
+ pytest.skip("TEST_DATABASE_URL is not set — integration tests are disabled")
+ return url
+
+
+@pytest.fixture(scope="session")
+def migrated_database(database_url: str) -> str:
+ """
+ Bring the test database to `head` once per session.
+
+ Downgrading to base first makes the fixture idempotent against a database left
+ behind by a previous run, so a developer does not have to drop the container to
+ get a clean session.
+ """
+ run_alembic("downgrade", "base", database_url=database_url)
+ run_alembic("upgrade", "head", database_url=database_url)
+ return database_url
+
+
+@pytest_asyncio.fixture
+async def db_session(migrated_database: str) -> AsyncGenerator[AsyncSession, None]:
+ """A session against the migrated test database, with the tables emptied first."""
+ # NullPool: each test gets its own connections and disposes of them, rather than
+ # leaving pooled ones open against a container that the next test truncates.
+ engine = create_async_engine(migrated_database, poolclass=NullPool)
+ sessionmaker = async_sessionmaker(engine, expire_on_commit=False)
+ async with engine.begin() as conn:
+ await conn.execute(text(f"TRUNCATE {', '.join(_TRUNCATE_TABLES)} RESTART IDENTITY CASCADE"))
+ try:
+ async with sessionmaker() as session:
+ yield session
+ finally:
+ await engine.dispose()
+
+
+@pytest_asyncio.fixture
+async def client(db_session: AsyncSession) -> AsyncGenerator[AsyncClient, None]:
+ """
+ An HTTP client bound to the real test database.
+
+ `get_db` is overridden to hand every request the one session the test also holds,
+ so a row the test writes is visible to the request and vice versa.
+ """
+ from app.api.dependencies import get_db
+ from app.main import app
+
+ async def _override_get_db() -> AsyncGenerator[AsyncSession, None]:
+ yield db_session
+
+ app.dependency_overrides[get_db] = _override_get_db
+ try:
+ async with AsyncClient(
+ transport=ASGITransport(app=app), base_url="http://test"
+ ) as http_client:
+ yield http_client
+ finally:
+ app.dependency_overrides.pop(get_db, None)
+
+
+@dataclass
+class SeededUser:
+ """A user that exists in the test database, plus the cookie that authenticates it."""
+
+ id: str
+ name: str
+ email: str
+ password: str
+ cookies: dict[str, str]
+
+
+@pytest_asyncio.fixture
+async def make_user(db_session: AsyncSession):
+ """
+ Factory for users that exist in the database and can make authenticated requests.
+
+ The session cookie is minted directly with `create_session` rather than by
+ posting to /auth/login, so that a test about document ownership fails for
+ ownership reasons and not because the login route changed.
+ """
+ from app.core.config import get_settings
+ from app.core.security import create_session, hash_password
+ from app.database.models import User
+
+ default_password = "correct horse battery staple"
+
+ async def _make(email: str, name: str = "Test User", password: str = default_password) -> SeededUser:
+ user = User(name=name, email=email, password_hash=hash_password(password))
+ db_session.add(user)
+ await db_session.commit()
+ await db_session.refresh(user)
+ return SeededUser(
+ id=user.id,
+ name=user.name,
+ email=user.email,
+ password=password,
+ cookies={get_settings().auth_cookie_name: create_session(user.id)},
+ )
+
+ return _make
+
+
+@pytest_asyncio.fixture
+async def make_document(db_session: AsyncSession):
+ """
+ Factory for an indexed document with one embedded chunk, owned by `user_id`.
+
+ The embedding is a fixed unit vector rather than a real one. These tests assert
+ on which rows a query is allowed to reach, not on ranking quality, and a constant
+ vector keeps the result order deterministic.
+ """
+ from app.database.models import EMBEDDING_DIM, Document, DocumentChunk, ProcessingStatus
+
+ async def _make(
+ user_id: str,
+ filename: str = "owned.pdf",
+ text: str = "The quarterly revenue figure was 4.2 million dollars.",
+ ) -> Document:
+ doc = Document(
+ user_id=user_id,
+ filename=filename,
+ original_filename=filename,
+ file_type="pdf",
+ file_size=1024,
+ storage_path=f"uploads/{filename}",
+ status=ProcessingStatus.indexed,
+ total_chunks=1,
+ )
+ db_session.add(doc)
+ await db_session.flush()
+ db_session.add(
+ DocumentChunk(
+ document_id=doc.id,
+ chunk_index=0,
+ page_number=1,
+ text=text,
+ embedding=[0.1] * EMBEDDING_DIM,
+ metadata_={},
+ )
+ )
+ await db_session.commit()
+ await db_session.refresh(doc)
+ return doc
+
+ return _make
diff --git a/backend/tests/test_integration_auth.py b/backend/tests/test_integration_auth.py
new file mode 100644
index 0000000..5f450bb
--- /dev/null
+++ b/backend/tests/test_integration_auth.py
@@ -0,0 +1,137 @@
+"""
+The /auth/* routes, against a real database.
+
+These paths depend on things a mock cannot have: the UNIQUE constraint on
+users.email that turns a duplicate signup into a 409, and the round trip through
+a real row for /auth/me. test_security.py already covers the hashing and session
+primitives in isolation; this covers the routes that use them.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+pytestmark = [pytest.mark.integration, pytest.mark.asyncio]
+
+PASSWORD = "correct horse battery staple"
+
+
+async def test_signup_creates_an_account_and_sets_a_session_cookie(client):
+ response = await client.post(
+ "/auth/signup",
+ json={"name": "Ada Lovelace", "email": "ada@example.com", "password": PASSWORD},
+ )
+
+ assert response.status_code == 201
+ body = response.json()
+ assert body["email"] == "ada@example.com"
+ assert body["name"] == "Ada Lovelace"
+ assert "password" not in body and "password_hash" not in body
+
+ cookie = response.cookies.get("docuquery_session")
+ assert cookie, "signup must establish a session"
+
+
+async def test_signup_normalizes_the_email_and_rejects_a_duplicate(client):
+ first = await client.post(
+ "/auth/signup",
+ json={"name": "Ada", "email": " Ada@Example.com ", "password": PASSWORD},
+ )
+ assert first.status_code == 201
+ assert first.json()["email"] == "ada@example.com", "email should be trimmed and lowercased"
+
+ # Same address in a different case must collide, which only the real UNIQUE
+ # constraint can demonstrate.
+ duplicate = await client.post(
+ "/auth/signup",
+ json={"name": "Imposter", "email": "ADA@EXAMPLE.COM", "password": PASSWORD},
+ )
+ assert duplicate.status_code == 409
+
+
+async def test_login_succeeds_with_the_right_password(client, make_user):
+ user = await make_user("ada@example.com", name="Ada", password=PASSWORD)
+
+ response = await client.post(
+ "/auth/login", json={"email": user.email, "password": PASSWORD}
+ )
+
+ assert response.status_code == 200
+ assert response.json()["id"] == user.id
+ assert response.cookies.get("docuquery_session")
+
+
+async def test_login_with_the_wrong_password_is_401_and_sets_no_cookie(client, make_user):
+ user = await make_user("ada@example.com", password=PASSWORD)
+
+ response = await client.post(
+ "/auth/login", json={"email": user.email, "password": "not-the-password"}
+ )
+
+ assert response.status_code == 401
+ assert not response.cookies.get("docuquery_session")
+
+
+async def test_login_for_an_unknown_email_is_401_with_the_same_message(client):
+ """A different message here would let an attacker enumerate registered emails."""
+ unknown = await client.post(
+ "/auth/login", json={"email": "nobody@example.com", "password": PASSWORD}
+ )
+ assert unknown.status_code == 401
+
+ known = await client.post(
+ "/auth/login", json={"email": "nobody@example.com", "password": "another-wrong-one"}
+ )
+ assert known.json()["detail"] == unknown.json()["detail"]
+
+
+async def test_me_returns_the_authenticated_user(client, make_user):
+ user = await make_user("ada@example.com", name="Ada Lovelace")
+
+ response = await client.get("/auth/me", cookies=user.cookies)
+
+ assert response.status_code == 200
+ assert response.json()["id"] == user.id
+ assert response.json()["email"] == "ada@example.com"
+
+
+async def test_me_without_a_cookie_is_401(client):
+ response = await client.get("/auth/me")
+ assert response.status_code == 401
+
+
+async def test_me_with_a_tampered_cookie_is_401(client, make_user):
+ user = await make_user("ada@example.com")
+ tampered = {"docuquery_session": user.cookies["docuquery_session"] + "x"}
+
+ response = await client.get("/auth/me", cookies=tampered)
+
+ assert response.status_code == 401
+
+
+async def test_me_is_401_after_the_user_row_is_deleted(client, db_session, make_user):
+ """
+ A signature that still verifies is not enough; the user must still exist.
+ Only a real database can delete the row out from under a valid cookie.
+ """
+ from app.database.models import User
+
+ user = await make_user("ada@example.com")
+ assert (await client.get("/auth/me", cookies=user.cookies)).status_code == 200
+
+ await db_session.execute(User.__table__.delete().where(User.id == user.id))
+ await db_session.commit()
+
+ assert (await client.get("/auth/me", cookies=user.cookies)).status_code == 401
+
+
+async def test_logout_clears_the_session_cookie(client, make_user):
+ user = await make_user("ada@example.com")
+
+ response = await client.post("/auth/logout", cookies=user.cookies)
+
+ assert response.status_code == 204
+ # Starlette clears a cookie by re-setting it empty with an expiry in the past.
+ set_cookie = response.headers.get("set-cookie", "")
+ assert "docuquery_session=" in set_cookie
+ assert 'docuquery_session=""' in set_cookie or "Max-Age=0" in set_cookie or "expires=" in set_cookie.lower()
diff --git a/backend/tests/test_integration_migrations.py b/backend/tests/test_integration_migrations.py
new file mode 100644
index 0000000..55e80e7
--- /dev/null
+++ b/backend/tests/test_integration_migrations.py
@@ -0,0 +1,167 @@
+"""
+Migration d5e2c8a7f1b3, executed against a real database.
+
+A migration that is only ever read is not known to work. This drives the actual
+Alembic CLI over real rows: it seeds the exact state a9f3d6c1b5e8 left behind --
+a document marked `indexed` whose chunk has no embedding -- then runs
+upgrade -> downgrade -> upgrade and asserts the status at each step.
+
+The round trip matters as much as the upgrade. railway.json runs
+`alembic upgrade head` on every boot, so a migration that cannot be reversed
+leaves no way back except a database restore.
+"""
+
+from __future__ import annotations
+
+import pytest
+from sqlalchemy import select, text
+
+from app.database.models import Document, DocumentChunk, ProcessingStatus
+
+from .conftest import run_alembic
+
+pytestmark = [pytest.mark.integration, pytest.mark.asyncio]
+
+# The revision immediately before the one under test.
+PREVIOUS_REVISION = "a9f3d6c1b5e8"
+TARGET_REVISION = "d5e2c8a7f1b3"
+
+
+async def _seed_stranded_document(db_session, *, filename: str, embedding) -> str:
+ """A document claiming to be indexed, with one chunk whose embedding is `embedding`."""
+ doc = Document(
+ filename=filename,
+ original_filename=filename,
+ file_type="pdf",
+ file_size=2048,
+ storage_path=f"uploads/{filename}",
+ status=ProcessingStatus.indexed,
+ total_chunks=1,
+ )
+ db_session.add(doc)
+ await db_session.flush()
+ db_session.add(
+ DocumentChunk(
+ document_id=doc.id,
+ chunk_index=0,
+ page_number=1,
+ text="Revenue for the period was 4.2 million dollars.",
+ embedding=embedding,
+ metadata_={},
+ )
+ )
+ await db_session.commit()
+ return doc.id
+
+
+async def _status_of(db_session, doc_id: str) -> str:
+ # Read through raw SQL: the migration changes the row underneath the session,
+ # and the ORM identity map would otherwise hand back the stale object.
+ return await db_session.scalar(
+ text("SELECT status::text FROM documents WHERE id = :id"), {"id": doc_id}
+ )
+
+
+async def test_upgrade_downgrade_upgrade_round_trip(db_session, migrated_database):
+ from app.database.models import EMBEDDING_DIM
+
+ stranded = await _seed_stranded_document(db_session, filename="stranded.pdf", embedding=None)
+ healthy = await _seed_stranded_document(
+ db_session, filename="healthy.pdf", embedding=[0.1] * EMBEDDING_DIM
+ )
+
+ try:
+ # The database is already at head, so step back over the migration first to
+ # observe the state it is meant to correct.
+ run_alembic("downgrade", PREVIOUS_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, stranded) == "indexed"
+
+ run_alembic("upgrade", TARGET_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, stranded) == "failed", (
+ "a document whose chunks lost their embeddings must stop claiming to be indexed"
+ )
+ assert await _status_of(db_session, healthy) == "indexed", (
+ "a document with real embeddings must be left alone"
+ )
+
+ run_alembic("downgrade", PREVIOUS_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, stranded) == "indexed", (
+ "downgrade must restore the previous status"
+ )
+ assert await _status_of(db_session, healthy) == "indexed"
+
+ run_alembic("upgrade", TARGET_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, stranded) == "failed", (
+ "re-applying must be deterministic, not accumulate state"
+ )
+ finally:
+ # Whatever happened above, leave the database at head for the other tests.
+ run_alembic("upgrade", "head", database_url=migrated_database)
+
+
+async def test_upgrade_leaves_unrelated_failures_alone(db_session, migrated_database):
+ """
+ A document that failed to parse has no chunks at all. The migration selects on
+ NULL-embedding chunk rows, so it must not touch one, in either direction.
+ """
+ doc = Document(
+ filename="unparseable.pdf",
+ original_filename="unparseable.pdf",
+ file_type="pdf",
+ file_size=10,
+ storage_path="uploads/unparseable.pdf",
+ status=ProcessingStatus.failed,
+ total_chunks=0,
+ )
+ db_session.add(doc)
+ await db_session.commit()
+
+ try:
+ run_alembic("downgrade", PREVIOUS_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, doc.id) == "failed", (
+ "downgrade must not promote a genuinely failed document to indexed"
+ )
+
+ run_alembic("upgrade", TARGET_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, doc.id) == "failed"
+ finally:
+ run_alembic("upgrade", "head", database_url=migrated_database)
+
+
+async def test_every_migration_reverses_all_the_way_to_base(migrated_database):
+ """
+ `downgrade base` then `upgrade head` over the whole chain.
+
+ This is the check that catches a migration whose downgrade was never written or
+ never run -- the cheapest possible guard on a project where every deploy runs
+ `alembic upgrade head` automatically.
+ """
+ try:
+ run_alembic("downgrade", "base", database_url=migrated_database)
+ run_alembic("upgrade", "head", database_url=migrated_database)
+ finally:
+ run_alembic("upgrade", "head", database_url=migrated_database)
+
+
+async def test_chunk_rows_survive_the_round_trip(db_session, migrated_database):
+ """The migration only rewrites `status`; it must not drop or alter chunk data."""
+ from app.database.models import EMBEDDING_DIM
+
+ doc_id = await _seed_stranded_document(
+ db_session, filename="preserved.pdf", embedding=[0.2] * EMBEDDING_DIM
+ )
+
+ try:
+ run_alembic("downgrade", PREVIOUS_REVISION, database_url=migrated_database)
+ run_alembic("upgrade", TARGET_REVISION, database_url=migrated_database)
+
+ chunks = (
+ await db_session.execute(
+ select(DocumentChunk).where(DocumentChunk.document_id == doc_id)
+ )
+ ).scalars().all()
+ assert len(chunks) == 1
+ assert chunks[0].text.startswith("Revenue for the period")
+ assert chunks[0].embedding is not None
+ finally:
+ run_alembic("upgrade", "head", database_url=migrated_database)
diff --git a/backend/tests/test_integration_tenant_isolation.py b/backend/tests/test_integration_tenant_isolation.py
new file mode 100644
index 0000000..1637bc3
--- /dev/null
+++ b/backend/tests/test_integration_tenant_isolation.py
@@ -0,0 +1,178 @@
+"""
+Tenant isolation, against a real database.
+
+Every document route and /retrieve scopes its query with `Document.user_id ==
+current_user.id`. Until now nothing executed that SQL: the rest of the suite hands
+the routes an AsyncMock, which returns whatever the test told it to return, so a
+query that had dropped its ownership filter would pass exactly as happily as one
+that had not.
+
+These tests seed two real users, give user A a real document with a real chunk,
+and assert that user B cannot reach it through any route that takes a document id
+-- and that a cross-tenant read is a 404, never a 403. 403 would confirm the
+document exists, which is itself a disclosure: it turns an opaque id into an
+oracle for "is this a real document belonging to someone else".
+"""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import pytest
+
+from app.database.models import EMBEDDING_DIM
+
+pytestmark = [pytest.mark.integration, pytest.mark.asyncio]
+
+
+@pytest.fixture
+def stub_query_embedding():
+ """
+ Replace query embedding with a constant vector.
+
+ /retrieve would otherwise load the ONNX model, which takes seconds and tests
+ nothing here. The vector search, its ownership filter and the SQL around it all
+ still run for real -- only the encoder is stubbed.
+ """
+ with patch(
+ "app.retrieval.retrieval_pipeline.embed_query",
+ return_value=[0.1] * EMBEDDING_DIM,
+ ) as stub:
+ yield stub
+
+
+async def test_document_list_shows_only_the_callers_documents(client, make_user, make_document):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ await make_document(alice.id, filename="alice-report.pdf")
+ await make_document(bob.id, filename="bob-notes.pdf")
+
+ response = await client.get("/documents", cookies=bob.cookies)
+
+ assert response.status_code == 200
+ filenames = [doc["filename"] for doc in response.json()]
+ assert filenames == ["bob-notes.pdf"]
+
+
+async def test_reading_another_users_document_is_404(client, make_user, make_document):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ doc = await make_document(alice.id)
+
+ response = await client.get(f"/documents/{doc.id}", cookies=bob.cookies)
+
+ assert response.status_code == 404, "a cross-tenant read must not confirm the document exists"
+
+
+async def test_reading_another_users_chunks_is_404(client, make_user, make_document):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ doc = await make_document(alice.id)
+
+ response = await client.get(f"/documents/{doc.id}/chunks", cookies=bob.cookies)
+
+ assert response.status_code == 404
+
+
+async def test_reading_another_users_debug_diagnostics_is_404(client, make_user, make_document):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ doc = await make_document(alice.id)
+
+ response = await client.get(f"/documents/{doc.id}/debug", cookies=bob.cookies)
+
+ assert response.status_code == 404
+
+
+async def test_deleting_another_users_document_is_404_and_leaves_it_intact(
+ client, make_user, make_document
+):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ doc = await make_document(alice.id)
+
+ response = await client.delete(f"/documents/{doc.id}", cookies=bob.cookies)
+ assert response.status_code == 404
+
+ # The 404 must mean "nothing happened", not "deleted, then reported missing".
+ still_there = await client.get(f"/documents/{doc.id}", cookies=alice.cookies)
+ assert still_there.status_code == 200
+
+
+async def test_retrieval_never_returns_another_users_chunks(
+ client, make_user, make_document, stub_query_embedding
+):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ await make_document(
+ alice.id,
+ filename="alice-secret.pdf",
+ text="The acquisition price agreed in private was 900 million dollars.",
+ )
+
+ response = await client.post(
+ "/retrieve", json={"query": "acquisition price"}, cookies=bob.cookies
+ )
+
+ # Bob owns nothing, so retrieval finds nothing. Either a 404 from the
+ # no-results path or an empty 200 is acceptable; a chunk of Alice's is not.
+ assert response.status_code in (200, 404)
+ if response.status_code == 200:
+ body = response.json()
+ assert body["chunks"] == []
+ assert "900 million" not in body.get("context", "")
+
+
+async def test_retrieval_returns_the_callers_own_chunks(
+ client, make_user, make_document, stub_query_embedding
+):
+ """
+ The counterpart to the test above.
+
+ Without this, every isolation assertion would still pass if retrieval were
+ broken outright and returned nothing to anybody.
+ """
+ alice = await make_user("alice@example.com")
+ await make_document(
+ alice.id,
+ filename="alice-report.pdf",
+ text="The acquisition price agreed in private was 900 million dollars.",
+ )
+
+ response = await client.post(
+ "/retrieve", json={"query": "acquisition price"}, cookies=alice.cookies
+ )
+
+ assert response.status_code == 200
+ body = response.json()
+ assert body["chunks"], "the owner must still be able to retrieve their own chunk"
+ assert any("900 million" in chunk["text"] for chunk in body["chunks"])
+
+
+async def test_deleting_a_user_cascades_to_documents_and_chunks(
+ client, db_session, make_user, make_document
+):
+ """
+ The FK cascades are declared in the ORM and the migration. Nothing had ever
+ executed them, so this asserts the database really drops the dependent rows
+ rather than raising a foreign-key violation or orphaning chunks.
+ """
+ from sqlalchemy import func, select
+
+ from app.database.models import Document, DocumentChunk, User
+
+ alice = await make_user("alice@example.com")
+ await make_document(alice.id)
+
+ await db_session.execute(
+ User.__table__.delete().where(User.id == alice.id)
+ )
+ await db_session.commit()
+
+ remaining_docs = await db_session.scalar(
+ select(func.count()).select_from(Document).where(Document.user_id == alice.id)
+ )
+ remaining_chunks = await db_session.scalar(select(func.count()).select_from(DocumentChunk))
+
+ assert remaining_docs == 0
+ assert remaining_chunks == 0
diff --git a/backend/tests/test_security.py b/backend/tests/test_security.py
index bc61900..2c2d3e2 100644
--- a/backend/tests/test_security.py
+++ b/backend/tests/test_security.py
@@ -1,3 +1,6 @@
+import pytest
+
+from app.core.config import DEV_AUTH_SECRET, Settings
from app.core.security import create_session, hash_password, read_session, verify_password
@@ -12,3 +15,19 @@ def test_session_round_trip_and_tamper_rejection():
token = create_session("user-123")
assert read_session(token) == "user-123"
assert read_session(token + "tampered") is None
+
+
+@pytest.mark.parametrize("secret", [DEV_AUTH_SECRET, "", " "])
+def test_production_settings_reject_a_guessable_auth_secret(secret):
+ """A non-debug build must not start with a session key anyone could know."""
+ with pytest.raises(ValueError, match="AUTH_SECRET"):
+ Settings(debug=False, auth_secret=secret)
+
+
+def test_production_settings_accept_a_real_auth_secret():
+ assert Settings(debug=False, auth_secret="s3kr1t-from-the-deploy-env").auth_secret
+
+
+def test_debug_settings_still_work_without_an_auth_secret():
+ """Local development should need no setup, so debug builds keep the default."""
+ assert Settings(debug=True, auth_secret=DEV_AUTH_SECRET).auth_secret == DEV_AUTH_SECRET
diff --git a/frontend/components/landing/useDemoLoop.ts b/frontend/components/landing/useDemoLoop.ts
index 0e273c3..db1248b 100644
--- a/frontend/components/landing/useDemoLoop.ts
+++ b/frontend/components/landing/useDemoLoop.ts
@@ -21,14 +21,17 @@ export function useDemoLoop(steps: number, { intervalMs = 1500, holdMs = 3200, a
/** Reveals `text` one character at a time while `active`; shows it in full otherwise. */
export function useTypewriter(text: string, active: boolean, speedMs = 26) {
- const [count, setCount] = useState(text.length)
+ const [count, setCount] = useState(active ? 0 : text.length)
+ // Rewinding belongs in render, not in an effect: an effect would let the browser
+ // paint the previous line's length for a frame before snapping back to zero.
+ const [inputs, setInputs] = useState({ text, active })
+ if (inputs.text !== text || inputs.active !== active) {
+ setInputs({ text, active })
+ setCount(active ? 0 : text.length)
+ }
useEffect(() => {
- if (!active) {
- setCount(text.length)
- return
- }
- setCount(0)
+ if (!active) return
let typed = 0
const id = setInterval(() => {
typed += 1
, HNSW index)"
+ },
+ "metrics": {
+ "document": {
+ "queries": 2,
+ "recall_at_1": 1.0,
+ "precision_at_1": 1.0,
+ "ndcg_at_1": 1.0,
+ "recall_at_3": 1.0,
+ "precision_at_3": 1.0,
+ "ndcg_at_3": 1.0,
+ "recall_at_5": 1.0,
+ "precision_at_5": 0.9,
+ "ndcg_at_5": 1.0,
+ "recall_at_10": 1.0,
+ "precision_at_10": 0.8,
+ "ndcg_at_10": 0.987,
+ "hit_rate_at_5": 1.0,
+ "hit_rate_at_5_note": "identical to recall_at_5: ground truth names one relevant page per query",
+ "mrr": 1.0
+ },
+ "page": {
+ "queries": 2,
+ "recall_at_1": 1.0,
+ "precision_at_1": 1.0,
+ "ndcg_at_1": 1.0,
+ "recall_at_3": 1.0,
+ "precision_at_3": 1.0,
+ "ndcg_at_3": 1.0,
+ "recall_at_5": 1.0,
+ "precision_at_5": 0.9,
+ "ndcg_at_5": 1.0,
+ "recall_at_10": 1.0,
+ "precision_at_10": 0.8,
+ "ndcg_at_10": 0.987,
+ "hit_rate_at_5": 1.0,
+ "hit_rate_at_5_note": "identical to recall_at_5: ground truth names one relevant page per query",
+ "mrr": 1.0
+ },
+ "answer": {
+ "queries": 2,
+ "recall_at_1": 1.0,
+ "precision_at_1": 1.0,
+ "ndcg_at_1": 1.0,
+ "recall_at_3": 1.0,
+ "precision_at_3": 0.3333,
+ "ndcg_at_3": 1.0,
+ "recall_at_5": 1.0,
+ "precision_at_5": 0.2,
+ "ndcg_at_5": 1.0,
+ "recall_at_10": 1.0,
+ "precision_at_10": 0.1,
+ "ndcg_at_10": 1.0,
+ "hit_rate_at_5": 1.0,
+ "hit_rate_at_5_note": "identical to recall_at_5: ground truth names one relevant page per query",
+ "mrr": 1.0,
+ "scorable_queries": 2,
+ "note": "scored only on queries whose generated answer span is present verbatim in the ground-truth passage"
+ }
+ },
+ "queries_with_no_results": 0,
+ "retrieval_latency_ms": {
+ "p50": 22.07,
+ "p95": 488.567,
+ "p99": 488.567,
+ "min": 22.07,
+ "max": 488.567,
+ "mean": 255.318,
+ "n": 2
+ },
+ "per_query": [
+ {
+ "query_id": "q001",
+ "query": "What is the maximum file size supported for uploads in DocuQuery?",
+ "gt_filename": "README.md",
+ "gt_page": 1,
+ "answer_span_scorable": true,
+ "retrieved": [
+ {
+ "rank": 1,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 3,
+ "similarity": 0.8436,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": true
+ },
+ {
+ "rank": 2,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 0,
+ "similarity": 0.8396,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 3,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 16,
+ "similarity": 0.8379,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 4,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 5,
+ "similarity": 0.8316,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 5,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 2,
+ "similarity": 0.8298,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 6,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 12,
+ "similarity": 0.816,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 7,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 13,
+ "similarity": 0.8143,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 8,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 4,
+ "similarity": 0.8138,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 9,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 7,
+ "similarity": 0.8077,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 10,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 18,
+ "similarity": 0.8073,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ }
+ ],
+ "first_relevant_rank": {
+ "document": 1,
+ "page": 1,
+ "answer": 1
+ },
+ "latency_ms": 488.57,
+ "n_retrieved": 10
+ },
+ {
+ "query_id": "q002",
+ "query": "Which package is required for FastAPI to parse multipart/form-data uploads but is never imported directly by the source code?",
+ "gt_filename": "DEPENDENCIES_VERIFIED.md",
+ "gt_page": 1,
+ "answer_span_scorable": true,
+ "retrieved": [
+ {
+ "rank": 1,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 4,
+ "similarity": 0.8855,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": true
+ },
+ {
+ "rank": 2,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 1,
+ "similarity": 0.8843,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 3,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 6,
+ "similarity": 0.8786,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 4,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 3,
+ "similarity": 0.8635,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 5,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 10,
+ "similarity": 0.8532,
+ "relevant_document": false,
+ "relevant_page": false,
+ "relevant_answer": false
+ },
+ {
+ "rank": 6,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 2,
+ "similarity": 0.8456,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 7,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 12,
+ "similarity": 0.8401,
+ "relevant_document": false,
+ "relevant_page": false,
+ "relevant_answer": false
+ },
+ {
+ "rank": 8,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 16,
+ "similarity": 0.8399,
+ "relevant_document": false,
+ "relevant_page": false,
+ "relevant_answer": false
+ },
+ {
+ "rank": 9,
+ "filename": "DEPENDENCIES_VERIFIED.md",
+ "page": 1,
+ "chunk_index": 0,
+ "similarity": 0.8327,
+ "relevant_document": true,
+ "relevant_page": true,
+ "relevant_answer": false
+ },
+ {
+ "rank": 10,
+ "filename": "README.md",
+ "page": 1,
+ "chunk_index": 11,
+ "similarity": 0.8276,
+ "relevant_document": false,
+ "relevant_page": false,
+ "relevant_answer": false
+ }
+ ],
+ "first_relevant_rank": {
+ "document": 1,
+ "page": 1,
+ "answer": 1
+ },
+ "latency_ms": 22.07,
+ "n_retrieved": 10
+ }
+ ]
+}
\ No newline at end of file
diff --git a/backend/evaluation/run_config_sweep.py b/backend/evaluation/run_config_sweep.py
new file mode 100644
index 0000000..5473486
--- /dev/null
+++ b/backend/evaluation/run_config_sweep.py
@@ -0,0 +1,231 @@
+"""
+Compare retrieval configurations on one fixed query set.
+
+Two kinds of comparison are run:
+
+ index-side chunk size and overlap. Each configuration re-ingests the whole
+ corpus under its own synthetic user, so the index really is
+ rebuilt -- nothing is simulated by re-slicing an existing index.
+
+ query-side similarity threshold and metadata filtering. These reuse the
+ baseline index, because neither of them changes what is stored.
+
+The query set is generated once from the baseline index and reused verbatim by
+every configuration. Ground truth is anchored to (document, page), which does
+not move when chunk size changes, so the comparison is like-for-like.
+
+Usage:
+ python evaluation/run_config_sweep.py --corpus ../corpus --baseline-user-id eval-xxxx
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+from _common import RESULTS_DIR, config_id, load, save
+
+BACKEND_ROOT = Path(__file__).resolve().parent.parent
+
+# (label, chunk_size, chunk_overlap). The first entry must be the shipped
+# default so every improvement is stated relative to what the product does now.
+INDEX_CONFIGS = [
+ ("current-800-120", 800, 120),
+ ("small-400-60", 400, 60),
+ ("large-1200-180", 1200, 180),
+ ("no-overlap-800-0", 800, 0),
+]
+
+# (label, extra flags for run_retrieval_eval.py)
+QUERY_CONFIGS = [
+ ("threshold-0.30-default", []),
+ ("threshold-0.00", ["--no-threshold"]),
+ ("metadata-filter-document", ["--filter-document"]),
+]
+
+
+def run(cmd: list[str]) -> None:
+ print(f"\n$ {' '.join(cmd)}")
+ # Run from backend/ so pydantic-settings finds .env, same as the app does.
+ result = subprocess.run(cmd, cwd=BACKEND_ROOT)
+ if result.returncode != 0:
+ raise SystemExit(f"command failed with exit code {result.returncode}")
+
+
+def headline(metrics: dict) -> dict:
+ """The comparable figures, pulled out of a full result file."""
+ out = {}
+ for level in ("document", "page", "answer"):
+ m = metrics.get(level, {})
+ out[level] = {
+ key: m.get(key)
+ for key in (
+ "queries",
+ "recall_at_1",
+ "recall_at_3",
+ "recall_at_5",
+ "precision_at_5",
+ "ndcg_at_5",
+ "mrr",
+ )
+ }
+ return out
+
+
+def delta(baseline: float | None, candidate: float | None) -> dict:
+ if baseline is None or candidate is None:
+ return {"absolute_pp": None, "relative_pct": None}
+ absolute = round((candidate - baseline) * 100, 2)
+ relative = round((candidate - baseline) / baseline * 100, 2) if baseline else None
+ return {"absolute_pp": absolute, "relative_pct": relative}
+
+
+def main() -> None:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--corpus", required=True)
+ ap.add_argument("--eval-set", default="eval_set.json")
+ ap.add_argument("--top-k", type=int, default=10)
+ ap.add_argument(
+ "--skip-ingest",
+ action="store_true",
+ help="assume every index configuration is already ingested",
+ )
+ ap.add_argument("--out", default="config_sweep.json")
+ args = ap.parse_args()
+
+ eval_set = load(args.eval_set)
+ embedding_model = eval_set.get("generator_model")
+ print(f"[sweep] query set: {args.eval_set} ({len(eval_set['queries'])} queries)")
+
+ from app.core.config import get_settings
+
+ model_label = get_settings().embedding_model
+
+ runs: dict[str, dict] = {}
+
+ for label, chunk_size, overlap in INDEX_CONFIGS:
+ user_id = config_id(
+ chunk_size=chunk_size,
+ chunk_overlap=overlap,
+ embedding_model=model_label,
+ label="",
+ )
+ if not args.skip_ingest:
+ run(
+ [
+ sys.executable,
+ "evaluation/ingest_corpus.py",
+ "--corpus",
+ str(Path(args.corpus).resolve()),
+ "--chunk-size",
+ str(chunk_size),
+ "--chunk-overlap",
+ str(overlap),
+ "--reset",
+ ]
+ )
+ run(
+ [
+ sys.executable,
+ "evaluation/run_retrieval_eval.py",
+ "--user-id",
+ user_id,
+ "--eval-set",
+ args.eval_set,
+ "--top-k",
+ str(args.top_k),
+ "--label",
+ label,
+ ]
+ )
+ result = load(f"retrieval_eval_{label}.json")
+ ingest = load(f"ingest_{user_id}.json")
+ runs[label] = {
+ "kind": "index",
+ "chunk_size": chunk_size,
+ "chunk_overlap": overlap,
+ "user_id": user_id,
+ "chunks_indexed": ingest["total_chunks"],
+ "documents_indexed": ingest["documents_indexed"],
+ "mean_chunk_chars": round(
+ sum(
+ d.get("chunk_chars_mean", 0) * d.get("chunks", 0)
+ for d in ingest["documents"]
+ if d["status"] == "indexed"
+ )
+ / max(1, ingest["total_chunks"]),
+ 1,
+ ),
+ "retrieval_latency_ms": result["retrieval_latency_ms"],
+ "queries_with_no_results": result["queries_with_no_results"],
+ "metrics": headline(result["metrics"]),
+ }
+
+ baseline_label = INDEX_CONFIGS[0][0]
+ baseline_user_id = runs[baseline_label]["user_id"]
+
+ for label, flags in QUERY_CONFIGS:
+ run(
+ [
+ sys.executable,
+ "evaluation/run_retrieval_eval.py",
+ "--user-id",
+ baseline_user_id,
+ "--eval-set",
+ args.eval_set,
+ "--top-k",
+ str(args.top_k),
+ "--label",
+ label,
+ *flags,
+ ]
+ )
+ result = load(f"retrieval_eval_{label}.json")
+ runs[label] = {
+ "kind": "query",
+ "flags": flags,
+ "user_id": baseline_user_id,
+ "retrieval_latency_ms": result["retrieval_latency_ms"],
+ "queries_with_no_results": result["queries_with_no_results"],
+ "metrics": headline(result["metrics"]),
+ }
+
+ baseline_metrics = runs[baseline_label]["metrics"]
+ comparison = {}
+ for label, entry in runs.items():
+ if label == baseline_label:
+ continue
+ comparison[label] = {
+ level: {
+ metric: delta(
+ baseline_metrics[level][metric], entry["metrics"][level][metric]
+ )
+ for metric in ("recall_at_1", "recall_at_3", "recall_at_5", "mrr", "ndcg_at_5")
+ }
+ for level in ("document", "page", "answer")
+ }
+
+ save(
+ args.out,
+ {
+ "eval_set": args.eval_set,
+ "eval_set_size": len(eval_set["queries"]),
+ "top_k": args.top_k,
+ "baseline": baseline_label,
+ "runs": runs,
+ "vs_baseline": comparison,
+ },
+ )
+
+ print("\n=== page-level recall@5 by configuration ===")
+ for label, entry in runs.items():
+ value = entry["metrics"]["page"]["recall_at_5"]
+ marker = " (baseline)" if label == baseline_label else ""
+ print(f"{label:28s} {value}{marker}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/backend/evaluation/run_perf_bench.py b/backend/evaluation/run_perf_bench.py
new file mode 100644
index 0000000..708145f
--- /dev/null
+++ b/backend/evaluation/run_perf_bench.py
@@ -0,0 +1,296 @@
+"""
+Performance benchmark: embedding throughput, retrieval latency, LLM latency.
+
+Ingestion-stage timings (parse / clean / chunk / embed / store) are produced by
+ingest_corpus.py and are not repeated here; this script covers the query side
+plus a standalone embedding throughput measurement.
+
+Every latency figure is reported as a distribution (min / P50 / P95 / P99 / max)
+over a stated number of runs, after a discarded warm-up, so no number here is a
+single lucky run.
+
+The --llm stage sends retrieved passages from the corpus to the configured LLM
+provider over the network and consumes API quota. It is off by default.
+
+Usage:
+ python evaluation/run_perf_bench.py --user-id eval-xxxx
+ python evaluation/run_perf_bench.py --user-id eval-xxxx --llm --llm-queries 20
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import os
+import statistics
+import time
+
+from _common import eval_database_url, load, percentiles, save
+
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+from app.core.config import get_settings
+from app.ingestion.embeddings import get_embedding_service
+from app.llm import get_response_generator
+from app.retrieval.embedding_query import embed_query
+from app.retrieval.exceptions import NoResultsError
+from app.retrieval.retrieval_pipeline import run_retrieval_pipeline
+
+
+def proc_sample() -> dict:
+ try:
+ import psutil
+ except ImportError:
+ return {}
+ p = psutil.Process(os.getpid())
+ return {
+ "rss_mb": round(p.memory_info().rss / 1024**2, 1),
+ "cpu_percent": p.cpu_percent(interval=None),
+ }
+
+
+def bench_embedding(texts: list[str], batch_sizes: tuple[int, ...]) -> dict:
+ """
+ Embedding throughput at several batch sizes.
+
+ The same text list is embedded each time so the only variable is batching.
+ """
+ settings = get_settings()
+ service = get_embedding_service()
+
+ t = time.perf_counter()
+ service.warm_up()
+ cold_load_ms = round((time.perf_counter() - t) * 1000, 2)
+
+ service.embed(texts[:8]) # warm-up, result discarded
+
+ results = {}
+ original = settings.embedding_batch_size
+ for batch_size in batch_sizes:
+ settings.embedding_batch_size = batch_size
+ t = time.perf_counter()
+ vectors = service.embed(texts)
+ elapsed = time.perf_counter() - t
+ results[f"batch_{batch_size}"] = {
+ "texts": len(texts),
+ "seconds": round(elapsed, 3),
+ "embeddings_per_second": round(len(texts) / elapsed, 1),
+ "ms_per_embedding": round(elapsed * 1000 / len(texts), 3),
+ "dim": len(vectors[0]),
+ }
+ print(f"[embed] batch={batch_size}: {results[f'batch_{batch_size}']}")
+ settings.embedding_batch_size = original
+
+ return {
+ "cold_load_ms": cold_load_ms,
+ "model": settings.embedding_model,
+ "by_batch_size": results,
+ "process_after": proc_sample(),
+ }
+
+
+async def bench_query_embedding(queries: list[str], repeats: int) -> dict:
+ samples = []
+ for _ in range(repeats):
+ for query in queries:
+ t = time.perf_counter()
+ await embed_query(query)
+ samples.append((time.perf_counter() - t) * 1000)
+ return percentiles(samples)
+
+
+async def bench_retrieval(db, user_id: str, queries: list[str], top_k: int, repeats: int) -> dict:
+ samples = []
+ empty = 0
+ chunk_counts = []
+ context_chars = []
+ for _ in range(repeats):
+ for query in queries:
+ t = time.perf_counter()
+ try:
+ result = await run_retrieval_pipeline(
+ query=query, db=db, user_id=user_id, top_k=top_k
+ )
+ chunk_counts.append(len(result.chunks))
+ context_chars.append(len(result.context))
+ except NoResultsError:
+ empty += 1
+ samples.append((time.perf_counter() - t) * 1000)
+ return {
+ "latency_ms": percentiles(samples),
+ "queries_per_second_serial": round(1000 / statistics.median(samples), 2) if samples else None,
+ "empty_results": empty,
+ "chunks_returned_mean": round(statistics.mean(chunk_counts), 2) if chunk_counts else None,
+ "context_chars_mean": round(statistics.mean(context_chars), 1) if context_chars else None,
+ "context_chars_max": max(context_chars) if context_chars else None,
+ }
+
+
+async def bench_concurrent_retrieval(
+ engine, user_id: str, queries: list[str], top_k: int, concurrency: int
+) -> dict:
+ """Wall-clock throughput with N retrievals in flight, each on its own session."""
+ Session = async_sessionmaker(engine, expire_on_commit=False)
+
+ async def one(query: str) -> float:
+ async with Session() as db:
+ t = time.perf_counter()
+ try:
+ await run_retrieval_pipeline(query=query, db=db, user_id=user_id, top_k=top_k)
+ except NoResultsError:
+ pass
+ return (time.perf_counter() - t) * 1000
+
+ semaphore = asyncio.Semaphore(concurrency)
+
+ async def guarded(query: str) -> float:
+ async with semaphore:
+ return await one(query)
+
+ t0 = time.perf_counter()
+ samples = await asyncio.gather(*(guarded(q) for q in queries))
+ wall = time.perf_counter() - t0
+
+ return {
+ "concurrency": concurrency,
+ "queries": len(queries),
+ "wall_seconds": round(wall, 3),
+ "queries_per_second": round(len(queries) / wall, 2),
+ "latency_ms": percentiles(list(samples)),
+ }
+
+
+async def bench_llm(db, user_id: str, queries: list[str], top_k: int) -> dict:
+ """
+ Time to first streamed token and full generation latency, over real calls.
+
+ Generation goes through ResponseGenerator.stream -- the same call POST /chat
+ makes -- so context truncation, the versioned system prompt and SSE framing
+ are all in the measured path. Retrieval is timed separately, so
+ time_to_first_token_ms excludes it and end_to_end_ms is what a user waits
+ for from question to last token.
+ """
+ settings = get_settings()
+ generator = get_response_generator()
+
+ ttft, generation, end_to_end, retrieval, chars, tokens, errors = [], [], [], [], [], [], []
+
+ for i, query in enumerate(queries, 1):
+ t_start = time.perf_counter()
+ try:
+ result = await run_retrieval_pipeline(
+ query=query, db=db, user_id=user_id, top_k=top_k
+ )
+ except NoResultsError:
+ continue
+ t_retrieved = time.perf_counter()
+
+ first_token_at = None
+ token_events = 0
+ answer_chars = 0
+ try:
+ async for event in generator.stream(query, result, mode="docuquery"):
+ if not event.startswith("event: token"):
+ continue
+ if first_token_at is None:
+ first_token_at = time.perf_counter()
+ token_events += 1
+ answer_chars += len(event) - len("event: token\ndata: \n\n")
+ except Exception as exc:
+ errors.append(f"{type(exc).__name__}: {exc}")
+ print(f"[{i}] llm error: {type(exc).__name__}: {exc}")
+ continue
+
+ t_end = time.perf_counter()
+ if first_token_at is None:
+ errors.append("stream produced no tokens")
+ continue
+
+ retrieval.append((t_retrieved - t_start) * 1000)
+ ttft.append((first_token_at - t_retrieved) * 1000)
+ generation.append((t_end - t_retrieved) * 1000)
+ end_to_end.append((t_end - t_start) * 1000)
+ chars.append(float(answer_chars))
+ tokens.append(float(token_events))
+ print(
+ f"[{i}/{len(queries)}] ttft={ttft[-1]:.0f}ms "
+ f"total={end_to_end[-1]:.0f}ms tokens={token_events}"
+ )
+
+ return {
+ "provider": settings.llm_provider,
+ "model": settings.llm_model,
+ "temperature": settings.llm_temperature,
+ "max_tokens": settings.llm_max_tokens,
+ "max_context_tokens": settings.llm_max_context_tokens,
+ "completed": len(ttft),
+ "errors": errors,
+ "retrieval_ms": percentiles(retrieval),
+ "time_to_first_token_ms": percentiles(ttft),
+ "generation_ms": percentiles(generation),
+ "end_to_end_ms": percentiles(end_to_end),
+ "streamed_token_events": percentiles(tokens),
+ "answer_chars": percentiles(chars),
+ "note": (
+ "streamed_token_events counts provider stream deltas, not tokeniser "
+ "tokens; answer_chars is JSON-escaped length and is an upper bound"
+ ),
+ }
+
+
+async def main() -> None:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--user-id", required=True)
+ ap.add_argument("--eval-set", default="eval_set.json")
+ ap.add_argument("--top-k", type=int, default=5)
+ ap.add_argument("--repeats", type=int, default=3)
+ ap.add_argument("--embed-texts", type=int, default=256)
+ ap.add_argument("--concurrency", type=int, default=8)
+ ap.add_argument("--llm", action="store_true", help="run real LLM calls (uses quota)")
+ ap.add_argument("--llm-queries", type=int, default=20)
+ ap.add_argument("--out", default="perf_bench.json")
+ args = ap.parse_args()
+
+ eval_set = load(args.eval_set)
+ queries = [q["query"] for q in eval_set["queries"]]
+ passages = [q["gt_passage"] for q in eval_set["queries"]]
+ print(f"[bench] {len(queries)} queries available")
+
+ payload: dict = {"config": vars(args), "process_start": proc_sample()}
+
+ texts = (passages * ((args.embed_texts // max(1, len(passages))) + 1))[: args.embed_texts]
+ payload["embedding"] = bench_embedding(texts, batch_sizes=(1, 8, 32, 64))
+
+ engine = create_async_engine(eval_database_url(), pool_pre_ping=True)
+ Session = async_sessionmaker(engine, expire_on_commit=False)
+
+ async with Session() as db:
+ payload["query_embedding_ms"] = await bench_query_embedding(queries[:20], args.repeats)
+ print(f"[query-embed] {payload['query_embedding_ms']}")
+
+ payload["retrieval"] = await bench_retrieval(
+ db, args.user_id, queries, args.top_k, args.repeats
+ )
+ print(f"[retrieval] {payload['retrieval']['latency_ms']}")
+
+ payload["retrieval_concurrent"] = await bench_concurrent_retrieval(
+ engine, args.user_id, queries, args.top_k, args.concurrency
+ )
+ print(f"[concurrent] {payload['retrieval_concurrent']}")
+
+ if args.llm:
+ async with Session() as db:
+ payload["llm"] = await bench_llm(
+ db, args.user_id, queries[: args.llm_queries], args.top_k
+ )
+ else:
+ payload["llm"] = {"skipped": "run with --llm to measure generation latency"}
+
+ await engine.dispose()
+
+ payload["process_end"] = proc_sample()
+ save(args.out, payload)
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/backend/evaluation/run_retrieval_eval.py b/backend/evaluation/run_retrieval_eval.py
new file mode 100644
index 0000000..2120273
--- /dev/null
+++ b/backend/evaluation/run_retrieval_eval.py
@@ -0,0 +1,264 @@
+"""
+Score the retrieval pipeline against a fixed ground-truth query set.
+
+Every query goes through app.retrieval.run_retrieval_pipeline -- the same
+function POST /retrieve and POST /chat call -- so what is measured is the
+deployed path: query embedding, the per-user SQL filter, the pgvector HNSW
+search, the similarity threshold, deduplication and ranking.
+
+Three relevance levels are scored, from loosest to strictest:
+
+ document the retrieved chunk came from the document the query was written from
+ page ... and from the same page -> page-level citation accuracy
+ answer the retrieved chunk text contains the answer span the generator
+ quoted -> answer-bearing chunk accuracy
+
+Ground truth names exactly one relevant page per query, so Recall@k and
+Hit Rate@k are the same quantity here: the share of queries with at least one
+relevant chunk in the top k. Both names are reported with that caveat attached
+rather than presented as two independent results.
+
+Usage:
+ python evaluation/run_retrieval_eval.py --user-id eval-xxxx
+ python evaluation/run_retrieval_eval.py --user-id eval-xxxx --no-threshold
+ python evaluation/run_retrieval_eval.py --user-id eval-xxxx --filter-document
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import math
+import re
+import time
+
+from _common import eval_database_url, load, percentiles, save
+
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+from app.core.config import get_settings
+from app.retrieval.exceptions import NoResultsError
+from app.retrieval.filters import RetrievalFilter
+from app.retrieval.retrieval_pipeline import run_retrieval_pipeline
+
+K_VALUES = (1, 3, 5, 10)
+LEVELS = ("document", "page", "answer")
+
+
+def normalize(text: str) -> str:
+ return re.sub(r"\s+", " ", text).strip().lower()
+
+
+def relevance(chunk, query: dict, level: str) -> bool:
+ if chunk.document_id != query["gt_document_id"]:
+ return False
+ if level == "document":
+ return True
+ if chunk.page != query["gt_page"]:
+ return False
+ if level == "page":
+ return True
+ span = normalize(query.get("answer_span", ""))
+ return bool(span) and span in normalize(chunk.text)
+
+
+def ndcg_at_k(relevances: list[bool], k: int) -> float:
+ """Binary-gain NDCG. Ideal ranking puts every relevant hit first."""
+ top = relevances[:k]
+ dcg = sum(1.0 / math.log2(i + 2) for i, rel in enumerate(top) if rel)
+ ideal_hits = min(sum(top), k)
+ if ideal_hits == 0:
+ return 0.0
+ idcg = sum(1.0 / math.log2(i + 2) for i in range(ideal_hits))
+ return dcg / idcg
+
+
+def score_level(per_query: list[list[bool]]) -> dict:
+ """Aggregate one relevance level across all queries."""
+ n = len(per_query)
+ if n == 0:
+ return {}
+
+ out: dict = {"queries": n}
+ for k in K_VALUES:
+ hits = sum(1 for rels in per_query if any(rels[:k]))
+ out[f"recall_at_{k}"] = round(hits / n, 4)
+ out[f"precision_at_{k}"] = round(
+ sum(sum(rels[:k]) / k for rels in per_query) / n, 4
+ )
+ out[f"ndcg_at_{k}"] = round(sum(ndcg_at_k(rels, k) for rels in per_query) / n, 4)
+
+ out["hit_rate_at_5"] = out["recall_at_5"]
+ out["hit_rate_at_5_note"] = (
+ "identical to recall_at_5: ground truth names one relevant page per query"
+ )
+
+ reciprocal = []
+ for rels in per_query:
+ rank = next((i + 1 for i, rel in enumerate(rels) if rel), None)
+ reciprocal.append(1.0 / rank if rank else 0.0)
+ out["mrr"] = round(sum(reciprocal) / n, 4)
+
+ return out
+
+
+async def main() -> None:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--user-id", required=True)
+ ap.add_argument("--eval-set", default="eval_set.json")
+ ap.add_argument("--top-k", type=int, default=10)
+ ap.add_argument(
+ "--no-threshold",
+ action="store_true",
+ help="set the similarity threshold to 0 for this run",
+ )
+ ap.add_argument(
+ "--filter-document",
+ action="store_true",
+ help="pass the ground-truth document_id as a metadata filter",
+ )
+ ap.add_argument("--label", default="")
+ ap.add_argument("--out", default=None)
+ args = ap.parse_args()
+
+ settings = get_settings()
+ original_threshold = settings.retrieval_similarity_threshold
+ if args.no_threshold:
+ settings.retrieval_similarity_threshold = 0.0
+ # The pipeline caps results at min(top_k, retrieval_max_context_chunks);
+ # raise the cap so a top_k of 10 is actually returned.
+ settings.retrieval_max_context_chunks = max(
+ settings.retrieval_max_context_chunks, args.top_k
+ )
+
+ eval_set = load(args.eval_set)
+ queries = eval_set["queries"]
+ print(f"[eval-set] {len(queries)} queries from {args.eval_set}")
+
+ engine = create_async_engine(eval_database_url(), pool_pre_ping=True)
+ Session = async_sessionmaker(engine, expire_on_commit=False)
+
+ rows: list[dict] = []
+ latencies: list[float] = []
+ per_level: dict[str, list[list[bool]]] = {level: [] for level in LEVELS}
+ answer_span_scorable = 0
+ empty_results = 0
+
+ async with Session() as db:
+ for i, query in enumerate(queries, 1):
+ filters = (
+ RetrievalFilter(document_id=query["gt_document_id"])
+ if args.filter_document
+ else None
+ )
+
+ t0 = time.perf_counter()
+ try:
+ result = await run_retrieval_pipeline(
+ query=query["query"],
+ db=db,
+ user_id=args.user_id,
+ top_k=args.top_k,
+ filters=filters,
+ )
+ chunks = result.chunks
+ except NoResultsError:
+ chunks = []
+ empty_results += 1
+ latency_ms = (time.perf_counter() - t0) * 1000
+ latencies.append(latency_ms)
+
+ rels = {level: [relevance(c, query, level) for c in chunks] for level in LEVELS}
+
+ # A query can only be scored at answer-span level if the generator
+ # actually quoted a span that appears in the passage it was given.
+ span = normalize(query.get("answer_span", ""))
+ span_scorable = bool(span) and span in normalize(query["gt_passage"])
+ if span_scorable:
+ answer_span_scorable += 1
+
+ for level in LEVELS:
+ if level == "answer" and not span_scorable:
+ continue
+ per_level[level].append(rels[level])
+
+ first_rank = {
+ level: next((j + 1 for j, r in enumerate(rels[level]) if r), None)
+ for level in LEVELS
+ }
+
+ rows.append(
+ {
+ "query_id": query["query_id"],
+ "query": query["query"],
+ "gt_filename": query["gt_filename"],
+ "gt_page": query["gt_page"],
+ "answer_span_scorable": span_scorable,
+ "retrieved": [
+ {
+ "rank": j + 1,
+ "filename": c.filename,
+ "page": c.page,
+ "chunk_index": c.chunk_index,
+ "similarity": round(c.similarity, 4),
+ "relevant_document": rels["document"][j],
+ "relevant_page": rels["page"][j],
+ "relevant_answer": rels["answer"][j],
+ }
+ for j, c in enumerate(chunks)
+ ],
+ "first_relevant_rank": first_rank,
+ "latency_ms": round(latency_ms, 2),
+ "n_retrieved": len(chunks),
+ }
+ )
+
+ if i % 10 == 0:
+ print(f"[{i}/{len(queries)}] scored")
+
+ await engine.dispose()
+
+ metrics = {level: score_level(per_level[level]) for level in LEVELS}
+ metrics["answer"]["scorable_queries"] = answer_span_scorable
+ metrics["answer"]["note"] = (
+ "scored only on queries whose generated answer span is present verbatim "
+ "in the ground-truth passage"
+ )
+
+ payload = {
+ "run": {
+ "label": args.label,
+ "user_id": args.user_id,
+ "eval_set": args.eval_set,
+ "eval_set_size": len(queries),
+ "top_k": args.top_k,
+ "similarity_threshold": settings.retrieval_similarity_threshold,
+ "similarity_threshold_default": original_threshold,
+ "metadata_filter": "document_id" if args.filter_document else "none",
+ "embedding_model": settings.embedding_model,
+ "vector_distance": "cosine (pgvector <=>, HNSW index)",
+ },
+ "metrics": metrics,
+ "queries_with_no_results": empty_results,
+ "retrieval_latency_ms": percentiles(latencies),
+ "per_query": rows,
+ }
+
+ name = args.out or f"retrieval_eval_{args.label or args.user_id}.json"
+ save(name, payload)
+
+ print("\n=== retrieval metrics ===")
+ for level in LEVELS:
+ m = metrics[level]
+ print(
+ f"{level:9s} n={m['queries']:3d} "
+ f"R@1={m['recall_at_1']:.3f} R@3={m['recall_at_3']:.3f} "
+ f"R@5={m['recall_at_5']:.3f} MRR={m['mrr']:.3f} "
+ f"NDCG@5={m['ndcg_at_5']:.3f} P@5={m['precision_at_5']:.3f}"
+ )
+ print(f"\nlatency ms: {payload['retrieval_latency_ms']}")
+ print(f"queries returning nothing: {empty_results}")
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/backend/evaluation/sets/manual.json b/backend/evaluation/sets/manual.json
new file mode 100644
index 0000000..dccf495
--- /dev/null
+++ b/backend/evaluation/sets/manual.json
@@ -0,0 +1,440 @@
+{
+ "status": "DRAFT - not reviewed, not used to produce any reported number.",
+ "description": "Hand-written evaluation slices covering what the LLM-generated set cannot: questions with no answer in the corpus, questions needing two documents, and two-turn exchanges whose second turn is not a standalone query.",
+ "corpus": "corpus/manifest.json",
+ "ground_truth_convention": "Documents are referenced by their manifest `id`, never by database document_id, because the latter is regenerated on every ingest.",
+ "caveats": [
+ "Written from document titles and scope, not by reading every page. Each item needs a human to confirm the answer really is where it is claimed to be, or really is absent.",
+ "The unanswerable slice was the fragile one and this proved correct: 4 of 15 draft items asked about frameworks the NIST and Federal Reserve documents genuinely discuss. Those were replaced and the slice re-verified; see `verification`.",
+ "Multi-hop items assert that two documents are needed. A single document may in practice answer some of them, which would make those items easy rather than wrong."
+ ],
+ "verification": {
+ "status": "verified",
+ "method": "Full text of all 27 corpus PDFs extracted with PyMuPDF and searched for each item's distinguishing terms, case-insensitively and on word boundaries. Word boundaries matter: a substring search reported 903 hits for 'Rust', every one of them inside 'trust'.",
+ "result": "4 of the original 15 were contaminated and were replaced (see each item's why_absent): PCI DSS, HIPAA, Basel III and NBER are all genuinely discussed in the corpus. The remaining 11 verify absent.",
+ "not_verified": "The multi_hop and followup slices are NOT mechanically verified. Their ground-truth document assignments come from document scope and still need a human to confirm the answer is on the page claimed."
+ },
+ "slices": {
+ "unanswerable": {
+ "description": "The corpus does not contain the answer. Measures whether retrieval abstains or returns confident-looking but irrelevant chunks above the similarity threshold.",
+ "expected": "no relevant chunk; ideally nothing above threshold",
+ "items": [
+ {
+ "id": "una-01",
+ "query": "What is the maximum response time an organisation has under GDPR Article 12 to answer a data subject access request?",
+ "why_absent": "EU data-protection law; the corpus is U.S. federal guidance and contains no GDPR article text."
+ },
+ {
+ "id": "una-02",
+ "query": "Under the EU's Digital Operational Resilience Act, how often must a financial entity carry out threat-led penetration testing?",
+ "why_absent": "EU financial-sector regulation. Replaced the original PCI DSS question: 'PCI DSS' and 'Payment Card Industry' occur 18 times across SP 800-92, 800-146, 800-209 and 800-218 as a compliance driver, which made it a poor probe for absence."
+ },
+ {
+ "id": "una-03",
+ "query": "What is the minimum observation window for a SOC 2 Type II examination?",
+ "why_absent": "AICPA attestation standard; absent from this corpus."
+ },
+ {
+ "id": "una-04",
+ "query": "Which risk tier does the EU AI Act assign to remote biometric identification in publicly accessible spaces?",
+ "why_absent": "EU legislation; the AI RMF here is a voluntary U.S. framework and predates it."
+ },
+ {
+ "id": "una-05",
+ "query": "What are the five technical controls required for UK Cyber Essentials certification?",
+ "why_absent": "UK NCSC scheme. 'Cyber Essentials' does not occur in the corpus. Note that the acronym NCSC does appear three times, but it refers to the U.S. ODNI National Counterintelligence and Security Center, not the UK National Cyber Security Centre."
+ },
+ {
+ "id": "una-06",
+ "query": "What maturity levels are defined for the Australian Essential Eight mitigation strategies?",
+ "why_absent": "Australian Signals Directorate guidance; absent from this corpus."
+ },
+ {
+ "id": "una-07",
+ "query": "What is the maximum fine for a data-protection breach under Brazil's LGPD?",
+ "why_absent": "Brazilian data-protection law. Replaced the original HIPAA question: 'HIPAA' occurs 33 times across the corpus as a compliance context, notably in IR 8259 and SP 800-207."
+ },
+ {
+ "id": "una-08",
+ "query": "What reserve requirement ratio does the People's Bank of China apply to large commercial banks?",
+ "why_absent": "Chinese monetary policy. Replaced the original Basel III question: 'Basel III' and 'liquidity coverage ratio' both appear in the 2022 and 2023 Federal Reserve annual reports."
+ },
+ {
+ "id": "una-09",
+ "query": "Which price index does the Bank of Japan use to measure its 2 percent inflation target?",
+ "why_absent": "Japanese monetary policy. Replaced the original NBER question: 'National Bureau of Economic Research' appears in both Federal Reserve annual reports, defining the recession shading on their charts."
+ },
+ {
+ "id": "una-10",
+ "query": "What is the recommended cold-water steeping time for a first-flush Darjeeling?",
+ "why_absent": "Entirely outside the corpus domain; a control for obviously irrelevant queries."
+ },
+ {
+ "id": "una-11",
+ "query": "How does the Rust borrow checker's lifetime elision algorithm decide when a reference outlives its owner?",
+ "why_absent": "Programming-language internals; no document here discusses Rust."
+ },
+ {
+ "id": "una-12",
+ "query": "What is the maximum transmit power permitted for a Wi-Fi 6E access point in the 6 GHz band under FCC rules?",
+ "why_absent": "FCC radio regulation; outside the scope of every document here."
+ },
+ {
+ "id": "una-13",
+ "query": "How many players are on the field per team during a standard rugby union match?",
+ "why_absent": "Entirely outside the corpus domain; a second control query."
+ },
+ {
+ "id": "una-14",
+ "query": "What is the current statutory corporation tax rate for large companies in the Republic of Ireland?",
+ "why_absent": "Foreign tax law; absent from the U.S. federal documents in this corpus."
+ },
+ {
+ "id": "una-15",
+ "query": "Which antibiotic class is first-line treatment for uncomplicated community-acquired pneumonia in adults?",
+ "why_absent": "Clinical medicine; entirely outside the corpus domain."
+ }
+ ]
+ },
+ "multi_hop": {
+ "description": "Answering requires combining two documents. Measures whether top-k retrieval surfaces both, rather than only the more lexically similar one.",
+ "items": [
+ {
+ "id": "mh-01",
+ "query": "How should an organisation verify third-party components in a container image before deploying it?",
+ "gt_document_ids": [
+ "nist-sp-800-190",
+ "nist-sp-800-218"
+ ],
+ "hop_note": "Image-level guidance is in the container security guide; verifying acquired third-party components is an SSDF practice."
+ },
+ {
+ "id": "mh-02",
+ "query": "How does a zero trust policy engine's access decision relate to the authorisation step of the Risk Management Framework?",
+ "gt_document_ids": [
+ "nist-sp-800-207",
+ "nist-sp-800-37r2"
+ ],
+ "hop_note": "Policy engine is defined in the ZTA document; authorise is an RMF step."
+ },
+ {
+ "id": "mh-03",
+ "query": "After a ransomware event, what should an organisation do about media that cannot be verified clean?",
+ "gt_document_ids": [
+ "nist-sp-1800-25",
+ "nist-sp-800-88r1"
+ ],
+ "hop_note": "Recovery context from the data-integrity practice guide; sanitisation decisions from the media sanitization guidelines."
+ },
+ {
+ "id": "mh-04",
+ "query": "Which cloud service model leaves the customer most responsible for securing the operating system, and what does that imply for patching?",
+ "gt_document_ids": [
+ "nist-sp-800-145",
+ "nist-sp-800-146"
+ ],
+ "hop_note": "Service-model definitions in the cloud definition; security implications in the synopsis."
+ },
+ {
+ "id": "mh-05",
+ "query": "Is personally identifiable information also controlled unclassified information, and do the two require different handling?",
+ "gt_document_ids": [
+ "nist-sp-800-122",
+ "nist-sp-800-171r2"
+ ],
+ "hop_note": "PII definition and handling in one document, CUI requirements in the other."
+ },
+ {
+ "id": "mh-06",
+ "query": "Which log sources should be retained to support the detection and analysis phase of incident response?",
+ "gt_document_ids": [
+ "nist-sp-800-92",
+ "nist-sp-800-61r2"
+ ],
+ "hop_note": "Log source guidance in the log management guide; incident response phases in the handling guide."
+ },
+ {
+ "id": "mh-07",
+ "query": "How do the governance activities in the AI risk framework map onto the Identify function of the Cybersecurity Framework?",
+ "gt_document_ids": [
+ "nist-ai-100-1",
+ "nist-cswp-cyberframework-1-1"
+ ],
+ "hop_note": "Govern is an AI RMF function; Identify is a CSF core function."
+ },
+ {
+ "id": "mh-08",
+ "query": "What should a federal agency expect an IoT device manufacturer to have already done before the agency procures the device?",
+ "gt_document_ids": [
+ "nist-ir-8259",
+ "nist-sp-800-213"
+ ],
+ "hop_note": "Manufacturer-side activities in the IR; the federal acquirer's expectations in the SP."
+ },
+ {
+ "id": "mh-09",
+ "query": "How do secure software development practices feed into supply chain risk management for acquired software?",
+ "gt_document_ids": [
+ "nist-sp-800-218",
+ "nist-sp-800-161r1"
+ ],
+ "hop_note": "SSDF practices in one, C-SCRM controls in the other."
+ },
+ {
+ "id": "mh-10",
+ "query": "What inputs does a risk assessment provide to the categorise and select steps of the Risk Management Framework?",
+ "gt_document_ids": [
+ "nist-sp-800-30r1",
+ "nist-sp-800-37r2"
+ ],
+ "hop_note": "Assessment process in the risk assessment guide; the steps in the RMF document."
+ },
+ {
+ "id": "mh-11",
+ "query": "How should backup integrity be validated as part of contingency planning for a destructive-malware event?",
+ "gt_document_ids": [
+ "nist-sp-800-34r1",
+ "nist-sp-1800-25"
+ ],
+ "hop_note": "Contingency planning and backup strategy in one; integrity validation against ransomware in the other."
+ },
+ {
+ "id": "mh-12",
+ "query": "What should happen to storage media at end of life in a storage area network environment?",
+ "gt_document_ids": [
+ "nist-sp-800-209",
+ "nist-sp-800-88r1"
+ ],
+ "hop_note": "Storage infrastructure context in one; the sanitisation decision matrix in the other."
+ },
+ {
+ "id": "mh-13",
+ "query": "How does technical security testing produce evidence for a control assessment during authorisation?",
+ "gt_document_ids": [
+ "nist-sp-800-115",
+ "nist-sp-800-37r2"
+ ],
+ "hop_note": "Testing techniques in one; assessment and authorisation in the other."
+ },
+ {
+ "id": "mh-14",
+ "query": "Which authenticator assurance level is appropriate for resources accessed through a zero trust architecture?",
+ "gt_document_ids": [
+ "nist-sp-800-63-3",
+ "nist-sp-800-207"
+ ],
+ "hop_note": "AAL definitions in the identity guidelines; ZTA access requirements in the other."
+ },
+ {
+ "id": "mh-15",
+ "query": "How did the Federal Reserve's reported balance sheet position change between its 2022 and 2023 annual reports?",
+ "gt_document_ids": [
+ "frb-annual-report-2022",
+ "frb-annual-report-2023"
+ ],
+ "hop_note": "Requires reading the same figure from two different annual reports."
+ }
+ ]
+ },
+ "followup": {
+ "description": "Two-turn exchanges where turn 2 is elliptical and meaningless alone. Measures the value of query condensing: turn-2 retrieval with the raw text vs. with a standalone rewrite.",
+ "items": [
+ {
+ "id": "fu-01",
+ "turn_1": {
+ "query": "What is the purpose of a policy enforcement point in a zero trust architecture?",
+ "gt_document_id": "nist-sp-800-207"
+ },
+ "turn_2": {
+ "query": "How does it differ from the policy engine?",
+ "gt_document_id": "nist-sp-800-207",
+ "depends_on": "'it' refers to the policy enforcement point"
+ },
+ "reference_rewrite": "How does a policy enforcement point differ from a policy engine in a zero trust architecture?"
+ },
+ {
+ "id": "fu-02",
+ "turn_1": {
+ "query": "What are the three cloud service models defined by NIST?",
+ "gt_document_id": "nist-sp-800-145"
+ },
+ "turn_2": {
+ "query": "Which of them gives the customer the least control?",
+ "gt_document_id": "nist-sp-800-145",
+ "depends_on": "'them' refers to the three cloud service models"
+ },
+ "reference_rewrite": "Which NIST cloud service model gives the customer the least control?"
+ },
+ {
+ "id": "fu-03",
+ "turn_1": {
+ "query": "What are the phases of the incident response life cycle?",
+ "gt_document_id": "nist-sp-800-61r2"
+ },
+ "turn_2": {
+ "query": "What happens in the third one?",
+ "gt_document_id": "nist-sp-800-61r2",
+ "depends_on": "'the third one' refers to the third incident response phase"
+ },
+ "reference_rewrite": "What happens during the containment, eradication and recovery phase of incident response?"
+ },
+ {
+ "id": "fu-04",
+ "turn_1": {
+ "query": "What is the difference between clearing and purging media?",
+ "gt_document_id": "nist-sp-800-88r1"
+ },
+ "turn_2": {
+ "query": "When is destruction required instead?",
+ "gt_document_id": "nist-sp-800-88r1",
+ "depends_on": "'instead' contrasts with clearing and purging"
+ },
+ "reference_rewrite": "When is physical destruction of media required instead of clearing or purging?"
+ },
+ {
+ "id": "fu-05",
+ "turn_1": {
+ "query": "What are the core functions of the Cybersecurity Framework?",
+ "gt_document_id": "nist-cswp-cyberframework-1-1"
+ },
+ "turn_2": {
+ "query": "Which one covers incident response?",
+ "gt_document_id": "nist-cswp-cyberframework-1-1",
+ "depends_on": "'one' refers to a core function"
+ },
+ "reference_rewrite": "Which Cybersecurity Framework core function covers incident response?"
+ },
+ {
+ "id": "fu-06",
+ "turn_1": {
+ "query": "What are the four functions of the AI Risk Management Framework?",
+ "gt_document_id": "nist-ai-100-1"
+ },
+ "turn_2": {
+ "query": "What does the first one involve?",
+ "gt_document_id": "nist-ai-100-1",
+ "depends_on": "'the first one' refers to the first AI RMF function"
+ },
+ "reference_rewrite": "What does the Govern function of the AI Risk Management Framework involve?"
+ },
+ {
+ "id": "fu-07",
+ "turn_1": {
+ "query": "What is a container image?",
+ "gt_document_id": "nist-sp-800-190"
+ },
+ "turn_2": {
+ "query": "What are the main risks associated with them?",
+ "gt_document_id": "nist-sp-800-190",
+ "depends_on": "'them' refers to container images"
+ },
+ "reference_rewrite": "What are the main security risks associated with container images?"
+ },
+ {
+ "id": "fu-08",
+ "turn_1": {
+ "query": "What are the steps of the Risk Management Framework?",
+ "gt_document_id": "nist-sp-800-37r2"
+ },
+ "turn_2": {
+ "query": "Which step comes immediately after categorisation?",
+ "gt_document_id": "nist-sp-800-37r2",
+ "depends_on": "depends on the RMF step sequence given in turn 1"
+ },
+ "reference_rewrite": "Which Risk Management Framework step comes immediately after the categorise step?"
+ },
+ {
+ "id": "fu-09",
+ "turn_1": {
+ "query": "What is controlled unclassified information?",
+ "gt_document_id": "nist-sp-800-171r2"
+ },
+ "turn_2": {
+ "query": "How many security requirements protect it?",
+ "gt_document_id": "nist-sp-800-171r2",
+ "depends_on": "'it' refers to controlled unclassified information"
+ },
+ "reference_rewrite": "How many security requirements does NIST specify for protecting controlled unclassified information?"
+ },
+ {
+ "id": "fu-10",
+ "turn_1": {
+ "query": "What is the definition of personally identifiable information?",
+ "gt_document_id": "nist-sp-800-122"
+ },
+ "turn_2": {
+ "query": "What are the confidentiality impact levels for it?",
+ "gt_document_id": "nist-sp-800-122",
+ "depends_on": "'it' refers to personally identifiable information"
+ },
+ "reference_rewrite": "What are the confidentiality impact levels for personally identifiable information?"
+ },
+ {
+ "id": "fu-11",
+ "turn_1": {
+ "query": "What are the practices of the Secure Software Development Framework?",
+ "gt_document_id": "nist-sp-800-218"
+ },
+ "turn_2": {
+ "query": "Which group of them deals with the organisation rather than the code?",
+ "gt_document_id": "nist-sp-800-218",
+ "depends_on": "'them' refers to the SSDF practice groups"
+ },
+ "reference_rewrite": "Which Secure Software Development Framework practice group deals with preparing the organisation rather than the code?"
+ },
+ {
+ "id": "fu-12",
+ "turn_1": {
+ "query": "What is the purpose of a business impact analysis?",
+ "gt_document_id": "nist-sp-800-34r1"
+ },
+ "turn_2": {
+ "query": "What outputs does it produce?",
+ "gt_document_id": "nist-sp-800-34r1",
+ "depends_on": "'it' refers to the business impact analysis"
+ },
+ "reference_rewrite": "What outputs does a business impact analysis produce?"
+ },
+ {
+ "id": "fu-13",
+ "turn_1": {
+ "query": "What are the authenticator assurance levels?",
+ "gt_document_id": "nist-sp-800-63-3"
+ },
+ "turn_2": {
+ "query": "What is required at the highest one?",
+ "gt_document_id": "nist-sp-800-63-3",
+ "depends_on": "'the highest one' refers to the highest AAL"
+ },
+ "reference_rewrite": "What is required at authenticator assurance level 3?"
+ },
+ {
+ "id": "fu-14",
+ "turn_1": {
+ "query": "What is log management?",
+ "gt_document_id": "nist-sp-800-92"
+ },
+ "turn_2": {
+ "query": "What are the main challenges in doing it?",
+ "gt_document_id": "nist-sp-800-92",
+ "depends_on": "'it' refers to log management"
+ },
+ "reference_rewrite": "What are the main challenges in log management?"
+ },
+ {
+ "id": "fu-15",
+ "turn_1": {
+ "query": "What does the Survey of Consumer Finances measure?",
+ "gt_document_id": "frb-scf-2023"
+ },
+ "turn_2": {
+ "query": "How did median family net worth change in the most recent period it covers?",
+ "gt_document_id": "frb-scf-2023",
+ "depends_on": "'it' refers to the Survey of Consumer Finances"
+ },
+ "reference_rewrite": "How did median family net worth change between 2019 and 2022 according to the Survey of Consumer Finances?"
+ }
+ ]
+ }
+ }
+}
diff --git a/backend/migrations/versions/d5e2c8a7f1b3_fail_documents_with_null_embeddings.py b/backend/migrations/versions/d5e2c8a7f1b3_fail_documents_with_null_embeddings.py
new file mode 100644
index 0000000..5a5fefe
--- /dev/null
+++ b/backend/migrations/versions/d5e2c8a7f1b3_fail_documents_with_null_embeddings.py
@@ -0,0 +1,81 @@
+"""mark documents unsearchable when their chunks lost embeddings
+
+Revision ID: d5e2c8a7f1b3
+Revises: a9f3d6c1b5e8
+Create Date: 2026-09-21 00:00:00.000000
+
+Migration a9f3d6c1b5e8 switched the embedding model from bge-base (768 dims) to
+bge-small (384 dims). Existing vectors were incompatible, so it ran
+
+ UPDATE document_chunks SET embedding = NULL
+
+but left `documents.status` alone. Every affected document therefore still reads
+`indexed` while none of its chunks can be matched by a vector search: the UI
+reports the file as ready, retrieval silently never returns it, and the user sees
+an assistant that cannot find something it was told it had ingested.
+
+a9f3d6c1b5e8 has already run in production, so it is not edited. This migration
+corrects the state it left behind, which is the only safe direction.
+
+Repairing the data is deliberately NOT part of this migration -- re-embedding
+requires loading the ONNX model and re-reading the source files, which is not
+work a migration should do while the application waits to boot. `status='failed'`
+is the honest description of these rows, and backend/scripts/reindex.py rebuilds
+them out of band.
+"""
+
+from alembic import op
+
+revision = "d5e2c8a7f1b3"
+down_revision = "a9f3d6c1b5e8"
+branch_labels = None
+depends_on = None
+
+
+# A document is unsearchable if it claims to be indexed but holds at least one
+# chunk with no embedding. Documents with no chunk rows at all are a different
+# defect and are intentionally left alone here.
+_AFFECTED = """
+ id IN (
+ SELECT DISTINCT document_id
+ FROM document_chunks
+ WHERE embedding IS NULL
+ )
+"""
+
+
+def upgrade() -> None:
+ op.execute(
+ f"""
+ UPDATE documents
+ SET status = 'failed'
+ WHERE status = 'indexed'
+ AND {_AFFECTED}
+ """
+ )
+
+
+def downgrade() -> None:
+ """
+ Put the affected documents back to 'indexed'.
+
+ This restores the state a9f3d6c1b5e8 left, which is the state this migration
+ found -- not a healthy state. It is an honest inverse rather than a good one:
+ the rows go back to claiming they are indexed while still holding NULL
+ embeddings.
+
+ The predicate is the same one `upgrade` selected on, so a document that failed
+ for an unrelated reason (a parse error, say) keeps its status, because its
+ chunks have embeddings or it has no chunks. The one case this cannot
+ distinguish is a document that was genuinely 'failed' with NULL-embedding
+ chunks before this migration ran; such a row is promoted to 'indexed' on
+ downgrade. Re-running `upgrade` returns it to 'failed'.
+ """
+ op.execute(
+ f"""
+ UPDATE documents
+ SET status = 'indexed'
+ WHERE status = 'failed'
+ AND {_AFFECTED}
+ """
+ )
diff --git a/backend/pytest.ini b/backend/pytest.ini
index 78c5011..4b7983b 100644
--- a/backend/pytest.ini
+++ b/backend/pytest.ini
@@ -1,3 +1,7 @@
[pytest]
asyncio_mode = auto
testpaths = tests
+markers =
+ integration: runs against a real PostgreSQL database with pgvector. Requires
+ TEST_DATABASE_URL; skipped automatically when it is unset. Select with
+ `-m integration`, exclude with `-m "not integration"`.
diff --git a/backend/requirements.txt b/backend/requirements.txt
index db1067f..c6762d1 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -47,3 +47,4 @@ google-generativeai==0.8.3
# --- Testing ---
pytest==9.1.1
pytest-asyncio==1.4.0
+pytest-cov==6.0.0
diff --git a/backend/scripts/reindex.py b/backend/scripts/reindex.py
new file mode 100644
index 0000000..c9d0737
--- /dev/null
+++ b/backend/scripts/reindex.py
@@ -0,0 +1,171 @@
+"""
+Re-run ingestion for documents whose chunks are no longer searchable.
+
+Migration a9f3d6c1b5e8 cleared every embedding when the model changed dimensions,
+and d5e2c8a7f1b3 marks the documents it stranded as `failed`. This script is the
+repair step: it rebuilds those documents' chunks and embeddings from the files
+still on disk.
+
+ # one document
+ python -m scripts.reindex --document-id 6da1be77-be1f-4d76-9b7d-1a4d19218b9c
+
+ # every failed document whose source file still exists
+ python -m scripts.reindex --all-failed
+
+ # see what would happen and touch nothing
+ python -m scripts.reindex --all-failed --dry-run
+
+Run it from the `backend/` directory with the same environment the API uses, so
+that DATABASE_URL and the embedding model directory resolve identically.
+
+Documents are processed one at a time on purpose. Embedding is CPU-bound and the
+Railway instance that runs this is the same one serving traffic; a parallel
+rebuild is how the earlier OOM crash happened.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+import sys
+from pathlib import Path
+
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.logging import configure_logging
+from app.database.models import Document, DocumentChunk, ProcessingStatus
+from app.database.session import AsyncSessionLocal
+from app.ingestion.pipeline import run_ingestion_pipeline
+
+logger = logging.getLogger("reindex")
+
+
+async def _select_documents(
+ db: AsyncSession, document_id: str | None, all_failed: bool
+) -> list[Document]:
+ if document_id:
+ doc = await db.scalar(select(Document).where(Document.id == document_id))
+ return [doc] if doc else []
+
+ if all_failed:
+ result = await db.execute(
+ select(Document)
+ .where(Document.status == ProcessingStatus.failed)
+ .order_by(Document.upload_time)
+ )
+ return list(result.scalars().all())
+
+ return []
+
+
+async def _reindex_one(db: AsyncSession, doc: Document, dry_run: bool) -> bool:
+ """Rebuild one document's chunks. Returns True when it ends up indexed."""
+ source = Path(doc.storage_path)
+ if not source.is_file():
+ # The row survives a lost file, but nothing can be rebuilt from it. Leaving
+ # the status at `failed` is accurate; the operator has to re-upload.
+ logger.warning(
+ "SKIP %s %r — file is gone (%s)", doc.id, doc.original_filename, doc.storage_path
+ )
+ return False
+
+ if dry_run:
+ logger.info(
+ "WOULD REINDEX %s %r status=%s chunks=%d",
+ doc.id, doc.original_filename, doc.status.value, doc.total_chunks,
+ )
+ return False
+
+ # store_chunks appends rather than replacing, so stale rows -- including the
+ # NULL-embedding ones this script exists to clear -- must go first or the
+ # document ends up with two generations of chunks and doubled retrieval hits.
+ deleted = await db.execute(
+ delete(DocumentChunk).where(DocumentChunk.document_id == doc.id)
+ )
+ doc.total_chunks = 0
+ await db.commit()
+ logger.info(
+ "REINDEX %s %r — cleared %d stale chunk(s)",
+ doc.id, doc.original_filename, deleted.rowcount or 0,
+ )
+
+ # The pipeline owns its own status transitions and swallows exceptions, so the
+ # outcome is read back from the row rather than caught here.
+ await run_ingestion_pipeline(
+ document_id=doc.id,
+ storage_path=doc.storage_path,
+ filename=doc.original_filename,
+ db=db,
+ )
+
+ await db.refresh(doc)
+ if doc.status == ProcessingStatus.indexed:
+ logger.info("OK %s %r — %d chunk(s)", doc.id, doc.original_filename, doc.total_chunks)
+ return True
+
+ logger.error(
+ "FAILED %s %r — status=%s after reindex; see the pipeline log above",
+ doc.id, doc.original_filename, doc.status.value,
+ )
+ return False
+
+
+async def reindex(document_id: str | None, all_failed: bool, dry_run: bool) -> int:
+ async with AsyncSessionLocal() as db:
+ documents = await _select_documents(db, document_id, all_failed)
+
+ if not documents:
+ if document_id:
+ logger.error("No document with id %s", document_id)
+ return 1
+ logger.info("Nothing to do — no documents with status='failed'.")
+ return 0
+
+ logger.info(
+ "%s %d document(s)%s",
+ "Would reindex" if dry_run else "Reindexing",
+ len(documents),
+ " (dry run)" if dry_run else "",
+ )
+
+ succeeded = 0
+ for doc in documents:
+ if await _reindex_one(db, doc, dry_run):
+ succeeded += 1
+
+ if dry_run:
+ return 0
+
+ failed = len(documents) - succeeded
+ logger.info("Done — %d succeeded, %d still failed.", succeeded, failed)
+ # A non-zero exit lets this be chained in a deploy script without the
+ # caller having to parse the log.
+ return 1 if failed else 0
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+ description="Rebuild chunks and embeddings for documents that are not searchable."
+ )
+ target = parser.add_mutually_exclusive_group(required=True)
+ target.add_argument("--document-id", help="Reindex exactly this document.")
+ target.add_argument(
+ "--all-failed",
+ action="store_true",
+ help="Reindex every document with status='failed' whose file still exists.",
+ )
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="List what would be reindexed without modifying anything.",
+ )
+ args = parser.parse_args()
+
+ configure_logging()
+ return asyncio.run(reindex(args.document_id, args.all_failed, args.dry_run))
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
new file mode 100644
index 0000000..56c086b
--- /dev/null
+++ b/backend/tests/conftest.py
@@ -0,0 +1,227 @@
+"""
+Shared pytest fixtures.
+
+The bulk of this suite runs against mocks and needs no services. The fixtures below
+add a second tier: tests marked `@pytest.mark.integration` run against a real
+PostgreSQL database with the pgvector extension, so that SQL, constraints, cascades
+and the ownership filters in the query layer are actually exercised.
+
+Integration tests are opt-in via TEST_DATABASE_URL. When it is unset every
+integration test is skipped, so `pytest -q` on a laptop with no database behaves
+exactly as it did before. CI sets it to a pgvector/pgvector:pg16 service container.
+
+ # local
+ docker run -d --name docuquery-test-db -p 5433:5432 \
+ -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=docuquery_test \
+ pgvector/pgvector:pg16
+ export TEST_DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5433/docuquery_test
+ pytest -q -m integration
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import AsyncGenerator
+
+import pytest
+import pytest_asyncio
+from httpx import ASGITransport, AsyncClient
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+
+BACKEND_DIR = Path(__file__).resolve().parent.parent
+
+# Tables emptied between integration tests. Ordering is irrelevant because the
+# TRUNCATE is a single statement with CASCADE, but document_chunks is listed first
+# to make the dependency direction obvious to a reader.
+_TRUNCATE_TABLES = ("document_chunks", "documents", "users")
+
+# Alembic's env.py reads the app Settings, which refuse to build without AUTH_SECRET
+# whenever DEBUG is false. The migration subprocess therefore needs one of its own;
+# it never signs a cookie that leaves the test process.
+_SUBPROCESS_AUTH_SECRET = "integration-test-secret-not-used-for-any-real-session"
+
+
+def integration_database_url() -> str | None:
+ """The database integration tests run against, or None when they are disabled."""
+ return os.environ.get("TEST_DATABASE_URL") or None
+
+
+def run_alembic(*args: str, database_url: str) -> subprocess.CompletedProcess[str]:
+ """
+ Invoke the Alembic CLI against `database_url` in a subprocess.
+
+ A subprocess rather than an in-process `alembic.command` call for two reasons:
+ migrations/env.py calls `asyncio.run()`, which raises when it is reached from
+ inside pytest-asyncio's already-running event loop; and it resolves the URL from
+ a `functools.lru_cache`d Settings object, so pointing it at the test database
+ in-process would leak that override into every later test in the session.
+ """
+ env = {
+ **os.environ,
+ "DATABASE_URL": database_url,
+ "AUTH_SECRET": os.environ.get("AUTH_SECRET") or _SUBPROCESS_AUTH_SECRET,
+ }
+ return subprocess.run(
+ [sys.executable, "-m", "alembic", *args],
+ cwd=BACKEND_DIR,
+ env=env,
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+
+
+@pytest.fixture(scope="session")
+def database_url() -> str:
+ """Skip the whole integration tier when no test database is configured."""
+ url = integration_database_url()
+ if not url:
+ pytest.skip("TEST_DATABASE_URL is not set — integration tests are disabled")
+ return url
+
+
+@pytest.fixture(scope="session")
+def migrated_database(database_url: str) -> str:
+ """
+ Bring the test database to `head` once per session.
+
+ Downgrading to base first makes the fixture idempotent against a database left
+ behind by a previous run, so a developer does not have to drop the container to
+ get a clean session.
+ """
+ run_alembic("downgrade", "base", database_url=database_url)
+ run_alembic("upgrade", "head", database_url=database_url)
+ return database_url
+
+
+@pytest_asyncio.fixture
+async def db_session(migrated_database: str) -> AsyncGenerator[AsyncSession, None]:
+ """A session against the migrated test database, with the tables emptied first."""
+ # NullPool: each test gets its own connections and disposes of them, rather than
+ # leaving pooled ones open against a container that the next test truncates.
+ engine = create_async_engine(migrated_database, poolclass=NullPool)
+ sessionmaker = async_sessionmaker(engine, expire_on_commit=False)
+ async with engine.begin() as conn:
+ await conn.execute(text(f"TRUNCATE {', '.join(_TRUNCATE_TABLES)} RESTART IDENTITY CASCADE"))
+ try:
+ async with sessionmaker() as session:
+ yield session
+ finally:
+ await engine.dispose()
+
+
+@pytest_asyncio.fixture
+async def client(db_session: AsyncSession) -> AsyncGenerator[AsyncClient, None]:
+ """
+ An HTTP client bound to the real test database.
+
+ `get_db` is overridden to hand every request the one session the test also holds,
+ so a row the test writes is visible to the request and vice versa.
+ """
+ from app.api.dependencies import get_db
+ from app.main import app
+
+ async def _override_get_db() -> AsyncGenerator[AsyncSession, None]:
+ yield db_session
+
+ app.dependency_overrides[get_db] = _override_get_db
+ try:
+ async with AsyncClient(
+ transport=ASGITransport(app=app), base_url="http://test"
+ ) as http_client:
+ yield http_client
+ finally:
+ app.dependency_overrides.pop(get_db, None)
+
+
+@dataclass
+class SeededUser:
+ """A user that exists in the test database, plus the cookie that authenticates it."""
+
+ id: str
+ name: str
+ email: str
+ password: str
+ cookies: dict[str, str]
+
+
+@pytest_asyncio.fixture
+async def make_user(db_session: AsyncSession):
+ """
+ Factory for users that exist in the database and can make authenticated requests.
+
+ The session cookie is minted directly with `create_session` rather than by
+ posting to /auth/login, so that a test about document ownership fails for
+ ownership reasons and not because the login route changed.
+ """
+ from app.core.config import get_settings
+ from app.core.security import create_session, hash_password
+ from app.database.models import User
+
+ default_password = "correct horse battery staple"
+
+ async def _make(email: str, name: str = "Test User", password: str = default_password) -> SeededUser:
+ user = User(name=name, email=email, password_hash=hash_password(password))
+ db_session.add(user)
+ await db_session.commit()
+ await db_session.refresh(user)
+ return SeededUser(
+ id=user.id,
+ name=user.name,
+ email=user.email,
+ password=password,
+ cookies={get_settings().auth_cookie_name: create_session(user.id)},
+ )
+
+ return _make
+
+
+@pytest_asyncio.fixture
+async def make_document(db_session: AsyncSession):
+ """
+ Factory for an indexed document with one embedded chunk, owned by `user_id`.
+
+ The embedding is a fixed unit vector rather than a real one. These tests assert
+ on which rows a query is allowed to reach, not on ranking quality, and a constant
+ vector keeps the result order deterministic.
+ """
+ from app.database.models import EMBEDDING_DIM, Document, DocumentChunk, ProcessingStatus
+
+ async def _make(
+ user_id: str,
+ filename: str = "owned.pdf",
+ text: str = "The quarterly revenue figure was 4.2 million dollars.",
+ ) -> Document:
+ doc = Document(
+ user_id=user_id,
+ filename=filename,
+ original_filename=filename,
+ file_type="pdf",
+ file_size=1024,
+ storage_path=f"uploads/{filename}",
+ status=ProcessingStatus.indexed,
+ total_chunks=1,
+ )
+ db_session.add(doc)
+ await db_session.flush()
+ db_session.add(
+ DocumentChunk(
+ document_id=doc.id,
+ chunk_index=0,
+ page_number=1,
+ text=text,
+ embedding=[0.1] * EMBEDDING_DIM,
+ metadata_={},
+ )
+ )
+ await db_session.commit()
+ await db_session.refresh(doc)
+ return doc
+
+ return _make
diff --git a/backend/tests/test_integration_auth.py b/backend/tests/test_integration_auth.py
new file mode 100644
index 0000000..5f450bb
--- /dev/null
+++ b/backend/tests/test_integration_auth.py
@@ -0,0 +1,137 @@
+"""
+The /auth/* routes, against a real database.
+
+These paths depend on things a mock cannot have: the UNIQUE constraint on
+users.email that turns a duplicate signup into a 409, and the round trip through
+a real row for /auth/me. test_security.py already covers the hashing and session
+primitives in isolation; this covers the routes that use them.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+pytestmark = [pytest.mark.integration, pytest.mark.asyncio]
+
+PASSWORD = "correct horse battery staple"
+
+
+async def test_signup_creates_an_account_and_sets_a_session_cookie(client):
+ response = await client.post(
+ "/auth/signup",
+ json={"name": "Ada Lovelace", "email": "ada@example.com", "password": PASSWORD},
+ )
+
+ assert response.status_code == 201
+ body = response.json()
+ assert body["email"] == "ada@example.com"
+ assert body["name"] == "Ada Lovelace"
+ assert "password" not in body and "password_hash" not in body
+
+ cookie = response.cookies.get("docuquery_session")
+ assert cookie, "signup must establish a session"
+
+
+async def test_signup_normalizes_the_email_and_rejects_a_duplicate(client):
+ first = await client.post(
+ "/auth/signup",
+ json={"name": "Ada", "email": " Ada@Example.com ", "password": PASSWORD},
+ )
+ assert first.status_code == 201
+ assert first.json()["email"] == "ada@example.com", "email should be trimmed and lowercased"
+
+ # Same address in a different case must collide, which only the real UNIQUE
+ # constraint can demonstrate.
+ duplicate = await client.post(
+ "/auth/signup",
+ json={"name": "Imposter", "email": "ADA@EXAMPLE.COM", "password": PASSWORD},
+ )
+ assert duplicate.status_code == 409
+
+
+async def test_login_succeeds_with_the_right_password(client, make_user):
+ user = await make_user("ada@example.com", name="Ada", password=PASSWORD)
+
+ response = await client.post(
+ "/auth/login", json={"email": user.email, "password": PASSWORD}
+ )
+
+ assert response.status_code == 200
+ assert response.json()["id"] == user.id
+ assert response.cookies.get("docuquery_session")
+
+
+async def test_login_with_the_wrong_password_is_401_and_sets_no_cookie(client, make_user):
+ user = await make_user("ada@example.com", password=PASSWORD)
+
+ response = await client.post(
+ "/auth/login", json={"email": user.email, "password": "not-the-password"}
+ )
+
+ assert response.status_code == 401
+ assert not response.cookies.get("docuquery_session")
+
+
+async def test_login_for_an_unknown_email_is_401_with_the_same_message(client):
+ """A different message here would let an attacker enumerate registered emails."""
+ unknown = await client.post(
+ "/auth/login", json={"email": "nobody@example.com", "password": PASSWORD}
+ )
+ assert unknown.status_code == 401
+
+ known = await client.post(
+ "/auth/login", json={"email": "nobody@example.com", "password": "another-wrong-one"}
+ )
+ assert known.json()["detail"] == unknown.json()["detail"]
+
+
+async def test_me_returns_the_authenticated_user(client, make_user):
+ user = await make_user("ada@example.com", name="Ada Lovelace")
+
+ response = await client.get("/auth/me", cookies=user.cookies)
+
+ assert response.status_code == 200
+ assert response.json()["id"] == user.id
+ assert response.json()["email"] == "ada@example.com"
+
+
+async def test_me_without_a_cookie_is_401(client):
+ response = await client.get("/auth/me")
+ assert response.status_code == 401
+
+
+async def test_me_with_a_tampered_cookie_is_401(client, make_user):
+ user = await make_user("ada@example.com")
+ tampered = {"docuquery_session": user.cookies["docuquery_session"] + "x"}
+
+ response = await client.get("/auth/me", cookies=tampered)
+
+ assert response.status_code == 401
+
+
+async def test_me_is_401_after_the_user_row_is_deleted(client, db_session, make_user):
+ """
+ A signature that still verifies is not enough; the user must still exist.
+ Only a real database can delete the row out from under a valid cookie.
+ """
+ from app.database.models import User
+
+ user = await make_user("ada@example.com")
+ assert (await client.get("/auth/me", cookies=user.cookies)).status_code == 200
+
+ await db_session.execute(User.__table__.delete().where(User.id == user.id))
+ await db_session.commit()
+
+ assert (await client.get("/auth/me", cookies=user.cookies)).status_code == 401
+
+
+async def test_logout_clears_the_session_cookie(client, make_user):
+ user = await make_user("ada@example.com")
+
+ response = await client.post("/auth/logout", cookies=user.cookies)
+
+ assert response.status_code == 204
+ # Starlette clears a cookie by re-setting it empty with an expiry in the past.
+ set_cookie = response.headers.get("set-cookie", "")
+ assert "docuquery_session=" in set_cookie
+ assert 'docuquery_session=""' in set_cookie or "Max-Age=0" in set_cookie or "expires=" in set_cookie.lower()
diff --git a/backend/tests/test_integration_migrations.py b/backend/tests/test_integration_migrations.py
new file mode 100644
index 0000000..55e80e7
--- /dev/null
+++ b/backend/tests/test_integration_migrations.py
@@ -0,0 +1,167 @@
+"""
+Migration d5e2c8a7f1b3, executed against a real database.
+
+A migration that is only ever read is not known to work. This drives the actual
+Alembic CLI over real rows: it seeds the exact state a9f3d6c1b5e8 left behind --
+a document marked `indexed` whose chunk has no embedding -- then runs
+upgrade -> downgrade -> upgrade and asserts the status at each step.
+
+The round trip matters as much as the upgrade. railway.json runs
+`alembic upgrade head` on every boot, so a migration that cannot be reversed
+leaves no way back except a database restore.
+"""
+
+from __future__ import annotations
+
+import pytest
+from sqlalchemy import select, text
+
+from app.database.models import Document, DocumentChunk, ProcessingStatus
+
+from .conftest import run_alembic
+
+pytestmark = [pytest.mark.integration, pytest.mark.asyncio]
+
+# The revision immediately before the one under test.
+PREVIOUS_REVISION = "a9f3d6c1b5e8"
+TARGET_REVISION = "d5e2c8a7f1b3"
+
+
+async def _seed_stranded_document(db_session, *, filename: str, embedding) -> str:
+ """A document claiming to be indexed, with one chunk whose embedding is `embedding`."""
+ doc = Document(
+ filename=filename,
+ original_filename=filename,
+ file_type="pdf",
+ file_size=2048,
+ storage_path=f"uploads/{filename}",
+ status=ProcessingStatus.indexed,
+ total_chunks=1,
+ )
+ db_session.add(doc)
+ await db_session.flush()
+ db_session.add(
+ DocumentChunk(
+ document_id=doc.id,
+ chunk_index=0,
+ page_number=1,
+ text="Revenue for the period was 4.2 million dollars.",
+ embedding=embedding,
+ metadata_={},
+ )
+ )
+ await db_session.commit()
+ return doc.id
+
+
+async def _status_of(db_session, doc_id: str) -> str:
+ # Read through raw SQL: the migration changes the row underneath the session,
+ # and the ORM identity map would otherwise hand back the stale object.
+ return await db_session.scalar(
+ text("SELECT status::text FROM documents WHERE id = :id"), {"id": doc_id}
+ )
+
+
+async def test_upgrade_downgrade_upgrade_round_trip(db_session, migrated_database):
+ from app.database.models import EMBEDDING_DIM
+
+ stranded = await _seed_stranded_document(db_session, filename="stranded.pdf", embedding=None)
+ healthy = await _seed_stranded_document(
+ db_session, filename="healthy.pdf", embedding=[0.1] * EMBEDDING_DIM
+ )
+
+ try:
+ # The database is already at head, so step back over the migration first to
+ # observe the state it is meant to correct.
+ run_alembic("downgrade", PREVIOUS_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, stranded) == "indexed"
+
+ run_alembic("upgrade", TARGET_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, stranded) == "failed", (
+ "a document whose chunks lost their embeddings must stop claiming to be indexed"
+ )
+ assert await _status_of(db_session, healthy) == "indexed", (
+ "a document with real embeddings must be left alone"
+ )
+
+ run_alembic("downgrade", PREVIOUS_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, stranded) == "indexed", (
+ "downgrade must restore the previous status"
+ )
+ assert await _status_of(db_session, healthy) == "indexed"
+
+ run_alembic("upgrade", TARGET_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, stranded) == "failed", (
+ "re-applying must be deterministic, not accumulate state"
+ )
+ finally:
+ # Whatever happened above, leave the database at head for the other tests.
+ run_alembic("upgrade", "head", database_url=migrated_database)
+
+
+async def test_upgrade_leaves_unrelated_failures_alone(db_session, migrated_database):
+ """
+ A document that failed to parse has no chunks at all. The migration selects on
+ NULL-embedding chunk rows, so it must not touch one, in either direction.
+ """
+ doc = Document(
+ filename="unparseable.pdf",
+ original_filename="unparseable.pdf",
+ file_type="pdf",
+ file_size=10,
+ storage_path="uploads/unparseable.pdf",
+ status=ProcessingStatus.failed,
+ total_chunks=0,
+ )
+ db_session.add(doc)
+ await db_session.commit()
+
+ try:
+ run_alembic("downgrade", PREVIOUS_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, doc.id) == "failed", (
+ "downgrade must not promote a genuinely failed document to indexed"
+ )
+
+ run_alembic("upgrade", TARGET_REVISION, database_url=migrated_database)
+ assert await _status_of(db_session, doc.id) == "failed"
+ finally:
+ run_alembic("upgrade", "head", database_url=migrated_database)
+
+
+async def test_every_migration_reverses_all_the_way_to_base(migrated_database):
+ """
+ `downgrade base` then `upgrade head` over the whole chain.
+
+ This is the check that catches a migration whose downgrade was never written or
+ never run -- the cheapest possible guard on a project where every deploy runs
+ `alembic upgrade head` automatically.
+ """
+ try:
+ run_alembic("downgrade", "base", database_url=migrated_database)
+ run_alembic("upgrade", "head", database_url=migrated_database)
+ finally:
+ run_alembic("upgrade", "head", database_url=migrated_database)
+
+
+async def test_chunk_rows_survive_the_round_trip(db_session, migrated_database):
+ """The migration only rewrites `status`; it must not drop or alter chunk data."""
+ from app.database.models import EMBEDDING_DIM
+
+ doc_id = await _seed_stranded_document(
+ db_session, filename="preserved.pdf", embedding=[0.2] * EMBEDDING_DIM
+ )
+
+ try:
+ run_alembic("downgrade", PREVIOUS_REVISION, database_url=migrated_database)
+ run_alembic("upgrade", TARGET_REVISION, database_url=migrated_database)
+
+ chunks = (
+ await db_session.execute(
+ select(DocumentChunk).where(DocumentChunk.document_id == doc_id)
+ )
+ ).scalars().all()
+ assert len(chunks) == 1
+ assert chunks[0].text.startswith("Revenue for the period")
+ assert chunks[0].embedding is not None
+ finally:
+ run_alembic("upgrade", "head", database_url=migrated_database)
diff --git a/backend/tests/test_integration_tenant_isolation.py b/backend/tests/test_integration_tenant_isolation.py
new file mode 100644
index 0000000..1637bc3
--- /dev/null
+++ b/backend/tests/test_integration_tenant_isolation.py
@@ -0,0 +1,178 @@
+"""
+Tenant isolation, against a real database.
+
+Every document route and /retrieve scopes its query with `Document.user_id ==
+current_user.id`. Until now nothing executed that SQL: the rest of the suite hands
+the routes an AsyncMock, which returns whatever the test told it to return, so a
+query that had dropped its ownership filter would pass exactly as happily as one
+that had not.
+
+These tests seed two real users, give user A a real document with a real chunk,
+and assert that user B cannot reach it through any route that takes a document id
+-- and that a cross-tenant read is a 404, never a 403. 403 would confirm the
+document exists, which is itself a disclosure: it turns an opaque id into an
+oracle for "is this a real document belonging to someone else".
+"""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import pytest
+
+from app.database.models import EMBEDDING_DIM
+
+pytestmark = [pytest.mark.integration, pytest.mark.asyncio]
+
+
+@pytest.fixture
+def stub_query_embedding():
+ """
+ Replace query embedding with a constant vector.
+
+ /retrieve would otherwise load the ONNX model, which takes seconds and tests
+ nothing here. The vector search, its ownership filter and the SQL around it all
+ still run for real -- only the encoder is stubbed.
+ """
+ with patch(
+ "app.retrieval.retrieval_pipeline.embed_query",
+ return_value=[0.1] * EMBEDDING_DIM,
+ ) as stub:
+ yield stub
+
+
+async def test_document_list_shows_only_the_callers_documents(client, make_user, make_document):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ await make_document(alice.id, filename="alice-report.pdf")
+ await make_document(bob.id, filename="bob-notes.pdf")
+
+ response = await client.get("/documents", cookies=bob.cookies)
+
+ assert response.status_code == 200
+ filenames = [doc["filename"] for doc in response.json()]
+ assert filenames == ["bob-notes.pdf"]
+
+
+async def test_reading_another_users_document_is_404(client, make_user, make_document):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ doc = await make_document(alice.id)
+
+ response = await client.get(f"/documents/{doc.id}", cookies=bob.cookies)
+
+ assert response.status_code == 404, "a cross-tenant read must not confirm the document exists"
+
+
+async def test_reading_another_users_chunks_is_404(client, make_user, make_document):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ doc = await make_document(alice.id)
+
+ response = await client.get(f"/documents/{doc.id}/chunks", cookies=bob.cookies)
+
+ assert response.status_code == 404
+
+
+async def test_reading_another_users_debug_diagnostics_is_404(client, make_user, make_document):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ doc = await make_document(alice.id)
+
+ response = await client.get(f"/documents/{doc.id}/debug", cookies=bob.cookies)
+
+ assert response.status_code == 404
+
+
+async def test_deleting_another_users_document_is_404_and_leaves_it_intact(
+ client, make_user, make_document
+):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ doc = await make_document(alice.id)
+
+ response = await client.delete(f"/documents/{doc.id}", cookies=bob.cookies)
+ assert response.status_code == 404
+
+ # The 404 must mean "nothing happened", not "deleted, then reported missing".
+ still_there = await client.get(f"/documents/{doc.id}", cookies=alice.cookies)
+ assert still_there.status_code == 200
+
+
+async def test_retrieval_never_returns_another_users_chunks(
+ client, make_user, make_document, stub_query_embedding
+):
+ alice = await make_user("alice@example.com")
+ bob = await make_user("bob@example.com")
+ await make_document(
+ alice.id,
+ filename="alice-secret.pdf",
+ text="The acquisition price agreed in private was 900 million dollars.",
+ )
+
+ response = await client.post(
+ "/retrieve", json={"query": "acquisition price"}, cookies=bob.cookies
+ )
+
+ # Bob owns nothing, so retrieval finds nothing. Either a 404 from the
+ # no-results path or an empty 200 is acceptable; a chunk of Alice's is not.
+ assert response.status_code in (200, 404)
+ if response.status_code == 200:
+ body = response.json()
+ assert body["chunks"] == []
+ assert "900 million" not in body.get("context", "")
+
+
+async def test_retrieval_returns_the_callers_own_chunks(
+ client, make_user, make_document, stub_query_embedding
+):
+ """
+ The counterpart to the test above.
+
+ Without this, every isolation assertion would still pass if retrieval were
+ broken outright and returned nothing to anybody.
+ """
+ alice = await make_user("alice@example.com")
+ await make_document(
+ alice.id,
+ filename="alice-report.pdf",
+ text="The acquisition price agreed in private was 900 million dollars.",
+ )
+
+ response = await client.post(
+ "/retrieve", json={"query": "acquisition price"}, cookies=alice.cookies
+ )
+
+ assert response.status_code == 200
+ body = response.json()
+ assert body["chunks"], "the owner must still be able to retrieve their own chunk"
+ assert any("900 million" in chunk["text"] for chunk in body["chunks"])
+
+
+async def test_deleting_a_user_cascades_to_documents_and_chunks(
+ client, db_session, make_user, make_document
+):
+ """
+ The FK cascades are declared in the ORM and the migration. Nothing had ever
+ executed them, so this asserts the database really drops the dependent rows
+ rather than raising a foreign-key violation or orphaning chunks.
+ """
+ from sqlalchemy import func, select
+
+ from app.database.models import Document, DocumentChunk, User
+
+ alice = await make_user("alice@example.com")
+ await make_document(alice.id)
+
+ await db_session.execute(
+ User.__table__.delete().where(User.id == alice.id)
+ )
+ await db_session.commit()
+
+ remaining_docs = await db_session.scalar(
+ select(func.count()).select_from(Document).where(Document.user_id == alice.id)
+ )
+ remaining_chunks = await db_session.scalar(select(func.count()).select_from(DocumentChunk))
+
+ assert remaining_docs == 0
+ assert remaining_chunks == 0
diff --git a/backend/tests/test_security.py b/backend/tests/test_security.py
index bc61900..2c2d3e2 100644
--- a/backend/tests/test_security.py
+++ b/backend/tests/test_security.py
@@ -1,3 +1,6 @@
+import pytest
+
+from app.core.config import DEV_AUTH_SECRET, Settings
from app.core.security import create_session, hash_password, read_session, verify_password
@@ -12,3 +15,19 @@ def test_session_round_trip_and_tamper_rejection():
token = create_session("user-123")
assert read_session(token) == "user-123"
assert read_session(token + "tampered") is None
+
+
+@pytest.mark.parametrize("secret", [DEV_AUTH_SECRET, "", " "])
+def test_production_settings_reject_a_guessable_auth_secret(secret):
+ """A non-debug build must not start with a session key anyone could know."""
+ with pytest.raises(ValueError, match="AUTH_SECRET"):
+ Settings(debug=False, auth_secret=secret)
+
+
+def test_production_settings_accept_a_real_auth_secret():
+ assert Settings(debug=False, auth_secret="s3kr1t-from-the-deploy-env").auth_secret
+
+
+def test_debug_settings_still_work_without_an_auth_secret():
+ """Local development should need no setup, so debug builds keep the default."""
+ assert Settings(debug=True, auth_secret=DEV_AUTH_SECRET).auth_secret == DEV_AUTH_SECRET
diff --git a/frontend/components/landing/useDemoLoop.ts b/frontend/components/landing/useDemoLoop.ts
index 0e273c3..db1248b 100644
--- a/frontend/components/landing/useDemoLoop.ts
+++ b/frontend/components/landing/useDemoLoop.ts
@@ -21,14 +21,17 @@ export function useDemoLoop(steps: number, { intervalMs = 1500, holdMs = 3200, a
/** Reveals `text` one character at a time while `active`; shows it in full otherwise. */
export function useTypewriter(text: string, active: boolean, speedMs = 26) {
- const [count, setCount] = useState(text.length)
+ const [count, setCount] = useState(active ? 0 : text.length)
+ // Rewinding belongs in render, not in an effect: an effect would let the browser
+ // paint the previous line's length for a frame before snapping back to zero.
+ const [inputs, setInputs] = useState({ text, active })
+ if (inputs.text !== text || inputs.active !== active) {
+ setInputs({ text, active })
+ setCount(active ? 0 : text.length)
+ }
useEffect(() => {
- if (!active) {
- setCount(text.length)
- return
- }
- setCount(0)
+ if (!active) return
let typed = 0
const id = setInterval(() => {
typed += 1