chore(phase-0): integration tier, embedding-status fix, evaluation corpus

.github/workflows/ci.yml

@@ -0,0 +1,87 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+# A new push to a PR makes the in-flight run irrelevant.
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  backend:
+    name: Backend — pytest
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: backend
+    services:
+      # The unit tier mocks the database, but the integration tier runs real SQL —
+      # ownership filters, cascades and pgvector columns are only meaningful against
+      # a real server. pg16 matches the Railway instance.
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: docuquery_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      # AUTH_SECRET is required because `from app.main import app` builds Settings at
+      # import time, and a non-debug Settings refuses the dev default.
+      AUTH_SECRET: ci-only-secret-not-used-for-any-real-session
+      # Presence of this variable is what enables the integration tier; without it
+      # those tests skip. See backend/tests/conftest.py.
+      TEST_DATABASE_URL: postgresql+asyncpg://postgres:postgres@localhost:5432/docuquery_test
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          cache: pip
+          cache-dependency-path: backend/requirements.txt
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      # Unit and integration run as one invocation so that a single coverage figure
+      # covers both. Coverage is reported, not gated — there is no --cov-fail-under
+      # until the baseline in BASELINE.md establishes what the number should be.
+      - name: Run tests
+        run: pytest -q --cov=app --cov-report=term-missing --cov-report=xml
+
+  frontend:
+    name: Frontend — typecheck, lint, build
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Typecheck
+        run: npx tsc --noEmit
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Build
+        run: npm run build

CLAUDE.md

@@ -0,0 +1,105 @@
+# DocuQuery — working rules
+
+Retrieval-augmented document search. Upload PDFs, DOCX, TXT or Markdown; they are
+parsed, chunked, embedded and stored in PostgreSQL with pgvector; questions are
+answered from the retrieved passages with citations back to filename and page.
+
+## Layout
+
+```
+backend/
+  app/
+    api/            FastAPI routes. Thin — no business logic here.
+    core/           Settings, logging, security primitives, middleware.
+    database/       Declarative Base, engine, session, ORM models.
+    ingestion/      parse → clean → chunk → embed → store. pipeline.py orchestrates.
+    retrieval/      embed query → vector search → score → build context → citations.
+    llm/            Providers, prompts, response generator, SSE framing.
+    services/       Document persistence used by the routes.
+  migrations/       Alembic. versions/ is the revision chain.
+  evaluation/       Corpus manifest, eval-set builder, retrieval and perf harnesses.
+  scripts/          Operational one-offs (reindex.py).
+  tests/            pytest. conftest.py holds the integration fixtures.
+frontend/           Next.js 16 App Router, React 19, TanStack Query, Zustand.
+```
+
+The API layer stays thin on purpose: routes validate, call into a service or
+pipeline, and translate exceptions into status codes. Anything that would be worth
+unit-testing belongs a layer down.
+
+## Running things
+
+All backend commands run from `backend/`, and need `AUTH_SECRET` set (see below).
+
+```bash
+# Unit tests — mocked database, no services needed. This is the default.
+pytest -q
+
+# Integration tests — real PostgreSQL. Skipped entirely when TEST_DATABASE_URL is unset.
+docker run -d --name docuquery-test-db -p 5434:5432 \
+    -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \
+    -e POSTGRES_DB=docuquery_test pgvector/pgvector:pg16
+export TEST_DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5434/docuquery_test
+pytest -q -m integration        # just the integration tier
+pytest -q                       # both tiers
+pytest -q --cov=app --cov-report=term-missing
+
+# Evaluation — see backend/evaluation/README.md for the full sequence.
+# Note these are invoked as paths, not with `-m`: the scripts import `_common`
+# flatly, and _common.py is what puts backend/ on sys.path for `app.*`.
+python evaluation/download_corpus.py
+python evaluation/ingest_corpus.py --corpus evaluation/corpus/files --reset
+python evaluation/build_eval_set.py --n 100
+python evaluation/run_retrieval_eval.py
+python evaluation/run_perf_bench.py
+```
+
+Point the harness at a throwaway database with `EVAL_DATABASE_URL`; it defaults
+to the application's own `DATABASE_URL`, which is rarely what you want.
+
+Use port **5434** for the test database. The project's own dev stack
+(`docker compose`) already binds **5433**, and the integration fixtures TRUNCATE
+whatever they are pointed at.
+
+`AUTH_SECRET` is mandatory whenever `DEBUG=false`: `Settings` refuses to build with
+the committed development default, because that value alone signs every session
+cookie. For local work either export a throwaway value or set `DEBUG=true`.
+
+```bash
+export AUTH_SECRET=$(python -c "import secrets; print(secrets.token_urlsafe(48))")
+```
+
+## Standing rules
+
+**One PR per phase.** A phase is a numbered body of work with its own branch and its
+own PR. Do not fold the next phase's changes into the current PR because they are
+small or convenient; they get reviewed in the phase that owns them.
+
+**Every retrieval change includes a before/after table against `BASELINE.md`.**
+Any change to chunking, embedding, the similarity threshold, top-k, filtering,
+scoring or context building is a retrieval change. Re-run
+`run_retrieval_eval.py` on the same query set and put both columns in the PR
+description. "It seems better" is not a result, and a retrieval change without a
+table is not reviewable.
+
+**Never edit a migration that has already been applied.** Alembic will not re-run a
+stamped revision, so an edit leaves every deployed database describing a history it
+does not have. Correct a bad migration with a new one. This is not hypothetical
+here: `a9f3d6c1b5e8` cleared every embedding without resetting document status, and
+the fix was `d5e2c8a7f1b3`, not a patch to the original.
+
+Write a real `downgrade()`, and keep migrations additive where the change allows it.
+`backend/railway.json` runs `alembic upgrade head` on **every boot**, so merging a
+PR that contains a migration *is* running that migration against production. Review
+it on those terms. The same applies to any fail-fast configuration guard: it takes
+the API down on the next deploy if the value is not already set in Railway.
+
+**Read `frontend/AGENTS.md` before touching Next.js code.** This is Next.js 16 with
+React 19. Framework behaviour has changed in ways that predate most training data —
+`params` is a Promise, route-level state preservation and `unstable_instant`
+validation depend on `cacheComponents`, which this project does **not** currently
+enable. The guides bundled at `frontend/node_modules/next/dist/docs/` are the
+authority for this exact version; prefer them over recollection.
+
+**Do not change retrieval behaviour and measure it in the same breath.** A baseline
+is only a baseline if it describes code that is actually deployed, bugs included.

README.md

@@ -11,9 +11,13 @@
 Upload your documents. DocuQuery reads them, indexes them, and answers questions with the
 exact page and passage the answer came from — no more skimming a 40-page report for one number.
 
-[**Live demo**](#-see-it-work) · [**Quick start**](#-quick-start) · [**Features**](#-features) · [**API reference**](#-api-reference) · [**Roadmap**](#-roadmap)
+### [▶ Try it live at docuqueryvc.vercel.app](https://docuqueryvc.vercel.app)
 
-![Build Status](https://img.shields.io/badge/build-passing-brightgreen?style=flat-square)
+<sub>Frontend on Vercel · API at [`docuqueryprod.up.railway.app`](https://docuqueryprod.up.railway.app/health) on Railway · sign up with any email to upload your own documents</sub>
+
+[**See it work**](#-see-it-work) · [**Quick start**](#-quick-start) · [**Features**](#-features) · [**API reference**](#-api-reference) · [**Roadmap**](#-roadmap)
+
+[![CI](https://github.com/VampiricCyborg/DocuQuery/actions/workflows/ci.yml/badge.svg)](https://github.com/VampiricCyborg/DocuQuery/actions/workflows/ci.yml)
 ![Version](https://img.shields.io/badge/version-0.1.0-blue?style=flat-square)
 ![Next.js](https://img.shields.io/badge/Next.js-16.2-black?style=flat-square&logo=next.js)
 ![React](https://img.shields.io/badge/React-19.2-61dafb?style=flat-square&logo=react)
@@ -47,6 +51,10 @@ and hands the model only that — then shows you exactly where the answer came f
 <img src="docs/screenshots/demo.gif" alt="A question is typed, DocuQuery searches the uploaded documents, streams back an answer, then shows the cited source" width="760">
 <br/>
 <sub>One question, start to finish — typed, searched, answered, cited. No cuts.</sub>
+
+<br/>
+
+<sub>Rather do it yourself? <a href="https://docuqueryvc.vercel.app">Open the live app</a> and upload a document.</sub>
 </div>
 
 <br/>
@@ -235,6 +243,7 @@ Every request to `POST /chat` carries a `mode`:
 | `GET` | `/documents` | List all documents with processing status |
 | `GET` | `/documents/{id}` | Get a single document |
 | `GET` | `/documents/{id}/chunks` | Inspect a document's indexed chunks |
+| `GET` | `/documents/{id}/debug` | Indexing diagnostics — recorded vs. actual chunk count, and how many carry an embedding |
 | `DELETE` | `/documents/{id}` | Delete a document and all its chunks |
 | `POST` | `/retrieve` | Vector similarity search — chunks + context, no LLM call |
 | `POST` | `/chat` | Full RAG pipeline — streams an answer with citations |
@@ -319,7 +328,12 @@ cd frontend && npx tsc --noEmit && npm run lint && npm run build
 ```
 
 The backend suite covers parsing, chunking, retrieval scoring, citations, prompt construction,
-provider abstraction, streaming, password hashing, and session validation.
+provider abstraction, streaming, password hashing, and session validation. It needs no database
+and no API keys — the database and the embedding service are mocked.
+
+Every push to `main` and every pull request runs the same commands on GitHub Actions
+([`.github/workflows/ci.yml`](.github/workflows/ci.yml)) — that is what the CI badge at the top
+reports.
 
 ---
 

backend/.env.example

@@ -16,6 +16,14 @@ POSTGRES_USER=docuquery
 POSTGRES_PASSWORD=<strong-random-password>
 POSTGRES_DB=docuquery
 
+# Integration tests only. When set, tests marked `integration` run Alembic against
+# this database and exercise real SQL; when unset they skip. Must point at a server
+# with the pgvector extension available, and must NOT be a database you care about —
+# the fixtures TRUNCATE it between tests.
+#   docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=postgres \
+#       -e POSTGRES_DB=docuquery_test pgvector/pgvector:pg16
+TEST_DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5433/docuquery_test
+
 # File Storage
 UPLOAD_DIR=uploads
 MAX_FILE_SIZE_MB=50
@@ -27,6 +35,18 @@ CHUNK_OVERLAP=120
 EMBEDDING_MODEL=BAAI/bge-small-en-v1.5
 EMBEDDING_BATCH_SIZE=32
 
+# Authentication
+# REQUIRED whenever DEBUG=false - the app refuses to start without it.
+# This value alone signs every session cookie; anyone who knows it can forge a
+# session for any user. Generate one with:
+#   python -c "import secrets; print(secrets.token_urlsafe(48))"
+AUTH_SECRET=<generate-a-strong-random-value>
+AUTH_COOKIE_NAME=docuquery_session
+AUTH_SESSION_DAYS=30
+# Cookies are Secure + SameSite=None by default, for a cross-site Vercel -> Railway
+# setup. Set to false only for local http development.
+AUTH_COOKIE_SECURE=true
+
 # LLM
 LLM_PROVIDER=groq
 LLM_MODEL=qwen/qwen3.8-27b

backend/.gitignore

@@ -33,3 +33,9 @@ models/
 .vscode/
 .idea/
 *.iml
+
+# Evaluation corpus PDFs. The manifest (corpus/manifest.json) IS committed —
+# it carries the URL, SHA-256 and licence for each document, and
+# `python -m evaluation.download_corpus` reproduces this directory from it.
+# The files themselves are ~95MB of third-party publications.
+evaluation/corpus/files/