feat(reranker): default to a small multilingual cross-encoder, keep bge-reranker-v2-m3 selectable

.github/workflows/release-models.yml

@@ -68,6 +68,20 @@ jobs:
       contains(github.event.inputs.targets, 'reranker') ||
       github.event.inputs.targets == 'all'
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Default: small multilingual cross-encoder that ships its own ONNX file.
+          - slug: mmarco-mminilmv2-l12-h384-v1
+            model_id: cross-encoder/mmarco-mMiniLMv2-L12-H384-v1
+            revision: 1427fd652930e4ba29e8149678df786c240d8825
+            push_latest: true
+          # Higher quality, much slower on CPU; exported to ONNX at build time.
+          - slug: bge-reranker-v2-m3
+            model_id: BAAI/bge-reranker-v2-m3
+            revision: ""
+            push_latest: false
     steps:
       - uses: actions/checkout@v7
 
@@ -79,19 +93,22 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set model tag
-        run: echo "MODEL_TAG=bge-reranker-v2-m3-$(date +%Y.%m)" >> $GITHUB_ENV
+        run: echo "MODEL_TAG=${{ matrix.slug }}-$(date +%Y.%m)" >> $GITHUB_ENV
 
       - name: Build and push TEI reranker model init (ONNX)
         run: |
           docker build \
             -f deploy/Containerfile.tei-reranker-model-init \
-            --build-arg MODEL_ID=BAAI/bge-reranker-v2-m3 \
+            --build-arg MODEL_ID=${{ matrix.model_id }} \
+            --build-arg MODEL_REVISION=${{ matrix.revision }} \
             --build-arg MODEL_DATE=$(date +%Y.%m) \
             -t ${{ env.REGISTRY }}/tei-reranker-model-init:${{ env.MODEL_TAG }} \
-            -t ${{ env.REGISTRY }}/tei-reranker-model-init:latest \
             .
           docker push ${{ env.REGISTRY }}/tei-reranker-model-init:${{ env.MODEL_TAG }}
-          docker push ${{ env.REGISTRY }}/tei-reranker-model-init:latest
+          if [ "${{ matrix.push_latest }}" = "true" ]; then
+            docker tag ${{ env.REGISTRY }}/tei-reranker-model-init:${{ env.MODEL_TAG }} ${{ env.REGISTRY }}/tei-reranker-model-init:latest
+            docker push ${{ env.REGISTRY }}/tei-reranker-model-init:latest
+          fi
 
       - name: Install cosign
         uses: sigstore/cosign-installer@v3

AGENTS.md

@@ -11,7 +11,7 @@ This file provides context for AI coding agents working in this repository.
 | Backend | Runtime | Model | Path prefix | Env vars |
 |---|---|---|---|---|
 | Embedding | TEI (Rust) | BAAI/bge-m3 | `/v1/embeddings`, `/embed`, `/embed_sparse` | `EMBEDDING_ENABLED`, `EMBEDDING_URL` |
-| Reranker | FlagEmbedding (Python) | BAAI/bge-reranker-v2-m3 | `/v1/rerank`, `/rerank` | `RERANKER_ENABLED`, `RERANKER_URL` |
+| Reranker | TEI (Rust, ONNX) | cross-encoder/mmarco-mMiniLMv2-L12-H384-v1 (default), BAAI/bge-reranker-v2-m3 | `/v1/rerank`, `/rerank` | `RERANKER_ENABLED`, `RERANKER_URL` |
 | Whisper ASR | faster-whisper (Python) | large-v3-turbo | `/v1/audio/transcriptions`, `/v1/audio/translations` | `WHISPER_ENABLED`, `WHISPER_URL` |
 
 `/v1/models` returns a merged list from all enabled backends (with static fallback).

Makefile

@@ -97,12 +97,14 @@ docker/tei-model-init-embedding:
 
 ## ── Docker / Track E: Reranker (TEI/ONNX) ──────────────────────────────────
 
-# tei-reranker-model-init — ONNX-exported bge-reranker-v2-m3
-# Uses optimum-cli to export SafeTensors → ONNX (required by TEI ORT backend).
+# tei-reranker-model-init — reranker as ONNX for TEI's ORT backend.
+# Uses the model's own onnx/model.onnx if it ships one, else optimum-cli exports it.
+# Pick the model with RERANKER_MODEL_ID, e.g. RERANKER_MODEL_ID=BAAI/bge-reranker-v2-m3
+RERANKER_MODEL_ID ?= cross-encoder/mmarco-mMiniLMv2-L12-H384-v1
 docker/tei-reranker-model-init:
 	docker build \
 	  -f deploy/Containerfile.tei-reranker-model-init \
-	  --build-arg MODEL_ID=BAAI/bge-reranker-v2-m3 \
+	  --build-arg MODEL_ID=$(RERANKER_MODEL_ID) \
 	  --build-arg MODEL_DATE=$(MODEL_DATE) \
 	  -t $(REGISTRY)/tei-reranker-model-init:$(VERSION) \
 	  .

README.md

@@ -11,7 +11,7 @@ Unified CPU-based inference stack — serving embeddings, reranking and transcri
 | -------------------- | ---------------------------------------------------------------------- | ----------------------- | --------------------------------------- |
 | **inference-router** | Go                                                                     | —                       | Unified OpenAI-compatible reverse proxy |
 | **embedding**        | [TEI](https://github.com/huggingface/text-embeddings-inference) (Rust) | BAAI/bge-m3             | Dense + sparse embeddings               |
-| **reranker**         | [FlagEmbedding](https://github.com/FlagOpen/FlagEmbedding) (Python)    | BAAI/bge-reranker-v2-m3 | Cross-encoder reranking                 |
+| **reranker**         | [TEI](https://github.com/huggingface/text-embeddings-inference) (Rust, ONNX) | mmarco-mMiniLMv2-L12-H384-v1 (default), bge-reranker-v2-m3 (option) | Cross-encoder reranking |
 | **whisper**          | [faster-whisper](https://github.com/SYSTRAN/faster-whisper) (Python)   | large-v3-turbo          | Audio transcription (ASR)               |
 
 All backends are disabled by default and enabled selectively via Helm values or env vars.
@@ -108,7 +108,7 @@ All images are published to `ghcr.io/layer87-labs/`:
 | `tei-base`            | Hardened TEI base (patchelf, non-root) |
 | `tei-runtime`         | TEI runtime (model via volume)         |
 | `tei-model-init`      | BGE-M3 model baked, copies to volume   |
-| `reranker-model-init` | BGE-reranker-v2-m3 baked               |
+| `tei-reranker-model-init` | Reranker model as ONNX (default: mmarco-mMiniLMv2-L12-H384-v1; option: BGE-reranker-v2-m3) |
 | `reranker-server`     | FlagEmbedding HTTP server              |
 | `whisper`             | Whisper ASR server, model baked in     |
 
@@ -170,12 +170,103 @@ Prometheus metrics on `:9090/metrics`:
 | 8192                 | 6-10Gi     | May OOM on 8Gi limit |
 | 16384                | 10-16Gi    | GPU-only             |
 
-### Reranker (FlagEmbedding + BGE-reranker-v2-m3)
+### Reranker model choice
+
+The reranker model is baked into the `tei-reranker-model-init` image and
+selected by its tag (`reranker.modelInit.image.tag`). Both models run on the
+same TEI runtime with the ONNX Runtime backend.
+
+Measured with TEI `cpu-1.9.4`, limited to 4 CPUs, `--max-batch-tokens 2048`,
+one `/rerank` request with one query and 20 documents of 800 characters each
+(German/English mix), 4 runs per model. The CPU was an older AVX2 machine, so
+only the ratios carry over.
+
+| Model                                                  | Format                    | Parameters | Latency (20 docs) |
+| ------------------------------------------------------ | ------------------------- | ---------- | ----------------- |
+| `cross-encoder/mmarco-mMiniLMv2-L12-H384-v1` (default) | fp32 ONNX                 | 118M       | 0.76 to 0.85 s    |
+| `BAAI/bge-reranker-v2-m3`                              | fp32 ONNX                 | 568M       | 8.3 to 11.9 s     |
+| `BAAI/bge-reranker-v2-m3`                              | int8 ONNX                 | 568M       | 7.9 to 8.2 s      |
+| `Alibaba-NLP/gte-multilingual-reranker-base`           | safetensors (candle path) | n/a        | 11.1 to 12.1 s    |
+
+That is about 0.04 s per document pair for the default against roughly 0.4 to
+0.5 s for `bge-reranker-v2-m3`, so reranking 40 candidates with the larger
+model does not fit an interactive latency budget on 4 CPUs.
+
+Quality, on a downstream 75-question German/English relevance set (hybrid
+retrieval, rerank the top 20 candidates, 800 characters per document):
+
+| Setup                                      | hit@1 | hit@5 | hit@10 |
+| ------------------------------------------ | ----- | ----- | ------ |
+| No rerank                                  | 77%   | 95%   | 96%    |
+| `BAAI/bge-reranker-v2-m3`                  | 92%   | 96%   | 96%    |
+| `cross-encoder/mmarco-mMiniLMv2-L12-H384-v1` | 92%   | 96%   | 96%    |
+
+This set is small and saturated (one question is 1.3 points), its documents
+were short, and the embeddings used for candidate retrieval were a simple
+stand-in, so it cannot separate the two models. Real-world German technical
+text may favour the larger model; measure on your own data if quality matters
+more than latency.
+
+#### Sizing the request for the default model
+
+Latency is dominated by the number of passages and their length, and one
+request already uses all CPUs, so concurrent requests queue. Measured with the
+default model on a 4-vCPU pod (TEI `cpu-1.9.4`, `--max-batch-tokens 2048`),
+75 queries, passages of title plus text cut to the given length (800
+characters are about 190 tokens, 500 about 120), p95 latency of one `/rerank`
+call:
+
+| Passages | 800 chars | 600 chars | 500 chars | 400 chars |
+| -------- | --------- | --------- | --------- | --------- |
+| 40       | 3.2 s     | 2.5 s     | 2.1 s     | 1.8 s     |
+| 30       | 2.4 s     | 1.9 s     | 1.7 s     | 1.4 s     |
+| 20       | 1.7 s     | 1.3 s     | 1.1 s     | 1.0 s     |
+| 15       | 1.3 s     | 1.0 s     | 0.9 s     | 0.7 s     |
+
+With four such requests in flight at once, p95 is about 3 to 4 times higher
+(for example 3.0 s for 15 passages of 500 characters, 6.4 s for 40 of 400).
+If the budget is about 1.5 s per request, send at most 15 to 20 passages of
+about 500 characters, and scale out (`reranker.replicaCount`) or add CPU for
+concurrent callers. Reranking a short list also bounds what it can gain: it can
+only reorder the passages it receives.
+
+Memory: the working set stayed at about 1.4 GiB, including four concurrent
+requests of 40 passages, so the chart defaults to a 1.5 GiB request and a 2 GiB
+limit. Inputs longer than the model's 512-token limit are truncated by default
+(`truncate: true`, or the TEI default when the field is omitted); with
+`truncate: false` the request fails with HTTP 422.
+
+The model repository also ships quantised ONNX variants (for example
+`onnx/model_qint8_avx512_vnni.onnx`, `onnx/model_quint8_avx2.onnx`). They are
+not used by this chart and were not measured; they are CPU-specific and
+would need to be baked as `onnx/model.onnx` for TEI.
+
+Licence: `cross-encoder/mmarco-mMiniLMv2-L12-H384-v1` is Apache-2.0,
+`BAAI/bge-reranker-v2-m3` is Apache-2.0 as well (as tagged on the Hugging Face hub); check the model card of
+anything else you select.
+
+To use the higher-quality model (more CPU or a GPU), build or pull the matching
+init image and set:
 
-| `MAX_LENGTH` | RSS (fp16) | Recommendation  |
-| ------------ | ---------- | --------------- |
-| 512          | 2-4Gi      | **CPU default** |
-| 1024         | 4-6Gi      | Larger context  |
+```yaml
+reranker:
+  modelName: "BAAI/bge-reranker-v2-m3"
+  modelInit:
+    image:
+      tag: "bge-reranker-v2-m3-<YYYY.MM>"   # see the published tags
+```
+
+The init step copies with `--ignore-existing`, so when you change the model on
+an existing release, delete the reranker model PVC (or install with
+`reranker.persistence.enabled=false`) so the old model files are not reused.
+`BAAI/bge-reranker-v2-m3` needs `requests: 4Gi` / `limits: 8Gi` memory with
+`--max-batch-tokens 2048`; the default model needs far less.
+
+To build an init image yourself: `make docker/tei-reranker-model-init
+RERANKER_MODEL_ID=<huggingface id>`. If the model repository ships
+`onnx/model.onnx` it is used as-is; otherwise the model is exported with
+Optimum. Only XLM-RoBERTa-style sequence-classification models that TEI
+supports will work.
 
 ### Whisper (faster-whisper + large-v3-turbo, int8)
 

deploy/Containerfile.tei-reranker-model-init

@@ -2,22 +2,30 @@
 #
 # Containerfile.tei-reranker-model-init — TEI-compatible reranker model init container
 #
-# Downloads bge-reranker-v2-m3 and exports it to ONNX via Optimum.
-# TEI requires onnx/model.onnx for XLMRoberta-based rerankers (ORT backend).
+# Downloads a cross-encoder reranker and prepares onnx/model.onnx for TEI's ORT
+# backend. If the model repository already ships onnx/model.onnx, that file is
+# used as-is; otherwise the model is exported to ONNX via Optimum.
 #
-# Build:
+# Build (default, small multilingual model that ships its own ONNX file):
+#   docker build -f deploy/Containerfile.tei-reranker-model-init \
+#     --build-arg MODEL_ID=cross-encoder/mmarco-mMiniLMv2-L12-H384-v1 \
+#     --build-arg MODEL_REVISION=1427fd652930e4ba29e8149678df786c240d8825 \
+#     -t ghcr.io/layer87-labs/tei-reranker-model-init:mmarco-mminilmv2-l12-h384-v1-2026.10 .
+#
+# Build (higher quality, much slower on CPU; ONNX export at build time):
 #   docker build -f deploy/Containerfile.tei-reranker-model-init \
 #     --build-arg MODEL_ID=BAAI/bge-reranker-v2-m3 \
-#     --build-arg MODEL_DATE=2026.06 \
-#     -t ghcr.io/layer87-labs/tei-reranker-model-init:bge-reranker-v2-m3-2026.06 .
+#     -t ghcr.io/layer87-labs/tei-reranker-model-init:bge-reranker-v2-m3-2026.10 .
 
-ARG MODEL_ID=BAAI/bge-reranker-v2-m3
-ARG MODEL_DATE=2026.06
+ARG MODEL_ID=cross-encoder/mmarco-mMiniLMv2-L12-H384-v1
+ARG MODEL_REVISION=
+ARG MODEL_DATE=2026.10
 
 # ── Stage 1: download + ONNX export ───────────────────────────────────────────
 FROM python:3.12-slim AS builder
 
 ARG MODEL_ID
+ARG MODEL_REVISION
 
 RUN pip install --quiet --no-cache-dir \
       "huggingface_hub[cli]>=0.24" \
@@ -26,20 +34,33 @@ RUN pip install --quiet --no-cache-dir \
       "torch>=2.2" \
       --extra-index-url https://download.pytorch.org/whl/cpu
 
-# Download model (safetensors + tokenizer, no PyTorch .bin needed for export)
-RUN hf download "${MODEL_ID}" \
-      --exclude "*.ot" \
-      --exclude "flax_model*" \
-      --exclude "tf_model*" \
+# Download model. Several patterns go into ONE --exclude: the option takes a
+# list, so a second --exclude would replace the first.
+RUN hf download "${MODEL_ID}" ${MODEL_REVISION:+--revision "${MODEL_REVISION}"} \
+      --exclude "*.ot" "flax_model*" "tf_model*" "pytorch_model.bin" \
+                "openvino/*" "onnx/model_*" \
       --local-dir /model-src
 
-# Export to ONNX — task=text-classification = cross-encoder (reranker)
-RUN optimum-cli export onnx \
-      --model /model-src \
-      --task text-classification \
-      --opset 14 \
-      --framework pt \
-      /model-onnx/
+# Use the repository's own onnx/model.onnx when present (copying only what TEI
+# needs), else export to ONNX. task=text-classification = cross-encoder (reranker)
+RUN if [ -f /model-src/onnx/model.onnx ]; then \
+      echo "using shipped onnx/model.onnx"; \
+      mkdir -p /model-onnx/onnx && \
+      cp /model-src/onnx/model.onnx /model-onnx/onnx/ && \
+      cd /model-src && \
+      for f in config.json tokenizer.json tokenizer_config.json \
+               special_tokens_map.json sentencepiece.bpe.model; do \
+        if [ -f "$f" ]; then cp "$f" /model-onnx/; fi; \
+      done; \
+    else \
+      optimum-cli export onnx \
+        --model /model-src \
+        --task text-classification \
+        --opset 14 \
+        --framework pt \
+        /model-onnx/; \
+    fi && ls -l /model-onnx && \
+    { test -s /model-onnx/onnx/model.onnx || test -s /model-onnx/model.onnx; }
 
 # ── Stage 2: minimal init container ──────────────────────────────────────────
 FROM debian:bookworm-slim
@@ -62,4 +83,4 @@ USER 1000
 ENTRYPOINT ["sh", "-c", "rsync -rlgoDv --ignore-existing /model-baked/ /model/ && echo 'tei-reranker-model-init: done'"]
 
 LABEL org.opencontainers.image.source="https://github.com/layer87-labs/inference-stack"
-LABEL org.opencontainers.image.description="TEI reranker model init — bge-reranker-v2-m3 exported to ONNX for TEI ORT backend"
+LABEL org.opencontainers.image.description="TEI reranker model init — cross-encoder reranker as ONNX for TEI ORT backend"

deploy/helm/Chart.yaml

@@ -3,7 +3,7 @@ name: inference-stack
 description: >
   Inference Stack — unified CPU-based model serving for embedding,
   reranking and transcription. Includes TEI (BGE-M3), TEI Reranker
-  (BGE-reranker-v2-m3 via ONNX/ORT), Whisper ASR and an inference-router that exposes
+  (cross-encoder via ONNX/ORT), Whisper ASR and an inference-router that exposes
   all models under a single OpenAI-compatible endpoint.
 
 type: application

deploy/helm/templates/NOTES.txt

@@ -10,7 +10,7 @@ Enabled backends:
   ✗ Embedding  (disabled)
 {{- end }}
 {{- if .Values.reranker.enabled }}
-  ✓ Reranker   (BAAI/bge-reranker-v2-m3)
+  ✓ Reranker   ({{ .Values.reranker.modelName }})
     Runtime:    {{ .Values.reranker.image.repository }}:{{ .Values.reranker.image.tag | default .Chart.AppVersion }}
     Model init: {{ .Values.reranker.modelInit.image.repository }}:{{ .Values.reranker.modelInit.image.tag | default .Chart.AppVersion }}
     Service:    {{ include "inference-stack.fullname" . }}-reranker:{{ .Values.reranker.service.port }}