feat(whisper): OpenAI-compatible transcription server, probes that survive load

.github/workflows/release-models.yml

@@ -15,6 +15,7 @@ on:
       - "deploy/Containerfile.tei-model-init"
       - "deploy/Containerfile.tei-reranker-model-init"
       - "deploy/Containerfile.whisper"
+      - "deploy/whisper/**"
 
 permissions:
   contents: read

AGENTS.md

@@ -40,7 +40,9 @@ deploy/
   Containerfile.tei-model-init  Embedding model init container (bakes BGE-M3)
   Containerfile.reranker-model-init  Reranker model init container
   Containerfile.reranker-server      FlagEmbedding FastAPI server (/v1/rerank)
-  Containerfile.whisper         Whisper ASR with model baked in
+  Containerfile.whisper         Whisper ASR server (deploy/whisper/server.py) with model baked in
+  whisper/
+    server.py                   faster-whisper FastAPI server (/v1/audio/transcriptions)
   reranker/
     server.py                   FlagEmbedding FastAPI server
   helm/                         Helm chart (Chart.yaml, values.yaml, templates/)
@@ -70,7 +72,7 @@ Key details:
 
 ### Whisper (Standalone)
 
-Model is baked directly into the image at build time. No init container, no PVC needed.
+Model is baked directly into the image at build time. No init container, no PVC needed. The image runs `deploy/whisper/server.py`, which speaks the OpenAI audio API itself (the router forwards paths unchanged). `/health` does not queue behind inference. Pin `av` (see Containerfile) — newer PyAV majors break faster-whisper.
 
 ### Router (Go Proxy)
 

Makefile

@@ -14,7 +14,6 @@ LDSYMS          ?= \
 # TEI and Whisper versions — read from Containerfile ARGs by default.
 # Override on the command line if needed: make docker/tei-runtime TEI_VERSION=cpu-1.9.4
 TEI_VERSION     ?= $(shell grep '^ARG TEI_VERSION=' deploy/Containerfile.tei-base | sed 's/ARG TEI_VERSION=//')
-WHISPER_VERSION ?= $(shell grep '^ARG WHISPER_VERSION=' deploy/Containerfile.whisper | sed 's/ARG WHISPER_VERSION=//')
 WHISPER_MODEL   ?= $(shell grep '^ARG WHISPER_MODEL=' deploy/Containerfile.whisper | sed 's/ARG WHISPER_MODEL=//')
 
 .PHONY: build build/router build/mockbackend tidy lint test test-local \
@@ -114,7 +113,6 @@ docker/tei-reranker-model-init:
 docker/whisper:
 	docker build \
 	  -f deploy/Containerfile.whisper \
-	  --build-arg WHISPER_VERSION=$(WHISPER_VERSION) \
 	  --build-arg WHISPER_MODEL=$(WHISPER_MODEL) \
 	  -t $(REGISTRY)/whisper:$(VERSION) \
 	  .

README.md

@@ -79,7 +79,12 @@ curl http://localhost:8080/v1/models
 
 ### Whisper (Standalone)
 
-Model baked into image at build time. No init container or PVC needed.
+Model baked into image at build time. No init container or PVC needed. The
+image runs `deploy/whisper/server.py` (faster-whisper behind a small FastAPI
+app) which implements `POST /v1/audio/transcriptions` and
+`/v1/audio/translations` natively, so the router forwards requests unchanged.
+`/health` is answered independently of inference and stays responsive while a
+transcription is running.
 
 ### Router (Go Proxy)
 
@@ -97,7 +102,7 @@ All images are published to `ghcr.io/layer87-labs/`:
 | `tei-model-init`      | BGE-M3 model baked, copies to volume   |
 | `reranker-model-init` | BGE-reranker-v2-m3 baked               |
 | `reranker-server`     | FlagEmbedding HTTP server              |
-| `whisper`             | Whisper ASR with model baked in        |
+| `whisper`             | Whisper ASR server, model baked in     |
 
 All images run as non-root with no privilege escalation.
 
@@ -164,6 +169,24 @@ Prometheus metrics on `:9090/metrics`:
 | 512          | 2-4Gi      | **CPU default** |
 | 1024         | 4-6Gi      | Larger context  |
 
+### Whisper (faster-whisper + large-v3-turbo, int8)
+
+Measured on CPU: ~2.0 GiB RSS peak; roughly real-time speed on 2 threads (a
+3.7 min recording took 3.3 min), faster with more threads.
+
+- `whisper.cpuThreads` must match the CPU limit (CTranslate2 sizes itself by
+  the node's cores, not by the cgroup limit).
+- Transcriptions run one at a time; further requests queue in the server. Keep
+  the router's `WHISPER_TIMEOUT` (default `300s`) above queue time + runtime,
+  e.g. via `router.extraEnv`.
+- Upload limit: `whisper.maxUploadMB` (default 100). Formats: anything FFmpeg
+  decodes (wav, mp3, m4a, ogg, flac, webm, mp4, ...).
+- `response_format`: `json` (default), `text`, `verbose_json`, `srt`, `vtt`.
+- Set `whisper.language` (e.g. `de`) to skip language auto-detection; a
+  request's own `language` field wins.
+- `/v1/audio/translations` (to English) is exposed, but large-v3-turbo is not
+  trained for translation; use a non-turbo model for that.
+
 ## Environment Variables
 
 | Variable               | Default | Description                   |

deploy/Containerfile.whisper

@@ -1,71 +1,60 @@
 # syntax=docker/dockerfile:1
 #
-# Containerfile.whisper — Whisper ASR HTTP server, hardened
+# Containerfile.whisper — Whisper ASR HTTP server (CPU), model baked in
 #
-# Exposes an OpenAI-compatible /v1/audio/transcriptions endpoint.
-# Uses faster-whisper (CTranslate2 backend) for CPU-efficient inference.
+# OpenAI-compatible POST /v1/audio/transcriptions (and /v1/audio/translations)
+# served by deploy/whisper/server.py on top of faster-whisper (CTranslate2).
 #
-# Model is baked into the image at build time (no init container needed —
-# Whisper large-v3-turbo is ~800 MB, small enough to keep in a single image).
+# Why a purpose-built server instead of an off-the-shelf ASR web service:
+#   - the router forwards OpenAI-style paths unchanged, so the backend must
+#     speak that API itself;
+#   - /health is served independently of inference, so a probe never queues
+#     behind a long transcription.
 #
-# Hardened:
-#   1. execstack cleared on libctranslate2*.so (fixes mprotect PROT_EXEC crash)
-#   2. Model baked into /home/whisper/.cache — no cold-start download
-#   3. Non-root (uid 1000)
-#   4. HF_HUB_OFFLINE=0 — huggingface_hub xet cache requires live API validation
-#      even with a fully-populated cache; offline mode breaks model loading.
-
-ARG WHISPER_VERSION=v1.9.1
-ARG WHISPER_MODEL=large-v3-turbo
-
-# ── Stage 1: patchelf binary ──────────────────────────────────────────────────
-FROM debian:bookworm-slim AS tools
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends patchelf && \
-    rm -rf /var/lib/apt/lists/*
-
-# ── Stage 2: Whisper runtime ──────────────────────────────────────────────────
-FROM docker.io/onerahmet/openai-whisper-asr-webservice:${WHISPER_VERSION}
+# Model is baked into the image (large-v3-turbo, ~1.6 GB) and loaded from a
+# local path — the container runs fully offline and needs no PVC.
+#
+# Hardened: non-root (uid 1000), no network access needed at runtime,
+# ctranslate2 4.x ships a non-executable stack, so the default seccomp
+# profile and allowPrivilegeEscalation=false work.
 
-ARG WHISPER_MODEL
+FROM python:3.12-slim
 
-USER root
+ARG WHISPER_MODEL=large-v3-turbo
 
-ENV ASR_MODEL=${WHISPER_MODEL}
-ENV ASR_ENGINE=faster_whisper
-# Upstream image hardcodes port 9000; UVICORN_PORT env is ignored by uvicorn.
-# We accept the default — the chart must map service port 9000.
-ENV UVICORN_PORT=9000
+# av is pinned: faster-whisper 1.2.x breaks with newer PyAV major versions
+# ("open() got an unexpected keyword argument 'metadata_errors'").
+RUN pip install --no-cache-dir \
+    "faster-whisper==1.2.1" \
+    "av==16.1.0" \
+    "fastapi>=0.110.0" \
+    "uvicorn[standard]>=0.27.0" \
+    "python-multipart>=0.0.9"
 
-# Use venv python for everything.
-ENV PATH="/app/.venv/bin:$PATH"
+RUN groupadd -g 1000 whisper && \
+    useradd -u 1000 -g whisper -m -s /bin/bash whisper && \
+    mkdir -p /opt/model && chown whisper:whisper /opt/model
 
-# Clear executable-stack ELF flag on ctranslate2 shared libraries.
-# Required for securityContext.allowPrivilegeEscalation=false.
-COPY --from=tools /usr/bin/patchelf /usr/bin/patchelf
-RUN find /app/.venv -name 'libctranslate2*' -exec patchelf --clear-execstack {} \; && \
-    rm -f /usr/bin/patchelf
+# Bake the model (needs network at build time only).
+USER 1000
+RUN python -c "from faster_whisper.utils import download_model; download_model('${WHISPER_MODEL}', output_dir='/opt/model')" && \
+    rm -rf /home/whisper/.cache
 
-# Create non-root user before downloading the model.
-RUN groupadd -g 1000 whisper && \
-    useradd -u 1000 -g whisper -m -s /bin/bash whisper
+WORKDIR /app
+COPY --chown=1000:1000 deploy/whisper/server.py /app/server.py
 
-# Download model at build time into the whisper user's home cache.
-RUN HF_HUB_OFFLINE=0 HUGGINGFACE_OFFLINE=0 \
-    HOME=/home/whisper \
-    python -c "from faster_whisper import WhisperModel; WhisperModel('${WHISPER_MODEL}', device='cpu', compute_type='int8')"
+ENV MODEL_PATH=/opt/model \
+    MODEL_NAME=whisper-${WHISPER_MODEL} \
+    COMPUTE_TYPE=int8 \
+    HF_HUB_OFFLINE=1 \
+    PORT=9000
 
-# Fix ownership (model was downloaded as root into /home/whisper/.cache).
-RUN chown -R whisper:whisper /home/whisper /app
+EXPOSE 9000
 
-# Don't enforce HF offline mode: huggingface_hub xet cache (used by
-# mobiuslabsgmbh/faster-whisper-large-v3-turbo) requires a live API call
-# even when the cache is fully populated. Model won't be re-downloaded
-# since it's already cached.
-ENV HF_HUB_OFFLINE=0
-ENV HUGGINGFACE_OFFLINE=0
+HEALTHCHECK --interval=15s --timeout=5s --start-period=60s --retries=8 \
+  CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')" || exit 1
 
-USER 1000
+CMD ["python", "/app/server.py"]
 
 LABEL org.opencontainers.image.source="https://github.com/layer87-labs/inference-stack"
-LABEL org.opencontainers.image.description="Whisper ASR HTTP server (CPU) — model baked in, hardened"
+LABEL org.opencontainers.image.description="Whisper ASR HTTP server (CPU) — OpenAI-compatible /v1/audio/transcriptions, model baked in"

deploy/helm/templates/router/deployment.yaml

@@ -71,6 +71,9 @@ spec:
         - name: WHISPER_URL
           value: "http://{{ include "inference-stack.fullname" . }}-whisper:{{ .Values.whisper.service.port }}"
         {{- end }}
+        {{- with .Values.router.extraEnv }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
         readinessProbe:
           httpGet:
             path: /readyz

deploy/helm/templates/whisper/deployment.yaml

@@ -38,30 +38,38 @@ spec:
         image: {{ include "inference-stack.image" (dict "registry" .Values.global.imageRegistry "repo" .Values.whisper.image.repository "tag" .Values.whisper.image.tag "appVersion" .Chart.AppVersion) }}
         imagePullPolicy: {{ .Values.whisper.image.pullPolicy }}
         env:
-        - name: ASR_MODEL
-          value: {{ .Values.whisper.modelName | quote }}
-        - name: ASR_ENGINE
-          value: "faster_whisper"
+        - name: MODEL_NAME
+          value: "whisper-{{ .Values.whisper.modelName }}"
+        - name: PORT
+          value: {{ .Values.whisper.service.port | quote }}
+        - name: CPU_THREADS
+          value: {{ .Values.whisper.cpuThreads | quote }}
+        - name: MAX_UPLOAD_MB
+          value: {{ .Values.whisper.maxUploadMB | quote }}
+        - name: MAX_CONCURRENT
+          value: {{ .Values.whisper.maxConcurrent | quote }}
+        - name: WHISPER_LANGUAGE
+          value: {{ .Values.whisper.language | quote }}
         ports:
         - name: http
           containerPort: {{ .Values.whisper.service.port }}
           protocol: TCP
+        # /health is answered independently of inference (see
+        # deploy/whisper/server.py), so a long transcription cannot fail it.
         readinessProbe:
           httpGet:
-            path: /docs
+            path: /health
             port: http
-          initialDelaySeconds: 30
           periodSeconds: 15
           timeoutSeconds: 5
           failureThreshold: 6
         livenessProbe:
           httpGet:
-            path: /docs
+            path: /health
             port: http
-          initialDelaySeconds: 60
           periodSeconds: 30
           timeoutSeconds: 10
-          failureThreshold: 3
+          failureThreshold: 6
         {{- if .Values.whisper.startupProbe.enabled }}
         startupProbe:
           httpGet:

deploy/helm/values.yaml

@@ -49,6 +49,10 @@ router:
 
   healthCheckInterval: "15s"
 
+  # Extra environment variables for the router, e.g. to tune per-backend
+  # timeouts (WHISPER_TIMEOUT, ROUTER_WRITE_TIMEOUT, ...). List of EnvVar.
+  extraEnv: []
+
   # Pod-level security context
   podSecurityContext:
     runAsNonRoot: true
@@ -290,34 +294,54 @@ reranker:
   tolerations: []
 
 # ── Whisper — OpenAI-compatible ASR ───────────────────────────────────────────
+# faster-whisper (CTranslate2, int8) behind deploy/whisper/server.py.
+# POST /v1/audio/transcriptions and /v1/audio/translations; /health is
+# answered independently of inference. The model is baked into the image, so
+# no PVC or init container is needed.
 whisper:
   enabled: false
 
-  # Whisper image has model baked in — versioned as <model>-<YYYY.MM>
+  # Image has the model baked in — versioned as <model>-<YYYY.MM>
   image:
     repository: whisper
-    tag: "large-v3-turbo-2026.06"
+    tag: "large-v3-turbo-2026.10"
     pullPolicy: IfNotPresent
 
   modelName: "large-v3-turbo"
 
+  # Default transcription language (ISO-639-1, e.g. "de"). Empty = auto-detect
+  # per request. A request's own `language` field always wins.
+  language: ""
+
+  # CTranslate2 threads. Must match the CPU limit: CTranslate2 sizes itself by
+  # the node's cores, not by the cgroup limit.
+  cpuThreads: 4
+
+  # Upload size limit per request (MB). Must stay below any body-size limit of
+  # the ingress/gateway in front of the router.
+  maxUploadMB: 100
+
+  # Transcriptions run one at a time (CPU-bound); further requests queue in
+  # the server. Keep the router's WHISPER_TIMEOUT above queue time + runtime.
+  maxConcurrent: 1
+
   service:
     port: 9000
 
+  # Measured with large-v3-turbo / int8: ~2.0 GiB RSS peak; ~1x real-time on
+  # 2 threads (a 3.7 min recording took 3.3 min), faster with more threads.
   resources:
     requests:
-      cpu: "500m"
-      memory: "2Gi"
-    limits:
       cpu: "2000m"
+      memory: "3Gi"
+    limits:
+      cpu: "4000m"
       memory: "4Gi"
 
-  # Pod-level security context. ctranslate2.so requires Unconfined seccomp
-  # (see container securityContext below), so the pod default matches.
   podSecurityContext:
     runAsNonRoot: true
     seccompProfile:
-      type: Unconfined
+      type: RuntimeDefault
 
   securityContext:
     runAsNonRoot: true
@@ -327,20 +351,17 @@ whisper:
     readOnlyRootFilesystem: false
     capabilities:
       drop: [ALL]
-    # ctranslate2.so (faster-whisper) uses mprotect(PROT_EXEC) — needs
-    # relaxed seccomp until ctranslate2 is rebuilt without execstack.
     seccompProfile:
-      type: Unconfined
+      type: RuntimeDefault
 
-  # Startup probe — whisper loads the model on first request and may
-  # need 30-60s to warm up on CPU.
+  # Startup probe — model load takes ~10-30s from local disk.
   startupProbe:
     enabled: true
     httpGet:
-      path: /docs
+      path: /health
       port: http
-    periodSeconds: 15
-    failureThreshold: 40
+    periodSeconds: 10
+    failureThreshold: 30
 
   imagePullSecrets: []
   nodeSelector: {}