fix(whisper): avoid repetition loops on long audio, add startup warmup

README.md

@@ -215,11 +215,33 @@ Measured on CPU: ~2.0 GiB RSS peak; roughly real-time speed on 2 threads (a
     `Release notes for Kubernetes, PostgreSQL and Grafana.`
   - Hints raise the odds of a spelling; they do not guarantee it. Keep them
     short and spell terms the way they should appear.
+- Repetition loops on long audio: the sequential decoder conditions every
+  window on the previous text and can get stuck repeating itself. The server
+  therefore (1) keeps `condition_on_previous_text` off by default
+  (`whisper.conditionOnPreviousText`), (2) sends audio longer than
+  `whisper.batchThresholdSeconds` (default 35 s, `WHISPER_BATCH_THRESHOLD_S`)
+  through faster-whisper's `BatchedInferencePipeline` (VAD-cut chunks decoded
+  independently, `whisper.batchSize` = 8, `WHISPER_BATCH_SIZE`), and (3) uses
+  faster-whisper's temperature fallback (0.0, 0.2, ... 1.0 with its default
+  compression-ratio, log-prob and no-speech thresholds) on the sequential
+  path. The batched path always uses VAD and only the first temperature (the
+  library does not fall back there). A request's explicit `temperature` form
+  field (including `0`) is used as given, without fallback; clients that always
+  send `temperature=0` therefore get no fallback on the sequential path.
+  Measured locally (FLEURS de, int8, 4 threads, beam 1, VAD): three 55-88 s
+  clips WER 3.1 % instead of 42 % (one clip looped to 256 instead of 120
+  words) and 56 s of audio in 17.8 s instead of 27.2 s; 40 single clips
+  5.6 % vs 5.2 % (noise); no speed difference on short clips.
+- Warmup: after loading the model the server runs a 2 s synthetic clip through
+  VAD and decoder (`WHISPER_WARMUP`, default `true`) so the first real request
+  does not pay the initialisation cost. `/health` returns 503 until model and
+  warmup are done; a failed warmup is logged and does not block startup.
 - Per-request log line (JSON after the prefix `whisper_request`), e.g.
-  `whisper_request {"status":"ok","audio_s":9.0,"prep_s":0.4,"infer_s":7.6,"total_s":8.1,"rtf":0.889,...}`:
+  `whisper_request {"status":"ok","path":"standard","audio_s":9.0,"prep_s":0.4,"infer_s":7.6,"total_s":8.1,"rtf":0.889,...}`:
   audio duration (`audio_s`, `audio_after_vad_s`), upload, queue wait, decode +
   VAD (`prep_s`) and inference (`infer_s`) time, real-time factor
-  (`rtf = (prep_s + infer_s) / audio_s`), model, language, `beam_size`, `vad`,
+  (`rtf = (prep_s + infer_s) / audio_s`), model, language, `beam_size`, `vad`, `path` (`standard|batched`),
+  `temperature_mode` (`fallback|fixed`),
   file size, and `prompt_chars` / `hotwords_chars` / `prompt_source`
   (`none|default|request|both`). Prompt, hotwords and transcript text are
   never logged.

deploy/helm/templates/whisper/deployment.yaml

@@ -62,12 +62,19 @@ spec:
           value: {{ .Values.whisper.initialPrompt | quote }}
         - name: WHISPER_HOTWORDS
           value: {{ .Values.whisper.hotwords | quote }}
+        - name: WHISPER_CONDITION_ON_PREVIOUS_TEXT
+          value: {{ .Values.whisper.conditionOnPreviousText | quote }}
+        - name: WHISPER_BATCH_THRESHOLD_S
+          value: {{ .Values.whisper.batchThresholdSeconds | quote }}
+        - name: WHISPER_BATCH_SIZE
+          value: {{ .Values.whisper.batchSize | quote }}
         ports:
         - name: http
           containerPort: {{ .Values.whisper.service.port }}
           protocol: TCP
         # /health is answered independently of inference (see
-        # deploy/whisper/server.py), so a long transcription cannot fail it.
+        # deploy/whisper/server.py), so a long transcription cannot fail it. It
+        # reports ready only after the model is loaded and the warmup has run.
         readinessProbe:
           httpGet:
             path: /health

deploy/helm/values.yaml

@@ -348,6 +348,16 @@ whisper:
   # `hotwords` field replaces it. Free text, e.g. "Kubernetes Grafana Postgres".
   hotwords: ""
 
+  # Condition each 30 s window on the previous text (WHISPER_CONDITION_ON_PREVIOUS_TEXT).
+  # Off by default: it lets a repetition loop carry on through long audio.
+  conditionOnPreviousText: false
+
+  # Audio longer than this (seconds) is transcribed with faster-whisper's
+  # BatchedInferencePipeline (VAD-cut chunks decoded independently, in batches
+  # of batchSize). Shorter audio uses the regular sequential decoder.
+  batchThresholdSeconds: 35
+  batchSize: 8
+
   # CTranslate2 threads. Must match the CPU limit: CTranslate2 sizes itself by
   # the node's cores, not by the cgroup limit.
   cpuThreads: 4

deploy/whisper/server.py

@@ -14,7 +14,17 @@
                    appended to WHISPER_INITIAL_PROMPT, see combine_prompt()
   hotwords         optional hint phrases; replaces WHISPER_HOTWORDS
   response_format  json (default) | text | verbose_json | srt | vtt
-  temperature      float, default 0
+  temperature      float; when omitted the server uses faster-whisper's temperature
+                   fallback (0.0 .. 1.0, see TEMPERATURES). An explicit value
+                   (including 0) is used as given, without fallback.
+
+Long audio (longer than WHISPER_BATCH_THRESHOLD_S) goes through faster-whisper's
+BatchedInferencePipeline, short audio through the regular transcribe(). The
+sequential decoder can fall into repetition loops on long recordings because
+every window is conditioned on the previous text; the batched pipeline decodes
+VAD-cut chunks independently and is also faster on long audio. The batched
+pipeline always uses VAD and only the first temperature (no fallback; it has no
+previous-text conditioning to loop on).
 
 Inference runs in a worker thread and is serialised by a semaphore, so the
 event loop (and with it /health) stays responsive while a transcription runs.
@@ -23,6 +33,10 @@
 audio duration, real-time factor and parameters. Prompt, hotwords and
 transcript are never logged, only whether and how long the prompt/hotwords
 were.
+
+At startup the model is loaded and one short synthetic clip is run through the
+pipeline (VAD and decoder) so the first real request does not pay the
+initialisation cost; /health reports ready only after that.
 """
 
 import asyncio
@@ -45,6 +59,17 @@
 BEAM_SIZE = int(os.environ.get("BEAM_SIZE", "1"))
 VAD_FILTER = os.environ.get("VAD_FILTER", "true").lower() == "true"
 PORT = int(os.environ.get("PORT", "9000"))
+# Audio longer than this (seconds) is transcribed with BatchedInferencePipeline.
+BATCH_THRESHOLD_S = float(os.environ.get("WHISPER_BATCH_THRESHOLD_S", "35"))
+BATCH_SIZE = int(os.environ.get("WHISPER_BATCH_SIZE", "8"))
+# Off by default: conditioning every window on the previous text is what lets
+# a repetition loop carry on.
+CONDITION_ON_PREVIOUS_TEXT = os.environ.get("WHISPER_CONDITION_ON_PREVIOUS_TEXT", "false").lower() == "true"
+WARMUP = os.environ.get("WHISPER_WARMUP", "true").lower() == "true"
+SAMPLE_RATE = 16000
+# faster-whisper's own fallback schedule; thresholds (compression ratio 2.4,
+# log-prob -1.0, no-speech 0.6) stay at its defaults.
+TEMPERATURES = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]
 # Server-side defaults; many STT clients send neither prompt nor hotwords.
 DEFAULT_PROMPT = os.environ.get("WHISPER_INITIAL_PROMPT", "").strip()
 DEFAULT_HOTWORDS = os.environ.get("WHISPER_HOTWORDS", "").strip()
@@ -56,6 +81,8 @@
 FORMATS = {"json", "text", "verbose_json", "srt", "vtt"}
 
 _model = None
+_batched = None
+_ready = False
 _sem: asyncio.Semaphore | None = None
 
 
@@ -69,11 +96,48 @@ def _load_model():
     return model
 
 
+def _load_batched(model):
+    from faster_whisper import BatchedInferencePipeline
+
+    return BatchedInferencePipeline(model=model)
+
+
+def _warmup_audio():
+    """Two seconds of a quiet tone plus noise (deterministic)."""
+    import numpy as np
+
+    n = 2 * SAMPLE_RATE
+    t = np.arange(n, dtype=np.float32) / SAMPLE_RATE
+    noise = np.random.default_rng(0).standard_normal(n).astype(np.float32)
+    return (0.05 * np.sin(2 * np.pi * 220 * t) + 0.02 * noise).astype(np.float32)
+
+
+def _warmup():
+    """Run a short clip through the pipeline and discard the result.
+
+    Pass 1 uses the configured VAD setting (initialises the VAD runtime), pass 2
+    runs with VAD off so the encoder and decoder are exercised even when the
+    VAD drops the synthetic audio. Failures are logged, never raised.
+    """
+    t0 = time.monotonic()
+    try:
+        audio = _warmup_audio()
+        for vad in dict.fromkeys((VAD_FILTER, False)):
+            _transcribe(audio, 2.0, "transcribe", DEFAULT_LANGUAGE, None, None, None, vad=vad)
+        print(f"Warmup done in {time.monotonic() - t0:.1f}s", flush=True)
+    except Exception as exc:
+        print(f"Warmup failed ({type(exc).__name__}: {exc}); continuing", flush=True)
+
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    global _model, _sem
+    global _model, _batched, _ready, _sem
     _sem = asyncio.Semaphore(MAX_CONCURRENT)
     _model = await asyncio.to_thread(_load_model)
+    _batched = await asyncio.to_thread(_load_batched, _model)
+    if WARMUP:
+        await asyncio.to_thread(_warmup)
+    _ready = True
     yield
 
 
@@ -82,7 +146,7 @@ async def lifespan(app: FastAPI):
 
 @app.get("/health")
 async def health():
-    if _model is None:
+    if _model is None or not _ready:
         return JSONResponse({"status": "loading"}, status_code=503)
     return {"status": "ok"}
 
@@ -125,34 +189,56 @@ def resolve_hotwords(default: str, request: str | None) -> str | None:
     return ((request or "").strip() or default)[:MAX_PROMPT_CHARS] or None
 
 
-def _transcribe(path: str, task: str, language: str | None, prompt: str | None, hotwords: str | None,
-                temperature: float):
-    t0 = time.monotonic()
-    # transcribe() decodes the whole file (PyAV/FFmpeg, resampling to 16 kHz),
-    # runs VAD and builds the features before it returns the lazy generator.
-    segments, info = _model.transcribe(
-        path,
+def _decode(path: str):
+    """Decode to 16 kHz mono float32 (PyAV/FFmpeg). Returns (audio, duration_s)."""
+    from faster_whisper.audio import decode_audio
+
+    audio = decode_audio(path, sampling_rate=SAMPLE_RATE)
+    return audio, len(audio) / SAMPLE_RATE
+
+
+def _transcribe(audio, duration_s: float, task: str, language: str | None, prompt: str | None,
+                hotwords: str | None, temperature: float | None, vad: bool | None = None):
+    """Transcribe decoded audio; returns (segments, info, timing, meta).
+
+    temperature None = faster-whisper's fallback schedule (TEMPERATURES); a
+    number is used as given.
+    """
+    vad = VAD_FILTER if vad is None else vad
+    batched = duration_s > BATCH_THRESHOLD_S and _batched is not None
+    kwargs = dict(
         task=task,
         language=language,
         initial_prompt=prompt,
         hotwords=hotwords,
-        temperature=temperature,
+        temperature=TEMPERATURES if temperature is None else temperature,
         beam_size=BEAM_SIZE,
-        vad_filter=VAD_FILTER,
+        condition_on_previous_text=CONDITION_ON_PREVIOUS_TEXT,
     )
+    t0 = time.monotonic()
+    # transcribe() runs VAD and builds the features before it returns the lazy
+    # generator.
+    if batched:
+        # The batched pipeline requires VAD (it cuts the chunks).
+        vad = True
+        segments, info = _batched.transcribe(audio, vad_filter=True, batch_size=BATCH_SIZE, **kwargs)
+    else:
+        segments, info = _model.transcribe(audio, vad_filter=vad, **kwargs)
     t1 = time.monotonic()
     # The actual decoding happens while iterating.
     segments = list(segments)
     t2 = time.monotonic()
-    return segments, info, {"prep_s": t1 - t0, "infer_s": t2 - t1}
+    meta = {"path": "batched" if batched else "standard", "vad": vad,
+            "temperature_mode": "fixed" if temperature is not None else "fallback"}
+    return segments, info, {"prep_s": t1 - t0, "infer_s": t2 - t1}, meta
 
 
 def _log_request(**fields):
     print("whisper_request " + json.dumps(fields, separators=(",", ":")), flush=True)
 
 
 async def _handle(file: UploadFile, task: str, language: str | None, prompt: str | None,
-                  hotwords: str | None, response_format: str, temperature: float):
+                  hotwords: str | None, response_format: str, temperature: float | None):
     if _model is None:
         raise HTTPException(status_code=503, detail="model not loaded yet")
     if response_format not in FORMATS:
@@ -178,8 +264,13 @@ async def _handle(file: UploadFile, task: str, language: str | None, prompt: str
         async with _sem:
             t_run = time.monotonic()
             try:
-                segments, info, timing = await asyncio.to_thread(
-                    _transcribe, tmp.name, task, language, prompt, hotwords, temperature)
+                t_dec = time.monotonic()
+                audio, duration_s = await asyncio.to_thread(_decode, tmp.name)
+                decode_s = time.monotonic() - t_dec
+                segments, info, timing, meta = await asyncio.to_thread(
+                    _transcribe, audio, duration_s, task, language, prompt, hotwords, temperature)
+                timing["prep_s"] += decode_s
+                del audio
             except Exception as exc:  # undecodable audio etc.
                 _log_request(status="error", error=type(exc).__name__, task=task, file_bytes=size,
                              total_s=round(time.monotonic() - t_start, 3))
@@ -197,8 +288,12 @@ async def _handle(file: UploadFile, task: str, language: str | None, prompt: str
         compute_type=COMPUTE_TYPE,
         cpu_threads=CPU_THREADS,
         beam_size=BEAM_SIZE,
-        vad=VAD_FILTER,
-        temperature=temperature,
+        vad=meta["vad"],
+        path=meta["path"],
+        batch_size=BATCH_SIZE if meta["path"] == "batched" else None,
+        temperature_mode=meta["temperature_mode"],
+        temperature=temperature if temperature is not None else TEMPERATURES,
+        condition_on_previous_text=CONDITION_ON_PREVIOUS_TEXT,
         prompt_chars=len(prompt or ""),
         prompt_source=prompt_source,
         hotwords_chars=len(hotwords or ""),
@@ -246,7 +341,7 @@ async def transcriptions(
     prompt: str | None = Form(None),
     hotwords: str | None = Form(None),
     response_format: str = Form("json"),
-    temperature: float = Form(0.0),
+    temperature: float | None = Form(None),
 ):
     return await _handle(file, "transcribe", language, prompt, hotwords, response_format, temperature)
 
@@ -258,7 +353,7 @@ async def translations(
     prompt: str | None = Form(None),
     hotwords: str | None = Form(None),
     response_format: str = Form("json"),
-    temperature: float = Form(0.0),
+    temperature: float | None = Form(None),
 ):
     return await _handle(file, "translate", None, prompt, hotwords, response_format, temperature)