From c943ab3644afe60686a8b3392fbc3c5ec484b476 Mon Sep 17 00:00:00 2001 From: Eike Haller <58111764+eksrha@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:06:20 +0200 Subject: [PATCH] fix(whisper): avoid repetition loops on long audio, add startup warmup Pass faster-whisper's temperature fallback schedule unless the request sets temperature explicitly, keep condition_on_previous_text off by default and transcribe audio longer than WHISPER_BATCH_THRESHOLD_S (default 35 s) with BatchedInferencePipeline. Run a short synthetic clip through VAD and decoder at startup and report /health ready only afterwards. The request log gains path, batch_size, temperature_mode and condition_on_previous_text. Closes #17 --- README.md | 26 +++- deploy/helm/templates/whisper/deployment.yaml | 9 +- deploy/helm/values.yaml | 10 ++ deploy/whisper/server.py | 135 ++++++++++++++--- deploy/whisper/tests/test_server.py | 143 +++++++++++++++++- 5 files changed, 297 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 3c92272..0bcc96b 100644 --- a/README.md +++ b/README.md @@ -215,11 +215,33 @@ Measured on CPU: ~2.0 GiB RSS peak; roughly real-time speed on 2 threads (a `Release notes for Kubernetes, PostgreSQL and Grafana.` - Hints raise the odds of a spelling; they do not guarantee it. Keep them short and spell terms the way they should appear. +- Repetition loops on long audio: the sequential decoder conditions every + window on the previous text and can get stuck repeating itself. The server + therefore (1) keeps `condition_on_previous_text` off by default + (`whisper.conditionOnPreviousText`), (2) sends audio longer than + `whisper.batchThresholdSeconds` (default 35 s, `WHISPER_BATCH_THRESHOLD_S`) + through faster-whisper's `BatchedInferencePipeline` (VAD-cut chunks decoded + independently, `whisper.batchSize` = 8, `WHISPER_BATCH_SIZE`), and (3) uses + faster-whisper's temperature fallback (0.0, 0.2, ... 1.0 with its default + compression-ratio, log-prob and no-speech thresholds) on the sequential + path. The batched path always uses VAD and only the first temperature (the + library does not fall back there). A request's explicit `temperature` form + field (including `0`) is used as given, without fallback; clients that always + send `temperature=0` therefore get no fallback on the sequential path. + Measured locally (FLEURS de, int8, 4 threads, beam 1, VAD): three 55-88 s + clips WER 3.1 % instead of 42 % (one clip looped to 256 instead of 120 + words) and 56 s of audio in 17.8 s instead of 27.2 s; 40 single clips + 5.6 % vs 5.2 % (noise); no speed difference on short clips. +- Warmup: after loading the model the server runs a 2 s synthetic clip through + VAD and decoder (`WHISPER_WARMUP`, default `true`) so the first real request + does not pay the initialisation cost. `/health` returns 503 until model and + warmup are done; a failed warmup is logged and does not block startup. - Per-request log line (JSON after the prefix `whisper_request`), e.g. - `whisper_request {"status":"ok","audio_s":9.0,"prep_s":0.4,"infer_s":7.6,"total_s":8.1,"rtf":0.889,...}`: + `whisper_request {"status":"ok","path":"standard","audio_s":9.0,"prep_s":0.4,"infer_s":7.6,"total_s":8.1,"rtf":0.889,...}`: audio duration (`audio_s`, `audio_after_vad_s`), upload, queue wait, decode + VAD (`prep_s`) and inference (`infer_s`) time, real-time factor - (`rtf = (prep_s + infer_s) / audio_s`), model, language, `beam_size`, `vad`, + (`rtf = (prep_s + infer_s) / audio_s`), model, language, `beam_size`, `vad`, `path` (`standard|batched`), + `temperature_mode` (`fallback|fixed`), file size, and `prompt_chars` / `hotwords_chars` / `prompt_source` (`none|default|request|both`). Prompt, hotwords and transcript text are never logged. diff --git a/deploy/helm/templates/whisper/deployment.yaml b/deploy/helm/templates/whisper/deployment.yaml index a296ef5..08e0827 100644 --- a/deploy/helm/templates/whisper/deployment.yaml +++ b/deploy/helm/templates/whisper/deployment.yaml @@ -62,12 +62,19 @@ spec: value: {{ .Values.whisper.initialPrompt | quote }} - name: WHISPER_HOTWORDS value: {{ .Values.whisper.hotwords | quote }} + - name: WHISPER_CONDITION_ON_PREVIOUS_TEXT + value: {{ .Values.whisper.conditionOnPreviousText | quote }} + - name: WHISPER_BATCH_THRESHOLD_S + value: {{ .Values.whisper.batchThresholdSeconds | quote }} + - name: WHISPER_BATCH_SIZE + value: {{ .Values.whisper.batchSize | quote }} ports: - name: http containerPort: {{ .Values.whisper.service.port }} protocol: TCP # /health is answered independently of inference (see - # deploy/whisper/server.py), so a long transcription cannot fail it. + # deploy/whisper/server.py), so a long transcription cannot fail it. It + # reports ready only after the model is loaded and the warmup has run. readinessProbe: httpGet: path: /health diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index c5b2779..9a06e47 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -348,6 +348,16 @@ whisper: # `hotwords` field replaces it. Free text, e.g. "Kubernetes Grafana Postgres". hotwords: "" + # Condition each 30 s window on the previous text (WHISPER_CONDITION_ON_PREVIOUS_TEXT). + # Off by default: it lets a repetition loop carry on through long audio. + conditionOnPreviousText: false + + # Audio longer than this (seconds) is transcribed with faster-whisper's + # BatchedInferencePipeline (VAD-cut chunks decoded independently, in batches + # of batchSize). Shorter audio uses the regular sequential decoder. + batchThresholdSeconds: 35 + batchSize: 8 + # CTranslate2 threads. Must match the CPU limit: CTranslate2 sizes itself by # the node's cores, not by the cgroup limit. cpuThreads: 4 diff --git a/deploy/whisper/server.py b/deploy/whisper/server.py index 768f70a..e812745 100644 --- a/deploy/whisper/server.py +++ b/deploy/whisper/server.py @@ -14,7 +14,17 @@ appended to WHISPER_INITIAL_PROMPT, see combine_prompt() hotwords optional hint phrases; replaces WHISPER_HOTWORDS response_format json (default) | text | verbose_json | srt | vtt - temperature float, default 0 + temperature float; when omitted the server uses faster-whisper's temperature + fallback (0.0 .. 1.0, see TEMPERATURES). An explicit value + (including 0) is used as given, without fallback. + +Long audio (longer than WHISPER_BATCH_THRESHOLD_S) goes through faster-whisper's +BatchedInferencePipeline, short audio through the regular transcribe(). The +sequential decoder can fall into repetition loops on long recordings because +every window is conditioned on the previous text; the batched pipeline decodes +VAD-cut chunks independently and is also faster on long audio. The batched +pipeline always uses VAD and only the first temperature (no fallback; it has no +previous-text conditioning to loop on). Inference runs in a worker thread and is serialised by a semaphore, so the event loop (and with it /health) stays responsive while a transcription runs. @@ -23,6 +33,10 @@ audio duration, real-time factor and parameters. Prompt, hotwords and transcript are never logged, only whether and how long the prompt/hotwords were. + +At startup the model is loaded and one short synthetic clip is run through the +pipeline (VAD and decoder) so the first real request does not pay the +initialisation cost; /health reports ready only after that. """ import asyncio @@ -45,6 +59,17 @@ BEAM_SIZE = int(os.environ.get("BEAM_SIZE", "1")) VAD_FILTER = os.environ.get("VAD_FILTER", "true").lower() == "true" PORT = int(os.environ.get("PORT", "9000")) +# Audio longer than this (seconds) is transcribed with BatchedInferencePipeline. +BATCH_THRESHOLD_S = float(os.environ.get("WHISPER_BATCH_THRESHOLD_S", "35")) +BATCH_SIZE = int(os.environ.get("WHISPER_BATCH_SIZE", "8")) +# Off by default: conditioning every window on the previous text is what lets +# a repetition loop carry on. +CONDITION_ON_PREVIOUS_TEXT = os.environ.get("WHISPER_CONDITION_ON_PREVIOUS_TEXT", "false").lower() == "true" +WARMUP = os.environ.get("WHISPER_WARMUP", "true").lower() == "true" +SAMPLE_RATE = 16000 +# faster-whisper's own fallback schedule; thresholds (compression ratio 2.4, +# log-prob -1.0, no-speech 0.6) stay at its defaults. +TEMPERATURES = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0] # Server-side defaults; many STT clients send neither prompt nor hotwords. DEFAULT_PROMPT = os.environ.get("WHISPER_INITIAL_PROMPT", "").strip() DEFAULT_HOTWORDS = os.environ.get("WHISPER_HOTWORDS", "").strip() @@ -56,6 +81,8 @@ FORMATS = {"json", "text", "verbose_json", "srt", "vtt"} _model = None +_batched = None +_ready = False _sem: asyncio.Semaphore | None = None @@ -69,11 +96,48 @@ def _load_model(): return model +def _load_batched(model): + from faster_whisper import BatchedInferencePipeline + + return BatchedInferencePipeline(model=model) + + +def _warmup_audio(): + """Two seconds of a quiet tone plus noise (deterministic).""" + import numpy as np + + n = 2 * SAMPLE_RATE + t = np.arange(n, dtype=np.float32) / SAMPLE_RATE + noise = np.random.default_rng(0).standard_normal(n).astype(np.float32) + return (0.05 * np.sin(2 * np.pi * 220 * t) + 0.02 * noise).astype(np.float32) + + +def _warmup(): + """Run a short clip through the pipeline and discard the result. + + Pass 1 uses the configured VAD setting (initialises the VAD runtime), pass 2 + runs with VAD off so the encoder and decoder are exercised even when the + VAD drops the synthetic audio. Failures are logged, never raised. + """ + t0 = time.monotonic() + try: + audio = _warmup_audio() + for vad in dict.fromkeys((VAD_FILTER, False)): + _transcribe(audio, 2.0, "transcribe", DEFAULT_LANGUAGE, None, None, None, vad=vad) + print(f"Warmup done in {time.monotonic() - t0:.1f}s", flush=True) + except Exception as exc: + print(f"Warmup failed ({type(exc).__name__}: {exc}); continuing", flush=True) + + @asynccontextmanager async def lifespan(app: FastAPI): - global _model, _sem + global _model, _batched, _ready, _sem _sem = asyncio.Semaphore(MAX_CONCURRENT) _model = await asyncio.to_thread(_load_model) + _batched = await asyncio.to_thread(_load_batched, _model) + if WARMUP: + await asyncio.to_thread(_warmup) + _ready = True yield @@ -82,7 +146,7 @@ async def lifespan(app: FastAPI): @app.get("/health") async def health(): - if _model is None: + if _model is None or not _ready: return JSONResponse({"status": "loading"}, status_code=503) return {"status": "ok"} @@ -125,26 +189,48 @@ def resolve_hotwords(default: str, request: str | None) -> str | None: return ((request or "").strip() or default)[:MAX_PROMPT_CHARS] or None -def _transcribe(path: str, task: str, language: str | None, prompt: str | None, hotwords: str | None, - temperature: float): - t0 = time.monotonic() - # transcribe() decodes the whole file (PyAV/FFmpeg, resampling to 16 kHz), - # runs VAD and builds the features before it returns the lazy generator. - segments, info = _model.transcribe( - path, +def _decode(path: str): + """Decode to 16 kHz mono float32 (PyAV/FFmpeg). Returns (audio, duration_s).""" + from faster_whisper.audio import decode_audio + + audio = decode_audio(path, sampling_rate=SAMPLE_RATE) + return audio, len(audio) / SAMPLE_RATE + + +def _transcribe(audio, duration_s: float, task: str, language: str | None, prompt: str | None, + hotwords: str | None, temperature: float | None, vad: bool | None = None): + """Transcribe decoded audio; returns (segments, info, timing, meta). + + temperature None = faster-whisper's fallback schedule (TEMPERATURES); a + number is used as given. + """ + vad = VAD_FILTER if vad is None else vad + batched = duration_s > BATCH_THRESHOLD_S and _batched is not None + kwargs = dict( task=task, language=language, initial_prompt=prompt, hotwords=hotwords, - temperature=temperature, + temperature=TEMPERATURES if temperature is None else temperature, beam_size=BEAM_SIZE, - vad_filter=VAD_FILTER, + condition_on_previous_text=CONDITION_ON_PREVIOUS_TEXT, ) + t0 = time.monotonic() + # transcribe() runs VAD and builds the features before it returns the lazy + # generator. + if batched: + # The batched pipeline requires VAD (it cuts the chunks). + vad = True + segments, info = _batched.transcribe(audio, vad_filter=True, batch_size=BATCH_SIZE, **kwargs) + else: + segments, info = _model.transcribe(audio, vad_filter=vad, **kwargs) t1 = time.monotonic() # The actual decoding happens while iterating. segments = list(segments) t2 = time.monotonic() - return segments, info, {"prep_s": t1 - t0, "infer_s": t2 - t1} + meta = {"path": "batched" if batched else "standard", "vad": vad, + "temperature_mode": "fixed" if temperature is not None else "fallback"} + return segments, info, {"prep_s": t1 - t0, "infer_s": t2 - t1}, meta def _log_request(**fields): @@ -152,7 +238,7 @@ def _log_request(**fields): async def _handle(file: UploadFile, task: str, language: str | None, prompt: str | None, - hotwords: str | None, response_format: str, temperature: float): + hotwords: str | None, response_format: str, temperature: float | None): if _model is None: raise HTTPException(status_code=503, detail="model not loaded yet") if response_format not in FORMATS: @@ -178,8 +264,13 @@ async def _handle(file: UploadFile, task: str, language: str | None, prompt: str async with _sem: t_run = time.monotonic() try: - segments, info, timing = await asyncio.to_thread( - _transcribe, tmp.name, task, language, prompt, hotwords, temperature) + t_dec = time.monotonic() + audio, duration_s = await asyncio.to_thread(_decode, tmp.name) + decode_s = time.monotonic() - t_dec + segments, info, timing, meta = await asyncio.to_thread( + _transcribe, audio, duration_s, task, language, prompt, hotwords, temperature) + timing["prep_s"] += decode_s + del audio except Exception as exc: # undecodable audio etc. _log_request(status="error", error=type(exc).__name__, task=task, file_bytes=size, total_s=round(time.monotonic() - t_start, 3)) @@ -197,8 +288,12 @@ async def _handle(file: UploadFile, task: str, language: str | None, prompt: str compute_type=COMPUTE_TYPE, cpu_threads=CPU_THREADS, beam_size=BEAM_SIZE, - vad=VAD_FILTER, - temperature=temperature, + vad=meta["vad"], + path=meta["path"], + batch_size=BATCH_SIZE if meta["path"] == "batched" else None, + temperature_mode=meta["temperature_mode"], + temperature=temperature if temperature is not None else TEMPERATURES, + condition_on_previous_text=CONDITION_ON_PREVIOUS_TEXT, prompt_chars=len(prompt or ""), prompt_source=prompt_source, hotwords_chars=len(hotwords or ""), @@ -246,7 +341,7 @@ async def transcriptions( prompt: str | None = Form(None), hotwords: str | None = Form(None), response_format: str = Form("json"), - temperature: float = Form(0.0), + temperature: float | None = Form(None), ): return await _handle(file, "transcribe", language, prompt, hotwords, response_format, temperature) @@ -258,7 +353,7 @@ async def translations( prompt: str | None = Form(None), hotwords: str | None = Form(None), response_format: str = Form("json"), - temperature: float = Form(0.0), + temperature: float | None = Form(None), ): return await _handle(file, "translate", None, prompt, hotwords, response_format, temperature) diff --git a/deploy/whisper/tests/test_server.py b/deploy/whisper/tests/test_server.py index a18c122..d95239d 100644 --- a/deploy/whisper/tests/test_server.py +++ b/deploy/whisper/tests/test_server.py @@ -24,8 +24,8 @@ class FakeModel: def __init__(self): self.calls = [] - def transcribe(self, path, **kwargs): - self.calls.append(kwargs) + def transcribe(self, audio, **kwargs): + self.calls.append({"audio": audio, **kwargs}) seg = SimpleNamespace(text=f" {SECRET_TEXT}", start=0.0, end=2.0, avg_logprob=-0.1, no_speech_prob=0.0) info = SimpleNamespace(language="de", duration=10.0, duration_after_vad=8.0) return iter([seg]), info @@ -34,7 +34,15 @@ def transcribe(self, path, **kwargs): @pytest.fixture def client(monkeypatch): model = FakeModel() + batched = FakeModel() + audio = {"duration": 10.0} monkeypatch.setattr(server, "_model", model) + monkeypatch.setattr(server, "_batched", batched) + monkeypatch.setattr(server, "_ready", True) + monkeypatch.setattr(server, "_decode", lambda path: ("audio", audio["duration"])) + monkeypatch.setattr(server, "BATCH_THRESHOLD_S", 35.0) + monkeypatch.setattr(server, "BATCH_SIZE", 8) + monkeypatch.setattr(server, "CONDITION_ON_PREVIOUS_TEXT", False) monkeypatch.setattr(server, "DEFAULT_PROMPT", "") monkeypatch.setattr(server, "DEFAULT_HOTWORDS", "") monkeypatch.setattr(server, "DEFAULT_LANGUAGE", None) @@ -43,6 +51,8 @@ def client(monkeypatch): monkeypatch.setattr(server, "_sem", asyncio.Semaphore(1)) c = TestClient(server.app) c.model = model + c.batched = batched + c.audio = audio return c @@ -132,7 +142,7 @@ def test_log_line_has_timings_and_no_content(client, capsys, monkeypatch): def test_error_log_without_content(client, capsys): - def boom(path, **kw): + def boom(audio, **kw): raise RuntimeError("decode failed") client.model.transcribe = boom @@ -140,3 +150,130 @@ def boom(path, **kw): assert r.status_code == 400 out = capsys.readouterr().out assert '"status":"error"' in out and SECRET_PROMPT not in out + + +def test_default_uses_temperature_fallback(client): + post(client) + kw = client.model.calls[0] + assert kw["temperature"] == [0.0, 0.2, 0.4, 0.6, 0.8, 1.0] + assert kw["condition_on_previous_text"] is False + assert "compression_ratio_threshold" not in kw # faster-whisper defaults apply + + +def test_explicit_temperature_wins_including_zero(client): + post(client, temperature="0") + assert client.model.calls[-1]["temperature"] == 0.0 + post(client, temperature="0.4") + assert client.model.calls[-1]["temperature"] == 0.4 + + +def test_condition_on_previous_text_switch(client, monkeypatch): + monkeypatch.setattr(server, "CONDITION_ON_PREVIOUS_TEXT", True) + post(client) + assert client.model.calls[0]["condition_on_previous_text"] is True + + +def test_short_audio_uses_standard_path(client): + client.audio["duration"] = 35.0 # threshold is exclusive + post(client) + assert len(client.model.calls) == 1 and client.batched.calls == [] + assert client.model.calls[0]["audio"] == "audio" + + +def test_long_audio_uses_batched_path(client): + client.audio["duration"] = 88.0 + post(client, prompt="Alpha", hotwords="Beta", language="de") + assert client.model.calls == [] + kw = client.batched.calls[0] + assert kw["batch_size"] == 8 and kw["vad_filter"] is True + assert kw["initial_prompt"] == "Alpha" and kw["hotwords"] == "Beta" and kw["language"] == "de" + assert kw["condition_on_previous_text"] is False + + +def test_batched_path_forces_vad_even_if_disabled(client, monkeypatch): + monkeypatch.setattr(server, "VAD_FILTER", False) + client.audio["duration"] = 60.0 + post(client) + assert client.batched.calls[0]["vad_filter"] is True + + +def test_threshold_configurable(client, monkeypatch): + monkeypatch.setattr(server, "BATCH_THRESHOLD_S", 5.0) + post(client) + assert len(client.batched.calls) == 1 + + +def test_response_formats_unchanged_on_batched_path(client): + client.audio["duration"] = 90.0 + assert post(client, response_format="json").json() == {"text": SECRET_TEXT} + assert post(client, response_format="text").text == SECRET_TEXT + "\n" + body = post(client, response_format="verbose_json").json() + assert body["segments"][0]["text"] == f" {SECRET_TEXT}" and body["language"] == "de" + assert "00:00:00,000 --> 00:00:02,000" in post(client, response_format="srt").text + assert post(client, response_format="vtt").text.startswith("WEBVTT") + + +def test_log_has_path_and_temperature_mode(client, capsys): + post(client) + client.audio["duration"] = 80.0 + post(client, temperature="0") + recs = [json.loads(l.split(" ", 1)[1]) for l in capsys.readouterr().out.splitlines() + if l.startswith("whisper_request ")] + assert (recs[0]["path"], recs[0]["temperature_mode"]) == ("standard", "fallback") + assert (recs[1]["path"], recs[1]["temperature_mode"]) == ("batched", "fixed") + assert recs[1]["batch_size"] == 8 and recs[1]["temperature"] == 0.0 + assert recs[0]["batch_size"] is None + + +def test_health_not_ready_until_warmup_done(client, monkeypatch): + assert client.get("/health").json() == {"status": "ok"} + monkeypatch.setattr(server, "_ready", False) + r = client.get("/health") + assert r.status_code == 503 and r.json() == {"status": "loading"} + + +def test_warmup_runs_standard_path_with_and_without_vad(client, monkeypatch): + monkeypatch.setattr(server, "_warmup_audio", lambda: "tone") + monkeypatch.setattr(server, "VAD_FILTER", True) + server._warmup() + assert [c["vad_filter"] for c in client.model.calls] == [True, False] + assert all(c["audio"] == "tone" for c in client.model.calls) + assert client.batched.calls == [] + + +def test_warmup_failure_is_only_logged(client, monkeypatch, capsys): + monkeypatch.setattr(server, "_warmup_audio", lambda: "tone") + + def boom(audio, **kw): + raise RuntimeError("vad init failed") + + client.model.transcribe = boom + server._warmup() # must not raise + assert "Warmup failed" in capsys.readouterr().out + + +def test_lifespan_ready_only_after_warmup(monkeypatch): + order = [] + monkeypatch.setattr(server, "_ready", False) + monkeypatch.setattr(server, "_model", None) + monkeypatch.setattr(server, "_load_model", lambda: order.append("load") or FakeModel()) + monkeypatch.setattr(server, "_load_batched", lambda m: order.append("batched") or FakeModel()) + monkeypatch.setattr(server, "WARMUP", True) + + def fake_warmup(): + order.append("warmup") + assert server._ready is False + + monkeypatch.setattr(server, "_warmup", fake_warmup) + with TestClient(server.app) as c: + assert order == ["load", "batched", "warmup"] + assert c.get("/health").json() == {"status": "ok"} + + +def test_warmup_can_be_disabled(monkeypatch): + monkeypatch.setattr(server, "_load_model", lambda: FakeModel()) + monkeypatch.setattr(server, "_load_batched", lambda m: FakeModel()) + monkeypatch.setattr(server, "WARMUP", False) + monkeypatch.setattr(server, "_warmup", lambda: pytest.fail("warmup must not run")) + with TestClient(server.app) as c: + assert c.get("/health").status_code == 200