diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54b109b..71ef238 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,9 @@ jobs: - name: Typecheck (electron) run: npm run typecheck:electron + - name: Source-language transcription tests + run: npm run test:transcription + - name: Build Next.js run: npm run build diff --git a/components/ModelSelector.tsx b/components/ModelSelector.tsx index 6b56c91..700055b 100644 --- a/components/ModelSelector.tsx +++ b/components/ModelSelector.tsx @@ -26,9 +26,10 @@ import { SignalBarsMedium, } from "./SignalBars"; import { - TRANSCRIPT_LANGUAGE_ORDER, + AUTO_TRANSCRIPT_LANGUAGE_INFO, + TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER, TRANSCRIPT_LANGUAGES, - type TranscriptLanguage, + type TranscriptLanguagePreference, } from "@/lib/languages"; import { MODEL_ORDER, @@ -207,11 +208,13 @@ export default function ModelSelector({ : source === "import" ? "Import transcript" : String(source)); - const languageInfo = TRANSCRIPT_LANGUAGES[transcriptLanguage]; + const languageInfo = + transcriptLanguage === "auto" + ? AUTO_TRANSCRIPT_LANGUAGE_INFO + : TRANSCRIPT_LANGUAGES[transcriptLanguage]; const showLanguageInTrigger = isWhisperModel(source) && - !activeTrigger?.busy && - transcriptLanguage !== "en"; + !activeTrigger?.busy; // Always mount options (hidden when closed) so custom triggers stay registered. const options = children ?? ( @@ -416,9 +419,12 @@ export function LanguageSection() { const selector = useSelectorCtx(); const [submenuOpen, setSubmenuOpen] = useState(false); const submenuId = useId(); - const active = TRANSCRIPT_LANGUAGES[language]; + const active = + language === "auto" + ? AUTO_TRANSCRIPT_LANGUAGE_INFO + : TRANSCRIPT_LANGUAGES[language]; - const select = (next: TranscriptLanguage) => { + const select = (next: TranscriptLanguagePreference) => { setLanguage(next); setSubmenuOpen(false); selector.closeMenu(); @@ -427,7 +433,7 @@ export function LanguageSection() { return (

- Language + Source language

{/* No portal: stay in the parent panel DOM so outside-click on the model menu still treats this flyout as inside the floating tree. */} @@ -472,11 +478,14 @@ export function LanguageSection() { - {TRANSCRIPT_LANGUAGE_ORDER.map((id) => { - const option = TRANSCRIPT_LANGUAGES[id]; + {TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER.map((id) => { + const option = + id === "auto" + ? AUTO_TRANSCRIPT_LANGUAGE_INFO + : TRANSCRIPT_LANGUAGES[id]; const selected = id === language; return (
+ {selector.value === "parakeet" && ( +

+ Parakeet detects the spoken language automatically; this setting is used only for word alignment. +

+ )} ); } diff --git a/hooks/useTranscriber.ts b/hooks/useTranscriber.ts index a52fd14..5a854ba 100644 --- a/hooks/useTranscriber.ts +++ b/hooks/useTranscriber.ts @@ -6,6 +6,7 @@ import { reportError } from "@/lib/sentry"; import { useEditorStore } from "@/lib/store"; import { trackEvent } from "@/lib/telemetry"; import type { WorkerResponse } from "@/lib/types"; +import { resolveTranscriptLanguage } from "@/lib/transcriptionOptions"; let activeWorker: Worker | null = null; @@ -33,7 +34,10 @@ export function useTranscriber() { return; } const model = store.source; - const transcriptLanguage = store.transcriptLanguage; + const transcriptLanguagePreference = store.transcriptLanguage; + const transcriptLanguage = resolveTranscriptLanguage( + transcriptLanguagePreference + ); store.setStatus("transcribing"); store.setProgress({ message: "Loading speech model…", value: null }); @@ -64,7 +68,7 @@ export function useTranscriber() { // Nothing about the media itself — not its length, not the text. trackEvent("transcription_completed", { model, - language: transcriptLanguage, + language: transcriptLanguagePreference, }); break; case "error": @@ -96,7 +100,12 @@ export function useTranscriber() { // long recording the copy this replaces was hundreds of megabytes held for // the length of the run. workerRef.current.postMessage( - { audio, duration, model, language: transcriptLanguage }, + { + audio, + duration, + model, + ...(transcriptLanguage ? { language: transcriptLanguage } : {}), + }, [audio.buffer] ); }, []); diff --git a/lib/languages.ts b/lib/languages.ts index 0074b66..2fb3743 100644 --- a/lib/languages.ts +++ b/lib/languages.ts @@ -1,4 +1,5 @@ export type TranscriptLanguage = "en" | "es" | "fr" | "de" | "zh"; +export type TranscriptLanguagePreference = "auto" | TranscriptLanguage; export interface TranscriptLanguageInfo { label: string; @@ -11,7 +12,14 @@ export interface TranscriptLanguageInfo { const LANGUAGE_STORAGE_KEY = "rescript.transcript-language"; -export const DEFAULT_TRANSCRIPT_LANGUAGE: TranscriptLanguage = "en"; +export const DEFAULT_TRANSCRIPT_LANGUAGE: TranscriptLanguagePreference = "auto"; + +export const AUTO_TRANSCRIPT_LANGUAGE_INFO = { + label: "Auto-detect", + nativeLabel: "Auto-detect", + flag: "🌐", + code: "AUTO", +} satisfies TranscriptLanguageInfo; export const TRANSCRIPT_LANGUAGES: Record< TranscriptLanguage, @@ -57,6 +65,11 @@ export const TRANSCRIPT_LANGUAGE_ORDER: TranscriptLanguage[] = [ "zh", ]; +export const TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER: TranscriptLanguagePreference[] = [ + "auto", + ...TRANSCRIPT_LANGUAGE_ORDER, +]; + export function isTranscriptLanguage( value: unknown ): value is TranscriptLanguage { @@ -69,12 +82,18 @@ export function isTranscriptLanguage( ); } +export function isTranscriptLanguagePreference( + value: unknown +): value is TranscriptLanguagePreference { + return value === "auto" || isTranscriptLanguage(value); +} + /** Read the last-selected transcript language from localStorage. */ -export function loadTranscriptLanguagePreference(): TranscriptLanguage { +export function loadTranscriptLanguagePreference(): TranscriptLanguagePreference { if (typeof window === "undefined") return DEFAULT_TRANSCRIPT_LANGUAGE; try { const raw = window.localStorage.getItem(LANGUAGE_STORAGE_KEY); - if (isTranscriptLanguage(raw)) return raw; + if (isTranscriptLanguagePreference(raw)) return raw; } catch { // private mode / disabled storage } @@ -82,7 +101,9 @@ export function loadTranscriptLanguagePreference(): TranscriptLanguage { } /** Persist the selected transcript language for the next visit. */ -export function saveTranscriptLanguagePreference(language: TranscriptLanguage) { +export function saveTranscriptLanguagePreference( + language: TranscriptLanguagePreference +) { if (typeof window === "undefined") return; try { window.localStorage.setItem(LANGUAGE_STORAGE_KEY, language); diff --git a/lib/projects.ts b/lib/projects.ts index f4e2719..c539d72 100644 --- a/lib/projects.ts +++ b/lib/projects.ts @@ -7,10 +7,10 @@ */ import { isTranscriptSource, type TranscriptSource } from "./source"; -import type { TranscriptLanguage } from "./languages"; +import type { TranscriptLanguagePreference } from "./languages"; import { DEFAULT_TRANSCRIPT_LANGUAGE, - isTranscriptLanguage, + isTranscriptLanguagePreference, } from "./languages"; import type { MediaKind } from "./media"; import type { ManualCut, SceneBoundary, SpeakerInfo, Word } from "./types"; @@ -26,7 +26,7 @@ export interface ProjectMeta { mediaKind: MediaKind; duration: number; source: TranscriptSource; - transcriptLanguage: TranscriptLanguage; + transcriptLanguage: TranscriptLanguagePreference; updatedAt: number; createdAt: number; } @@ -137,7 +137,7 @@ export async function listProjects(): Promise { mediaKind: r.mediaKind, duration: r.duration, source: projectSource(r), - transcriptLanguage: isTranscriptLanguage(r.transcriptLanguage) + transcriptLanguage: isTranscriptLanguagePreference(r.transcriptLanguage) ? r.transcriptLanguage : DEFAULT_TRANSCRIPT_LANGUAGE, updatedAt: r.updatedAt, @@ -181,7 +181,7 @@ export async function putProject(input: ProjectWrite): Promise { mediaKind: input.mediaKind, duration: input.duration, source: isTranscriptSource(input.source) ? input.source : "base", - transcriptLanguage: isTranscriptLanguage(input.transcriptLanguage) + transcriptLanguage: isTranscriptLanguagePreference(input.transcriptLanguage) ? input.transcriptLanguage : DEFAULT_TRANSCRIPT_LANGUAGE, words: input.words, diff --git a/lib/store.ts b/lib/store.ts index 4c7e867..77c5d21 100644 --- a/lib/store.ts +++ b/lib/store.ts @@ -31,10 +31,10 @@ import { isTranscriptSource, type TranscriptSource } from "./source"; import { trackEvent } from "./telemetry"; import { DEFAULT_TRANSCRIPT_LANGUAGE, - isTranscriptLanguage, + isTranscriptLanguagePreference, loadTranscriptLanguagePreference, saveTranscriptLanguagePreference, - type TranscriptLanguage, + type TranscriptLanguagePreference, } from "./languages"; import { detectMediaKind, type MediaKind } from "./media"; import { buildWaveformPeaks, type WaveformPeaks } from "./waveform"; @@ -80,7 +80,7 @@ interface EditorState { /** Transcript source selected on the upload screen (speech model or import). */ source: TranscriptSource; /** Language hint sent to Whisper when transcribing (Parakeet auto-detects). */ - transcriptLanguage: TranscriptLanguage; + transcriptLanguage: TranscriptLanguagePreference; /** * Caption file parsed on the upload screen when source is "import". * Cleared when switching back to a speech model or after media loads. @@ -150,7 +150,7 @@ interface EditorState { /** Delete a saved project; if it is the active one, resets to the home screen. */ removeProject: (id: string) => Promise; setSource: (s: TranscriptSource) => void; - setTranscriptLanguage: (language: TranscriptLanguage) => void; + setTranscriptLanguage: (language: TranscriptLanguagePreference) => void; setPendingTranscript: (t: PendingTranscript | null) => void; setDuration: (d: number) => void; /** @@ -428,7 +428,7 @@ export const useEditorStore = create((set, get) => ({ mediaKind: record.mediaKind, duration: record.duration, source: isTranscriptSource(record.source) ? record.source : "base", - transcriptLanguage: isTranscriptLanguage(record.transcriptLanguage) + transcriptLanguage: isTranscriptLanguagePreference(record.transcriptLanguage) ? record.transcriptLanguage : DEFAULT_TRANSCRIPT_LANGUAGE, projectId: record.id, diff --git a/lib/transcriptionOptions.ts b/lib/transcriptionOptions.ts new file mode 100644 index 0000000..24a0c4d --- /dev/null +++ b/lib/transcriptionOptions.ts @@ -0,0 +1,28 @@ +import type { + TranscriptLanguage, + TranscriptLanguagePreference, +} from "./languages"; + +export interface WhisperLanguageOptions { + task: "transcribe"; + language?: string; +} + +/** Resolve a saved UI preference to the concrete hint accepted by ASR backends. */ +export function resolveTranscriptLanguage( + preference: TranscriptLanguagePreference +): TranscriptLanguage | undefined { + return preference === "auto" ? undefined : preference; +} + +/** + * Whisper must always transcribe in the source language. In automatic mode the + * language key is omitted so Whisper can detect it from the audio. + */ +export function whisperLanguageOptions( + language: string | undefined +): WhisperLanguageOptions { + return language + ? { task: "transcribe", language } + : { task: "transcribe" }; +} diff --git a/lib/types.ts b/lib/types.ts index cca974f..9beb9cd 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -103,5 +103,5 @@ export interface WorkerRequest { * Transcript language: conditions Whisper decoding, selects the CTC aligner, * and is passed through for Parakeet alignment (Parakeet ASR still auto-detects). */ - language: import("./languages").TranscriptLanguage; + language?: import("./languages").TranscriptLanguage; } diff --git a/package.json b/package.json index 1c5dfc9..89b781e 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "build": "next build", "start": "next start", "lint": "eslint", + "test:transcription": "tsx tests/languages-test.ts && tsx tests/transcription-options-test.ts", "postinstall": "patch-package && node scripts/copy-assets.mjs", "build:electron": "node scripts/build-electron.mjs", "typecheck:electron": "tsc -p electron/tsconfig.json --noEmit", diff --git a/tests/languages-test.ts b/tests/languages-test.ts index b10e6b9..d3a9151 100644 --- a/tests/languages-test.ts +++ b/tests/languages-test.ts @@ -1,15 +1,20 @@ import { DEFAULT_TRANSCRIPT_LANGUAGE, isTranscriptLanguage, + isTranscriptLanguagePreference, + TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER, TRANSCRIPT_LANGUAGE_ORDER, TRANSCRIPT_LANGUAGES, } from "../lib/languages"; { - if (DEFAULT_TRANSCRIPT_LANGUAGE !== "en") { - throw new Error("expected default language to be en"); + if (DEFAULT_TRANSCRIPT_LANGUAGE !== "auto") { + throw new Error("expected default language preference to be auto"); } if (isTranscriptLanguage("auto")) throw new Error("did not expect auto to be valid"); + if (!isTranscriptLanguagePreference("auto")) { + throw new Error("expected auto to be a valid preference"); + } if (!isTranscriptLanguage("en")) throw new Error("expected en to be valid"); if (!isTranscriptLanguage("es")) throw new Error("expected es to be valid"); if (!isTranscriptLanguage("fr")) throw new Error("expected fr to be valid"); @@ -18,6 +23,14 @@ import { if (isTranscriptLanguage("pt")) throw new Error("did not expect pt to be valid"); } +{ + if (TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER.join(",") !== "auto,en,es,fr,de,zh") { + throw new Error( + `unexpected preference order: ${TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER.join(",")}` + ); + } +} + { const labels = TRANSCRIPT_LANGUAGE_ORDER.map( (id) => TRANSCRIPT_LANGUAGES[id].nativeLabel diff --git a/tests/transcription-options-test.ts b/tests/transcription-options-test.ts new file mode 100644 index 0000000..4d1f0b5 --- /dev/null +++ b/tests/transcription-options-test.ts @@ -0,0 +1,29 @@ +import { + resolveTranscriptLanguage, + whisperLanguageOptions, +} from "../lib/transcriptionOptions"; +import type { TranscriptLanguage } from "../lib/languages"; + +function assert(value: unknown, message: string): asserts value { + if (!value) throw new Error(message); +} + +assert(resolveTranscriptLanguage("auto") === undefined, "auto must omit language"); +assert(resolveTranscriptLanguage("zh") === "zh", "zh must remain concrete"); + +const automatic = whisperLanguageOptions(undefined); +assert(automatic.task === "transcribe", "auto must transcribe"); +assert(!("language" in automatic), "auto must not contain a language key"); + +for (const language of ["en", "es", "fr", "de", "zh"] satisfies TranscriptLanguage[]) { + const options = whisperLanguageOptions(language); + assert(options.task === "transcribe", `${language} must transcribe`); + assert(options.language === language, `${language} must be passed through`); + assert(options.task !== ("translate" as "transcribe"), "translate must never be used"); +} + +const detected = whisperLanguageOptions("ja"); +assert(detected.task === "transcribe", "a detected language must still transcribe"); +assert(detected.language === "ja", "a detected Whisper language code must pass through"); + +console.log("ALL TRANSCRIPTION OPTIONS TESTS PASSED"); diff --git a/workers/transcription.worker.ts b/workers/transcription.worker.ts index 8da612f..237680c 100644 --- a/workers/transcription.worker.ts +++ b/workers/transcription.worker.ts @@ -23,6 +23,8 @@ import { AutoModelForCTC, AutoTokenizer, WhisperTextStreamer, + LogitsProcessor, + LogitsProcessorList, Tensor, env, type AutomaticSpeechRecognitionPipeline, @@ -72,6 +74,7 @@ import { type CtcVocab, } from "@/lib/forcedAlign"; import type { TranscriptLanguage } from "@/lib/languages"; +import { whisperLanguageOptions } from "@/lib/transcriptionOptions"; import { VAD_FRAME_SIZE, VAD_SAMPLE_RATE, @@ -149,6 +152,75 @@ const WHISPER_LEAD_PAD_S = 0.5; type AsrChunk = { text: string; timestamp: [number, number | null] }; +/** Restrict Whisper's first generated token to its language-token vocabulary. */ +class WhisperLanguageTokenProcessor extends LogitsProcessor { + private readonly allowed: Set; + + constructor(tokenIds: number[]) { + super(); + this.allowed = new Set(tokenIds); + } + + _call(_inputIds: bigint[][], logits: Tensor): Tensor { + const values = logits.data as Float32Array; + const vocabularySize = logits.dims.at(-1) ?? 0; + for (let index = 0; index < values.length; index++) { + if (!this.allowed.has(index % vocabularySize)) values[index] = -Infinity; + } + return logits; + } +} + +/** + * Transformers.js 4.2 does not yet implement Whisper language detection: when + * `language` is omitted it explicitly defaults to English. Run Whisper's + * standard first-token probe instead, constrained to every language token in + * the checkpoint, then feed the detected code back into source-language + * transcription. + */ +async function detectWhisperLanguage( + transcriber: AutomaticSpeechRecognitionPipeline, + audio: Float32Array +): Promise { + const generationConfig = transcriber.model.generation_config as { + decoder_start_token_id?: number | null; + lang_to_id?: Record | null; + }; + const startToken = generationConfig.decoder_start_token_id; + const langToId = generationConfig.lang_to_id; + if (startToken == null || !langToId) { + throw new Error("This Whisper model does not expose language detection tokens."); + } + + const tokenToLanguage = new Map(); + for (const [token, id] of Object.entries(langToId)) { + const match = /^<\|([a-z]{2,3})\|>$/.exec(token); + if (match) tokenToLanguage.set(id, match[1]); + } + if (tokenToLanguage.size === 0) { + throw new Error("This Whisper model has no usable language tokens."); + } + + const { input_features } = await transcriber.processor(audio); + const languageProcessors = new LogitsProcessorList(); + languageProcessors.push(new WhisperLanguageTokenProcessor([...tokenToLanguage.keys()])); + const generated = (await transcriber.model.generate({ + inputs: input_features, + decoder_input_ids: new Tensor( + "int64", + BigInt64Array.of(BigInt(startToken)), + [1, 1] + ), + max_new_tokens: 1, + do_sample: false, + logits_processor: languageProcessors, + })) as Tensor; + const sequence = generated.tolist()[0] as bigint[]; + const language = tokenToLanguage.get(Number(sequence.at(-1))); + if (!language) throw new Error("Whisper could not detect the source language."); + return language; +} + const post = (msg: WorkerResponse, transfer: Transferable[] = []) => (self as unknown as Worker).postMessage(msg, transfer); @@ -953,7 +1025,7 @@ async function refineWordTimestamps( speechFrames: boolean[], audio: Float32Array, duration: number, - language: TranscriptLanguage + language: TranscriptLanguage | undefined ): Promise { let out = alignWordsToSpeech(words, speechFrames, { duration, @@ -961,7 +1033,7 @@ async function refineWordTimestamps( sampleRate: VAD_SAMPLE_RATE, }); - if (alignModelFor(language)) { + if (language && alignModelFor(language)) { try { const measured = await forceAlign(out, audio, duration, language); out = expandToAcoustics(measured, speechEnvelope(audio, VAD_SAMPLE_RATE)); @@ -1122,11 +1194,11 @@ async function finishWithDiarization( async function runParakeet( audio: Float32Array, duration: number, - transcriptLanguage: TranscriptLanguage + transcriptLanguage: TranscriptLanguage | undefined ): Promise { // Overlap diarizer (+ language-matched aligner) with Parakeet load. getDiarizer().catch(() => {}); - if (alignModelFor(transcriptLanguage)) { + if (transcriptLanguage && alignModelFor(transcriptLanguage)) { getAligner(transcriptLanguage).catch(() => {}); } const [loaded, vad] = await Promise.all([getParakeet(), getVad()]); @@ -1208,12 +1280,12 @@ async function runWhisper( audio: Float32Array, duration: number, choice: WhisperModel, - transcriptLanguage: TranscriptLanguage + transcriptLanguage: TranscriptLanguage | undefined ): Promise { // Overlap Whisper + Silero downloads; diarizer and language-matched aligner // warm in the background so both are cached by the time the transcript lands. getDiarizer().catch(() => {}); - if (alignModelFor(transcriptLanguage)) { + if (transcriptLanguage && alignModelFor(transcriptLanguage)) { getAligner(transcriptLanguage).catch(() => {}); } const [asr, vad] = await Promise.all([getAsr(choice), getVad()]); @@ -1228,6 +1300,21 @@ async function runWhisper( 0 ); + let whisperLanguage: string | undefined = transcriptLanguage; + if (!whisperLanguage && speechSegments.length > 0) { + post({ type: "progress", message: "Detecting language…", value: null }); + const probe = speechSegments.reduce((longest, segment) => + segment.endSample - segment.startSample > longest.endSample - longest.startSample + ? segment + : longest + ); + const probeEnd = Math.min(probe.endSample, probe.startSample + 30 * VAD_SAMPLE_RATE); + whisperLanguage = await detectWhisperLanguage( + transcriber, + audio.slice(probe.startSample, probeEnd) + ); + } + post({ type: "progress", message: "Transcribing…", value: 0 }); let partial = ""; @@ -1294,7 +1381,7 @@ async function runWhisper( // short VAD segments on every model here, whether with Whisper's // <|startofprev|> filler prompt or CrisperWhisper's mode tags. See the note // above MODELS in lib/models.ts; vad-regression-test.ts guards it. - language: transcriptLanguage, + ...whisperLanguageOptions(whisperLanguage), }; const rawWords: Word[] = []; @@ -1409,7 +1496,7 @@ self.onmessage = async (event: MessageEvent) => { const { audio, duration, model, language } = event.data; try { const choice: ModelId = model ?? "base"; - const transcriptLanguage: TranscriptLanguage = language ?? "en"; + const transcriptLanguage: TranscriptLanguage | undefined = language; let words: Word[]; if (isParakeetModel(choice)) {