feat(transcription): preserve the detected source language

.github/workflows/ci.yml

@@ -38,6 +38,9 @@ jobs:
       - name: Typecheck (electron)
         run: npm run typecheck:electron
 
+      - name: Source-language transcription tests
+        run: npm run test:transcription
+
       - name: Build Next.js
         run: npm run build
 

components/ModelSelector.tsx

@@ -26,9 +26,10 @@ import {
   SignalBarsMedium,
 } from "./SignalBars";
 import {
-  TRANSCRIPT_LANGUAGE_ORDER,
+  AUTO_TRANSCRIPT_LANGUAGE_INFO,
+  TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER,
   TRANSCRIPT_LANGUAGES,
-  type TranscriptLanguage,
+  type TranscriptLanguagePreference,
 } from "@/lib/languages";
 import {
   MODEL_ORDER,
@@ -207,11 +208,13 @@ export default function ModelSelector({
       : source === "import"
         ? "Import transcript"
         : String(source));
-  const languageInfo = TRANSCRIPT_LANGUAGES[transcriptLanguage];
+  const languageInfo =
+    transcriptLanguage === "auto"
+      ? AUTO_TRANSCRIPT_LANGUAGE_INFO
+      : TRANSCRIPT_LANGUAGES[transcriptLanguage];
   const showLanguageInTrigger =
     isWhisperModel(source) &&
-    !activeTrigger?.busy &&
-    transcriptLanguage !== "en";
+    !activeTrigger?.busy;
 
   // Always mount options (hidden when closed) so custom triggers stay registered.
   const options = children ?? (
@@ -416,9 +419,12 @@ export function LanguageSection() {
   const selector = useSelectorCtx();
   const [submenuOpen, setSubmenuOpen] = useState(false);
   const submenuId = useId();
-  const active = TRANSCRIPT_LANGUAGES[language];
+  const active =
+    language === "auto"
+      ? AUTO_TRANSCRIPT_LANGUAGE_INFO
+      : TRANSCRIPT_LANGUAGES[language];
 
-  const select = (next: TranscriptLanguage) => {
+  const select = (next: TranscriptLanguagePreference) => {
     setLanguage(next);
     setSubmenuOpen(false);
     selector.closeMenu();
@@ -427,7 +433,7 @@ export function LanguageSection() {
   return (
     <div>
       <p className="px-2.5 pb-1 pt-1.5 text-[11px] font-medium tracking-wide text-zinc-400 dark:text-zinc-500">
-        Language
+        Source language
       </p>
       {/* No portal: stay in the parent panel DOM so outside-click on the model
           menu still treats this flyout as inside the floating tree. */}
@@ -472,11 +478,14 @@ export function LanguageSection() {
           <PopoverContent
             id={submenuId}
             role="menu"
-            aria-label="Transcript language"
+            aria-label="Source language"
             className="z-50 w-44 overflow-hidden p-1"
           >
-            {TRANSCRIPT_LANGUAGE_ORDER.map((id) => {
-              const option = TRANSCRIPT_LANGUAGES[id];
+            {TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER.map((id) => {
+              const option =
+                id === "auto"
+                  ? AUTO_TRANSCRIPT_LANGUAGE_INFO
+                  : TRANSCRIPT_LANGUAGES[id];
               const selected = id === language;
               return (
                 <button
@@ -508,6 +517,11 @@ export function LanguageSection() {
           </PopoverContent>
         </div>
       </Popover>
+      {selector.value === "parakeet" && (
+        <p className="px-2.5 pb-1 pt-1 text-[11px] leading-relaxed text-zinc-400 dark:text-zinc-500">
+          Parakeet detects the spoken language automatically; this setting is used only for word alignment.
+        </p>
+      )}
     </div>
   );
 }

hooks/useTranscriber.ts

@@ -6,6 +6,7 @@ import { reportError } from "@/lib/sentry";
 import { useEditorStore } from "@/lib/store";
 import { trackEvent } from "@/lib/telemetry";
 import type { WorkerResponse } from "@/lib/types";
+import { resolveTranscriptLanguage } from "@/lib/transcriptionOptions";
 
 let activeWorker: Worker | null = null;
 
@@ -33,7 +34,10 @@ export function useTranscriber() {
       return;
     }
     const model = store.source;
-    const transcriptLanguage = store.transcriptLanguage;
+    const transcriptLanguagePreference = store.transcriptLanguage;
+    const transcriptLanguage = resolveTranscriptLanguage(
+      transcriptLanguagePreference
+    );
     store.setStatus("transcribing");
     store.setProgress({ message: "Loading speech model…", value: null });
 
@@ -64,7 +68,7 @@ export function useTranscriber() {
           // Nothing about the media itself — not its length, not the text.
           trackEvent("transcription_completed", {
             model,
-            language: transcriptLanguage,
+            language: transcriptLanguagePreference,
           });
           break;
         case "error":
@@ -96,7 +100,12 @@ export function useTranscriber() {
     // long recording the copy this replaces was hundreds of megabytes held for
     // the length of the run.
     workerRef.current.postMessage(
-      { audio, duration, model, language: transcriptLanguage },
+      {
+        audio,
+        duration,
+        model,
+        ...(transcriptLanguage ? { language: transcriptLanguage } : {}),
+      },
       [audio.buffer]
     );
   }, []);

lib/languages.ts

@@ -1,4 +1,5 @@
 export type TranscriptLanguage = "en" | "es" | "fr" | "de" | "zh";
+export type TranscriptLanguagePreference = "auto" | TranscriptLanguage;
 
 export interface TranscriptLanguageInfo {
   label: string;
@@ -11,7 +12,14 @@ export interface TranscriptLanguageInfo {
 
 const LANGUAGE_STORAGE_KEY = "rescript.transcript-language";
 
-export const DEFAULT_TRANSCRIPT_LANGUAGE: TranscriptLanguage = "en";
+export const DEFAULT_TRANSCRIPT_LANGUAGE: TranscriptLanguagePreference = "auto";
+
+export const AUTO_TRANSCRIPT_LANGUAGE_INFO = {
+  label: "Auto-detect",
+  nativeLabel: "Auto-detect",
+  flag: "🌐",
+  code: "AUTO",
+} satisfies TranscriptLanguageInfo;
 
 export const TRANSCRIPT_LANGUAGES: Record<
   TranscriptLanguage,
@@ -57,6 +65,11 @@ export const TRANSCRIPT_LANGUAGE_ORDER: TranscriptLanguage[] = [
   "zh",
 ];
 
+export const TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER: TranscriptLanguagePreference[] = [
+  "auto",
+  ...TRANSCRIPT_LANGUAGE_ORDER,
+];
+
 export function isTranscriptLanguage(
   value: unknown
 ): value is TranscriptLanguage {
@@ -69,20 +82,28 @@ export function isTranscriptLanguage(
   );
 }
 
+export function isTranscriptLanguagePreference(
+  value: unknown
+): value is TranscriptLanguagePreference {
+  return value === "auto" || isTranscriptLanguage(value);
+}
+
 /** Read the last-selected transcript language from localStorage. */
-export function loadTranscriptLanguagePreference(): TranscriptLanguage {
+export function loadTranscriptLanguagePreference(): TranscriptLanguagePreference {
   if (typeof window === "undefined") return DEFAULT_TRANSCRIPT_LANGUAGE;
   try {
     const raw = window.localStorage.getItem(LANGUAGE_STORAGE_KEY);
-    if (isTranscriptLanguage(raw)) return raw;
+    if (isTranscriptLanguagePreference(raw)) return raw;
   } catch {
     // private mode / disabled storage
   }
   return DEFAULT_TRANSCRIPT_LANGUAGE;
 }
 
 /** Persist the selected transcript language for the next visit. */
-export function saveTranscriptLanguagePreference(language: TranscriptLanguage) {
+export function saveTranscriptLanguagePreference(
+  language: TranscriptLanguagePreference
+) {
   if (typeof window === "undefined") return;
   try {
     window.localStorage.setItem(LANGUAGE_STORAGE_KEY, language);

lib/projects.ts

@@ -7,10 +7,10 @@
  */
 
 import { isTranscriptSource, type TranscriptSource } from "./source";
-import type { TranscriptLanguage } from "./languages";
+import type { TranscriptLanguagePreference } from "./languages";
 import {
   DEFAULT_TRANSCRIPT_LANGUAGE,
-  isTranscriptLanguage,
+  isTranscriptLanguagePreference,
 } from "./languages";
 import type { MediaKind } from "./media";
 import type { ManualCut, SceneBoundary, SpeakerInfo, Word } from "./types";
@@ -26,7 +26,7 @@ export interface ProjectMeta {
   mediaKind: MediaKind;
   duration: number;
   source: TranscriptSource;
-  transcriptLanguage: TranscriptLanguage;
+  transcriptLanguage: TranscriptLanguagePreference;
   updatedAt: number;
   createdAt: number;
 }
@@ -137,7 +137,7 @@ export async function listProjects(): Promise<ProjectMeta[]> {
       mediaKind: r.mediaKind,
       duration: r.duration,
       source: projectSource(r),
-      transcriptLanguage: isTranscriptLanguage(r.transcriptLanguage)
+      transcriptLanguage: isTranscriptLanguagePreference(r.transcriptLanguage)
         ? r.transcriptLanguage
         : DEFAULT_TRANSCRIPT_LANGUAGE,
       updatedAt: r.updatedAt,
@@ -181,7 +181,7 @@ export async function putProject(input: ProjectWrite): Promise<string> {
     mediaKind: input.mediaKind,
     duration: input.duration,
     source: isTranscriptSource(input.source) ? input.source : "base",
-    transcriptLanguage: isTranscriptLanguage(input.transcriptLanguage)
+    transcriptLanguage: isTranscriptLanguagePreference(input.transcriptLanguage)
       ? input.transcriptLanguage
       : DEFAULT_TRANSCRIPT_LANGUAGE,
     words: input.words,

lib/store.ts

@@ -31,10 +31,10 @@ import { isTranscriptSource, type TranscriptSource } from "./source";
 import { trackEvent } from "./telemetry";
 import {
   DEFAULT_TRANSCRIPT_LANGUAGE,
-  isTranscriptLanguage,
+  isTranscriptLanguagePreference,
   loadTranscriptLanguagePreference,
   saveTranscriptLanguagePreference,
-  type TranscriptLanguage,
+  type TranscriptLanguagePreference,
 } from "./languages";
 import { detectMediaKind, type MediaKind } from "./media";
 import { buildWaveformPeaks, type WaveformPeaks } from "./waveform";
@@ -80,7 +80,7 @@ interface EditorState {
   /** Transcript source selected on the upload screen (speech model or import). */
   source: TranscriptSource;
   /** Language hint sent to Whisper when transcribing (Parakeet auto-detects). */
-  transcriptLanguage: TranscriptLanguage;
+  transcriptLanguage: TranscriptLanguagePreference;
   /**
    * Caption file parsed on the upload screen when source is "import".
    * Cleared when switching back to a speech model or after media loads.
@@ -150,7 +150,7 @@ interface EditorState {
   /** Delete a saved project; if it is the active one, resets to the home screen. */
   removeProject: (id: string) => Promise<void>;
   setSource: (s: TranscriptSource) => void;
-  setTranscriptLanguage: (language: TranscriptLanguage) => void;
+  setTranscriptLanguage: (language: TranscriptLanguagePreference) => void;
   setPendingTranscript: (t: PendingTranscript | null) => void;
   setDuration: (d: number) => void;
   /**
@@ -428,7 +428,7 @@ export const useEditorStore = create<EditorState>((set, get) => ({
       mediaKind: record.mediaKind,
       duration: record.duration,
       source: isTranscriptSource(record.source) ? record.source : "base",
-      transcriptLanguage: isTranscriptLanguage(record.transcriptLanguage)
+      transcriptLanguage: isTranscriptLanguagePreference(record.transcriptLanguage)
         ? record.transcriptLanguage
         : DEFAULT_TRANSCRIPT_LANGUAGE,
       projectId: record.id,

lib/transcriptionOptions.ts

@@ -0,0 +1,28 @@
+import type {
+  TranscriptLanguage,
+  TranscriptLanguagePreference,
+} from "./languages";
+
+export interface WhisperLanguageOptions {
+  task: "transcribe";
+  language?: string;
+}
+
+/** Resolve a saved UI preference to the concrete hint accepted by ASR backends. */
+export function resolveTranscriptLanguage(
+  preference: TranscriptLanguagePreference
+): TranscriptLanguage | undefined {
+  return preference === "auto" ? undefined : preference;
+}
+
+/**
+ * Whisper must always transcribe in the source language. In automatic mode the
+ * language key is omitted so Whisper can detect it from the audio.
+ */
+export function whisperLanguageOptions(
+  language: string | undefined
+): WhisperLanguageOptions {
+  return language
+    ? { task: "transcribe", language }
+    : { task: "transcribe" };
+}

lib/types.ts

@@ -103,5 +103,5 @@ export interface WorkerRequest {
    * Transcript language: conditions Whisper decoding, selects the CTC aligner,
    * and is passed through for Parakeet alignment (Parakeet ASR still auto-detects).
    */
-  language: import("./languages").TranscriptLanguage;
+  language?: import("./languages").TranscriptLanguage;
 }

package.json

@@ -19,6 +19,7 @@
     "build": "next build",
     "start": "next start",
     "lint": "eslint",
+    "test:transcription": "tsx tests/languages-test.ts && tsx tests/transcription-options-test.ts",
     "postinstall": "patch-package && node scripts/copy-assets.mjs",
     "build:electron": "node scripts/build-electron.mjs",
     "typecheck:electron": "tsc -p electron/tsconfig.json --noEmit",

tests/languages-test.ts

@@ -1,15 +1,20 @@
 import {
   DEFAULT_TRANSCRIPT_LANGUAGE,
   isTranscriptLanguage,
+  isTranscriptLanguagePreference,
+  TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER,
   TRANSCRIPT_LANGUAGE_ORDER,
   TRANSCRIPT_LANGUAGES,
 } from "../lib/languages";
 
 {
-  if (DEFAULT_TRANSCRIPT_LANGUAGE !== "en") {
-    throw new Error("expected default language to be en");
+  if (DEFAULT_TRANSCRIPT_LANGUAGE !== "auto") {
+    throw new Error("expected default language preference to be auto");
   }
   if (isTranscriptLanguage("auto")) throw new Error("did not expect auto to be valid");
+  if (!isTranscriptLanguagePreference("auto")) {
+    throw new Error("expected auto to be a valid preference");
+  }
   if (!isTranscriptLanguage("en")) throw new Error("expected en to be valid");
   if (!isTranscriptLanguage("es")) throw new Error("expected es to be valid");
   if (!isTranscriptLanguage("fr")) throw new Error("expected fr to be valid");
@@ -18,6 +23,14 @@ import {
   if (isTranscriptLanguage("pt")) throw new Error("did not expect pt to be valid");
 }
 
+{
+  if (TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER.join(",") !== "auto,en,es,fr,de,zh") {
+    throw new Error(
+      `unexpected preference order: ${TRANSCRIPT_LANGUAGE_PREFERENCE_ORDER.join(",")}`
+    );
+  }
+}
+
 {
   const labels = TRANSCRIPT_LANGUAGE_ORDER.map(
     (id) => TRANSCRIPT_LANGUAGES[id].nativeLabel

tests/transcription-options-test.ts

@@ -0,0 +1,29 @@
+import {
+  resolveTranscriptLanguage,
+  whisperLanguageOptions,
+} from "../lib/transcriptionOptions";
+import type { TranscriptLanguage } from "../lib/languages";
+
+function assert(value: unknown, message: string): asserts value {
+  if (!value) throw new Error(message);
+}
+
+assert(resolveTranscriptLanguage("auto") === undefined, "auto must omit language");
+assert(resolveTranscriptLanguage("zh") === "zh", "zh must remain concrete");
+
+const automatic = whisperLanguageOptions(undefined);
+assert(automatic.task === "transcribe", "auto must transcribe");
+assert(!("language" in automatic), "auto must not contain a language key");
+
+for (const language of ["en", "es", "fr", "de", "zh"] satisfies TranscriptLanguage[]) {
+  const options = whisperLanguageOptions(language);
+  assert(options.task === "transcribe", `${language} must transcribe`);
+  assert(options.language === language, `${language} must be passed through`);
+  assert(options.task !== ("translate" as "transcribe"), "translate must never be used");
+}
+
+const detected = whisperLanguageOptions("ja");
+assert(detected.task === "transcribe", "a detected language must still transcribe");
+assert(detected.language === "ja", "a detected Whisper language code must pass through");
+
+console.log("ALL TRANSCRIPTION OPTIONS TESTS PASSED");