heygen-com · sumit-uit · Sep 12, 2026
diff --git a/skills/media-use/audio/references/tts.md b/skills/media-use/audio/references/tts.md
@@ -39,8 +39,16 @@ Use another voice only for a documented reason, and write the reason down.
 | Order | Provider          | Env trigger                                 | Voice IDs                                   | Word timestamps                           | Audio format         |
 | ----- | ----------------- | ------------------------------------------- | ------------------------------------------- | ----------------------------------------- | -------------------- |
 | 1     | HeyGen (Starfish) | `$HEYGEN_API_KEY` / `~/.heygen/credentials` | UUIDs from `GET /v3/voices?engine=starfish` | **Yes** (`word_timestamps[]` in response) | mp3 → wav via ffmpeg |
-| 2     | ElevenLabs        | `$ELEVENLABS_API_KEY`                       | UUIDs from elevenlabs.io dashboard          | No                                        | mp3 → wav via ffmpeg |
-| 3     | Kokoro-82M        | always (local fallback)                     | `am_michael`, `af_heart`, … (54 voices)     | No                                        | wav direct           |
+| 2     | Chatterbox (local, self-hosted) | live health check at `$CHATTERBOX_BASE_URL` (default `http://127.0.0.1:4123/v1`) | `default` (server's configured zero-shot reference clip — no catalog) | No | wav direct |
+| 3     | ElevenLabs        | `$ELEVENLABS_API_KEY`                       | UUIDs from elevenlabs.io dashboard          | No                                        | mp3 → wav via ffmpeg |
+| 4     | Kokoro-82M        | always (local fallback)                     | `am_michael`, `af_heart`, … (54 voices)     | No                                        | wav direct           |
+
+Chatterbox ranks above ElevenLabs/Kokoro once its local server is reachable — a
+cloned owner voice beats a generic one whenever it's up. It's a zero-shot voice
+clone server (no fixed voice catalog — swap the server's `VOICE_SAMPLE_PATH` or
+its voice-library endpoint to change whose voice comes out); pass `--voice` to
+override the default "default" id if the server exposes named voices. No word
+timings, same as ElevenLabs/Kokoro — the caller chains a Whisper/Parakeet pass.
 
 ```bash
 # Local Kokoro CLI
@@ -86,6 +94,7 @@ node skills/media-use/audio/scripts/heygen-tts.mjs --list   # public starfish vo
 | Goal                                                      | Use                                                 |
 | --------------------------------------------------------- | --------------------------------------------------- |
 | Best voice quality + word timestamps in one call          | **HeyGen**                                          |
+| A specific cloned voice (e.g. the channel owner's own)    | **Chatterbox** (local server must be running)       |
 | Drop-in cloud TTS, big voice catalog                      | **ElevenLabs**                                      |
 | Offline, no API key, fast iteration                       | **Kokoro**                                          |
 | Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) |

diff --git a/skills/media-use/audio/scripts/audio.mjs b/skills/media-use/audio/scripts/audio.mjs
@@ -15,7 +15,7 @@
 //
 // ── audio_request.json (input) ────────────────────────────────────────────────
 //   {
-//     "provider": "auto",          // auto|heygen|elevenlabs|kokoro (override: --provider)
+//     "provider": "auto",          // auto|heygen|chatterbox|elevenlabs|kokoro (override: --provider)
 //     "lang": "en", "speed": 1.0,
 //     "lines": [                   // one TTS unit each; id joins back to the caller's model
 //       { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
@@ -126,7 +126,7 @@ let ttsProvider = prev.tts_provider ?? null;
 let voiceId = prev.voice_id ?? null;
 if (only.has("tts") && lines.length) {
   try {
-    ttsProvider = pickProvider(
+    ttsProvider = await pickProvider(
       providerOverride || (request.provider === "auto" ? null : request.provider),
     );
   } catch (e) {

diff --git a/skills/media-use/audio/scripts/lib/tts.mjs b/skills/media-use/audio/scripts/lib/tts.mjs
@@ -6,13 +6,21 @@ import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs";
 //        Direct v3 REST (NOT `hyperframes tts`, which in the published build is
 //        Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps
 //        in the same call, so no separate transcribe pass.
-//   2. ElevenLabs         — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
+//   2. Chatterbox (local, self-hosted) — $CHATTERBOX_BASE_URL (default
+//        http://127.0.0.1:4123/v1), health-checked at pick time. Zero-shot
+//        voice clone server (see moneyturbo's chatterbox_tts / config.toml
+//        [chatterbox]) — takes priority over ElevenLabs/Kokoro once reachable,
+//        since a cloned owner voice beats a generic one whenever it's up. No
 //        word timings → caller chains transcribeWav().
-//   3. Kokoro-82M (local) — always available, via the published `hyperframes tts`
+//   3. ElevenLabs         — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
+//        word timings → caller chains transcribeWav().
+//   4. Kokoro-82M (local) — always available, via the published `hyperframes tts`
 //        CLI. No word timings → caller chains transcribeWav().
 //
 // "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never
-// by the CLI — see the note above.
+// by the CLI — see the note above. "Chatterbox available" is decided by a live
+// health check (best-effort, short timeout) since it's a local server that may
+// or may not be running.
 
 import { spawn, spawnSync } from "node:child_process";
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
@@ -34,20 +42,50 @@ export function elevenlabsAvailable() {
   return r.status === 0;
 }
 
+export function chatterboxBaseUrl() {
+  return (process.env.CHATTERBOX_BASE_URL || "http://127.0.0.1:4123/v1").replace(/\/$/, "");
+}
+
+// Live health check, short timeout — this is a local server that may or may
+// not be running, unlike HeyGen (credential presence) or ElevenLabs (env +
+// python package). Best-effort: any failure (timeout, connection refused,
+// non-200) means "not available", never throws.
+export async function chatterboxAvailable() {
+  try {
+    const base = chatterboxBaseUrl().replace(/\/v1$/, "");
+    const res = await fetch(`${base}/health`, { signal: AbortSignal.timeout(1500) });
+    if (!res.ok) return false;
+    const body = await res.json();
+    return body?.status === "healthy" && body?.model_loaded === true;
+  } catch {
+    return false;
+  }
+}
+
 // First available provider wins; an explicit choice is honored (and validated).
-export function pickProvider(userProvider) {
+// Chatterbox ranks above ElevenLabs/Kokoro (once reachable) since a cloned
+// owner voice is preferred over a generic one whenever the local server is up.
+export async function pickProvider(userProvider) {
   if (userProvider) {
-    if (!["heygen", "elevenlabs", "kokoro"].includes(userProvider))
-      throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro)`);
+    if (!["heygen", "chatterbox", "elevenlabs", "kokoro"].includes(userProvider))
+      throw new Error(
+        `invalid provider "${userProvider}" (heygen | chatterbox | elevenlabs | kokoro)`,
+      );
     if (userProvider === "heygen" && !heygenAvailable())
       throw new Error(
         "provider=heygen but no HeyGen credentials (set $HEYGEN_API_KEY or run `npx hyperframes auth login`)",
       );
+    if (userProvider === "chatterbox" && !(await chatterboxAvailable()))
+      throw new Error(
+        `provider=chatterbox but no healthy server at ${chatterboxBaseUrl()} (start it, or set $CHATTERBOX_BASE_URL)`,
+      );
     if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY)
       throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set");
     return userProvider;
   }
-  return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro";
+  if (heygenAvailable()) return "heygen";
+  if (await chatterboxAvailable()) return "chatterbox";
+  return elevenlabsAvailable() ? "elevenlabs" : "kokoro";
 }
 
 // ── voice resolution ──────────────────────────────────────────────────────────
@@ -56,6 +94,7 @@ export function pickProvider(userProvider) {
 // their own defaults.
 export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
   if (userVoice) return userVoice;
+  if (provider === "chatterbox") return "default"; // server's configured VOICE_SAMPLE_PATH clone
   if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
   if (provider === "kokoro") {
     if (lang === "en") return "am_michael";
@@ -252,6 +291,7 @@ export async function synthesizeOne({
   hyperframesDir,
 }) {
   if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
+  if (provider === "chatterbox") return synthesizeChatterbox({ text, speed, wavAbs });
   if (provider === "elevenlabs") {
     // The Python helper writes straight to wavAbs; unlike heygen (transcodeToWav)
     // and kokoro (the `hyperframes tts` CLI), it does NOT create the parent dir,
@@ -344,6 +384,44 @@ export async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }, d
   }
 }
 
+// Chatterbox's OpenAI-compatible /audio/speech endpoint returns real WAV bytes
+// regardless of the requested response_format (confirmed against
+// travisvn/chatterbox-tts-api) — no ffmpeg transcode needed, unlike HeyGen's
+// mp3-over-a-CDN-URL path. No word timings → caller chains transcribeWav().
+// `deps` is injectable for tests; production uses the real fetch/fs impls.
+export async function synthesizeChatterbox({ text, speed, wavAbs }, deps = {}) {
+  const fetchImpl = deps.fetch ?? fetch;
+  const mkdir = deps.mkdirSync ?? mkdirSync;
+  const write = deps.writeFileSync ?? writeFileSync;
+  try {
+    mkdir(dirname(wavAbs), { recursive: true });
+    const res = await fetchImpl(`${chatterboxBaseUrl()}/audio/speech`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model: "chatterbox",
+        input: text,
+        voice: "default-Female", // label only — server clones its configured reference clip
+        response_format: "wav",
+        speed: Math.max(0.25, Math.min(4.0, Number(speed) || 1.0)),
+      }),
+      signal: AbortSignal.timeout(600_000),
+    });
+    if (!res.ok) {
+      return {
+        ok: false,
+        words: null,
+        error: `chatterbox /audio/speech HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`,
+      };
+    }
+    const bytes = Buffer.from(await res.arrayBuffer());
+    write(wavAbs, bytes);
+    return { ok: true, words: null };
+  } catch (e) {
+    return { ok: false, words: null, error: e?.message ? String(e.message) : String(e) };
+  }
+}
+
 // ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the
 // flat [{id,text,start,end}] word array, or null. Each call uses a throwaway
 // --dir so parallel scenes don't collide on transcript.json.

diff --git a/skills/media-use/audio/scripts/lib/tts.test.mjs b/skills/media-use/audio/scripts/lib/tts.test.mjs
@@ -8,6 +8,10 @@ import {
   ffprobeDuration,
   synthesizeOne,
   synthesizeHeygen,
+  synthesizeChatterbox,
+  chatterboxAvailable,
+  chatterboxBaseUrl,
+  pickProvider,
   synthResult,
 } from "./tts.mjs";
 
@@ -155,3 +159,95 @@ test("synthResult names a non-zero subprocess exit", () => {
   assert.equal(res.ok, false);
   assert.match(res.error, /kokoro .* exited with status 2/);
 });
+
+test("chatterboxBaseUrl defaults to the local server, trims a trailing slash from the override", () => {
+  const saved = process.env.CHATTERBOX_BASE_URL;
+  try {
+    delete process.env.CHATTERBOX_BASE_URL;
+    assert.equal(chatterboxBaseUrl(), "http://127.0.0.1:4123/v1");
+    process.env.CHATTERBOX_BASE_URL = "http://example.internal:9000/v1/";
+    assert.equal(chatterboxBaseUrl(), "http://example.internal:9000/v1");
+  } finally {
+    if (saved === undefined) delete process.env.CHATTERBOX_BASE_URL;
+    else process.env.CHATTERBOX_BASE_URL = saved;
+  }
+});
+
+test("chatterboxAvailable is false on a connection failure (no server running)", async () => {
+  const saved = process.env.CHATTERBOX_BASE_URL;
+  try {
+    // Port 1 is reserved and nothing will ever answer on it — a fast, reliable "down" server.
+    process.env.CHATTERBOX_BASE_URL = "http://127.0.0.1:1/v1";
+    assert.equal(await chatterboxAvailable(), false);
+  } finally {
+    if (saved === undefined) delete process.env.CHATTERBOX_BASE_URL;
+    else process.env.CHATTERBOX_BASE_URL = saved;
+  }
+});
+
+test("pickProvider(chatterbox) rejects when no server is reachable", async () => {
+  const saved = process.env.CHATTERBOX_BASE_URL;
+  try {
+    process.env.CHATTERBOX_BASE_URL = "http://127.0.0.1:1/v1";
+    await assert.rejects(() => pickProvider("chatterbox"), /no healthy server/);
+  } finally {
+    if (saved === undefined) delete process.env.CHATTERBOX_BASE_URL;
+    else process.env.CHATTERBOX_BASE_URL = saved;
+  }
+});
+
+test("pickProvider rejects an unknown provider name, listing chatterbox in the valid set", async () => {
+  await assert.rejects(() => pickProvider("bogus"), /heygen \| chatterbox \| elevenlabs \| kokoro/);
+});
+
+test("synthesizeChatterbox creates the output dir and writes the response bytes", async () => {
+  const dir = mkdtempSync(join(tmpdir(), "tts-chatterbox-mkdir-"));
+  try {
+    const wavAbs = join(dir, "assets", "voice", "line-0.wav"); // nested, not yet created
+    const fakeBytes = new Uint8Array([1, 2, 3, 4]);
+    const res = await synthesizeChatterbox(
+      { text: "hi", speed: 1, wavAbs },
+      {
+        fetch: async () => ({
+          ok: true,
+          status: 200,
+          arrayBuffer: async () => fakeBytes.buffer,
+        }),
+      },
+    );
+    assert.equal(res.ok, true);
+    assert.equal(res.words, null);
+    assert.ok(existsSync(wavAbs), "wav file should be written");
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test("synthesizeChatterbox surfaces a non-200 response with its status and body", async () => {
+  const res = await synthesizeChatterbox(
+    { text: "hi", speed: 1, wavAbs: "/tmp/does-not-matter.wav" },
+    {
+      fetch: async () => ({
+        ok: false,
+        status: 503,
+        text: async () => "model not loaded",
+      }),
+    },
+  );
+  assert.equal(res.ok, false);
+  assert.match(res.error, /HTTP 503/);
+  assert.match(res.error, /model not loaded/);
+});
+
+test("synthesizeChatterbox surfaces a thrown network error", async () => {
+  const res = await synthesizeChatterbox(
+    { text: "hi", speed: 1, wavAbs: "/tmp/does-not-matter.wav" },
+    {
+      fetch: async () => {
+        throw new Error("fetch failed: ECONNREFUSED");
+      },
+    },
+  );
+  assert.equal(res.ok, false);
+  assert.match(res.error, /ECONNREFUSED/);
+});