diff --git a/skills/media-use/audio/references/tts.md b/skills/media-use/audio/references/tts.md index d9fbd43349..39975e58cb 100644 --- a/skills/media-use/audio/references/tts.md +++ b/skills/media-use/audio/references/tts.md @@ -39,8 +39,16 @@ Use another voice only for a documented reason, and write the reason down. | Order | Provider | Env trigger | Voice IDs | Word timestamps | Audio format | | ----- | ----------------- | ------------------------------------------- | ------------------------------------------- | ----------------------------------------- | -------------------- | | 1 | HeyGen (Starfish) | `$HEYGEN_API_KEY` / `~/.heygen/credentials` | UUIDs from `GET /v3/voices?engine=starfish` | **Yes** (`word_timestamps[]` in response) | mp3 → wav via ffmpeg | -| 2 | ElevenLabs | `$ELEVENLABS_API_KEY` | UUIDs from elevenlabs.io dashboard | No | mp3 → wav via ffmpeg | -| 3 | Kokoro-82M | always (local fallback) | `am_michael`, `af_heart`, … (54 voices) | No | wav direct | +| 2 | Chatterbox (local, self-hosted) | live health check at `$CHATTERBOX_BASE_URL` (default `http://127.0.0.1:4123/v1`) | `default` (server's configured zero-shot reference clip — no catalog) | No | wav direct | +| 3 | ElevenLabs | `$ELEVENLABS_API_KEY` | UUIDs from elevenlabs.io dashboard | No | mp3 → wav via ffmpeg | +| 4 | Kokoro-82M | always (local fallback) | `am_michael`, `af_heart`, … (54 voices) | No | wav direct | + +Chatterbox ranks above ElevenLabs/Kokoro once its local server is reachable — a +cloned owner voice beats a generic one whenever it's up. It's a zero-shot voice +clone server (no fixed voice catalog — swap the server's `VOICE_SAMPLE_PATH` or +its voice-library endpoint to change whose voice comes out); pass `--voice` to +override the default "default" id if the server exposes named voices. No word +timings, same as ElevenLabs/Kokoro — the caller chains a Whisper/Parakeet pass. ```bash # Local Kokoro CLI @@ -86,6 +94,7 @@ node skills/media-use/audio/scripts/heygen-tts.mjs --list # public starfish vo | Goal | Use | | --------------------------------------------------------- | --------------------------------------------------- | | Best voice quality + word timestamps in one call | **HeyGen** | +| A specific cloned voice (e.g. the channel owner's own) | **Chatterbox** (local server must be running) | | Drop-in cloud TTS, big voice catalog | **ElevenLabs** | | Offline, no API key, fast iteration | **Kokoro** | | Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) | diff --git a/skills/media-use/audio/scripts/audio.mjs b/skills/media-use/audio/scripts/audio.mjs index 167eb679ca..4e96c5c491 100644 --- a/skills/media-use/audio/scripts/audio.mjs +++ b/skills/media-use/audio/scripts/audio.mjs @@ -15,7 +15,7 @@ // // ── audio_request.json (input) ──────────────────────────────────────────────── // { -// "provider": "auto", // auto|heygen|elevenlabs|kokoro (override: --provider) +// "provider": "auto", // auto|heygen|chatterbox|elevenlabs|kokoro (override: --provider) // "lang": "en", "speed": 1.0, // "lines": [ // one TTS unit each; id joins back to the caller's model // { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] } @@ -126,7 +126,7 @@ let ttsProvider = prev.tts_provider ?? null; let voiceId = prev.voice_id ?? null; if (only.has("tts") && lines.length) { try { - ttsProvider = pickProvider( + ttsProvider = await pickProvider( providerOverride || (request.provider === "auto" ? null : request.provider), ); } catch (e) { diff --git a/skills/media-use/audio/scripts/lib/tts.mjs b/skills/media-use/audio/scripts/lib/tts.mjs index e341bf8edb..33452db711 100644 --- a/skills/media-use/audio/scripts/lib/tts.mjs +++ b/skills/media-use/audio/scripts/lib/tts.mjs @@ -6,13 +6,21 @@ import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs"; // Direct v3 REST (NOT `hyperframes tts`, which in the published build is // Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps // in the same call, so no separate transcribe pass. -// 2. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No +// 2. Chatterbox (local, self-hosted) — $CHATTERBOX_BASE_URL (default +// http://127.0.0.1:4123/v1), health-checked at pick time. Zero-shot +// voice clone server (see moneyturbo's chatterbox_tts / config.toml +// [chatterbox]) — takes priority over ElevenLabs/Kokoro once reachable, +// since a cloned owner voice beats a generic one whenever it's up. No // word timings → caller chains transcribeWav(). -// 3. Kokoro-82M (local) — always available, via the published `hyperframes tts` +// 3. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No +// word timings → caller chains transcribeWav(). +// 4. Kokoro-82M (local) — always available, via the published `hyperframes tts` // CLI. No word timings → caller chains transcribeWav(). // // "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never -// by the CLI — see the note above. +// by the CLI — see the note above. "Chatterbox available" is decided by a live +// health check (best-effort, short timeout) since it's a local server that may +// or may not be running. import { spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; @@ -34,20 +42,50 @@ export function elevenlabsAvailable() { return r.status === 0; } +export function chatterboxBaseUrl() { + return (process.env.CHATTERBOX_BASE_URL || "http://127.0.0.1:4123/v1").replace(/\/$/, ""); +} + +// Live health check, short timeout — this is a local server that may or may +// not be running, unlike HeyGen (credential presence) or ElevenLabs (env + +// python package). Best-effort: any failure (timeout, connection refused, +// non-200) means "not available", never throws. +export async function chatterboxAvailable() { + try { + const base = chatterboxBaseUrl().replace(/\/v1$/, ""); + const res = await fetch(`${base}/health`, { signal: AbortSignal.timeout(1500) }); + if (!res.ok) return false; + const body = await res.json(); + return body?.status === "healthy" && body?.model_loaded === true; + } catch { + return false; + } +} + // First available provider wins; an explicit choice is honored (and validated). -export function pickProvider(userProvider) { +// Chatterbox ranks above ElevenLabs/Kokoro (once reachable) since a cloned +// owner voice is preferred over a generic one whenever the local server is up. +export async function pickProvider(userProvider) { if (userProvider) { - if (!["heygen", "elevenlabs", "kokoro"].includes(userProvider)) - throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro)`); + if (!["heygen", "chatterbox", "elevenlabs", "kokoro"].includes(userProvider)) + throw new Error( + `invalid provider "${userProvider}" (heygen | chatterbox | elevenlabs | kokoro)`, + ); if (userProvider === "heygen" && !heygenAvailable()) throw new Error( "provider=heygen but no HeyGen credentials (set $HEYGEN_API_KEY or run `npx hyperframes auth login`)", ); + if (userProvider === "chatterbox" && !(await chatterboxAvailable())) + throw new Error( + `provider=chatterbox but no healthy server at ${chatterboxBaseUrl()} (start it, or set $CHATTERBOX_BASE_URL)`, + ); if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY) throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set"); return userProvider; } - return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro"; + if (heygenAvailable()) return "heygen"; + if (await chatterboxAvailable()) return "chatterbox"; + return elevenlabsAvailable() ? "elevenlabs" : "kokoro"; } // ── voice resolution ────────────────────────────────────────────────────────── @@ -56,6 +94,7 @@ export function pickProvider(userProvider) { // their own defaults. export async function resolveVoiceId({ provider, userVoice, lang = "en" }) { if (userVoice) return userVoice; + if (provider === "chatterbox") return "default"; // server's configured VOICE_SAMPLE_PATH clone if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel if (provider === "kokoro") { if (lang === "en") return "am_michael"; @@ -252,6 +291,7 @@ export async function synthesizeOne({ hyperframesDir, }) { if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }); + if (provider === "chatterbox") return synthesizeChatterbox({ text, speed, wavAbs }); if (provider === "elevenlabs") { // The Python helper writes straight to wavAbs; unlike heygen (transcodeToWav) // and kokoro (the `hyperframes tts` CLI), it does NOT create the parent dir, @@ -344,6 +384,44 @@ export async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }, d } } +// Chatterbox's OpenAI-compatible /audio/speech endpoint returns real WAV bytes +// regardless of the requested response_format (confirmed against +// travisvn/chatterbox-tts-api) — no ffmpeg transcode needed, unlike HeyGen's +// mp3-over-a-CDN-URL path. No word timings → caller chains transcribeWav(). +// `deps` is injectable for tests; production uses the real fetch/fs impls. +export async function synthesizeChatterbox({ text, speed, wavAbs }, deps = {}) { + const fetchImpl = deps.fetch ?? fetch; + const mkdir = deps.mkdirSync ?? mkdirSync; + const write = deps.writeFileSync ?? writeFileSync; + try { + mkdir(dirname(wavAbs), { recursive: true }); + const res = await fetchImpl(`${chatterboxBaseUrl()}/audio/speech`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: "chatterbox", + input: text, + voice: "default-Female", // label only — server clones its configured reference clip + response_format: "wav", + speed: Math.max(0.25, Math.min(4.0, Number(speed) || 1.0)), + }), + signal: AbortSignal.timeout(600_000), + }); + if (!res.ok) { + return { + ok: false, + words: null, + error: `chatterbox /audio/speech HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`, + }; + } + const bytes = Buffer.from(await res.arrayBuffer()); + write(wavAbs, bytes); + return { ok: true, words: null }; + } catch (e) { + return { ok: false, words: null, error: e?.message ? String(e.message) : String(e) }; + } +} + // ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the // flat [{id,text,start,end}] word array, or null. Each call uses a throwaway // --dir so parallel scenes don't collide on transcript.json. diff --git a/skills/media-use/audio/scripts/lib/tts.test.mjs b/skills/media-use/audio/scripts/lib/tts.test.mjs index a4f651908d..fa7e5391ef 100644 --- a/skills/media-use/audio/scripts/lib/tts.test.mjs +++ b/skills/media-use/audio/scripts/lib/tts.test.mjs @@ -8,6 +8,10 @@ import { ffprobeDuration, synthesizeOne, synthesizeHeygen, + synthesizeChatterbox, + chatterboxAvailable, + chatterboxBaseUrl, + pickProvider, synthResult, } from "./tts.mjs"; @@ -155,3 +159,95 @@ test("synthResult names a non-zero subprocess exit", () => { assert.equal(res.ok, false); assert.match(res.error, /kokoro .* exited with status 2/); }); + +test("chatterboxBaseUrl defaults to the local server, trims a trailing slash from the override", () => { + const saved = process.env.CHATTERBOX_BASE_URL; + try { + delete process.env.CHATTERBOX_BASE_URL; + assert.equal(chatterboxBaseUrl(), "http://127.0.0.1:4123/v1"); + process.env.CHATTERBOX_BASE_URL = "http://example.internal:9000/v1/"; + assert.equal(chatterboxBaseUrl(), "http://example.internal:9000/v1"); + } finally { + if (saved === undefined) delete process.env.CHATTERBOX_BASE_URL; + else process.env.CHATTERBOX_BASE_URL = saved; + } +}); + +test("chatterboxAvailable is false on a connection failure (no server running)", async () => { + const saved = process.env.CHATTERBOX_BASE_URL; + try { + // Port 1 is reserved and nothing will ever answer on it — a fast, reliable "down" server. + process.env.CHATTERBOX_BASE_URL = "http://127.0.0.1:1/v1"; + assert.equal(await chatterboxAvailable(), false); + } finally { + if (saved === undefined) delete process.env.CHATTERBOX_BASE_URL; + else process.env.CHATTERBOX_BASE_URL = saved; + } +}); + +test("pickProvider(chatterbox) rejects when no server is reachable", async () => { + const saved = process.env.CHATTERBOX_BASE_URL; + try { + process.env.CHATTERBOX_BASE_URL = "http://127.0.0.1:1/v1"; + await assert.rejects(() => pickProvider("chatterbox"), /no healthy server/); + } finally { + if (saved === undefined) delete process.env.CHATTERBOX_BASE_URL; + else process.env.CHATTERBOX_BASE_URL = saved; + } +}); + +test("pickProvider rejects an unknown provider name, listing chatterbox in the valid set", async () => { + await assert.rejects(() => pickProvider("bogus"), /heygen \| chatterbox \| elevenlabs \| kokoro/); +}); + +test("synthesizeChatterbox creates the output dir and writes the response bytes", async () => { + const dir = mkdtempSync(join(tmpdir(), "tts-chatterbox-mkdir-")); + try { + const wavAbs = join(dir, "assets", "voice", "line-0.wav"); // nested, not yet created + const fakeBytes = new Uint8Array([1, 2, 3, 4]); + const res = await synthesizeChatterbox( + { text: "hi", speed: 1, wavAbs }, + { + fetch: async () => ({ + ok: true, + status: 200, + arrayBuffer: async () => fakeBytes.buffer, + }), + }, + ); + assert.equal(res.ok, true); + assert.equal(res.words, null); + assert.ok(existsSync(wavAbs), "wav file should be written"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("synthesizeChatterbox surfaces a non-200 response with its status and body", async () => { + const res = await synthesizeChatterbox( + { text: "hi", speed: 1, wavAbs: "/tmp/does-not-matter.wav" }, + { + fetch: async () => ({ + ok: false, + status: 503, + text: async () => "model not loaded", + }), + }, + ); + assert.equal(res.ok, false); + assert.match(res.error, /HTTP 503/); + assert.match(res.error, /model not loaded/); +}); + +test("synthesizeChatterbox surfaces a thrown network error", async () => { + const res = await synthesizeChatterbox( + { text: "hi", speed: 1, wavAbs: "/tmp/does-not-matter.wav" }, + { + fetch: async () => { + throw new Error("fetch failed: ECONNREFUSED"); + }, + }, + ); + assert.equal(res.ok, false); + assert.match(res.error, /ECONNREFUSED/); +});