diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 7fb847dccd..5458d54022 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -1638,6 +1638,7 @@ "api_base": "https://api.xiaomimimo.com/v1", "model": "mimo-v2-tts", "mimo-tts-voice": "mimo_default", + "mimo-tts-voiceclone-audio": "", "mimo-tts-format": "wav", "mimo-tts-style-prompt": "", "mimo-tts-dialect": "", @@ -2604,7 +2605,12 @@ "mimo-tts-voice": { "description": "音色", "type": "string", - "hint": "MiMo TTS 的音色名称。可选值包括 'mimo_default'、'default_en'、'default_zh'。", + "hint": "MiMo TTS 的音色名称。可选值包括 'mimo_default'、'default_en'、'default_zh'。仅用于 mimo-v2.5-tts / mimo-v2-tts 等预置音色模型；使用 mimo-v2.5-tts-voiceclone 模型时本字段会被忽略，请改填下方的'音色复刻参考音频'。", + }, + "mimo-tts-voiceclone-audio": { + "description": "音色复刻参考音频", + "type": "string", + "hint": "仅在模型为 mimo-v2.5-tts-voiceclone（音色复刻）时需要填写。支持本地文件路径、http(s) 链接、或 base64/data URI，音频格式仅支持 mp3、wav，转换后大小不超过 10 MB。留空且选用该模型时会报错。", }, "mimo-tts-format": { "description": "输出格式", diff --git a/astrbot/core/provider/sources/mimo_api_common.py b/astrbot/core/provider/sources/mimo_api_common.py index e2dc6f92bd..837e99e2d2 100644 --- a/astrbot/core/provider/sources/mimo_api_common.py +++ b/astrbot/core/provider/sources/mimo_api_common.py @@ -56,14 +56,20 @@ def build_api_url(api_base: str) -> str: return normalized_api_base + "/chat/completions" -async def prepare_audio_input(audio_source: str) -> tuple[str, list[Path]]: +async def prepare_audio_input( + audio_source: str, + *, + target_format: str | None = "wav", + preserve_mp3: bool = False, +) -> tuple[str, list[Path]]: audio_data = await MediaResolver( audio_source, media_type="audio", default_suffix=".wav", ).to_base64_data( strict=True, - target_format="wav", + target_format=target_format, + preserve_mp3=preserve_mp3, ) if audio_data is None: raise ValueError(f"Invalid audio data: {describe_media_ref(audio_source)}") diff --git a/astrbot/core/provider/sources/mimo_tts_api_source.py b/astrbot/core/provider/sources/mimo_tts_api_source.py index 63389a10b6..2ef5e1fdfb 100644 --- a/astrbot/core/provider/sources/mimo_tts_api_source.py +++ b/astrbot/core/provider/sources/mimo_tts_api_source.py @@ -1,5 +1,7 @@ +import asyncio import base64 import uuid +from pathlib import Path from ..entities import ProviderType from ..provider import TTSProvider @@ -12,9 +14,11 @@ MiMoAPIError, build_api_url, build_headers, + cleanup_files, create_http_client, get_temp_dir, normalize_timeout, + prepare_audio_input, ) @@ -35,6 +39,9 @@ def __init__( self.proxy = provider_config.get("proxy", "") self.timeout = normalize_timeout(provider_config.get("timeout", 20)) self.voice = provider_config.get("mimo-tts-voice", DEFAULT_MIMO_TTS_VOICE) + self.voiceclone_audio_source = provider_config.get( + "mimo-tts-voiceclone-audio", "" + ).strip() self.audio_format = provider_config.get("mimo-tts-format", "wav") self.style_prompt = provider_config.get("mimo-tts-style-prompt", "") self.dialect = provider_config.get("mimo-tts-dialect", "") @@ -43,6 +50,13 @@ def __init__( ) self.set_model(provider_config.get("model", DEFAULT_MIMO_TTS_MODEL)) self.client = create_http_client(self.timeout, self.proxy) + # 音色复刻(voiceclone)参考音频转换结果缓存，避免每次合成都重新读取/转码同一份样本 + self._voiceclone_cache_source: str | None = None + self._voiceclone_cache_data_url: str | None = None + self._voiceclone_cleanup_paths: list[Path] = [] + # TTS provider 实例在管线中是长期共享的，可能被并发请求同时调用； + # 用锁序列化"检查缓存 -> 转换 -> 写入缓存"这一过程，避免重复转换和临时文件泄漏。 + self._voiceclone_lock = asyncio.Lock() def _build_user_prompt(self) -> str | None: seed_text = self.seed_text.strip() @@ -69,7 +83,52 @@ def _build_style_prefix(self) -> str: def _build_assistant_content(self, text: str) -> str: return f"{self._build_style_prefix()}{text}" - def _build_payload(self, text: str) -> dict: + def _is_voiceclone_model(self) -> bool: + return "voiceclone" in self.model_name + + async def _resolve_voiceclone_voice(self) -> str: + """将配置的参考音频样本转换为 voiceclone 所需的 data URL。 + + 结果会按音频来源缓存，避免每次合成请求都重新读取/转码同一份样本。 + 加锁是为了在并发请求下避免重复转换、避免临时文件泄漏或被错误清理。 + """ + if not self.voiceclone_audio_source: + raise MiMoAPIError( + "MiMo TTS voiceclone model (mimo-v2.5-tts-voiceclone) requires a " + "reference audio sample. Please set 'mimo-tts-voiceclone-audio' to " + "a local path, URL, or base64/data URI." + ) + + async with self._voiceclone_lock: + if ( + self._voiceclone_cache_data_url is not None + and self._voiceclone_cache_source == self.voiceclone_audio_source + ): + return self._voiceclone_cache_data_url + + try: + data_url, cleanup_paths = await prepare_audio_input( + self.voiceclone_audio_source, + # MiMo voiceclone 接受 mp3 或 wav；保留原始 mp3 而不强制转 wav， + # 可避免未压缩 PCM 带来的体积膨胀（更容易撞到官方 10 MB 上限）。 + # 其他格式（ogg/flac/silk 等）仍会兜底转换为 wav。 + target_format=None, + preserve_mp3=True, + ) + except Exception as exc: + raise MiMoAPIError( + f"Failed to prepare MiMo TTS voiceclone reference audio " + f"'{self.voiceclone_audio_source}': {exc}" + ) from exc + + # 旧缓存的临时文件不再需要，先清理掉再写入新结果 + cleanup_files(self._voiceclone_cleanup_paths) + self._voiceclone_cleanup_paths = cleanup_paths + self._voiceclone_cache_source = self.voiceclone_audio_source + self._voiceclone_cache_data_url = data_url + return data_url + + def _build_payload(self, text: str, voice_value: str | None = None) -> dict: messages: list[dict[str, str]] = [] user_prompt = self._build_user_prompt() @@ -91,7 +150,9 @@ def _build_payload(self, text: str) -> dict: audio_params = {"format": self.audio_format} # voice design 模型不支持 audio.voice 参数 if "voicedesign" not in self.model_name: - audio_params["voice"] = self.voice + audio_params["voice"] = ( + voice_value if voice_value is not None else self.voice + ) return { "model": self.model_name, @@ -100,10 +161,14 @@ def _build_payload(self, text: str) -> dict: } async def get_audio(self, text: str) -> str: + voice_value = None + if self._is_voiceclone_model(): + voice_value = await self._resolve_voiceclone_voice() + response = await self.client.post( build_api_url(self.api_base), headers=build_headers(self.chosen_api_key), - json=self._build_payload(text), + json=self._build_payload(text, voice_value), ) try: @@ -129,5 +194,7 @@ async def get_audio(self, text: str) -> str: return str(output_path) async def terminate(self): + cleanup_files(self._voiceclone_cleanup_paths) + self._voiceclone_cleanup_paths = [] if self.client: await self.client.aclose() diff --git a/dashboard/src/i18n/locales/en-US/features/config-metadata.json b/dashboard/src/i18n/locales/en-US/features/config-metadata.json index 31d1362b26..dda8a5632b 100644 --- a/dashboard/src/i18n/locales/en-US/features/config-metadata.json +++ b/dashboard/src/i18n/locales/en-US/features/config-metadata.json @@ -1616,7 +1616,11 @@ }, "mimo-tts-voice": { "description": "Voice", - "hint": "MiMo TTS voice name. Supported values include 'mimo_default', 'default_en', and 'default_zh'." + "hint": "MiMo TTS voice name. Supported values include 'mimo_default', 'default_en', and 'default_zh'. Only used by preset-voice models; ignored when the model is mimo-v2.5-tts-voiceclone (use 'Voice Clone Reference Audio' instead)." + }, + "mimo-tts-voiceclone-audio": { + "description": "Voice Clone Reference Audio", + "hint": "Required only when the model is mimo-v2.5-tts-voiceclone. Accepts a local file path, an http(s) URL, or a base64/data URI. Only mp3 and wav are supported, and the encoded size must not exceed 10 MB." }, "mimo-tts-format": { "description": "Output format", diff --git a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json index f0a1294d7c..bcdc44fa3d 100644 --- a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json +++ b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json @@ -1613,7 +1613,11 @@ }, "mimo-tts-voice": { "description": "Голос", - "hint": "Имя голоса MiMo TTS. Поддерживаются значения 'mimo_default', 'default_en' и 'default_zh'." + "hint": "Имя голоса MiMo TTS. Поддерживаются значения 'mimo_default', 'default_en' и 'default_zh'. Используется только для моделей с предустановленными голосами; игнорируется для модели mimo-v2.5-tts-voiceclone (используйте 'Референсное аудио для клонирования голоса')." + }, + "mimo-tts-voiceclone-audio": { + "description": "Референсное аудио для клонирования голоса", + "hint": "Требуется только для модели mimo-v2.5-tts-voiceclone. Принимает локальный путь к файлу, ссылку http(s) или base64/data URI. Поддерживаются только mp3 и wav, размер после кодирования не должен превышать 10 МБ." }, "mimo-tts-format": { "description": "Формат вывода", diff --git a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json index 6ae988383f..81b6f5eb52 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json +++ b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json @@ -1618,7 +1618,11 @@ }, "mimo-tts-voice": { "description": "音色", - "hint": "MiMo TTS 的音色名称。可选值包括 'mimo_default'、'default_en'、'default_zh'。" + "hint": "MiMo TTS 的音色名称。可选值包括 'mimo_default'、'default_en'、'default_zh'。仅用于预置音色模型；使用 mimo-v2.5-tts-voiceclone 模型时本字段会被忽略，请改填'音色复刻参考音频'。" + }, + "mimo-tts-voiceclone-audio": { + "description": "音色复刻参考音频", + "hint": "仅在模型为 mimo-v2.5-tts-voiceclone（音色复刻）时需要填写。支持本地文件路径、http(s) 链接、或 base64/data URI，音频格式仅支持 mp3、wav，转换后大小不超过 10 MB。" }, "mimo-tts-format": { "description": "输出格式", diff --git a/tests/test_mimo_api_sources.py b/tests/test_mimo_api_sources.py index c0b7abc7c2..b41a8503a5 100644 --- a/tests/test_mimo_api_sources.py +++ b/tests/test_mimo_api_sources.py @@ -21,6 +21,7 @@ def _make_tts_provider(overrides: dict | None = None) -> ProviderMiMoTTSAPI: "model": "mimo-v2-tts", "api_key": "test-key", "mimo-tts-voice": "mimo_default", + "mimo-tts-voiceclone-audio": "", "mimo-tts-format": "wav", "mimo-tts-seed-text": "seed text", } @@ -175,6 +176,211 @@ def test_mimo_headers_use_single_authorization_method(): } +def test_mimo_tts_build_payload_voice_value_override_takes_precedence(): + """voice_value 显式传入时应覆盖 self.voice (用于 voiceclone data URL)""" + provider = _make_tts_provider( + { + "model": "mimo-v2.5-tts-voiceclone", + "mimo-tts-voice": "mimo_default", + "mimo-tts-seed-text": "", + } + ) + try: + payload = provider._build_payload( + "hello", voice_value=MIMO_STT_TEST_AUDIO_DATA_URL + ) + assert payload["audio"]["voice"] == MIMO_STT_TEST_AUDIO_DATA_URL + finally: + asyncio.run(provider.terminate()) + + +def test_mimo_tts_is_voiceclone_model_detection(): + provider = _make_tts_provider({"model": "mimo-v2.5-tts-voiceclone"}) + try: + assert provider._is_voiceclone_model() is True + finally: + asyncio.run(provider.terminate()) + + provider2 = _make_tts_provider({"model": "mimo-v2.5-tts"}) + try: + assert provider2._is_voiceclone_model() is False + finally: + asyncio.run(provider2.terminate()) + + +@pytest.mark.asyncio +async def test_mimo_tts_voiceclone_without_audio_raises_clear_error(): + """选用 voiceclone 模型但未配置参考音频时应给出清晰报错，而不是悄悄发出错误请求""" + provider = _make_tts_provider( + { + "model": "mimo-v2.5-tts-voiceclone", + "mimo-tts-voiceclone-audio": "", + } + ) + try: + with pytest.raises(MiMoAPIError, match="mimo-tts-voiceclone-audio"): + await provider.get_audio("hello") + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_mimo_tts_voiceclone_sends_audio_data_url_as_voice(monkeypatch): + """voiceclone 模型应将参考音频转换的 data URL 填入 audio.voice 字段""" + provider = _make_tts_provider( + { + "model": "mimo-v2.5-tts-voiceclone", + "mimo-tts-voiceclone-audio": "/tmp/reference_voice.mp3", + "mimo-tts-voice": "mimo_default", # 应被忽略 + "mimo-tts-seed-text": "", + } + ) + + call_count = {"n": 0} + + async def fake_prepare_audio_input(audio_source: str, **kwargs): + call_count["n"] += 1 + assert audio_source == "/tmp/reference_voice.mp3" + assert kwargs == {"target_format": None, "preserve_mp3": True} + return MIMO_STT_TEST_AUDIO_DATA_URL, [] + + monkeypatch.setattr( + "astrbot.core.provider.sources.mimo_tts_api_source.prepare_audio_input", + fake_prepare_audio_input, + ) + + captured: dict = {} + + class _Response: + status_code = 200 + text = '{"choices":[{"message":{"audio":{"data":"ZmFrZQ=="}}}]}' + + def raise_for_status(self): + return None + + def json(self): + return {"choices": [{"message": {"audio": {"data": "ZmFrZQ=="}}}]} + + async def fake_post(_url, headers=None, json=None): + captured["json"] = json + return _Response() + + async def fake_aclose(): + return None + + provider.client = SimpleNamespace(post=fake_post, aclose=fake_aclose) + + try: + output_path = await provider.get_audio("hello") + assert captured["json"]["audio"]["voice"] == MIMO_STT_TEST_AUDIO_DATA_URL + assert output_path.endswith(".wav") + assert call_count["n"] == 1 + + # 第二次调用同一来源的音频应使用缓存，不应重复转换 + await provider.get_audio("hello again") + assert call_count["n"] == 1 + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_mimo_tts_voiceclone_cache_refreshes_on_source_change(monkeypatch): + """更换参考音频来源后应重新转换，而不是继续复用旧的缓存""" + provider = _make_tts_provider( + { + "model": "mimo-v2.5-tts-voiceclone", + "mimo-tts-voiceclone-audio": "/tmp/voice_a.mp3", + "mimo-tts-seed-text": "", + } + ) + + seen_sources: list[str] = [] + + async def fake_prepare_audio_input(audio_source: str, **kwargs): + seen_sources.append(audio_source) + return f"data:audio/mpeg;base64,{audio_source}", [] + + monkeypatch.setattr( + "astrbot.core.provider.sources.mimo_tts_api_source.prepare_audio_input", + fake_prepare_audio_input, + ) + + try: + first_voice = await provider._resolve_voiceclone_voice() + assert first_voice == "data:audio/mpeg;base64,/tmp/voice_a.mp3" + + provider.voiceclone_audio_source = "/tmp/voice_b.mp3" + second_voice = await provider._resolve_voiceclone_voice() + assert second_voice == "data:audio/mpeg;base64,/tmp/voice_b.mp3" + assert seen_sources == ["/tmp/voice_a.mp3", "/tmp/voice_b.mp3"] + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_mimo_tts_voiceclone_preserves_mp3_instead_of_forcing_wav(monkeypatch): + """voiceclone 应保留原始 mp3 而不强制转 wav,避免体积不必要地膨胀""" + provider = _make_tts_provider( + { + "model": "mimo-v2.5-tts-voiceclone", + "mimo-tts-voiceclone-audio": "/tmp/reference_voice.mp3", + "mimo-tts-seed-text": "", + } + ) + + captured_kwargs: dict = {} + + async def fake_prepare_audio_input(audio_source: str, **kwargs): + captured_kwargs.update(kwargs) + return "data:audio/mp3;base64,ZmFrZQ==", [] + + monkeypatch.setattr( + "astrbot.core.provider.sources.mimo_tts_api_source.prepare_audio_input", + fake_prepare_audio_input, + ) + + try: + voice = await provider._resolve_voiceclone_voice() + assert voice == "data:audio/mp3;base64,ZmFrZQ==" + assert captured_kwargs == {"target_format": None, "preserve_mp3": True} + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_mimo_tts_voiceclone_concurrent_calls_convert_audio_once(monkeypatch): + """并发调用 get_audio 时，参考音频只应被转换一次，不应出现重复转换或临时文件泄漏""" + provider = _make_tts_provider( + { + "model": "mimo-v2.5-tts-voiceclone", + "mimo-tts-voiceclone-audio": "/tmp/reference_voice.mp3", + "mimo-tts-seed-text": "", + } + ) + + call_count = {"n": 0} + + async def fake_prepare_audio_input(audio_source: str, **kwargs): + call_count["n"] += 1 + # 模拟较慢的转码过程，放大并发窗口 + await asyncio.sleep(0.05) + return MIMO_STT_TEST_AUDIO_DATA_URL, [] + + monkeypatch.setattr( + "astrbot.core.provider.sources.mimo_tts_api_source.prepare_audio_input", + fake_prepare_audio_input, + ) + + try: + results = await asyncio.gather( + *[provider._resolve_voiceclone_voice() for _ in range(5)] + ) + assert results == [MIMO_STT_TEST_AUDIO_DATA_URL] * 5 + assert call_count["n"] == 1 + finally: + await provider.terminate() + + @pytest.mark.asyncio async def test_mimo_tts_get_audio_handles_empty_choices(): provider = _make_tts_provider() @@ -270,6 +476,7 @@ async def to_base64_data(self, **kwargs): assert kwargs == { "strict": True, "target_format": "wav", + "preserve_mp3": False, } return _ResolvedAudio()