diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index 4a7bcec5c9..6e708f0f45 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -9,6 +9,9 @@ on: - "tests/test_cli.py" - "funasr/auto/auto_model.py" - "funasr/utils/load_utils.py" + - "funasr/bin/realtime_ws.py" + - "tests/test_realtime_ws_service.py" + - "tests/test_realtime_ws_hf_overrides.py" - "funasr/utils/timestamp_tools.py" - "funasr/models/paraformer/model.py" - "funasr/models/contextual_paraformer/model.py" @@ -40,6 +43,9 @@ on: - "tests/test_cli.py" - "funasr/auto/auto_model.py" - "funasr/utils/load_utils.py" + - "funasr/bin/realtime_ws.py" + - "tests/test_realtime_ws_service.py" + - "tests/test_realtime_ws_hf_overrides.py" - "funasr/utils/timestamp_tools.py" - "funasr/models/paraformer/model.py" - "funasr/models/contextual_paraformer/model.py" @@ -96,7 +102,7 @@ jobs: - name: Install CPU runtime and requested NumPy version run: | python -m pip install torch==2.10.0 torchaudio==2.10.0 --index-url https://download.pytorch.org/whl/cpu - python -m pip install "numpy==$NUMPY_VERSION" -e . pytest einops pytorch-wpe --report install-report.json + python -m pip install "numpy==$NUMPY_VERSION" -e . pytest websockets einops pytorch-wpe --report install-report.json - name: Record resolved environment and validate dependencies run: | git rev-parse HEAD | tee source-commit.txt @@ -113,6 +119,8 @@ jobs: tests/test_pypi_metadata.py \ tests/test_sensevoice_word_timestamps.py \ tests/test_pcm_input_format.py \ + tests/test_realtime_ws_service.py \ + tests/test_realtime_ws_hf_overrides.py \ tests/test_load_audio_bytes.py \ tests/test_timestamp_tools.py \ tests/test_punc_model_none.py \ diff --git a/funasr/bin/realtime_ws.py b/funasr/bin/realtime_ws.py index bce661b762..b6f96b54de 100644 --- a/funasr/bin/realtime_ws.py +++ b/funasr/bin/realtime_ws.py @@ -1348,6 +1348,9 @@ def load_models(args): gpu_memory_utilization=getattr(args, 'gpu_memory_utilization', 0.8), max_model_len=getattr(args, 'max_model_len', 2048), enforce_eager=getattr(args, 'enforce_eager', False), + vllm_kwargs={"hf_overrides": getattr(args, "hf_overrides", None)} + if getattr(args, "hf_overrides", None) + else {}, ) _vllm_engine = RealtimeBatchingEngine( engine, @@ -1614,12 +1617,25 @@ async def main(args): await asyncio.Future() +def _parse_hf_overrides(value): + overrides = json.loads(value) + if not isinstance(overrides, dict): + raise argparse.ArgumentTypeError("--hf-overrides must be a JSON object") + return overrides + + def build_arg_parser(): parser = argparse.ArgumentParser(description="Fun-ASR-Nano Streaming WebSocket Server") parser.add_argument("--port", type=int, default=10095) parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512") parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"]) parser.add_argument("--device", type=str, default="cuda:0") + parser.add_argument( + "--hf-overrides", + type=_parse_hf_overrides, + default=None, + help=("JSON object forwarded to the vLLM model configuration."), + ) parser.add_argument( "--vad-device", type=str, diff --git a/tests/test_realtime_ws_hf_overrides.py b/tests/test_realtime_ws_hf_overrides.py new file mode 100644 index 0000000000..f4ff6b7aa6 --- /dev/null +++ b/tests/test_realtime_ws_hf_overrides.py @@ -0,0 +1,41 @@ +import pytest + +from test_realtime_ws_service import load_service_module + + +@pytest.mark.parametrize("value", ["[]", "[1]", '"model"', "1", "true", "false", "null"]) +def test_cli_rejects_non_object_hf_overrides(value, capsys): + parser = load_service_module().build_arg_parser() + + with pytest.raises(SystemExit) as exc: + parser.parse_args(["--hf-overrides", value]) + + assert exc.value.code == 2 + assert "JSON object" in capsys.readouterr().err + + +@pytest.mark.parametrize("value", ["{", "{bad}", '{"architectures":}']) +def test_cli_rejects_malformed_hf_overrides(value): + parser = load_service_module().build_arg_parser() + + with pytest.raises(SystemExit) as exc: + parser.parse_args(["--hf-overrides", value]) + + assert exc.value.code == 2 + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("{}", {}), + ('{"architectures":["Qwen2ForCausalLM"]}', {"architectures": ["Qwen2ForCausalLM"]}), + ('{"rope_scaling":{"factor":2.0},"use_cache":false}', + {"rope_scaling": {"factor": 2.0}, "use_cache": False}), + ], +) +def test_cli_preserves_hf_override_objects(value, expected): + parser = load_service_module().build_arg_parser() + + args = parser.parse_args(["--hf-overrides", value]) + + assert args.hf_overrides == expected diff --git a/tests/test_realtime_ws_service.py b/tests/test_realtime_ws_service.py index 82292ecd5d..21e5cd416f 100644 --- a/tests/test_realtime_ws_service.py +++ b/tests/test_realtime_ws_service.py @@ -119,6 +119,17 @@ def test_cli_defaults_disable_speaker_and_bound_partial_window(): assert args.log_decode_profile is False assert args.vad_device == "cpu" assert args.vad_ncpu == 1 + assert args.hf_overrides is None + + +def test_cli_accepts_hf_overrides_json_object(): + module = load_service_module() + + args = module.build_arg_parser().parse_args( + ["--hf-overrides", '{"architectures":["FunASRForCausalLM"]}'] + ) + + assert args.hf_overrides == {"architectures": ["FunASRForCausalLM"]} def test_cli_accepts_client_endpoint_mode(): @@ -1806,6 +1817,39 @@ def test_load_models_reads_postprocess_hotword_file(monkeypatch, tmp_path): assert auto_model_calls == [] +def test_load_models_forwards_hf_overrides_to_vllm(monkeypatch): + module = load_service_module() + module._vllm_engine = None + module._asr_kwargs = None + module._vad_model = None + module._spk_model = None + vllm_calls = [] + + import funasr + + monkeypatch.setattr(funasr, "AutoModel", lambda *args, **kwargs: object()) + vllm_stub = types.ModuleType("funasr.auto.auto_model_vllm") + vllm_stub.AutoModelVLLM = ( + lambda **kwargs: vllm_calls.append(kwargs) or object() + ) + monkeypatch.setitem(sys.modules, "funasr.auto.auto_model_vllm", vllm_stub) + + args = module.build_arg_parser().parse_args( + [ + "--endpoint-mode", + "client", + "--hf-overrides", + '{"architectures":["FunASRForCausalLM"]}', + ] + ) + + module.load_models(args) + + assert vllm_calls[0]["vllm_kwargs"] == { + "hf_overrides": {"architectures": ["FunASRForCausalLM"]} + } + + def test_two_hour_session_keeps_audio_bounded_and_duration_absolute(): module = load_service_module() sample_rate = 10