From 51b1a149a82d4866b746b32febc0e953742b76b5 Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Sun, 1 Nov 2026 11:42:22 +0800 Subject: [PATCH] fix(runtime): repair evaluation CLIs and honor GPU device selection Keep runtime evaluation import-safe and use the requested inference device. Signed-off-by: LauraGPT --- .github/workflows/test-runtime-eval.yml | 58 +++++ .../libtorch/funasr_torch/paraformer_bin.py | 14 +- runtime/python/utils/README.md | 50 ++++ runtime/python/utils/runtime_eval.py | 160 ++++++++++++ runtime/python/utils/test_cer.py | 60 +---- runtime/python/utils/test_rtf.py | 83 +----- runtime/python/utils/test_rtf_gpu.py | 86 +----- tests/test_runtime_eval_cli.py | 246 ++++++++++++++++++ tests/test_runtime_eval_native.py | 163 ++++++++++++ 9 files changed, 711 insertions(+), 209 deletions(-) create mode 100644 .github/workflows/test-runtime-eval.yml create mode 100644 runtime/python/utils/README.md create mode 100644 runtime/python/utils/runtime_eval.py create mode 100644 tests/test_runtime_eval_cli.py create mode 100644 tests/test_runtime_eval_native.py diff --git a/.github/workflows/test-runtime-eval.yml b/.github/workflows/test-runtime-eval.yml new file mode 100644 index 0000000000..2e3a1901ea --- /dev/null +++ b/.github/workflows/test-runtime-eval.yml @@ -0,0 +1,58 @@ +name: Validate runtime evaluation tools + +on: + pull_request: + paths: + - "runtime/python/utils/**" + - "runtime/python/libtorch/funasr_torch/**" + - "runtime/python/onnxruntime/funasr_onnx/**" + - "tests/test_runtime_eval_*.py" + - ".github/workflows/test-runtime-eval.yml" + push: + branches: [main] + paths: + - "runtime/python/utils/**" + - "runtime/python/libtorch/funasr_torch/**" + - "runtime/python/onnxruntime/funasr_onnx/**" + - "tests/test_runtime_eval_*.py" + - ".github/workflows/test-runtime-eval.yml" + +permissions: + contents: read + +jobs: + runtime-eval: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + python: ["3.10", "3.12"] + env: + HF_HUB_OFFLINE: "1" + MODELSCOPE_OFFLINE: "1" + OMP_NUM_THREADS: "1" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install CPU runtime test dependencies + run: | + python -m pip install torch==2.8.0 --index-url https://download.pytorch.org/whl/cpu + python -m pip install pytest "numpy<2" onnx==1.19.0 onnxruntime==1.23.2 librosa kaldi-native-fbank PyYAML jieba sentencepiece + - name: Run CLI and native offline runtime regressions + run: | + git rev-parse HEAD > source-commit.txt + python -m pip freeze > resolved-dependencies.txt + python -m pytest -q tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py --junitxml=runtime-eval-results.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: runtime-eval-${{ matrix.python }} + path: | + source-commit.txt + resolved-dependencies.txt + runtime-eval-results.xml diff --git a/runtime/python/libtorch/funasr_torch/paraformer_bin.py b/runtime/python/libtorch/funasr_torch/paraformer_bin.py index 8e404126a7..fe405753cd 100644 --- a/runtime/python/libtorch/funasr_torch/paraformer_bin.py +++ b/runtime/python/libtorch/funasr_torch/paraformer_bin.py @@ -75,7 +75,8 @@ def __init__( self.converter = TokenIDConverter(token_list) self.tokenizer = CharTokenizer() self.frontend = WavFrontend(cmvn_file=cmvn_file, **config["frontend_conf"]) - self.ort_infer = torch.jit.load(model_file) + self.device = torch.device("cpu" if int(device_id) == -1 else f"cuda:{int(device_id)}") + self.ort_infer = torch.jit.load(model_file, map_location=self.device).eval() self.batch_size = batch_size self.device_id = device_id self.plot_timestamp_to = plot_timestamp_to @@ -98,12 +99,11 @@ def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs) -> feats, feats_len = self.extract_feat(waveform_list[beg_idx:end_idx]) try: with torch.no_grad(): - if int(self.device_id) == -1: - outputs = self.ort_infer(feats, feats_len) - am_scores, valid_token_lens = outputs[0], outputs[1] - else: - outputs = self.ort_infer(feats.cuda(), feats_len.cuda()) - am_scores, valid_token_lens = outputs[0].cpu(), outputs[1].cpu() + outputs = self.ort_infer( + feats.to(self.device), feats_len.to(self.device) + ) + outputs = tuple(output.cpu() for output in outputs) + am_scores, valid_token_lens = outputs[0], outputs[1] if len(outputs) == 4: # for BiCifParaformer Inference us_alphas, us_peaks = outputs[2], outputs[3] diff --git a/runtime/python/utils/README.md b/runtime/python/utils/README.md new file mode 100644 index 0000000000..e0beee747b --- /dev/null +++ b/runtime/python/utils/README.md @@ -0,0 +1,50 @@ +# Runtime Evaluation Tools + +Run these commands from a FunASR checkout with the chosen runtime and its +dependencies available. Importing the scripts or using `--help` does not load a +runtime, audio, or model. ONNX and Torch backends are selected independently. + +Use the checkout's runtime modules when evaluating a source change: + +```sh +export PYTHONPATH="$PWD/runtime/python/onnxruntime:$PWD/runtime/python/libtorch${PYTHONPATH:+:$PYTHONPATH}" +python runtime/python/utils/test_rtf.py --model_dir /path/to/exported-model --wav_file wav.scp +python runtime/python/utils/test_rtf_gpu.py --model_dir /path/to/exported-model --wav_file wav.scp --device_id 0 --batch_size 8 +python runtime/python/utils/test_cer.py --backend torch --model_dir /path/to/exported-model --wav_file wav.scp --output_dir transcripts +``` + +`wav.scp` contains an utterance ID followed by whitespace and an audio path. +Paths may contain spaces. Blank lines are ignored; empty or malformed lists +are rejected. Supply existing local exported models to avoid runtime download +or export behavior. Torch uses `model.torchscript`; ONNX uses `model.onnx`. +The corresponding quantized names are selected with `--quantize true`. + +The serial RTF and transcript tools default to CPU (`--device_id -1`). The +batched GPU tool defaults to CUDA device 0 and rejects negative device IDs. +It requires CUDA-capable Torch or ONNX Runtime, with compatible CUDA/cuDNN +libraries. Missing ONNX CUDA providers cause an error instead of a CPU timing +being reported as a GPU benchmark. ONNX may still place supported graph +operations on CPU; provider availability alone is not proof that every node +runs on CUDA. Use runtime profiling when auditing actual kernel placement. + +RTF is elapsed wall time divided by actual input audio duration. There is an +untimed audio-duration prepass and, by default, 30 warmups on the first audio +file (`--warmup 0` disables warmups). Timed model calls include file loading, +frontend, inference and decoding; Torch CUDA is synchronized around timing. +The serial tool invokes each file separately. The GPU tool passes the list to +the backend, whose `--batch_size` controls batching. These numbers are not +model-only kernel timings and depend on cache state and hardware. + +The transcript tool writes UTF-8 `text` and space-delimited `token` files. It +does not compute CER/WER or alter references. A missing inference result is an +error, not a successful empty transcript or benchmark. + +Offline regression tests use small synthetic graphs and generated audio: + +```sh +python -m pytest tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py +FUNASR_TEST_DEVICE=0 python -m pytest tests/test_runtime_eval_native.py +``` + +Native tests require Torch, ONNX, ONNX Runtime and the runtimes' audio/tokenizer +dependencies. They test device and CLI mechanics, not pretrained ASR accuracy. diff --git a/runtime/python/utils/runtime_eval.py b/runtime/python/utils/runtime_eval.py new file mode 100644 index 0000000000..a689cb4688 --- /dev/null +++ b/runtime/python/utils/runtime_eval.py @@ -0,0 +1,160 @@ +"""Shared command-line boundaries for the standalone runtime evaluation tools.""" + +import argparse +import importlib +import math +from pathlib import Path +import time + + +def _boolean(value): + lowered = value.lower() + if lowered in {"true", "yes", "t", "y", "1"}: + return True + if lowered in {"false", "no", "f", "n", "0"}: + return False + raise argparse.ArgumentTypeError("expected a boolean") + + +def _positive(value): + number = int(value) + if number < 1: + raise argparse.ArgumentTypeError("must be positive") + return number + + +def _nonnegative(value): + number = int(value) + if number < 0: + raise argparse.ArgumentTypeError("must be nonnegative") + return number + + +def parse_args(argv=None, *, gpu=False, cer=False): + parser = argparse.ArgumentParser() + parser.add_argument("--model_dir", required=True) + parser.add_argument("--backend", choices=["onnx", "torch"], default="onnx") + parser.add_argument("--wav_file", required=True, help="WAV SCP: utterance ID and audio path") + parser.add_argument("--quantize", type=_boolean, default=False) + parser.add_argument("--intra_op_num_threads", type=_positive, default=1) + parser.add_argument( + "--device_id", + type=_nonnegative if gpu else int, + default=0 if gpu else -1, + help="CUDA device index; -1 for CPU", + ) + parser.add_argument("--batch_size", type=_positive, default=1) + if cer: + parser.add_argument("--output_dir", required=True) + else: + parser.add_argument("--warmup", type=_nonnegative, default=30) + args = parser.parse_args(argv) + if args.device_id < -1: + parser.error("--device_id must be -1 or a CUDA device index") + return args + + +def read_wav_scp(path): + entries = [] + with open(path, encoding="utf-8") as source: + for number, line in enumerate(source, 1): + if not line.strip(): + continue + fields = line.strip().split(maxsplit=1) + if len(fields) != 2: + raise ValueError(f"{path}:{number}: expected utterance ID and audio path") + entries.append(tuple(fields)) + if not entries: + raise ValueError(f"{path}: empty WAV list") + return entries + + +def load_model(args): + backend = importlib.import_module("funasr_" + args.backend) + model = backend.Paraformer( + args.model_dir, + batch_size=args.batch_size, + device_id=str(args.device_id), + quantize=args.quantize, + intra_op_num_threads=args.intra_op_num_threads, + ) + if args.backend == "onnx" and args.device_id >= 0: + providers = model.ort_infer.session.get_providers() + if "CUDAExecutionProvider" not in providers: + raise RuntimeError("CUDAExecutionProvider is unavailable; refusing a CPU GPU benchmark") + return model + + +def _infer(model, paths): + result = model(paths) + expected = 1 if isinstance(paths, str) else len(paths) + if len(result) != expected: + raise RuntimeError(f"inference returned {len(result)} results for {expected} audio files") + return result + + +def transcribe(args): + entries = read_wav_scp(args.wav_file) + model = load_model(args) + output = Path(args.output_dir) + output.mkdir(parents=True, exist_ok=True) + with (output / "text").open("w", encoding="utf-8") as text_file, (output / "token").open( + "w", encoding="utf-8" + ) as token_file: + for utterance, path in entries: + row = _infer(model, path)[0] + if isinstance(row["preds"], str): + text, tokens = row["preds"], row["raw_tokens"] + else: + text, tokens = row["preds"] + if not isinstance(tokens, str): + tokens = " ".join(tokens) + text_file.write(f"{utterance} {text}\n") + token_file.write(f"{utterance} {tokens}\n") + + +def benchmark(args, *, batched=False): + entries = read_wav_scp(args.wav_file) + import librosa + + durations = [] + for _, path in entries: + audio, sample_rate = librosa.load(path, sr=None) + duration = len(audio) / sample_rate + if not math.isfinite(duration) or duration <= 0: + raise ValueError(f"{path}: audio duration must be positive") + durations.append(duration) + model = load_model(args) + print(f"backend: {args.backend}, device_id: {args.device_id}") + + def synchronize(): + if args.backend == "torch" and args.device_id >= 0: + import torch + + torch.cuda.synchronize(args.device_id) + + warmup_elapsed = 0.0 + for index in range(args.warmup): + synchronize() + start = time.perf_counter() + result = _infer(model, entries[0][1]) + synchronize() + elapsed = time.perf_counter() - start + warmup_elapsed += elapsed + average = warmup_elapsed / (index + 1) + print(result) + print(f"num: 1, time, {elapsed}, avg: {average}, rtf: {average / durations[0]}") + + synchronize() + start = time.perf_counter() + if batched: + _infer(model, [path for _, path in entries]) + else: + for _, path in entries: + _infer(model, path) + synchronize() + elapsed = time.perf_counter() - start + duration = sum(durations) + print(f"total_time_comput_ms: {elapsed * 1000:.6f}") + print(f"total_time_wav_ms: {duration * 1000:.6f}") + print(f"total_rtf: {elapsed / duration}") diff --git a/runtime/python/utils/test_cer.py b/runtime/python/utils/test_cer.py index 58285c2488..06a57a9a6d 100644 --- a/runtime/python/utils/test_cer.py +++ b/runtime/python/utils/test_cer.py @@ -1,56 +1,14 @@ -import os -import time -import sys -import librosa -from funasr.utils.types import str2bool +"""Write runtime transcripts and tokens for external CER/WER scoring.""" -import argparse -parser = argparse.ArgumentParser() -parser.add_argument("--model_dir", type=str, required=True) -parser.add_argument("--backend", type=str, default="onnx", help='["onnx", "torch"]') -parser.add_argument("--wav_file", type=str, default=None, help="amp fallback number") -parser.add_argument("--quantize", type=str2bool, default=False, help="quantized model") -parser.add_argument( - "--intra_op_num_threads", type=int, default=1, help="intra_op_num_threads for onnx" -) -parser.add_argument("--output_dir", type=str, default=None, help="amp fallback number") -args = parser.parse_args() +def main(argv=None): + if __package__: + from .runtime_eval import parse_args, transcribe + else: + from runtime_eval import parse_args, transcribe + transcribe(parse_args(argv, cer=True)) -from funasr.runtime.python.libtorch.funasr_torch import Paraformer -if args.backend == "onnx": - from funasr.runtime.python.onnxruntime.funasr_onnx import Paraformer - -model = Paraformer( - args.model_dir, - batch_size=1, - quantize=args.quantize, - intra_op_num_threads=args.intra_op_num_threads, -) - -wav_file_f = open(args.wav_file, "r") -wav_files = wav_file_f.readlines() - -output_dir = args.output_dir -if not os.path.exists(output_dir): - os.makedirs(output_dir) -if os.name == "nt": # Windows - newline = "\r\n" -else: # Linux Mac - newline = "\n" -text_f = open(os.path.join(output_dir, "text"), "w", newline=newline) -token_f = open(os.path.join(output_dir, "token"), "w", newline=newline) - -for i, wav_path_i in enumerate(wav_files): - wav_name, wav_path = wav_path_i.strip().split() - result = model(wav_path) - text_i = "{} {}\n".format(wav_name, result[0]["preds"][0]) - token_i = "{} {}\n".format(wav_name, result[0]["preds"][1]) - text_f.write(text_i) - text_f.flush() - token_f.write(token_i) - token_f.flush() -text_f.close() -token_f.close() +if __name__ == "__main__": + main() diff --git a/runtime/python/utils/test_rtf.py b/runtime/python/utils/test_rtf.py index 8e50c34f87..16c899c223 100644 --- a/runtime/python/utils/test_rtf.py +++ b/runtime/python/utils/test_rtf.py @@ -1,79 +1,14 @@ -import time -import sys -import librosa -from funasr.utils.type_utils import str2bool +"""Measure serial end-to-end runtime RTF, including frontend and audio loading.""" -import argparse -parser = argparse.ArgumentParser() -parser.add_argument("--model_dir", type=str, required=True) -parser.add_argument("--backend", type=str, default="onnx", help='["onnx", "torch"]') -parser.add_argument("--wav_file", type=str, default=None, help="amp fallback number") -parser.add_argument("--quantize", type=str2bool, default=False, help="quantized model") -parser.add_argument( - "--intra_op_num_threads", type=int, default=1, help="intra_op_num_threads for onnx" -) -args = parser.parse_args() +def main(argv=None): + if __package__: + from .runtime_eval import benchmark, parse_args + else: + from runtime_eval import benchmark, parse_args + benchmark(parse_args(argv)) -from funasr.runtime.python.libtorch.funasr_torch import Paraformer -if args.backend == "onnx": - from funasr.runtime.python.onnxruntime.funasr_onnx import Paraformer - -model = Paraformer( - args.model_dir, - batch_size=1, - quantize=args.quantize, - intra_op_num_threads=args.intra_op_num_threads, -) - -wav_file_f = open(args.wav_file, "r") -wav_files = wav_file_f.readlines() - -# warm-up -total = 0.0 -num = 30 -wav_path = ( - wav_files[0].split("\t")[1].strip() - if "\t" in wav_files[0] - else wav_files[0].split(" ")[1].strip() -) -for i in range(num): - beg_time = time.time() - result = model(wav_path) - end_time = time.time() - duration = end_time - beg_time - total += duration - print(result) - print( - "num: {}, time, {}, avg: {}, rtf: {}".format( - len(wav_path), duration, total / (i + 1), (total / (i + 1)) / 5.53 - ) - ) - -# infer time -beg_time = time.time() -for i, wav_path_i in enumerate(wav_files): - wav_path = ( - wav_path_i.split("\t")[1].strip() - if "\t" in wav_path_i - else wav_path_i.split(" ")[1].strip() - ) - result = model(wav_path) -end_time = time.time() -duration = (end_time - beg_time) * 1000 -print("total_time_comput_ms: {}".format(int(duration))) - -duration_time = 0.0 -for i, wav_path_i in enumerate(wav_files): - wav_path = ( - wav_path_i.split("\t")[1].strip() - if "\t" in wav_path_i - else wav_path_i.split(" ")[1].strip() - ) - waveform, _ = librosa.load(wav_path, sr=16000) - duration_time += len(waveform) / 16.0 -print("total_time_wav_ms: {}".format(int(duration_time))) - -print("total_rtf: {:.5}".format(duration / duration_time)) +if __name__ == "__main__": + main() diff --git a/runtime/python/utils/test_rtf_gpu.py b/runtime/python/utils/test_rtf_gpu.py index 38d4294011..172400d033 100644 --- a/runtime/python/utils/test_rtf_gpu.py +++ b/runtime/python/utils/test_rtf_gpu.py @@ -1,82 +1,14 @@ -import time -import sys -import librosa -from funasr.utils.types import str2bool +"""Measure batched GPU runtime RTF without silently benchmarking CPU fallback.""" -import argparse -parser = argparse.ArgumentParser() -parser.add_argument("--model_dir", type=str, required=True) -parser.add_argument("--backend", type=str, default="onnx", help='["onnx", "torch"]') -parser.add_argument("--wav_file", type=str, default=None, help="amp fallback number") -parser.add_argument("--quantize", type=str2bool, default=False, help="quantized model") -parser.add_argument( - "--intra_op_num_threads", type=int, default=1, help="intra_op_num_threads for onnx" -) -parser.add_argument("--batch_size", type=int, default=1, help="batch_size for onnx") -args = parser.parse_args() +def main(argv=None): + if __package__: + from .runtime_eval import benchmark, parse_args + else: + from runtime_eval import benchmark, parse_args + benchmark(parse_args(argv, gpu=True), batched=True) -from funasr.runtime.python.libtorch.funasr_torch import Paraformer -if args.backend == "onnx": - from funasr.runtime.python.onnxruntime.funasr_onnx import Paraformer - -model = Paraformer( - args.model_dir, - batch_size=args.batch_size, - quantize=args.quantize, - intra_op_num_threads=args.intra_op_num_threads, -) - -wav_file_f = open(args.wav_file, "r") -wav_files = wav_file_f.readlines() - -# warm-up -total = 0.0 -num = 30 -wav_path = ( - wav_files[0].split("\t")[1].strip() - if "\t" in wav_files[0] - else wav_files[0].split(" ")[1].strip() -) -for i in range(num): - beg_time = time.time() - result = model(wav_path) - end_time = time.time() - duration = end_time - beg_time - total += duration - print(result) - print( - "num: {}, time, {}, avg: {}, rtf: {}".format( - len(wav_path), duration, total / (i + 1), (total / (i + 1)) / 5.53 - ) - ) - -# infer time -wav_path = [] -beg_time = time.time() -for i, wav_path_i in enumerate(wav_files): - wav_path_i = ( - wav_path_i.split("\t")[1].strip() - if "\t" in wav_path_i - else wav_path_i.split(" ")[1].strip() - ) - wav_path += [wav_path_i] -result = model(wav_path) -end_time = time.time() -duration = (end_time - beg_time) * 1000 -print("total_time_comput_ms: {}".format(int(duration))) - -duration_time = 0.0 -for i, wav_path_i in enumerate(wav_files): - wav_path = ( - wav_path_i.split("\t")[1].strip() - if "\t" in wav_path_i - else wav_path_i.split(" ")[1].strip() - ) - waveform, _ = librosa.load(wav_path, sr=16000) - duration_time += len(waveform) / 16.0 -print("total_time_wav_ms: {}".format(int(duration_time))) - -print("total_rtf: {:.5}".format(duration / duration_time)) +if __name__ == "__main__": + main() diff --git a/tests/test_runtime_eval_cli.py b/tests/test_runtime_eval_cli.py new file mode 100644 index 0000000000..9eca635a2a --- /dev/null +++ b/tests/test_runtime_eval_cli.py @@ -0,0 +1,246 @@ +"""CLI boundary tests; backend stand-ins do not claim native ASR/GPU coverage.""" + +import importlib.util +import os +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "runtime/python/utils" +NAMES = ["test_cer.py", "test_rtf.py", "test_rtf_gpu.py"] + + +def invoke(name, *args, env=None): + return subprocess.run( + [sys.executable, "-S", str(SCRIPTS / name), *map(str, args)], + text=True, + capture_output=True, + timeout=30, + env=env, + ) + + +@pytest.mark.parametrize("name", NAMES) +def test_import_without_runtime_dependencies_or_cli_arguments(name): + result = subprocess.run( + [ + sys.executable, + "-S", + "-c", + "import runpy; runpy.run_path(" + repr(str(SCRIPTS / name)) + ")", + ], + text=True, + capture_output=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + assert not result.stdout + + +@pytest.mark.parametrize("name", NAMES) +def test_help_without_runtime_dependencies(name): + result = invoke(name, "--help") + assert result.returncode == 0, result.stderr + assert "--model_dir" in result.stdout and "--wav_file" in result.stdout + + +@pytest.mark.parametrize("name", NAMES) +def test_missing_required_arguments_are_parser_errors(name): + result = invoke(name) + assert result.returncode == 2 + assert "--model_dir" in result.stderr and "--wav_file" in result.stderr + + +def helper(): + path = SCRIPTS / "runtime_eval.py" + assert path.exists(), "Shared runtime CLI boundary is missing" + spec = importlib.util.spec_from_file_location("runtime_eval_under_test", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_wav_list_preserves_spaces_and_tabs(tmp_path): + scp = tmp_path / "wav.scp" + scp.write_text("first\t/some path/a.wav\n\nsecond /b path/b.wav\n") + assert helper().read_wav_scp(scp) == [ + ("first", "/some path/a.wav"), + ("second", "/b path/b.wav"), + ] + + +@pytest.mark.parametrize("content", ["", "\n \n", "missing-path\n"]) +def test_invalid_wav_list_is_rejected(tmp_path, content): + scp = tmp_path / "wav.scp" + scp.write_text(content) + with pytest.raises(ValueError): + helper().read_wav_scp(scp) + + +@pytest.fixture +def backends(tmp_path): + # Only model execution and media decoding are external here. Native device + # placement and inference have separate real-backend validation. + code = """ +import os +class Session: + def get_providers(self): + return ["CPUExecutionProvider"] if os.environ.get("CPU_FALLBACK") else ["CUDAExecutionProvider", "CPUExecutionProvider"] +class Engine: + session = Session() +class Paraformer: + def __init__(self, model_dir, batch_size=1, device_id="-1", **kwargs): + assert str(device_id) == os.environ["EXPECTED_DEVICE"], device_id + assert batch_size == int(os.environ.get("EXPECTED_BATCH", "1")) + self.ort_infer = Engine() + def __call__(self, paths): + if os.environ.get("EMPTY_RESULT"): + return [] + if isinstance(paths, str): + paths = [paths] + assert all(path.endswith("audio file.wav") for path in paths), paths + return [{"preds": ("fixture text", ["fixture", "text"])} for _ in paths] +""" + packages = tmp_path / "packages" + packages.mkdir() + for name in ["funasr_onnx", "funasr_torch"]: + target = packages / name + target.mkdir() + (target / "__init__.py").write_text(code) + (packages / "librosa.py").write_text("def load(path, sr=None):\n return [0] * 16000, 8000\n") + (packages / "torch.py").write_text( + "class CUDA:\n def synchronize(self, device=None): pass\n" "cuda = CUDA()\n" + ) + scp = tmp_path / "wav.scp" + scp.write_text("a /some path/audio file.wav\nb\t/other/audio file.wav\n") + env = dict(os.environ, PYTHONPATH=str(packages), EXPECTED_DEVICE="-1") + return packages, scp, env + + +@pytest.mark.parametrize("name", NAMES) +@pytest.mark.parametrize("backend", ["onnx", "torch"]) +def test_cli_uses_only_selected_backend_and_passes_device(tmp_path, backends, name, backend): + packages, scp, env = backends + other = "funasr_torch" if backend == "onnx" else "funasr_onnx" + (packages / other / "__init__.py").write_text('raise RuntimeError("wrong backend imported")\n') + args = ["--model_dir", "local-model", "--wav_file", scp, "--backend", backend] + if name == "test_cer.py": + args += ["--output_dir", tmp_path / "output"] + else: + args += ["--warmup", "1"] + if name == "test_rtf_gpu.py": + env.update(EXPECTED_DEVICE="2", EXPECTED_BATCH="2") + args += ["--device_id", "2", "--batch_size", "2"] + result = invoke(name, *args, env=env) + assert result.returncode == 0, result.stderr + if name == "test_cer.py": + assert (tmp_path / "output/text").read_text() == "a fixture text\nb fixture text\n" + assert (tmp_path / "output/token").read_text() == "a fixture text\nb fixture text\n" + else: + assert "total_time_wav_ms: 4000" in result.stdout + assert "num: 1," in result.stdout + assert "total_rtf:" in result.stdout + + +def test_gpu_cli_defaults_to_device_zero(backends): + _, scp, env = backends + env["EXPECTED_DEVICE"] = "0" + result = invoke( + "test_rtf_gpu.py", "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env + ) + assert result.returncode == 0, result.stderr + + +def test_gpu_cli_refuses_silent_onnx_cpu_fallback(backends): + _, scp, env = backends + env.update(EXPECTED_DEVICE="0", CPU_FALLBACK="1") + result = invoke( + "test_rtf_gpu.py", "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env + ) + assert result.returncode != 0 + assert "CUDAExecutionProvider" in result.stderr + assert "total_rtf:" not in result.stdout + + +@pytest.mark.parametrize( + "option,value", + [("--batch_size", "0"), ("--warmup", "-1"), ("--device_id", "-1"), ("--backend", "bogus")], +) +def test_gpu_cli_rejects_invalid_arguments(option, value, backends): + _, scp, env = backends + result = invoke( + "test_rtf_gpu.py", "--model_dir", "local-model", "--wav_file", scp, option, value, env=env + ) + assert result.returncode == 2, result.stderr + + +@pytest.mark.parametrize("name", ["test_rtf.py", "test_rtf_gpu.py"]) +def test_incomplete_inference_is_not_a_successful_benchmark(name, backends): + _, scp, env = backends + env.update(EMPTY_RESULT="1", EXPECTED_DEVICE="0" if "gpu" in name else "-1") + result = invoke(name, "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env) + assert result.returncode != 0 + assert "inference returned" in result.stderr + assert "total_rtf:" not in result.stdout + + +def test_zero_duration_is_rejected(backends): + packages, scp, env = backends + (packages / "librosa.py").write_text("def load(path, sr=None): return [], 16000\n") + result = invoke( + "test_rtf.py", "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env + ) + assert result.returncode != 0 + assert "duration" in result.stderr + assert "total_rtf:" not in result.stdout + + +@pytest.mark.parametrize( + "value,wanted", + [("t", True), ("f", False), ("True", True), ("False", False), ("1", True), ("0", False)], +) +def test_quantize_retains_legacy_boolean_spellings(value, wanted): + args = helper().parse_args( + ["--model_dir", "local-model", "--wav_file", "wav.scp", "--quantize", value] + ) + assert args.quantize is wanted + + +def test_rtf_uses_measured_duration_for_warmup_and_total(backends, monkeypatch, capsys): + _, scp, _ = backends + module = helper() + ticks = iter([0.0, 1.0, 2.0, 3.0]) + monkeypatch.setattr(module.time, "perf_counter", lambda: next(ticks)) + monkeypatch.setitem( + sys.modules, "librosa", SimpleNamespace(load=lambda *a, **k: ([0] * 16000, 8000)) + ) + monkeypatch.setattr( + module, + "load_model", + lambda args: lambda paths: [{}] * (1 if isinstance(paths, str) else len(paths)), + ) + args = module.parse_args( + ["--model_dir", "local-model", "--wav_file", str(scp), "--warmup", "1"] + ) + module.benchmark(args, batched=True) + output = capsys.readouterr().out + assert "avg: 1.0, rtf: 0.5" in output + assert "total_rtf: 0.25" in output + + +@pytest.mark.parametrize("name", NAMES) +def test_module_mode_help(name): + result = subprocess.run( + [sys.executable, "-S", "-m", "runtime.python.utils." + Path(name).stem, "--help"], + cwd=ROOT, + text=True, + capture_output=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + assert "--model_dir" in result.stdout diff --git a/tests/test_runtime_eval_native.py b/tests/test_runtime_eval_native.py new file mode 100644 index 0000000000..c3b60a8ccb --- /dev/null +++ b/tests/test_runtime_eval_native.py @@ -0,0 +1,163 @@ +"""Offline synthetic graphs exercise real frontends, devices and CLI inference. + +These fixed weights are test fixtures, not pretrained ASR or quality benchmarks. +Set FUNASR_TEST_DEVICE to a CUDA index for the same tests on real GPU hardware. +""" + +import importlib +import json +import os +from pathlib import Path +import subprocess +import sys +import wave + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +DEVICE = os.environ.get("FUNASR_TEST_DEVICE", "-1") + + +@pytest.fixture(scope="module") +def native_fixture(tmp_path_factory): + torch = pytest.importorskip("torch") + np = pytest.importorskip("numpy") + onnx = pytest.importorskip("onnx") + pytest.importorskip("onnxruntime") + pytest.importorskip("kaldi_native_fbank") + directory = tmp_path_factory.mktemp("runtime-native") + + class AcousticFixture(torch.nn.Module): + def __init__(self): + super().__init__() + self.register_buffer("weight", torch.arange(20).reshape(4, 5).float() * 0.00001) + self.register_buffer("bias", torch.tensor([0.0, 0.0, 0.0, 5.0, 0.0])) + + def forward(self, speech, lengths): + return speech[:, :2] @ self.weight + self.bias, torch.full_like(lengths, 2) + + model = AcousticFixture().eval() + speech = torch.zeros(2, 10, 4) + lengths = torch.tensor([10, 8], dtype=torch.int32) + torch.jit.trace(model, (speech, lengths)).save(str(directory / "model.torchscript")) + torch.onnx.export( + model, + (speech, lengths), + str(directory / "model.onnx"), + dynamo=False, + input_names=["speech", "speech_lengths"], + output_names=["logits", "lengths"], + dynamic_axes={ + "speech": {0: "B", 1: "T"}, + "speech_lengths": {0: "B"}, + "logits": {0: "B"}, + "lengths": {0: "B"}, + }, + opset_version=17, + ) + onnx.checker.check_model(str(directory / "model.onnx")) + (directory / "tokens.json").write_text( + json.dumps(["", "", "", "fixture", "unused"]) + ) + (directory / "config.yaml").write_text( + json.dumps( + { + "frontend_conf": {"n_mels": 4, "lfr_m": 1, "lfr_n": 1, "dither": 0.0}, + "model_conf": {"predictor_bias": 0}, + } + ) + ) + (directory / "am.mvn").write_text( + " 4 4\n 1 [ 0 0 0 0 ]\n" + " 4 4\n 1 [ 1 1 1 1 ]\n" + ) + paths = [] + for index, count in enumerate([16000, 8000]): + path = directory / f"audio {index}.wav" + signal = ( + np.zeros(count, dtype="