fix(runtime): repair evaluation CLIs and honor GPU device selection

.github/workflows/test-runtime-eval.yml

@@ -0,0 +1,58 @@
+name: Validate runtime evaluation tools
+
+on:
+  pull_request:
+    paths:
+      - "runtime/python/utils/**"
+      - "runtime/python/libtorch/funasr_torch/**"
+      - "runtime/python/onnxruntime/funasr_onnx/**"
+      - "tests/test_runtime_eval_*.py"
+      - ".github/workflows/test-runtime-eval.yml"
+  push:
+    branches: [main]
+    paths:
+      - "runtime/python/utils/**"
+      - "runtime/python/libtorch/funasr_torch/**"
+      - "runtime/python/onnxruntime/funasr_onnx/**"
+      - "tests/test_runtime_eval_*.py"
+      - ".github/workflows/test-runtime-eval.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  runtime-eval:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.10", "3.12"]
+    env:
+      HF_HUB_OFFLINE: "1"
+      MODELSCOPE_OFFLINE: "1"
+      OMP_NUM_THREADS: "1"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install CPU runtime test dependencies
+        run: |
+          python -m pip install torch==2.8.0 --index-url https://download.pytorch.org/whl/cpu
+          python -m pip install pytest "numpy<2" onnx==1.19.0 onnxruntime==1.23.2 librosa kaldi-native-fbank PyYAML jieba sentencepiece
+      - name: Run CLI and native offline runtime regressions
+        run: |
+          git rev-parse HEAD > source-commit.txt
+          python -m pip freeze > resolved-dependencies.txt
+          python -m pytest -q tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py --junitxml=runtime-eval-results.xml
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: runtime-eval-${{ matrix.python }}
+          path: |
+            source-commit.txt
+            resolved-dependencies.txt
+            runtime-eval-results.xml

runtime/python/libtorch/funasr_torch/paraformer_bin.py

@@ -75,7 +75,8 @@ def __init__(
         self.converter = TokenIDConverter(token_list)
         self.tokenizer = CharTokenizer()
         self.frontend = WavFrontend(cmvn_file=cmvn_file, **config["frontend_conf"])
-        self.ort_infer = torch.jit.load(model_file)
+        self.device = torch.device("cpu" if int(device_id) == -1 else f"cuda:{int(device_id)}")
+        self.ort_infer = torch.jit.load(model_file, map_location=self.device).eval()
         self.batch_size = batch_size
         self.device_id = device_id
         self.plot_timestamp_to = plot_timestamp_to
@@ -98,12 +99,11 @@ def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs) ->
             feats, feats_len = self.extract_feat(waveform_list[beg_idx:end_idx])
             try:
                 with torch.no_grad():
-                    if int(self.device_id) == -1:
-                        outputs = self.ort_infer(feats, feats_len)
-                        am_scores, valid_token_lens = outputs[0], outputs[1]
-                    else:
-                        outputs = self.ort_infer(feats.cuda(), feats_len.cuda())
-                        am_scores, valid_token_lens = outputs[0].cpu(), outputs[1].cpu()
+                    outputs = self.ort_infer(
+                        feats.to(self.device), feats_len.to(self.device)
+                    )
+                    outputs = tuple(output.cpu() for output in outputs)
+                    am_scores, valid_token_lens = outputs[0], outputs[1]
                 if len(outputs) == 4:
                     # for BiCifParaformer Inference
                     us_alphas, us_peaks = outputs[2], outputs[3]

runtime/python/utils/README.md

@@ -0,0 +1,50 @@
+# Runtime Evaluation Tools
+
+Run these commands from a FunASR checkout with the chosen runtime and its
+dependencies available. Importing the scripts or using `--help` does not load a
+runtime, audio, or model. ONNX and Torch backends are selected independently.
+
+Use the checkout's runtime modules when evaluating a source change:
+
+```sh
+export PYTHONPATH="$PWD/runtime/python/onnxruntime:$PWD/runtime/python/libtorch${PYTHONPATH:+:$PYTHONPATH}"
+python runtime/python/utils/test_rtf.py --model_dir /path/to/exported-model --wav_file wav.scp
+python runtime/python/utils/test_rtf_gpu.py --model_dir /path/to/exported-model --wav_file wav.scp --device_id 0 --batch_size 8
+python runtime/python/utils/test_cer.py --backend torch --model_dir /path/to/exported-model --wav_file wav.scp --output_dir transcripts
+```
+
+`wav.scp` contains an utterance ID followed by whitespace and an audio path.
+Paths may contain spaces. Blank lines are ignored; empty or malformed lists
+are rejected. Supply existing local exported models to avoid runtime download
+or export behavior. Torch uses `model.torchscript`; ONNX uses `model.onnx`.
+The corresponding quantized names are selected with `--quantize true`.
+
+The serial RTF and transcript tools default to CPU (`--device_id -1`). The
+batched GPU tool defaults to CUDA device 0 and rejects negative device IDs.
+It requires CUDA-capable Torch or ONNX Runtime, with compatible CUDA/cuDNN
+libraries. Missing ONNX CUDA providers cause an error instead of a CPU timing
+being reported as a GPU benchmark. ONNX may still place supported graph
+operations on CPU; provider availability alone is not proof that every node
+runs on CUDA. Use runtime profiling when auditing actual kernel placement.
+
+RTF is elapsed wall time divided by actual input audio duration. There is an
+untimed audio-duration prepass and, by default, 30 warmups on the first audio
+file (`--warmup 0` disables warmups). Timed model calls include file loading,
+frontend, inference and decoding; Torch CUDA is synchronized around timing.
+The serial tool invokes each file separately. The GPU tool passes the list to
+the backend, whose `--batch_size` controls batching. These numbers are not
+model-only kernel timings and depend on cache state and hardware.
+
+The transcript tool writes UTF-8 `text` and space-delimited `token` files. It
+does not compute CER/WER or alter references. A missing inference result is an
+error, not a successful empty transcript or benchmark.
+
+Offline regression tests use small synthetic graphs and generated audio:
+
+```sh
+python -m pytest tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py
+FUNASR_TEST_DEVICE=0 python -m pytest tests/test_runtime_eval_native.py
+```
+
+Native tests require Torch, ONNX, ONNX Runtime and the runtimes' audio/tokenizer
+dependencies. They test device and CLI mechanics, not pretrained ASR accuracy.

runtime/python/utils/runtime_eval.py

@@ -0,0 +1,160 @@
+"""Shared command-line boundaries for the standalone runtime evaluation tools."""
+
+import argparse
+import importlib
+import math
+from pathlib import Path
+import time
+
+
+def _boolean(value):
+    lowered = value.lower()
+    if lowered in {"true", "yes", "t", "y", "1"}:
+        return True
+    if lowered in {"false", "no", "f", "n", "0"}:
+        return False
+    raise argparse.ArgumentTypeError("expected a boolean")
+
+
+def _positive(value):
+    number = int(value)
+    if number < 1:
+        raise argparse.ArgumentTypeError("must be positive")
+    return number
+
+
+def _nonnegative(value):
+    number = int(value)
+    if number < 0:
+        raise argparse.ArgumentTypeError("must be nonnegative")
+    return number
+
+
+def parse_args(argv=None, *, gpu=False, cer=False):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_dir", required=True)
+    parser.add_argument("--backend", choices=["onnx", "torch"], default="onnx")
+    parser.add_argument("--wav_file", required=True, help="WAV SCP: utterance ID and audio path")
+    parser.add_argument("--quantize", type=_boolean, default=False)
+    parser.add_argument("--intra_op_num_threads", type=_positive, default=1)
+    parser.add_argument(
+        "--device_id",
+        type=_nonnegative if gpu else int,
+        default=0 if gpu else -1,
+        help="CUDA device index; -1 for CPU",
+    )
+    parser.add_argument("--batch_size", type=_positive, default=1)
+    if cer:
+        parser.add_argument("--output_dir", required=True)
+    else:
+        parser.add_argument("--warmup", type=_nonnegative, default=30)
+    args = parser.parse_args(argv)
+    if args.device_id < -1:
+        parser.error("--device_id must be -1 or a CUDA device index")
+    return args
+
+
+def read_wav_scp(path):
+    entries = []
+    with open(path, encoding="utf-8") as source:
+        for number, line in enumerate(source, 1):
+            if not line.strip():
+                continue
+            fields = line.strip().split(maxsplit=1)
+            if len(fields) != 2:
+                raise ValueError(f"{path}:{number}: expected utterance ID and audio path")
+            entries.append(tuple(fields))
+    if not entries:
+        raise ValueError(f"{path}: empty WAV list")
+    return entries
+
+
+def load_model(args):
+    backend = importlib.import_module("funasr_" + args.backend)
+    model = backend.Paraformer(
+        args.model_dir,
+        batch_size=args.batch_size,
+        device_id=str(args.device_id),
+        quantize=args.quantize,
+        intra_op_num_threads=args.intra_op_num_threads,
+    )
+    if args.backend == "onnx" and args.device_id >= 0:
+        providers = model.ort_infer.session.get_providers()
+        if "CUDAExecutionProvider" not in providers:
+            raise RuntimeError("CUDAExecutionProvider is unavailable; refusing a CPU GPU benchmark")
+    return model
+
+
+def _infer(model, paths):
+    result = model(paths)
+    expected = 1 if isinstance(paths, str) else len(paths)
+    if len(result) != expected:
+        raise RuntimeError(f"inference returned {len(result)} results for {expected} audio files")
+    return result
+
+
+def transcribe(args):
+    entries = read_wav_scp(args.wav_file)
+    model = load_model(args)
+    output = Path(args.output_dir)
+    output.mkdir(parents=True, exist_ok=True)
+    with (output / "text").open("w", encoding="utf-8") as text_file, (output / "token").open(
+        "w", encoding="utf-8"
+    ) as token_file:
+        for utterance, path in entries:
+            row = _infer(model, path)[0]
+            if isinstance(row["preds"], str):
+                text, tokens = row["preds"], row["raw_tokens"]
+            else:
+                text, tokens = row["preds"]
+            if not isinstance(tokens, str):
+                tokens = " ".join(tokens)
+            text_file.write(f"{utterance} {text}\n")
+            token_file.write(f"{utterance} {tokens}\n")
+
+
+def benchmark(args, *, batched=False):
+    entries = read_wav_scp(args.wav_file)
+    import librosa
+
+    durations = []
+    for _, path in entries:
+        audio, sample_rate = librosa.load(path, sr=None)
+        duration = len(audio) / sample_rate
+        if not math.isfinite(duration) or duration <= 0:
+            raise ValueError(f"{path}: audio duration must be positive")
+        durations.append(duration)
+    model = load_model(args)
+    print(f"backend: {args.backend}, device_id: {args.device_id}")
+
+    def synchronize():
+        if args.backend == "torch" and args.device_id >= 0:
+            import torch
+
+            torch.cuda.synchronize(args.device_id)
+
+    warmup_elapsed = 0.0
+    for index in range(args.warmup):
+        synchronize()
+        start = time.perf_counter()
+        result = _infer(model, entries[0][1])
+        synchronize()
+        elapsed = time.perf_counter() - start
+        warmup_elapsed += elapsed
+        average = warmup_elapsed / (index + 1)
+        print(result)
+        print(f"num: 1, time, {elapsed}, avg: {average}, rtf: {average / durations[0]}")
+
+    synchronize()
+    start = time.perf_counter()
+    if batched:
+        _infer(model, [path for _, path in entries])
+    else:
+        for _, path in entries:
+            _infer(model, path)
+    synchronize()
+    elapsed = time.perf_counter() - start
+    duration = sum(durations)
+    print(f"total_time_comput_ms: {elapsed * 1000:.6f}")
+    print(f"total_time_wav_ms: {duration * 1000:.6f}")
+    print(f"total_rtf: {elapsed / duration}")

runtime/python/utils/test_cer.py

@@ -1,56 +1,14 @@
-import os
-import time
-import sys
-import librosa
-from funasr.utils.types import str2bool
+"""Write runtime transcripts and tokens for external CER/WER scoring."""
 
-import argparse
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--model_dir", type=str, required=True)
-parser.add_argument("--backend", type=str, default="onnx", help='["onnx", "torch"]')
-parser.add_argument("--wav_file", type=str, default=None, help="amp fallback number")
-parser.add_argument("--quantize", type=str2bool, default=False, help="quantized model")
-parser.add_argument(
-    "--intra_op_num_threads", type=int, default=1, help="intra_op_num_threads for onnx"
-)
-parser.add_argument("--output_dir", type=str, default=None, help="amp fallback number")
-args = parser.parse_args()
+def main(argv=None):
+    if __package__:
+        from .runtime_eval import parse_args, transcribe
+    else:
+        from runtime_eval import parse_args, transcribe
 
+    transcribe(parse_args(argv, cer=True))
 
-from funasr.runtime.python.libtorch.funasr_torch import Paraformer
 
-if args.backend == "onnx":
-    from funasr.runtime.python.onnxruntime.funasr_onnx import Paraformer
-
-model = Paraformer(
-    args.model_dir,
-    batch_size=1,
-    quantize=args.quantize,
-    intra_op_num_threads=args.intra_op_num_threads,
-)
-
-wav_file_f = open(args.wav_file, "r")
-wav_files = wav_file_f.readlines()
-
-output_dir = args.output_dir
-if not os.path.exists(output_dir):
-    os.makedirs(output_dir)
-if os.name == "nt":  # Windows
-    newline = "\r\n"
-else:  # Linux Mac
-    newline = "\n"
-text_f = open(os.path.join(output_dir, "text"), "w", newline=newline)
-token_f = open(os.path.join(output_dir, "token"), "w", newline=newline)
-
-for i, wav_path_i in enumerate(wav_files):
-    wav_name, wav_path = wav_path_i.strip().split()
-    result = model(wav_path)
-    text_i = "{} {}\n".format(wav_name, result[0]["preds"][0])
-    token_i = "{} {}\n".format(wav_name, result[0]["preds"][1])
-    text_f.write(text_i)
-    text_f.flush()
-    token_f.write(token_i)
-    token_f.flush()
-text_f.close()
-token_f.close()
+if __name__ == "__main__":
+    main()