From c2ddbf378b9d17aef74e50f0956b1fbfe0f92b2a Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Sat, 30 May 2026 23:05:10 +0800 Subject: [PATCH] feat: add FunASR speech recognition backend --- backend/index.yaml | 108 ++++++++++++++ backend/python/funasr/Makefile | 23 +++ backend/python/funasr/backend.py | 133 ++++++++++++++++++ backend/python/funasr/install.sh | 21 +++ backend/python/funasr/requirements-cpu.txt | 3 + .../python/funasr/requirements-cublas12.txt | 3 + .../python/funasr/requirements-cublas13.txt | 3 + .../python/funasr/requirements-hipblas.txt | 3 + backend/python/funasr/requirements-intel.txt | 2 + backend/python/funasr/requirements-l4t12.txt | 1 + backend/python/funasr/requirements-l4t13.txt | 1 + backend/python/funasr/requirements-mps.txt | 2 + backend/python/funasr/requirements.txt | 5 + backend/python/funasr/run.sh | 9 ++ backend/python/funasr/test.sh | 13 ++ 15 files changed, 330 insertions(+) create mode 100644 backend/python/funasr/Makefile create mode 100644 backend/python/funasr/backend.py create mode 100755 backend/python/funasr/install.sh create mode 100644 backend/python/funasr/requirements-cpu.txt create mode 100644 backend/python/funasr/requirements-cublas12.txt create mode 100644 backend/python/funasr/requirements-cublas13.txt create mode 100644 backend/python/funasr/requirements-hipblas.txt create mode 100644 backend/python/funasr/requirements-intel.txt create mode 100644 backend/python/funasr/requirements-l4t12.txt create mode 100644 backend/python/funasr/requirements-l4t13.txt create mode 100644 backend/python/funasr/requirements-mps.txt create mode 100644 backend/python/funasr/requirements.txt create mode 100755 backend/python/funasr/run.sh create mode 100755 backend/python/funasr/test.sh diff --git a/backend/index.yaml b/backend/index.yaml index 887e2e57e24d..e75c9fb0cc3e 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -1002,6 +1002,31 @@ nvidia-l4t-cuda-12: "nvidia-l4t-faster-qwen3-tts" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-faster-qwen3-tts" icon: https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png +- &funasr + urls: + - https://github.com/modelscope/FunASR + description: | + FunASR is an industrial-grade speech recognition toolkit supporting 50+ languages. + Includes SenseVoice (170x realtime, emotion detection), Paraformer (highest Chinese + accuracy), and built-in VAD, punctuation restoration, and speaker diarization. + tags: + - speech-recognition + - ASR + - multilingual + license: mit + name: "funasr" + alias: "funasr" + capabilities: + nvidia: "cuda12-funasr" + amd: "rocm-funasr" + metal: "metal-funasr" + default: "cpu-funasr" + nvidia-cuda-13: "cuda13-funasr" + nvidia-cuda-12: "cuda12-funasr" + nvidia-l4t: "nvidia-l4t-funasr" + nvidia-l4t-cuda-12: "nvidia-l4t-funasr" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-funasr" + icon: https://avatars.githubusercontent.com/u/109454077 - &qwen-asr urls: - https://github.com/QwenLM/Qwen3-ASR @@ -4586,3 +4611,86 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-sherpa-onnx" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-12-sherpa-onnx +## funasr +- !!merge <<: *funasr + name: "funasr-development" + capabilities: + nvidia: "cuda12-funasr-development" + amd: "rocm-funasr-development" + nvidia-l4t: "nvidia-l4t-funasr-development" + metal: "metal-funasr-development" + default: "cpu-funasr-development" + nvidia-cuda-13: "cuda13-funasr-development" + nvidia-cuda-12: "cuda12-funasr-development" + nvidia-l4t-cuda-12: "nvidia-l4t-funasr-development" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-funasr-development" +- !!merge <<: *funasr + name: "cpu-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-funasr" + mirrors: + - localai/localai-backends:latest-cpu-funasr +- !!merge <<: *funasr + name: "cpu-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-funasr" + mirrors: + - localai/localai-backends:master-cpu-funasr +- !!merge <<: *funasr + name: "cuda12-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-funasr" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-funasr +- !!merge <<: *funasr + name: "cuda12-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-funasr" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-funasr +- !!merge <<: *funasr + name: "cuda13-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-funasr" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-13-funasr +- !!merge <<: *funasr + name: "cuda13-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-funasr" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-13-funasr +- !!merge <<: *funasr + name: "rocm-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-funasr" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-funasr +- !!merge <<: *funasr + name: "rocm-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-funasr" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-funasr +- !!merge <<: *funasr + name: "nvidia-l4t-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-funasr" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-funasr +- !!merge <<: *funasr + name: "nvidia-l4t-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-funasr" + mirrors: + - localai/localai-backends:master-nvidia-l4t-funasr +- !!merge <<: *funasr + name: "cuda13-nvidia-l4t-arm64-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-funasr" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-funasr +- !!merge <<: *funasr + name: "cuda13-nvidia-l4t-arm64-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-funasr" + mirrors: + - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-funasr +- !!merge <<: *funasr + name: "metal-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-funasr" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-funasr +- !!merge <<: *funasr + name: "metal-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-funasr" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-funasr diff --git a/backend/python/funasr/Makefile b/backend/python/funasr/Makefile new file mode 100644 index 000000000000..1bb15fe3af55 --- /dev/null +++ b/backend/python/funasr/Makefile @@ -0,0 +1,23 @@ +.PHONY: funasr +funasr: + bash install.sh + +.PHONY: run +run: funasr + @echo "Running funasr..." + bash run.sh + @echo "funasr run." + +.PHONY: test +test: funasr + @echo "Testing funasr..." + bash test.sh + @echo "funasr tested." + +.PHONY: protogen-clean +protogen-clean: + $(RM) backend_pb2_grpc.py backend_pb2.py + +.PHONY: clean +clean: protogen-clean + rm -rf venv __pycache__ diff --git a/backend/python/funasr/backend.py b/backend/python/funasr/backend.py new file mode 100644 index 000000000000..a18fa7281f38 --- /dev/null +++ b/backend/python/funasr/backend.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +""" +gRPC backend for LocalAI wrapping FunASR (SenseVoice / Paraformer). +""" +from concurrent import futures +import time +import argparse +import signal +import sys +import os +import backend_pb2 +import backend_pb2_grpc +import torch + +import grpc +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) +from grpc_auth import get_auth_interceptors + + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 +MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1')) + + +class BackendServicer(backend_pb2_grpc.BackendServicer): + def Health(self, request, context): + return backend_pb2.Reply(message=bytes("OK", 'utf-8')) + + def LoadModel(self, request, context): + from funasr import AutoModel + + device = "cpu" + if request.CUDA and torch.cuda.is_available(): + device = "cuda" + mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available() + if mps_available: + device = "mps" + + model_id = request.Model or "iic/SenseVoiceSmall" + + try: + print(f"Loading FunASR model: {model_id} on {device}", file=sys.stderr) + self.model = AutoModel( + model=model_id, + vad_model="fsmn-vad", + device=device, + disable_update=True, + ) + print("FunASR model loaded successfully", file=sys.stderr) + except Exception as err: + print(f"[ERROR] LoadModel failed: {err}", file=sys.stderr) + import traceback + traceback.print_exc(file=sys.stderr) + return backend_pb2.Result(success=False, message=str(err)) + + return backend_pb2.Result(message="Model loaded successfully", success=True) + + def AudioTranscription(self, request, context): + result_segments = [] + text = "" + try: + audio_path = request.dst + if not audio_path or not os.path.exists(audio_path): + print(f"Error: Audio file not found: {audio_path}", file=sys.stderr) + return backend_pb2.TranscriptResult(segments=[], text="") + + language = None + if request.language and request.language.strip(): + language = request.language.strip() + + kwargs = {} + if language: + kwargs["language"] = language + + results = self.model.generate(input=audio_path, **kwargs) + + if not results: + return backend_pb2.TranscriptResult(segments=[], text="") + + for idx, r in enumerate(results): + seg_text = r.get("text", "") if isinstance(r, dict) else str(r) + text += seg_text + result_segments.append(backend_pb2.TranscriptSegment( + id=idx, + start=0, + end=0, + text=seg_text, + )) + + except Exception as err: + print(f"Error in AudioTranscription: {err}", file=sys.stderr) + import traceback + traceback.print_exc(file=sys.stderr) + return backend_pb2.TranscriptResult(segments=[], text="") + + return backend_pb2.TranscriptResult(segments=result_segments, text=text) + + +def serve(address): + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=MAX_WORKERS), + options=[ + ('grpc.max_message_length', 50 * 1024 * 1024), + ('grpc.max_send_message_length', 50 * 1024 * 1024), + ('grpc.max_receive_message_length', 50 * 1024 * 1024), + ], + interceptors=get_auth_interceptors(), + ) + backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server) + server.add_insecure_port(address) + server.start() + print("Server started. Listening on: " + address, file=sys.stderr) + + def signal_handler(sig, frame): + print("Received termination signal. Shutting down...") + server.stop(0) + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run the gRPC server.") + parser.add_argument("--addr", default="localhost:50051", help="The address to bind the server to.") + args = parser.parse_args() + serve(args.addr) diff --git a/backend/python/funasr/install.sh b/backend/python/funasr/install.sh new file mode 100755 index 000000000000..71c9e79a93f9 --- /dev/null +++ b/backend/python/funasr/install.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +EXTRA_PIP_INSTALL_FLAGS="--no-build-isolation" + +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +if [ "x${BUILD_PROFILE}" == "xintel" ]; then + EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" +fi + +PYTHON_VERSION="3.12" +PYTHON_PATCH="12" +PY_STANDALONE_TAG="20251120" + +installRequirements diff --git a/backend/python/funasr/requirements-cpu.txt b/backend/python/funasr/requirements-cpu.txt new file mode 100644 index 000000000000..12aca055d1ad --- /dev/null +++ b/backend/python/funasr/requirements-cpu.txt @@ -0,0 +1,3 @@ +--extra-index-url https://download.pytorch.org/whl/cpu +torch +funasr diff --git a/backend/python/funasr/requirements-cublas12.txt b/backend/python/funasr/requirements-cublas12.txt new file mode 100644 index 000000000000..dde59b888642 --- /dev/null +++ b/backend/python/funasr/requirements-cublas12.txt @@ -0,0 +1,3 @@ +--extra-index-url https://download.pytorch.org/whl/cu121 +torch +funasr diff --git a/backend/python/funasr/requirements-cublas13.txt b/backend/python/funasr/requirements-cublas13.txt new file mode 100644 index 000000000000..84ec494212f1 --- /dev/null +++ b/backend/python/funasr/requirements-cublas13.txt @@ -0,0 +1,3 @@ +--extra-index-url https://download.pytorch.org/whl/cu131 +torch +funasr diff --git a/backend/python/funasr/requirements-hipblas.txt b/backend/python/funasr/requirements-hipblas.txt new file mode 100644 index 000000000000..f03062f69eee --- /dev/null +++ b/backend/python/funasr/requirements-hipblas.txt @@ -0,0 +1,3 @@ +--extra-index-url https://download.pytorch.org/whl/rocm6.1 +torch +funasr diff --git a/backend/python/funasr/requirements-intel.txt b/backend/python/funasr/requirements-intel.txt new file mode 100644 index 000000000000..8bd073084a41 --- /dev/null +++ b/backend/python/funasr/requirements-intel.txt @@ -0,0 +1,2 @@ +torch +funasr diff --git a/backend/python/funasr/requirements-l4t12.txt b/backend/python/funasr/requirements-l4t12.txt new file mode 100644 index 000000000000..279ccacfb239 --- /dev/null +++ b/backend/python/funasr/requirements-l4t12.txt @@ -0,0 +1 @@ +funasr diff --git a/backend/python/funasr/requirements-l4t13.txt b/backend/python/funasr/requirements-l4t13.txt new file mode 100644 index 000000000000..279ccacfb239 --- /dev/null +++ b/backend/python/funasr/requirements-l4t13.txt @@ -0,0 +1 @@ +funasr diff --git a/backend/python/funasr/requirements-mps.txt b/backend/python/funasr/requirements-mps.txt new file mode 100644 index 000000000000..d36c7ba25f1f --- /dev/null +++ b/backend/python/funasr/requirements-mps.txt @@ -0,0 +1,2 @@ +torch==2.7.1 +funasr diff --git a/backend/python/funasr/requirements.txt b/backend/python/funasr/requirements.txt new file mode 100644 index 000000000000..9ce0da738095 --- /dev/null +++ b/backend/python/funasr/requirements.txt @@ -0,0 +1,5 @@ +grpcio==1.71.0 +protobuf +certifi +packaging==24.1 +setuptools diff --git a/backend/python/funasr/run.sh b/backend/python/funasr/run.sh new file mode 100755 index 000000000000..eae121f37b0b --- /dev/null +++ b/backend/python/funasr/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +startBackend $@ diff --git a/backend/python/funasr/test.sh b/backend/python/funasr/test.sh new file mode 100755 index 000000000000..e958ef3eae87 --- /dev/null +++ b/backend/python/funasr/test.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +ensureVenv + +python -m pytest -x -s test.py "$@"