Add Whisper runner under examples/audio (#23904)

backends/native/CMakeLists.txt

@@ -46,7 +46,10 @@ install(
   PATTERN "ggml" EXCLUDE
 )
 
-if(EXECUTORCH_BUILD_NATIVE_VISION_RUNNER OR EXECUTORCH_BUILD_NATIVE_LLM_RUNNER)
+if(EXECUTORCH_BUILD_NATIVE_VISION_RUNNER
+   OR EXECUTORCH_BUILD_NATIVE_AUDIO_RUNNER
+   OR EXECUTORCH_BUILD_NATIVE_LLM_RUNNER
+)
   add_subdirectory(examples)
 endif()
 

backends/native/README.md

@@ -204,7 +204,7 @@ target_link_libraries(my_app PRIVATE native_api native_vulkan_engine)
 With `EXECUTORCH_BUILD_TESTS=ON`, `native_runtime_test` covers the shared runtime
 and API. `native_cpu_engine_test` covers the CPU engine when
 `EXECUTORCH_BUILD_NATIVE_CPU` is enabled. `native_vulkan_passes_test` covers the
-Vulkan graph rewrites without requiring a GPU. When either runner below is
+Vulkan graph rewrites without requiring a GPU. When any runner below is
 enabled, `native_runner_utils_test` covers their shared runner utilities. Run these
 with `ctest --test-dir cmake-out -R native_`.
 
@@ -310,6 +310,35 @@ and prints the selected engine and device and the top-5 logits. When
 invalid arguments, 2 for a loading or execution error, and 3 for a reference
 mismatch.
 
+## Whisper speech recognition runner
+
+`EXECUTORCH_BUILD_NATIVE_AUDIO_RUNNER=ON` builds `run_whisper_ggml_cpu` from
+`examples/audio` with Native GGML on CPU. It requires gflags. Initialize the
+GGML submodule as described above, then:
+
+```sh
+cmake -S . -B cmake-out -DCMAKE_BUILD_TYPE=Release \
+  -DEXECUTORCH_BUILD_NATIVE=ON -DEXECUTORCH_BUILD_NATIVE_GGML=ON \
+  -DEXECUTORCH_BUILD_NATIVE_AUDIO_RUNNER=ON
+cmake --build cmake-out --target run_whisper_ggml_cpu
+cmake --install cmake-out --prefix install
+install/bin/run_whisper_ggml_cpu --cpu_threads 4 \
+  --package whisper_tiny.ptn --input input_features.bin \
+  --prompt prompt_ids.txt --expected expected_ids.txt --eos_ids 50257
+```
+
+Export the `.ptn` with
+[`export_hf_whisper.py`](examples/audio/export_hf_whisper.py); `--dump_dir` also
+writes the log-mel input, forced prompt, and eager greedy ids, and the exporter
+prints the model's EOS ids. The runner projects cross-attention K/V once, then
+preserves those inputs and its self-attention cache across prefill and decode,
+including GGML graph rebuilds. It greedily decodes until EOS or
+`--max_new_tokens` (default 32, matching the exporter), capped by the remaining
+decoder context. `--max_cached_graphs` defaults to 1; 0 rebuilds each call.
+When `--expected` is given, it exits nonzero if the generated ids differ from
+the reference. Use the same `--max_new_tokens` value when exporting reference
+vectors and running the comparison.
+
 ## Native LLM runner
 
 `EXECUTORCH_BUILD_NATIVE_LLM_RUNNER=ON` builds `run_hf_llm_vulkan` with Native

backends/native/examples/CMakeLists.txt

@@ -17,6 +17,9 @@ add_subdirectory(runner_utils)
 if(EXECUTORCH_BUILD_NATIVE_VISION_RUNNER)
   add_subdirectory(vision)
 endif()
+if(EXECUTORCH_BUILD_NATIVE_AUDIO_RUNNER)
+  add_subdirectory(audio)
+endif()
 if(EXECUTORCH_BUILD_NATIVE_LLM_RUNNER)
   add_subdirectory(hf_llm/runtime)
 endif()

backends/native/examples/audio/BUCK

@@ -26,3 +26,30 @@ fbcode_target(
         "fbsource//third-party/pypi/safetensors:safetensors",
     ],
 )
+
+runtime.cxx_library(
+    name = "whisper_runner",
+    srcs = ["WhisperRunner.cpp"],
+    exported_headers = ["WhisperRunner.h"],
+    deps = [
+        "//executorch/backends/native/examples/runner_utils:runner_utils",
+        "//executorch/backends/native/runtime:runtime",
+        "//executorch/backends/native/runtime/deserialize:package",
+        "//executorch/backends/native/runtime/graph:scalar_type",
+    ],
+    exported_deps = ["//executorch/backends/native/runtime/engine:engine"],
+    external_deps = ["gflags"],
+    visibility = ["PUBLIC"],
+)
+
+fbcode_target(
+    _kind = runtime.cxx_binary,
+    name = "run_whisper_ggml_cpu",
+    srcs = ["run_whisper_ggml_cpu.cpp"],
+    deps = [
+        ":whisper_runner",
+        "//executorch/backends/native/runtime/engine:ggml_engine_cpu",
+    ],
+    external_deps = ["gflags"],
+    visibility = ["PUBLIC"],
+)

backends/native/examples/audio/CMakeLists.txt

@@ -0,0 +1,26 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+add_library(native_whisper_runner STATIC WhisperRunner.cpp)
+target_link_libraries(
+  native_whisper_runner
+  PUBLIC native_engine
+  PRIVATE ${_native_runner_gflags} native_package native_runner_utils
+          native_runtime
+)
+
+add_executable(run_whisper_ggml_cpu run_whisper_ggml_cpu.cpp)
+target_link_libraries(
+  run_whisper_ggml_cpu PRIVATE native_whisper_runner native_ggml_engine
+                               ${_native_runner_gflags}
+)
+foreach(target native_whisper_runner run_whisper_ggml_cpu)
+  target_compile_options(
+    ${target} PRIVATE $<$<CXX_COMPILER_ID:MSVC>:/EHsc>
+                      $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-fexceptions>
+  )
+endforeach()
+install(TARGETS run_whisper_ggml_cpu RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})