diff --git a/bazel/dependencies.bzl b/bazel/dependencies.bzl index e0439bb07..9e80e355a 100644 --- a/bazel/dependencies.bzl +++ b/bazel/dependencies.bzl @@ -14,6 +14,7 @@ load("@aspect_rules_lint//format:repositories.bzl", "rules_lint_dependencies") load("@bazel_lib//lib:repositories.bzl", "bazel_lib_dependencies", "bazel_lib_register_toolchains") +load("@com_google_benchmark//:bazel/benchmark_deps.bzl", "benchmark_deps") load("@com_google_googletest//:googletest_deps.bzl", "googletest_deps") load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") load("@envoy_toolshed//sysroot:sysroot.bzl", "setup_sysroots") @@ -31,6 +32,7 @@ def proxy_wasm_cpp_host_dependencies(): # Bazel extensions. googletest_deps() rules_foreign_cc_dependencies() + benchmark_deps() rules_lint_dependencies() diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 4f0937c2a..2c0930545 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -193,6 +193,14 @@ def proxy_wasm_cpp_host_repositories(): }, ) + maybe( + http_archive, + name = "com_google_benchmark", + sha256 = "9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340", + strip_prefix = "benchmark-1.9.5", + urls = ["https://github.com/google/benchmark/archive/refs/tags/v1.9.5.tar.gz"], + ) + # NullVM dependencies. maybe( diff --git a/test/BUILD b/test/BUILD index 1b45b1318..17cb81b6b 100644 --- a/test/BUILD +++ b/test/BUILD @@ -225,6 +225,7 @@ cc_test( "//:lib", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", + "@com_google_benchmark//:benchmark", ], ) diff --git a/test/wasm_vm_test.cc b/test/wasm_vm_test.cc index d792c63df..cc2daadc6 100644 --- a/test/wasm_vm_test.cc +++ b/test/wasm_vm_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "gtest/gtest.h" +#include "benchmark/benchmark.h" #include #include @@ -31,10 +32,12 @@ INSTANTIATE_TEST_SUITE_P(WasmEngines, TestVm, testing::ValuesIn(getWasmEngines() }); TEST_P(TestVm, Init) { + std::unique_ptr vm1 = makeVm(engine_); + std::unique_ptr vm2 = makeVm(engine_); auto time1 = std::chrono::steady_clock::now(); - vm_->warm(); + vm1->warm(); auto time2 = std::chrono::steady_clock::now(); - vm_->warm(); + vm2->warm(); auto time3 = std::chrono::steady_clock::now(); auto cold = std::chrono::duration_cast(time2 - time1).count(); @@ -43,24 +46,37 @@ TEST_P(TestVm, Init) { std::cout << "[" << engine_ << "] \"cold\" engine time: " << cold << "ns" << std::endl; std::cout << "[" << engine_ << "] \"warm\" engine time: " << warm << "ns" << std::endl; - // Default warm time in nanoseconds. - int warm_time_ns_limit = 10000; - -#if defined(__linux__) && defined(__s390x__) - // Linux 390x is significantly slower, so we use a more lenient limit. - warm_time_ns_limit = 75000; -#endif - - // Verify that getting a "warm" engine takes less than 10us. - EXPECT_LE(warm, warm_time_ns_limit); - // Verify that getting a "warm" engine takes at least 50x less time than getting a "cold" one. // We skip NullVM because warm() is a noop. if (engine_ == "null") { std::cout << "Skipping warm() performance assertions for NullVM." << std::endl; return; } - EXPECT_LE(warm * 50, cold); + int expected_warm_time_ns = 0; + double expected_warm_time_speedup_factor = 0; + if (engine_ == "v8") { + expected_warm_time_ns = 20000000; + expected_warm_time_speedup_factor = 1.5; + } else if (engine_ == "wamr") { + expected_warm_time_ns = 4000; + expected_warm_time_speedup_factor = 2000; + } else if (engine_ == "wasmedge") { + expected_warm_time_ns = 2000; + expected_warm_time_speedup_factor = 1.5; + } else if (engine_ == "wasmtime") { + expected_warm_time_ns = 8000; + expected_warm_time_speedup_factor = 6; + } + // Linux 390x is significantly slower, so we use a more lenient limit. +#if defined(__linux__) && defined(__s390x__) + expected_warm_time_ns *= 10; +#endif +#if defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(ADDRESS_SANITIZER) || \ + defined(HWADDRESS_SANITIZER) || defined(THREAD_SANITIZER) + expected_warm_time_ns *= 10; +#endif + EXPECT_LE(warm * expected_warm_time_speedup_factor, cold); + EXPECT_LT(warm, expected_warm_time_ns); } TEST_P(TestVm, Basic) { @@ -164,5 +180,40 @@ TEST_P(TestVm, DISABLED_CloneUntilOutOfMemory) { #endif +void BM_WarmVmStart(benchmark::State &state, auto makeVm) { + std::unique_ptr cold_vm = makeVm(); + cold_vm->warm(); + for (auto _ : state) { + std::unique_ptr warm_vm = makeVm(); + warm_vm->warm(); + } +} +#ifdef PROXY_WASM_HOST_ENGINE_V8 +static void BM_V8WarmVmStart(benchmark::State &state) { + BM_WarmVmStart(state, []() { return proxy_wasm::createV8Vm(); }); +} +BENCHMARK(BM_V8WarmVmStart); +#endif +#ifdef PROXY_WASM_HOST_ENGINE_WASMTIME +static void BM_WasmtimeWarmVmStart(benchmark::State &state) { + BM_WarmVmStart(state, []() { return proxy_wasm::createWasmtimeVm(); }); +} +BENCHMARK(BM_WasmtimeWarmVmStart); +#endif +#ifdef PROXY_WASM_HOST_ENGINE_WASMEDGE +static void BM_WasmEdgeWarmVmStart(benchmark::State &state) { + BM_WarmVmStart(state, []() { return proxy_wasm::createWasmEdgeVm(); }); +} +BENCHMARK(BM_WasmEdgeWarmVmStart); +#endif +#ifdef PROXY_WASM_HOST_ENGINE_WAMR +static void BM_WamrWarmVmStart(benchmark::State &state) { + BM_WarmVmStart(state, []() { return proxy_wasm::createWamrVm(); }); +} +BENCHMARK(BM_WamrWarmVmStart); +#endif + +TEST(TestVm, Benchmarks) { benchmark::RunSpecifiedBenchmarks(); } + } // namespace } // namespace proxy_wasm