Fix TestVm::Init test to reflect true time to create a warmed thread

bazel/dependencies.bzl

@@ -14,6 +14,7 @@
 
 load("@aspect_rules_lint//format:repositories.bzl", "rules_lint_dependencies")
 load("@bazel_lib//lib:repositories.bzl", "bazel_lib_dependencies", "bazel_lib_register_toolchains")
+load("@com_google_benchmark//:bazel/benchmark_deps.bzl", "benchmark_deps")
 load("@com_google_googletest//:googletest_deps.bzl", "googletest_deps")
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 load("@envoy_toolshed//sysroot:sysroot.bzl", "setup_sysroots")
@@ -31,6 +32,7 @@ def proxy_wasm_cpp_host_dependencies():
     # Bazel extensions.
     googletest_deps()
     rules_foreign_cc_dependencies()
+    benchmark_deps()
 
     rules_lint_dependencies()
 

bazel/repositories.bzl

@@ -193,6 +193,14 @@ def proxy_wasm_cpp_host_repositories():
         },
     )
 
+    maybe(
+        http_archive,
+        name = "com_google_benchmark",
+        sha256 = "9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340",
+        strip_prefix = "benchmark-1.9.5",
+        urls = ["https://github.com/google/benchmark/archive/refs/tags/v1.9.5.tar.gz"],
+    )
+
     # NullVM dependencies.
 
     maybe(

test/BUILD

@@ -225,6 +225,7 @@ cc_test(
         "//:lib",
         "@com_google_googletest//:gtest",
         "@com_google_googletest//:gtest_main",
+        "@com_google_benchmark//:benchmark",
     ],
 )
 

test/wasm_vm_test.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "gtest/gtest.h"
+#include "benchmark/benchmark.h"
 
 #include <memory>
 #include <string>
@@ -31,10 +32,12 @@ INSTANTIATE_TEST_SUITE_P(WasmEngines, TestVm, testing::ValuesIn(getWasmEngines()
                          });
 
 TEST_P(TestVm, Init) {
+  std::unique_ptr<WasmVm> vm1 = makeVm(engine_);
+  std::unique_ptr<WasmVm> vm2 = makeVm(engine_);
   auto time1 = std::chrono::steady_clock::now();
-  vm_->warm();
+  vm1->warm();
   auto time2 = std::chrono::steady_clock::now();
-  vm_->warm();
+  vm2->warm();
   auto time3 = std::chrono::steady_clock::now();
 
   auto cold = std::chrono::duration_cast<std::chrono::nanoseconds>(time2 - time1).count();
@@ -43,24 +46,37 @@ TEST_P(TestVm, Init) {
   std::cout << "[" << engine_ << "] \"cold\" engine time: " << cold << "ns" << std::endl;
   std::cout << "[" << engine_ << "] \"warm\" engine time: " << warm << "ns" << std::endl;
 
-  // Default warm time in nanoseconds.
-  int warm_time_ns_limit = 10000;
-
-#if defined(__linux__) && defined(__s390x__)
-  // Linux 390x is significantly slower, so we use a more lenient limit.
-  warm_time_ns_limit = 75000;
-#endif
-
-  // Verify that getting a "warm" engine takes less than 10us.
-  EXPECT_LE(warm, warm_time_ns_limit);
-
   // Verify that getting a "warm" engine takes at least 50x less time than getting a "cold" one.
   // We skip NullVM because warm() is a noop.
   if (engine_ == "null") {
     std::cout << "Skipping warm() performance assertions for NullVM." << std::endl;
     return;
   }
-  EXPECT_LE(warm * 50, cold);
+  int expected_warm_time_ns = 0;
+  double expected_warm_time_speedup_factor = 0;
+  if (engine_ == "v8") {
+    expected_warm_time_ns = 20000000;
+    expected_warm_time_speedup_factor = 1.5;
+  } else if (engine_ == "wamr") {
+    expected_warm_time_ns = 4000;
+    expected_warm_time_speedup_factor = 2000;
+  } else if (engine_ == "wasmedge") {
+    expected_warm_time_ns = 2000;
+    expected_warm_time_speedup_factor = 1.5;
+  } else if (engine_ == "wasmtime") {
+    expected_warm_time_ns = 8000;
+    expected_warm_time_speedup_factor = 6;
+  }
+  // Linux 390x is significantly slower, so we use a more lenient limit.
+#if defined(__linux__) && defined(__s390x__)
+  expected_warm_time_ns *= 10;
+#endif
+#if defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(ADDRESS_SANITIZER) ||        \
+    defined(HWADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
+  expected_warm_time_ns *= 10;
+#endif
+  EXPECT_LE(warm * expected_warm_time_speedup_factor, cold);
+  EXPECT_LT(warm, expected_warm_time_ns);
 }
 
 TEST_P(TestVm, Basic) {
@@ -164,5 +180,40 @@ TEST_P(TestVm, DISABLED_CloneUntilOutOfMemory) {
 
 #endif
 
+void BM_WarmVmStart(benchmark::State &state, auto makeVm) {
+  std::unique_ptr<WasmVm> cold_vm = makeVm();
+  cold_vm->warm();
+  for (auto _ : state) {
+    std::unique_ptr<WasmVm> warm_vm = makeVm();
+    warm_vm->warm();
+  }
+}
+#ifdef PROXY_WASM_HOST_ENGINE_V8
+static void BM_V8WarmVmStart(benchmark::State &state) {
+  BM_WarmVmStart(state, []() { return proxy_wasm::createV8Vm(); });
+}
+BENCHMARK(BM_V8WarmVmStart);
+#endif
+#ifdef PROXY_WASM_HOST_ENGINE_WASMTIME
+static void BM_WasmtimeWarmVmStart(benchmark::State &state) {
+  BM_WarmVmStart(state, []() { return proxy_wasm::createWasmtimeVm(); });
+}
+BENCHMARK(BM_WasmtimeWarmVmStart);
+#endif
+#ifdef PROXY_WASM_HOST_ENGINE_WASMEDGE
+static void BM_WasmEdgeWarmVmStart(benchmark::State &state) {
+  BM_WarmVmStart(state, []() { return proxy_wasm::createWasmEdgeVm(); });
+}
+BENCHMARK(BM_WasmEdgeWarmVmStart);
+#endif
+#ifdef PROXY_WASM_HOST_ENGINE_WAMR
+static void BM_WamrWarmVmStart(benchmark::State &state) {
+  BM_WarmVmStart(state, []() { return proxy_wasm::createWamrVm(); });
+}
+BENCHMARK(BM_WamrWarmVmStart);
+#endif
+
+TEST(TestVm, Benchmarks) { benchmark::RunSpecifiedBenchmarks(); }
+
 } // namespace
 } // namespace proxy_wasm