From 27fd34b124d18c46e9f51fec6fcb7e879c7ed170 Mon Sep 17 00:00:00 2001 From: Thomas Debesse Date: Fri, 26 Jun 2026 03:13:19 +0200 Subject: [PATCH] cmake,VirtualMachine: append the architecture name to loader binaries, implement box64 fallback, detect armhf multiarch support --- CMakeLists.txt | 14 +- src/engine/framework/VirtualMachine.cpp | 493 +++++++++++++----- .../nacl_multiarch.cpp} | 20 +- 3 files changed, 371 insertions(+), 156 deletions(-) rename tools/{nacl_helper_bootstrap-armhf/nacl_helper_bootstrap-armhf.cpp => nacl_multiarch/nacl_multiarch.cpp} (84%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6af566d9bf..d5e4b209dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -974,7 +974,7 @@ if (DEPS_DIR AND HAS_NACL AND (BUILD_CLIENT OR BUILD_TTY_CLIENT OR BUILD_SERVER add_custom_command(TARGET runtime_deps PRE_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${DEPS_DIR}/nacl_loader${CMAKE_EXECUTABLE_SUFFIX} - ${FULL_OUTPUT_DIR}/nacl_loader${CMAKE_EXECUTABLE_SUFFIX} + ${FULL_OUTPUT_DIR}/nacl_loader-${DAEMON_NACL_ARCH_NAME}${CMAKE_EXECUTABLE_SUFFIX} ) add_custom_command(TARGET runtime_deps PRE_BUILD @@ -986,13 +986,11 @@ if (DEPS_DIR AND HAS_NACL AND (BUILD_CLIENT OR BUILD_TTY_CLIENT OR BUILD_SERVER # Linux uses a bootstrap program to reserve address space if (YOKAI_TARGET_SYSTEM_LINUX_COMPATIBILITY) if (YOKAI_TARGET_ARCH_ARM64) - add_executable(nacl_helper_bootstrap-armhf tools/nacl_helper_bootstrap-armhf/nacl_helper_bootstrap-armhf.cpp) - add_dependencies(runtime_deps nacl_helper_bootstrap-armhf) - - add_custom_command(TARGET runtime_deps PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${FULL_OUTPUT_DIR}/lib-armhf + add_executable(nacl_multiarch-${DAEMON_NACL_ARCH_NAME} + tools/nacl_multiarch/nacl_multiarch.cpp + ${BUILDINFOLIST} ) + add_dependencies(runtime_deps nacl_multiarch-${DAEMON_NACL_ARCH_NAME}) add_custom_command(TARGET runtime_deps PRE_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory @@ -1004,7 +1002,7 @@ if (DEPS_DIR AND HAS_NACL AND (BUILD_CLIENT OR BUILD_TTY_CLIENT OR BUILD_SERVER add_custom_command(TARGET runtime_deps PRE_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${DEPS_DIR}/nacl_helper_bootstrap - ${FULL_OUTPUT_DIR}/nacl_helper_bootstrap + ${FULL_OUTPUT_DIR}/nacl_helper_bootstrap-${DAEMON_NACL_ARCH_NAME} ) endif() diff --git a/src/engine/framework/VirtualMachine.cpp b/src/engine/framework/VirtualMachine.cpp index df2739b9f3..16ed47901e 100644 --- a/src/engine/framework/VirtualMachine.cpp +++ b/src/engine/framework/VirtualMachine.cpp @@ -75,6 +75,15 @@ static Cvar::Cvar workaround_naclSystem_freebsd_disableQualification( "Disable platform qualification when running Linux NaCl loader on FreeBSD through Linuxulator", Cvar::NONE, true); +#if defined(DAEMON_NACL_RUNTIME_ENABLED) +// Many BSDs have a Linuxulator, for now we only tested one. +#if defined(__linux__) || defined(__FreeBSD__) +#define DAEMON_NACL_RUNTIME_LINUX +#define DAEMON_NACL_BOOSTRAP_ENABLED +#endif // defined(__linux__) || defined(DAEMON_LINUXULATOR) +#endif // defined(DAEMON_NACL_RUNTIME_ENABLED) + +#if defined(DAEMON_NACL_RUNTIME_ENABLED) #if defined(DAEMON_NACL_BOX64_EMULATION) static Cvar::Cvar workaround_box64_disableQualification( "workaround.box64.disableQualification", @@ -88,14 +97,41 @@ static Cvar::Cvar workaround_box64_disableBootstrap( static Cvar::Cvar vm_box64_path( "vm.box64.path", - "Path to the box64 binary for NaCl emulation (empty = search PATH)", + "Path to the box64 binary for NaCl emulation (empty = use provided one or search PATH if missing)", Cvar::NONE, ""); -// Resolve box64 binary path by searching PATH if not explicitly set. -static std::string ResolveBox64Path() { - std::string path = vm_box64_path.Get(); - if (!path.empty()) { - return path; +// Resolve box64 binary path by looking for a provided one or searching PATH if not explicitly set. +static std::string ResolveBox64Path(std::string naclPath) { + std::string requestedPath = vm_box64_path.Get(); + + if (requestedPath.empty()) { + std::string providedPath = FS::Path::Build(naclPath, "box64"); + + if (FS::RawPath::FileExists(providedPath)) { + const std::string libs_linux_amd64 = "libs-linux-amd64"; + const std::string libgcc_linux_amd64 = "libgcc_s.so.1"; + + std::string libsDirPath = FS::Path::Build(naclPath, libs_linux_amd64); + std::string libGccPath = FS::Path::Build(libsDirPath, libgcc_linux_amd64); + + if (FS::RawPath::FileExists(libGccPath)) { + Sys::SetEnv("BOX64_LD_LIBRARY_PATH", libsDirPath.c_str()); + } + else { + Log::Warn("Using provided Box64 executable but the libraries are missing."); + } + + return providedPath; + } + + Log::Warn("Box64 emulation is enabled but the executable is not provided with the engine: %s", providedPath); + } + + if (!FS::RawPath::FileExists(requestedPath)) { + Sys::Error("Box64 emulation is enabled but the requested executable is missing: %s", requestedPath); + } + else { + return requestedPath; } const char* envPath = getenv("PATH"); @@ -121,34 +157,47 @@ static std::string ResolveBox64Path() { "Install Box64 or set vm.box64.path to the full path of the box64 binary."); return ""; // unreachable } -#endif +#endif // DAEMON_NACL_BOX64_EMULATION +#endif // DAEMON_NACL_RUNTIME_ENABLED + +#if defined(DAEMON_NACL_RUNTIME_ENABLED) +static constexpr bool vmAvailable = true; +#else // !defined(DAEMON_NACL_RUNTIME_ENABLED) +static constexpr bool vmAvailable = false; +#endif // !defined(DAEMON_NACL_RUNTIME_ENABLED) + +static Cvar::Cvar vm_nacl_available( + "vm.nacl.available", + "Whether NaCl runtime is available on this platform", + Cvar::ROM, vmAvailable); + +#if defined(DAEMON_NACL_RUNTIME_ENABLED) +static Cvar::Cvar vm_timeout( + "vm.timeout", + "Receive timeout in seconds", + Cvar::NONE, 2); static Cvar::Cvar vm_nacl_qualification( "vm.nacl.qualification", "Enable NaCl loader platform qualification", Cvar::INIT, true); +#endif // defined(DAEMON_NACL_RUNTIME_ENABLED) + +#if defined(DAEMON_NACL_RUNTIME_ENABLED) +#if defined(DAEMON_NACL_RUNTIME_LINUX) && defined(YOKAI_ARCH_ARM64) +static Cvar::Cvar vm_nacl_multiarch( + "vm.nacl.multiarch", + "Use multiarch to run the NaCl loader when needed and available", + Cvar::INIT, true); +#endif // defined(DAEMON_NACL_RUNTIME_LINUX) && defined(YOKAI_ARCH_ARM64) +#endif // defined(DAEMON_NACL_RUNTIME_ENABLED) +#if defined(DAEMON_NACL_BOOSTRAP_ENABLED) static Cvar::Cvar vm_nacl_bootstrap( "vm.nacl.bootstrap", "Use NaCl bootstrap helper", Cvar::INIT, true); - -static Cvar::Cvar vm_timeout( - "vm.timeout", - "Receive timeout in seconds", - Cvar::NONE, 2); - -#if defined(DAEMON_NACL_RUNTIME_ENABLED) -static Cvar::Cvar vm_nacl_available( - "vm.nacl.available", - "Whether NaCl runtime is available on this platform", - Cvar::ROM, true); -#else -static Cvar::Cvar vm_nacl_available( - "vm.nacl.available", - "Whether NaCl runtime is available on this platform", - Cvar::ROM, false); -#endif +#endif // defined(DAEMON_NACL_BOOSTRAP_ENABLED) namespace VM { @@ -300,57 +349,225 @@ static std::pair InternalLoadModule(std::pair(probe.data()), + nullptr, + }; + + execve(probe.data(), argv, environ); + + switch (errno) + { + case ENOEXEC: + // The kernel cannot execute the probe. + _exit(100); + + case ENOENT: + case EACCES: + // Missing or inaccessible probe. + _exit(101); + + default: + _exit(102); + } + } + + int status; + if (waitpid(pid, &status, 0) < 0) { + return ExeSupport::Unknown; + } + + if (!WIFEXITED(status)) { + return ExeSupport::Unknown; + } + + switch (WEXITSTATUS(status)) + { + case 100: + return ExeSupport::Unsupported; + + case 101: + case 102: + return ExeSupport::Unknown; + + default: + // The probe has been executed properly. + return ExeSupport::Supported; + } +} +#endif // defined(DAEMON_NACL_RUNTIME_LINUX) && defined(NACL_ARCH_ARM64) + +#if defined(DAEMON_NACL_RUNTIME_ENABLED) static std::pair CreateNaClVM(std::pair pair, Str::StringRef name, bool debug, bool extract, int debugLoader) { CheckMinAddressSysctlTooLarge(); const std::string& libPath = FS::GetLibPath(); -#ifdef DAEMON_NACL_RUNTIME_PATH - const char* naclPath = DAEMON_NACL_RUNTIME_PATH_STRING; -#else - const std::string& naclPath = libPath; -#endif std::vector args; char rootSocketRedir[32]; - std::string module, nacl_loader, irt, bootstrap, modulePath, verbosity; FS::File stderrRedirect; -#if defined(DAEMON_NACL_BOX64_EMULATION) - std::string box64Path; - bool usingBox64 = false; -#endif + bool inheritEnvironment = false; + +#if defined(DAEMON_NACL_RUNTIME_PATH) + const char* naclPath = DAEMON_NACL_RUNTIME_PATH_STRING; +#else // !defined(DAEMON_NACL_RUNTIME_PATH) + const std::string& naclPath = libPath; +#endif // !defined(DAEMON_NACL_RUNTIME_PATH) + #if !defined(_WIN32) || defined(_WIN64) - constexpr bool win32Force64Bit = false; -#else + constexpr bool i686ForceAmd64 = false; +#else // !( !defined(_WIN32) || defined(_WIN64) ) // On Windows, even if we are running a 32-bit engine, we must use the // 64-bit nacl_loader if the host operating system is 64-bit. SYSTEM_INFO systemInfo; GetNativeSystemInfo(&systemInfo); - const bool win32Force64Bit = systemInfo.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_AMD64; + bool i686ForceAmd64 = systemInfo.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_AMD64; +#endif // !( !defined(_WIN32) || defined(_WIN64) ) + + std::string multiarchPath; + +#if defined(DAEMON_NACL_RUNTIME_LINUX) && defined(YOKAI_ARCH_ARM64) + bool hasMultiarch = vm_nacl_multiarch.Get(); +#else + constexpr bool hasMultiarch = false; + Q_UNUSED(multiarchPath); #endif + std::string bootstrapPath; + +#if defined(DAEMON_NACL_BOOSTRAP_ENABLED) + bool hasBootstrap = vm_nacl_bootstrap.Get(); +#else // !defined(DAEMON_NACL_BOOSTRAP_ENABLED) + constexpr bool hasBootstrap = false; + Q_UNUSED(bootstrapPath); +#endif // !defined(DAEMON_NACL_BOOSTRAP_ENABLED) + + std::string box64Path; + +#if defined(DAEMON_NACL_BOX64_EMULATION) + constexpr bool hasBox64 = true; +#else // !defined(DAEMON_NACL_BOX64_EMULATION) + constexpr bool hasBox64 = false; + Q_UNUSED(box64Path); +#endif // !defined(DAEMON_NACL_BOX64_EMULATION) + +#if defined(DAEMON_NACL_RUNTIME_LINUX) +#if defined(YOKAI_ARCH_ARM64) + /* We later turn this boolean off when we detect armhf support is not + provided on arm64. */ + bool hasArmhf = true; +#elif defined(YOKAI_ARCH_ARMHF) + constexpr bool hasArmhf = false; +#endif // !defined(YOKAI_ARCH_ARMHF) +#endif // !defined(DAEMON_NACL_RUNTIME_LINUX) + + bool useMultiarch = hasMultiarch; + bool useBox64 = hasBox64; + +#if defined(DAEMON_NACL_RUNTIME_LINUX) && defined(YOKAI_ARCH_ARM64) + const std::string libs_linux_armhf = "lib-armhf"; + const std::string libld_linux_armhf = "ld-linux-armhf"; + + std::string libsDirPath = FS::Path::Build(libPath, libs_linux_armhf); + std::string libLoaderPath = FS::Path::Build(libsDirPath, libld_linux_armhf); + + switch (CheckExeSupport(libLoaderPath)) + { + case ExeSupport::Supported: + Log::Notice("Working multiarch support."); + break; + + case ExeSupport::Unsupported: + Log::Notice("Unsupported multiarch."); + hasArmhf = false; + break; + + case ExeSupport::Unknown: + Log::Warn("Cannot determine multiarch support."); + hasArmhf = false; + break; + } + + if (!hasArmhf) { + useMultiarch = false; + } +#endif // defined(DAEMON_NACL_RUNTIME_LINUX) && defined(YOKAI_ARCH_ARM64) + + if (useMultiarch && useBox64) { + Log::Notice("Preferring multiarch over Box64 emulation."); + useBox64 = false; + } + + bool useBootstrap = hasBootstrap; + +#if defined(DAEMON_NACL_BOX64_EMULATION) + if (useBootstrap) { + /* Use Box64 to run the x86_64 NaCl loader on non-x86 architectures. + The bootstrap helper uses a double-exec pattern that Box64 cannot handle, + so we skip it and prepend "box64" to the nacl_loader command instead. */ + if (useBox64 + && workaround_box64_disableBootstrap.Get()) { + useBootstrap = false; + } + } +#endif // defined(DAEMON_NACL_BOX64_EMULATION) + + std::string arch = (useBox64 || i686ForceAmd64) ? "amd64" : DAEMON_NACL_ARCH_STRING; + + std::string moduleName = Str::Format("%s-%s.nexe", name, arch); + std::string irt = FS::Path::Build(naclPath, Str::Format("irt_core-%s.nexe", arch)); + std::string nacl_loader = FS::Path::Build(naclPath, Str::Format("nacl_loader-%s%s", arch, EXE_EXT)); + + std::string modulePath; + // Extract the nexe from the pak so that nacl_loader can load it - module = win32Force64Bit - ? name + "-amd64.nexe" - : Str::Format("%s-%s.nexe", name, DAEMON_NACL_ARCH_STRING); if (extract) { try { - FS::File out = FS::HomePath::OpenWrite(module); - if (const FS::LoadedPakInfo* pak = FS::PakPath::LocateFile(module)) - Log::Notice("Extracting VM module %s from %s...", module.c_str(), pak->path.c_str()); - FS::PakPath::CopyFile(module, out); + FS::File out = FS::HomePath::OpenWrite(moduleName); + if (const FS::LoadedPakInfo* pak = FS::PakPath::LocateFile(moduleName)) + Log::Notice("Extracting VM module %s from %s...", moduleName.c_str(), pak->path.c_str()); + FS::PakPath::CopyFile(moduleName, out); out.Close(); } catch (std::system_error& err) { - Sys::Drop("VM: Failed to extract VM module %s: %s", module, err.what()); + Sys::Drop("VM: Failed to extract VM module %s: %s", moduleName, err.what()); } - modulePath = FS::Path::Build(FS::GetHomePath(), module); + modulePath = FS::Path::Build(FS::GetHomePath(), moduleName); } else { - modulePath = FS::Path::Build(libPath, module); + modulePath = FS::Path::Build(libPath, moduleName); } // Generate command line Q_snprintf(rootSocketRedir, sizeof(rootSocketRedir), "%d:%d", ROOT_SOCKET_FD, (int)(intptr_t)pair.second.GetHandle()); - irt = FS::Path::Build(naclPath, win32Force64Bit - ? "irt_core-amd64.nexe" - : Str::Format("irt_core-%s.nexe", DAEMON_NACL_ARCH_STRING)); - nacl_loader = FS::Path::Build(naclPath, win32Force64Bit ? "nacl_loader-amd64" EXE_EXT : "nacl_loader" EXE_EXT); if (!FS::RawPath::FileExists(modulePath)) { Sys::Error("VM module file not found: %s", modulePath); @@ -364,99 +581,108 @@ static std::pair CreateNaClVM(std::pair Error while loading "sgame-armhf.nexe": CPU model is not supported + > Error while loading "sgame-armhf.nexe": CPU model is not supported - From nacl_loader --help we can read: + From nacl_loader --help we can read: - > -Q disable platform qualification (dangerous!) + > -Q disable platform qualification (dangerous!) - When this option is enabled, nacl_loader will print: + When this option is enabled, nacl_loader will print: - > PLATFORM QUALIFICATION DISABLED BY -Q - Native Client's sandbox will be unreliable! + > PLATFORM QUALIFICATION DISABLED BY -Q - Native Client's sandbox will be unreliable! - But the nexe will load and run. */ + But the nexe will load and run. */ - if (onArm64) { - Log::Warn("Disabling NaCL platform qualification on arm64 kernel architecture."); - enableQualification = false; - } + if (onArm64) { + Log::Warn("Disabling NaCl platform qualification on arm64 kernel architecture."); + enableQualification = false; } -#endif + } +#endif // defined(__linux__) && (defined(YOKAI_ARCH_ARM64) || defined(YOKAI_ARCH_ARMHF)) + +#if defined(DAEMON_NACL_BOX64_EMULATION) + if (enableQualification + && useBox64 + && workaround_box64_disableQualification.Get()) { + /* When running the amd64 NaCl loader under Box64, the loader's + platform qualification will fail because the CPU is not actually x86_64. + Disabling qualification allows the emulated loader to proceed. */ + + Log::Warn("Disabling NaCl platform qualification for Box64 emulation."); + enableQualification = false; + } +#endif // defined(DAEMON_NACL_BOX64_EMULATION) #if defined(__FreeBSD__) + if (enableQualification + && workaround_naclSystem_freebsd_disableQualification.Get()) { /* While it is possible to build a native FreeBSD engine, the only available NaCl loader is the Linux one, which can run on Linuxulator (the FreeBSD Linux compatibility layer). @@ -466,26 +692,10 @@ static std::pair CreateNaClVM(std::pair CreateNaClVM(std::pair CreateNaClVM(std::pair CreateNaClVM(std::pair CreateNativeVM(std::pair pair, Str::StringRef name, bool debug) { const std::string& libPath = FS::GetLibPath(); @@ -622,7 +831,9 @@ void VMBase::Create() } #endif if (type == TYPE_NACL || type == TYPE_NACL_LIBPATH) { +#if defined(DAEMON_NACL_RUNTIME_ENABLED) std::tie(processHandle, rootSocket) = CreateNaClVM(std::move(pair), name, params.debug.Get(), type == TYPE_NACL, params.debugLoader.Get()); +#endif // defined(DAEMON_NACL_RUNTIME_ENABLED) } else if (type == TYPE_NATIVE_EXE) { std::tie(processHandle, rootSocket) = CreateNativeVM(std::move(pair), name, params.debug.Get()); } else { diff --git a/tools/nacl_helper_bootstrap-armhf/nacl_helper_bootstrap-armhf.cpp b/tools/nacl_multiarch/nacl_multiarch.cpp similarity index 84% rename from tools/nacl_helper_bootstrap-armhf/nacl_helper_bootstrap-armhf.cpp rename to tools/nacl_multiarch/nacl_multiarch.cpp index ec16447482..84c4b28bfe 100644 --- a/tools/nacl_helper_bootstrap-armhf/nacl_helper_bootstrap-armhf.cpp +++ b/tools/nacl_multiarch/nacl_multiarch.cpp @@ -35,7 +35,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include "YokaiBuildInfo/DaemonEngine.h" + int main(int argc, char *argv[]) { + if (argc < 2) { + printf("nacl_multiarch-%s: missing command line\n", DAEMON_NACL_ARCH_STRING); + return 1; + } + char *directory = dirname(strdup(argv[0])); int err = chdir(directory); @@ -44,19 +51,18 @@ int main(int argc, char *argv[]) { return err; } - err = putenv(strdup("LD_LIBRARY_PATH=lib-armhf")); + char env[100]; + sprintf(env, "LD_LIBRARY_PATH=lib-%s", DAEMON_NACL_ARCH_STRING); + + err = putenv(strdup(env)); if (err != 0) { return err; } - char *helper = strdup("./nacl_helper_bootstrap"); - argv[0] = helper; - - execv(helper, argv); + execv(argv[1], &argv[1]); // The execv() function returns only if an error has occurred. - printf("nacl_helper_bootstrap-armhf: %s\n", strerror(errno)); - + printf("nacl_multiarch-%s: %s\n", DAEMON_NACL_ARCH_STRING, strerror(errno)); return 1; }