diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f49c64..4971e10 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -161,3 +161,39 @@ jobs: - name: Verify Logging Pipeline run: | python -u tests/verify_pipeline.py + + build-macos: + name: Build macOS Metal monitor + runs-on: macos-14 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install build tools + run: python -m pip install cmake==3.31.6 + + - name: Build native Metal monitor + run: ./build.sh --monitor --build-dir build-macos-ci + + - name: Configure C++ tests + run: | + cmake -S . -B build-macos-ci \ + -DBUILD_TESTING=ON \ + -DBUILD_GPUFL_EXAMPLE=OFF \ + -DBUILD_PYTHON=OFF \ + -DBUILD_GPUFL_MONITOR=ON \ + -DBUILD_GPUFL_LAUNCHER=OFF \ + -DBUILD_GPUFL_INJECT=OFF \ + -DGPUFL_ENABLE_NVIDIA=OFF \ + -DGPUFL_ENABLE_AMD=OFF \ + -DGPUFL_ENABLE_METAL=ON + + - name: Run C++ unit tests + run: | + cmake --build build-macos-ci --target gpufl_tests --parallel + ctest --test-dir build-macos-ci --output-on-failure --verbose --timeout 60 diff --git a/.gitignore b/.gitignore index d23e90c..4283c06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,15 @@ ### ai .claude/ .junie/ +.cache/ ### idea .idea/** build/ build-*/ +build_metal_stub/ build_tests/ +gpufl-monitor-macos/ cmake-build-*/ CMakeFiles/ CMakeCache.txt @@ -92,6 +95,8 @@ dist/ *.log +AGENTS.md + ### Python # Byte-compiled / optimized files __pycache__/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a1e2af..2de9247 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,8 +26,17 @@ set(CMAKE_CXX_EXTENSIONS OFF) # ----------------------- # Options # ----------------------- -option(GPUFL_ENABLE_NVIDIA "Enable NVIDIA backends (CUDA + NVML when available)" ON) +if(APPLE) + set(GPUFL_ENABLE_NVIDIA_DEFAULT OFF) + set(GPUFL_ENABLE_METAL_DEFAULT ON) +else() + set(GPUFL_ENABLE_NVIDIA_DEFAULT ON) + set(GPUFL_ENABLE_METAL_DEFAULT OFF) +endif() + +option(GPUFL_ENABLE_NVIDIA "Enable NVIDIA backends (CUDA + NVML when available)" ${GPUFL_ENABLE_NVIDIA_DEFAULT}) option(GPUFL_ENABLE_AMD "Enable AMD backends (ROCm when available)" OFF) +option(GPUFL_ENABLE_METAL "Enable Apple Metal backend (macOS basic monitoring)" ${GPUFL_ENABLE_METAL_DEFAULT}) option(GPUFL_ENABLE_AMD_ROCPROFILER "Enable AMD rocprofiler-sdk tracing backend when available" ON) # OFF builds the configuration release wheels ship (manylinux has no nvperf), # so CI can catch #if GPUFL_HAS_PERFWORKS mistakes. @@ -226,6 +235,7 @@ set(GPUFL_HAS_ROCM 0) set(GPUFL_HAS_ROCM_SMI 0) set(GPUFL_HAS_HIP 0) set(GPUFL_HAS_ROCPROFILER_SDK 0) +set(GPUFL_HAS_METAL 0) set(GPUFL_HAS_CUPTI 0) set(GPUFL_HAS_PERFWORKS 0) # ZLIB - try system install first, fall back to FetchContent so every platform @@ -285,6 +295,12 @@ target_sources(gpufl PRIVATE include/gpufl/core/logger/file_compressor.cpp) # and log a warning so the user knows HTTPS endpoints will fail. # ----------------------- include(FetchContent) + +# Resolve OpenSSL in this directory before cpp-httplib configures itself. +# Imported targets created only inside FetchContent's subdirectory are not +# guaranteed to be visible here, even though OpenSSL_FOUND can remain true. +find_package(OpenSSL 3.0 QUIET COMPONENTS SSL Crypto) + FetchContent_Declare( httplib GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git @@ -302,8 +318,7 @@ set(HTTPLIB_COMPILE OFF CACHE BOOL "" FORCE) set(HTTPLIB_INSTALL ON CACHE BOOL "" FORCE) FetchContent_MakeAvailable(httplib) -find_package(OpenSSL QUIET) -if(OpenSSL_FOUND) +if(TARGET OpenSSL::SSL AND TARGET OpenSSL::Crypto) message(STATUS "Found OpenSSL: ${OPENSSL_VERSION} - HTTPS upload enabled") target_compile_definitions(gpufl PRIVATE CPPHTTPLIB_OPENSSL_SUPPORT=1) target_link_libraries(gpufl PRIVATE OpenSSL::SSL OpenSSL::Crypto) @@ -312,7 +327,7 @@ else() message(WARNING "OpenSSL not found - gpufl::uploadLogs will support HTTP only. " "Pointing backend_url at an https:// endpoint will fail to verify. " - "Install OpenSSL (apt: libssl-dev, vcpkg: openssl, brew: openssl) " + "Install OpenSSL (apt: libssl-dev, vcpkg: openssl, brew: openssl@3) " "to enable TLS.") set(GPUFL_HTTPLIB_TLS 0) endif() @@ -353,6 +368,12 @@ else() target_compile_definitions(gpufl PUBLIC GPUFL_ENABLE_AMD=0) endif() +if(GPUFL_ENABLE_METAL) + target_compile_definitions(gpufl PUBLIC GPUFL_ENABLE_METAL=1) +else() + target_compile_definitions(gpufl PUBLIC GPUFL_ENABLE_METAL=0) +endif() + if(GPUFL_ENABLE_NVIDIA) # # CUDA capability: only if CUDA toolkit is available @@ -729,6 +750,32 @@ if(GPUFL_ENABLE_AMD) endif() endif() +# ----------------------- +# Metal backend (basic native monitoring) +# ----------------------- +if(GPUFL_ENABLE_METAL) + if(APPLE) + enable_language(OBJCXX) + find_library(METAL_FRAMEWORK Metal) + find_library(FOUNDATION_FRAMEWORK Foundation) + if(METAL_FRAMEWORK AND FOUNDATION_FRAMEWORK) + set(GPUFL_HAS_METAL 1) + target_sources(gpufl PRIVATE + include/gpufl/backends/metal/metal_collector.mm + ) + target_link_libraries(gpufl PRIVATE + ${METAL_FRAMEWORK} + ${FOUNDATION_FRAMEWORK} + ) + message(STATUS "Found Metal framework: ${METAL_FRAMEWORK}") + else() + message(WARNING "GPUFL_ENABLE_METAL=ON but Metal/Foundation framework not found; GPUFL_HAS_METAL=0.") + endif() + else() + message(WARNING "GPUFL_ENABLE_METAL=ON is only supported on macOS; GPUFL_HAS_METAL=0.") + endif() +endif() + target_compile_definitions(gpufl PUBLIC GPUFL_HAS_CUDA=${GPUFL_HAS_CUDA} GPUFL_HAS_NVML=${GPUFL_HAS_NVML} @@ -739,6 +786,7 @@ target_compile_definitions(gpufl PUBLIC GPUFL_HAS_ROCM_SMI=${GPUFL_HAS_ROCM_SMI} GPUFL_HAS_HIP=${GPUFL_HAS_HIP} GPUFL_HAS_ROCPROFILER_SDK=${GPUFL_HAS_ROCPROFILER_SDK} + GPUFL_HAS_METAL=${GPUFL_HAS_METAL} ) # ----------------------- diff --git a/README.md b/README.md index 5feccd3..7147ba8 100644 --- a/README.md +++ b/README.md @@ -64,11 +64,11 @@ CMAKE_ARGS="-DBUILD_TESTING=OFF" pip install "./gpufl-client[analyzer,viz]" The repository includes platform-specific helper scripts for local source builds. Use these scripts when you need to build against a specific CUDA -Toolkit, Python virtual environment, or wheel ABI. +Toolkit, Python virtual environment, Metal backend, or wheel ABI. #### Ubuntu / Linux -`build.sh` is the Linux entrypoint. It delegates to `build-ubuntu.sh`. +On Linux, `build.sh` delegates to `build-ubuntu.sh`. ```bash # Install into the active Python environment @@ -97,6 +97,43 @@ Useful options: | `--cuda-root PATH` | CUDA Toolkit root, for example `/usr/local/cuda-13.2`. | | `--wheel-dir PATH` | Output directory for built wheels. | +#### macOS / Apple Metal + +On macOS, `build.sh` delegates to `build-macos.sh`. The script builds with +Metal enabled and NVIDIA/AMD disabled. When Homebrew's `openssl@3` is +installed, the script passes its keg-only prefix to CMake automatically. + +The Metal backend currently supports native monitoring only. CUDA trace +injection and the `--trace` build mode are not available on macOS. + +```bash +# Install into the active Python environment +./build.sh + +# Build a wheel into ./dist +./build.sh --wheel + +# Build the native monitor binary +./build.sh --monitor + +# Build and run the monitor with a writable local log directory +./scripts/run-monitor-macos.sh + +# Use an explicit Python venv +./build-macos.sh --wheel --python .venv/bin/python +``` + +Useful options: + +| Option | Meaning | +|---|---| +| `--install` | Install the package into the selected Python environment. This is the default. | +| `--wheel` | Build a wheel into `./dist` or `--wheel-dir`. | +| `--monitor` | Build the native `gpufl-monitor` binary with the Metal backend. | +| `--python PATH` | Python executable to use for install or wheel mode. | +| `--wheel-dir PATH` | Output directory for built wheels. | +| `--build-dir PATH` | CMake build directory for `--monitor`. | + #### Windows Use `build-windows.ps1` from PowerShell. The script imports the Visual Studio @@ -128,7 +165,7 @@ Useful parameters: | `-WheelDir PATH` | Output directory for built wheels. | | `-NoVcVars` | Skip importing `vcvars64.bat` and use the current shell environment. | -Both platform scripts pass the current CMake options: +Linux and Windows Python builds pass the current CUDA CMake options: ```text BUILD_PYTHON=ON @@ -141,6 +178,18 @@ CUDAToolkit_ROOT= CMAKE_CUDA_COMPILER= ``` +macOS Python builds pass: + +```text +BUILD_PYTHON=ON +BUILD_GPUFL_EXAMPLE=OFF +BUILD_TESTING=OFF +PYBIND11_FINDPYTHON=ON +GPUFL_ENABLE_NVIDIA=OFF +GPUFL_ENABLE_AMD=OFF +GPUFL_ENABLE_METAL=ON +``` + ### C++ (CMake FetchContent) ```cmake cmake_minimum_required(VERSION 3.31) diff --git a/build-macos.sh b/build-macos.sh new file mode 100755 index 0000000..38b4528 --- /dev/null +++ b/build-macos.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PYTHON_BIN="${PYTHON:-python3}" +MODE="install" +WHEEL_DIR="$ROOT_DIR/dist" +BUILD_DIR="$ROOT_DIR/build-macos-monitor" +MONITOR_LOG_DIR="${GPUFL_MONITOR_LOG_DIR:-$ROOT_DIR/gpufl-monitor-macos/session}" +OPENSSL_ROOT="${OPENSSL_ROOT_DIR:-}" + +if [[ -z "$OPENSSL_ROOT" ]] && command -v brew >/dev/null 2>&1; then + OPENSSL_ROOT="$(brew --prefix openssl@3 2>/dev/null || true)" +fi + +usage() { + cat <<'EOF' +Usage: ./build-macos.sh [--install|--wheel|--monitor] [--python PATH] [--wheel-dir PATH] [--build-dir PATH] + +Defaults: + --install + --python ${PYTHON:-python3} + --wheel-dir ./dist + --build-dir ./build-macos-monitor + +Examples: + ./build-macos.sh + ./build-macos.sh --wheel + ./build-macos.sh --python .venv/bin/python + ./build-macos.sh --monitor +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --install) + MODE="install" + shift + ;; + --wheel) + MODE="wheel" + shift + ;; + --monitor) + MODE="monitor" + shift + ;; + --python) + PYTHON_BIN="$2" + shift 2 + ;; + --wheel-dir) + WHEEL_DIR="$2" + shift 2 + ;; + --build-dir) + BUILD_DIR="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ "$(uname -s 2>/dev/null || echo unknown)" != "Darwin" ]]; then + echo "build-macos.sh is only supported on macOS." >&2 + exit 2 +fi + +COMMON_CONFIG=( + -C cmake.define.BUILD_PYTHON=ON + -C cmake.define.BUILD_GPUFL_EXAMPLE=OFF + -C cmake.define.BUILD_TESTING=OFF + -C cmake.define.PYBIND11_FINDPYTHON=ON + -C cmake.define.GPUFL_ENABLE_NVIDIA=OFF + -C cmake.define.GPUFL_ENABLE_AMD=OFF + -C cmake.define.GPUFL_ENABLE_METAL=ON +) + +OPENSSL_CMAKE_ARG=() +if [[ -n "$OPENSSL_ROOT" ]]; then + COMMON_CONFIG+=( + -C "cmake.define.OPENSSL_ROOT_DIR=$OPENSSL_ROOT" + -C "cmake.define.OPENSSL_INCLUDE_DIR=$OPENSSL_ROOT/include" + -C "cmake.define.OPENSSL_SSL_LIBRARY=$OPENSSL_ROOT/lib/libssl.dylib" + -C "cmake.define.OPENSSL_CRYPTO_LIBRARY=$OPENSSL_ROOT/lib/libcrypto.dylib" + ) + OPENSSL_CMAKE_ARG=( + "-DOPENSSL_ROOT_DIR=$OPENSSL_ROOT" + "-DOPENSSL_INCLUDE_DIR=$OPENSSL_ROOT/include" + "-DOPENSSL_SSL_LIBRARY=$OPENSSL_ROOT/lib/libssl.dylib" + "-DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_ROOT/lib/libcrypto.dylib" + ) +fi + +echo "GPUFlight macOS build" +echo " mode: $MODE" +echo " python: $PYTHON_BIN" +if [[ -n "$OPENSSL_ROOT" ]]; then + echo " OpenSSL: $OPENSSL_ROOT" +fi + +if [[ "$MODE" == "wheel" ]]; then + mkdir -p "$WHEEL_DIR" + "$PYTHON_BIN" -m pip wheel "$ROOT_DIR" -w "$WHEEL_DIR" --no-deps -v "${COMMON_CONFIG[@]}" +elif [[ "$MODE" == "monitor" ]]; then + echo " build: $BUILD_DIR" + cmake -S "$ROOT_DIR" -B "$BUILD_DIR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_GPUFL_EXAMPLE=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_PYTHON=OFF \ + -DBUILD_GPUFL_MONITOR=ON \ + -DBUILD_GPUFL_LAUNCHER=OFF \ + -DBUILD_GPUFL_INJECT=OFF \ + -DGPUFL_ENABLE_NVIDIA=OFF \ + -DGPUFL_ENABLE_AMD=OFF \ + -DGPUFL_ENABLE_METAL=ON \ + "${OPENSSL_CMAKE_ARG[@]}" + cmake --build "$BUILD_DIR" --target gpufl-monitor --parallel + echo "" + echo "Built native monitor:" + echo " $BUILD_DIR/daemon/monitor/gpufl-monitor" + echo "" + echo "Run:" + echo " GPUFL_MONITOR_LOG_DIR=\"$MONITOR_LOG_DIR\" \\" + echo " GPUFL_MONITOR_BACKEND=metal \"$BUILD_DIR/daemon/monitor/gpufl-monitor\"" + echo "" + echo "Build and run with defaults:" + echo " $ROOT_DIR/scripts/run-monitor-macos.sh" +else + "$PYTHON_BIN" -m pip install "$ROOT_DIR" -v "${COMMON_CONFIG[@]}" +fi diff --git a/build.sh b/build.sh index 181fbe8..05bb5b1 100755 --- a/build.sh +++ b/build.sh @@ -8,9 +8,17 @@ case "$UNAME" in Linux*) exec "$ROOT_DIR/build-ubuntu.sh" "$@" ;; + Darwin*) + exec "$ROOT_DIR/build-macos.sh" "$@" + ;; *) cat >&2 <<'EOF' -build.sh is the Ubuntu/Linux entrypoint. +build.sh supports Linux and macOS. + +On macOS, use: + ./build-macos.sh + ./build-macos.sh --wheel + ./build-macos.sh --monitor On Windows, use: powershell -ExecutionPolicy Bypass -File .\build-windows.ps1 diff --git a/daemon/README.md b/daemon/README.md index 83dea77..a062e41 100644 --- a/daemon/README.md +++ b/daemon/README.md @@ -1,15 +1,16 @@ # gpufl-monitor - Standalone GPU Monitoring Daemon -`gpufl-monitor` is a **low-overhead, always-on** daemon that continuously samples GPU and host metrics (utilization, memory, temperature, power, CPU, RAM) via NVML (NVIDIA) or ROCm SMI (AMD) and writes them as JSONL event logs. A bundled Java agent (`gpufl-agent`) tails those logs and ships the data to a GPUFlight backend. +`gpufl-monitor` is a **low-overhead, always-on** daemon that continuously samples GPU and host metrics and writes them as JSONL event logs. On Linux it samples NVIDIA via NVML and AMD via ROCm SMI. On macOS it runs natively and samples Apple GPU inventory plus explicitly scoped Metal working-set information; public Metal APIs do not expose NVML-style global utilization, temperature, power, or clocks. -Both processes run inside a single Docker container managed by `supervisord`. +For NVIDIA and AMD Linux deployments, the C++ daemon and bundled Java agent (`gpufl-agent`) run inside a single Docker container managed by `supervisord`. For macOS, the Metal collector must run natively because Docker Desktop containers run inside a Linux VM and cannot access `Metal.framework`. GPU vendor support: -| Vendor | Dockerfile | Compose file | +| Vendor | Runtime | Entry point | |---|---|---| -| NVIDIA | `Dockerfile.monitor` | `docker-compose.monitor.yml` | -| AMD | `Dockerfile.monitor.amd` | `docker-compose.monitor.amd.yml` | +| NVIDIA | Linux Docker | `docker-compose.monitor.yml` | +| AMD | Linux Docker | `docker-compose.monitor.amd.yml` | +| Apple Metal | Native macOS | `scripts/run-monitor-macos.sh` | --- @@ -42,6 +43,14 @@ docker run --rm --device /dev/kfd --device /dev/dri \ rocm/dev-ubuntu-24.04:6.4-complete rocm-smi ``` +### macOS / Apple Metal + +- macOS with Xcode Command Line Tools or Xcode installed +- A Metal-capable Apple GPU +- CMake 3.31+ + +The monitor runs natively on macOS. Docker is useful only for a log-upload agent, not for Metal collection. + --- ## Building the image @@ -70,6 +79,16 @@ docker build \ . ``` +### macOS / Apple Metal + +From the **repository root**: + +```bash +./scripts/run-monitor-macos.sh +``` + +The script configures and builds `gpufl-monitor` with `GPUFL_ENABLE_METAL=ON`, `GPUFL_ENABLE_NVIDIA=OFF`, and `GPUFL_ENABLE_AMD=OFF`, then runs the native daemon. + --- ## Running with docker compose (recommended) @@ -116,6 +135,35 @@ Stop: docker compose -f docker-compose.monitor.amd.yml down ``` +### macOS / Apple Metal + +Run the native monitor: + +```bash +GPUFL_MONITOR_LOG_DIR="$PWD/gpufl-monitor-macos/session" \ +./scripts/run-monitor-macos.sh +``` + +Stop it with `Ctrl-C`. To upload macOS logs, run the Java agent natively or mount the log directory into an agent-only container. The collector itself must remain native. + +The macOS `job_start` record identifies `telemetry_backend` as `metal` and +uses `profiling_engine: metal.none` for the telemetry-only daemon. Each Metal +device includes: + +- `telemetry_capabilities`, which separates available observations from fixed + batch metrics that public Metal cannot provide. +- `process_allocated_mib`, scoped to the current monitor process rather than + system-wide GPU memory. +- `recommended_max_working_set_mib`, Metal's performance recommendation rather + than total or free VRAM. +- A nested `metal` inventory containing public architecture, registry, unified + memory, location, threadgroup, buffer, GPU-family, and counter-set metadata. + +The legacy numeric device batch columns are retained for wire compatibility. +For unsupported Metal metrics they remain zero; consumers must use +`telemetry_capabilities.unavailable` to treat those values as unavailable, not +as measured zero. + --- ## Running with docker run @@ -160,6 +208,7 @@ The named volume persists the agent's read cursor so it resumes from where it le | `GPUFL_MONITOR_APP` | `gpufl-monitor` | App name tag written into every event | | `GPUFL_MONITOR_LOG_DIR` | `/var/gpufl/monitor/session` | Directory where JSONL log files are written | | `GPUFL_MONITOR_INTERVAL_MS` | `5000` | Sampling interval in milliseconds | +| `GPUFL_MONITOR_BACKEND` | `auto` on Linux, `metal` on macOS | Backend selector: `auto`, `nvidia`, `amd`, `metal`, or `none` | ### Agent - log source diff --git a/daemon/monitor/main.cpp b/daemon/monitor/main.cpp index 062d8d5..6d630b0 100644 --- a/daemon/monitor/main.cpp +++ b/daemon/monitor/main.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include #include @@ -22,6 +24,32 @@ int getenv_int_or(const char* var, int fallback) { } } +gpufl::BackendKind parseBackend() { + const char* raw = std::getenv(gpufl::env::kMonitorBackend); + if (!raw || raw[0] == '\0') { +#if defined(__APPLE__) + return gpufl::BackendKind::Metal; +#else + return gpufl::BackendKind::Auto; +#endif + } + + std::string value(raw); + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + + if (value == "auto") return gpufl::BackendKind::Auto; + if (value == "nvidia") return gpufl::BackendKind::Nvidia; + if (value == "amd") return gpufl::BackendKind::Amd; + if (value == "metal") return gpufl::BackendKind::Metal; + if (value == "none") return gpufl::BackendKind::None; + + std::cerr << "Unrecognized " << gpufl::env::kMonitorBackend << "='" << raw + << "'; expected auto, nvidia, amd, metal, or none. Using auto.\n"; + return gpufl::BackendKind::Auto; +} + } // namespace int main() { @@ -30,6 +58,7 @@ int main() { opts.log_path = getenv_or(gpufl::env::kMonitorLogDir, "/var/gpufl/monitor/session"); opts.interval_ms = getenv_int_or(gpufl::env::kMonitorIntervalMs, 5000); + opts.backend = parseBackend(); if (opts.interval_ms <= 0) { std::cerr << "GPUFL_MONITOR_INTERVAL_MS must be positive.\n"; diff --git a/daemon/monitor/monitor_runner.cpp b/daemon/monitor/monitor_runner.cpp index 32cee44..48ee11c 100644 --- a/daemon/monitor/monitor_runner.cpp +++ b/daemon/monitor/monitor_runner.cpp @@ -37,17 +37,17 @@ bool waitForShutdownSignal() { #else bool setupSignalHandling() { sigset_t set; - if (::sigemptyset(&set) != 0) return false; - if (::sigaddset(&set, SIGINT) != 0) return false; - if (::sigaddset(&set, SIGTERM) != 0) return false; + if (sigemptyset(&set) != 0) return false; + if (sigaddset(&set, SIGINT) != 0) return false; + if (sigaddset(&set, SIGTERM) != 0) return false; return ::pthread_sigmask(SIG_BLOCK, &set, nullptr) == 0; } bool waitForShutdownSignal() { sigset_t set; - if (::sigemptyset(&set) != 0) return false; - if (::sigaddset(&set, SIGINT) != 0) return false; - if (::sigaddset(&set, SIGTERM) != 0) return false; + if (sigemptyset(&set) != 0) return false; + if (sigaddset(&set, SIGINT) != 0) return false; + if (sigaddset(&set, SIGTERM) != 0) return false; int sig = 0; const int rc = ::sigwait(&set, &sig); @@ -73,6 +73,7 @@ int runMonitorForeground(const MonitorRunOptions& opts) { init.app_name = opts.app_name.empty() ? "gpufl-monitor" : opts.app_name; init.log_path = opts.log_path; init.profiling_engine = gpufl::ProfilingEngine::Monitor; + init.backend = opts.backend; init.continuous_system_sampling = true; init.system_sample_rate_ms = opts.interval_ms; init.enable_debug_output = false; diff --git a/daemon/monitor/monitor_runner.hpp b/daemon/monitor/monitor_runner.hpp index 2318827..2677607 100644 --- a/daemon/monitor/monitor_runner.hpp +++ b/daemon/monitor/monitor_runner.hpp @@ -2,12 +2,15 @@ #include +#include "gpufl/gpufl.hpp" + namespace gpufl::daemon { struct MonitorRunOptions { std::string app_name = "gpufl-monitor"; std::string log_path; int interval_ms = 5000; + BackendKind backend = BackendKind::Auto; bool quiet = false; }; diff --git a/include/gpufl/backends/host_collector.hpp b/include/gpufl/backends/host_collector.hpp index 25d07c6..c879dd6 100644 --- a/include/gpufl/backends/host_collector.hpp +++ b/include/gpufl/backends/host_collector.hpp @@ -3,6 +3,11 @@ #if defined(_WIN32) #include +#elif defined(__APPLE__) +#include +#include +#include +#include #else #include @@ -86,6 +91,73 @@ class HostCollector { } } +#elif defined(__APPLE__) + // --- MACOS IMPLEMENTATION --- + struct CpuTicks { + uint64_t user = 0; + uint64_t system = 0; + uint64_t idle = 0; + uint64_t nice = 0; + }; + CpuTicks prev_; + + static bool readCpuTicks(CpuTicks& ticks) { + host_cpu_load_info_data_t info{}; + mach_msg_type_number_t count = HOST_CPU_LOAD_INFO_COUNT; + const kern_return_t rc = host_statistics( + mach_host_self(), HOST_CPU_LOAD_INFO, + reinterpret_cast(&info), &count); + if (rc != KERN_SUCCESS) return false; + + ticks.user = info.cpu_ticks[CPU_STATE_USER]; + ticks.system = info.cpu_ticks[CPU_STATE_SYSTEM]; + ticks.idle = info.cpu_ticks[CPU_STATE_IDLE]; + ticks.nice = info.cpu_ticks[CPU_STATE_NICE]; + return true; + } + + double sampleCpu() { + CpuTicks cur; + if (!readCpuTicks(cur)) return 0.0; + + const uint64_t prevTotal = + prev_.user + prev_.system + prev_.idle + prev_.nice; + const uint64_t curTotal = cur.user + cur.system + cur.idle + cur.nice; + const uint64_t totalDiff = curTotal - prevTotal; + const uint64_t idleDiff = cur.idle - prev_.idle; + + double percent = 0.0; + if (totalDiff > 0) { + percent = static_cast(totalDiff - idleDiff) / + static_cast(totalDiff) * 100.0; + } + + prev_ = cur; + return percent; + } + + static void sampleRam(HostSample& s) { + uint64_t totalBytes = 0; + size_t totalSize = sizeof(totalBytes); + if (sysctlbyname("hw.memsize", &totalBytes, &totalSize, nullptr, 0) == 0) { + s.ram_total_mib = totalBytes / (1024 * 1024); + } + + vm_statistics64_data_t vmStats{}; + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT; + const kern_return_t rc = host_statistics64( + mach_host_self(), HOST_VM_INFO64, + reinterpret_cast(&vmStats), &count); + if (rc == KERN_SUCCESS && s.ram_total_mib > 0) { + const uint64_t pageSize = static_cast(getpagesize()); + const uint64_t freeBytes = + static_cast(vmStats.free_count) * pageSize; + const uint64_t totalMiB = s.ram_total_mib; + const uint64_t freeMiB = freeBytes / (1024 * 1024); + s.ram_used_mib = totalMiB > freeMiB ? totalMiB - freeMiB : 0; + } + } + #else // --- LINUX IMPLEMENTATION --- struct CpuTicks { @@ -147,4 +219,4 @@ class HostCollector { } #endif }; -} // namespace gpufl \ No newline at end of file +} // namespace gpufl diff --git a/include/gpufl/backends/metal/metal_collector.hpp b/include/gpufl/backends/metal/metal_collector.hpp new file mode 100644 index 0000000..a6e45b1 --- /dev/null +++ b/include/gpufl/backends/metal/metal_collector.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +#include "gpufl/core/backend_interfaces.hpp" +#include "gpufl/core/events.hpp" + +namespace gpufl::metal { + +class MetalCollector : public IUnifiedGpuCollector { + public: + MetalCollector(); + ~MetalCollector() override; + + std::vector sampleAll() override; + std::vector sampleStaticInfo() override; + + bool canSampleTelemetry() const override { return available_; } + bool canSampleStaticInfo() const override { return available_; } + + static bool IsAvailable(std::string* reason = nullptr); + + private: + bool available_ = false; +}; + +} // namespace gpufl::metal diff --git a/include/gpufl/backends/metal/metal_collector.mm b/include/gpufl/backends/metal/metal_collector.mm new file mode 100644 index 0000000..53a3777 --- /dev/null +++ b/include/gpufl/backends/metal/metal_collector.mm @@ -0,0 +1,229 @@ +#if !(GPUFL_ENABLE_METAL && GPUFL_HAS_METAL) +#error "metal_collector.mm requires GPUFL_ENABLE_METAL && GPUFL_HAS_METAL" +#endif + +#include "gpufl/backends/metal/metal_collector.hpp" + +#import +#import + +#include +#include +#include +#include +#include + +namespace gpufl::metal { +namespace { + +constexpr uint64_t kMiB = 1024ULL * 1024ULL; + +std::string NSStringToString(NSString* value) { + if (value == nil) return {}; + const char* utf8 = [value UTF8String]; + return utf8 ? std::string(utf8) : std::string{}; +} + +std::string RegistryId(id device) { + std::ostringstream oss; + oss << "0x" << std::hex << std::setw(16) << std::setfill('0') + << [device registryID]; + return oss.str(); +} + +std::string RegistryUuid(id device) { + return "metal-registry-" + RegistryId(device); +} + +std::string DeviceLocationName(const MTLDeviceLocation location) { + switch (location) { + case MTLDeviceLocationBuiltIn: return "built_in"; + case MTLDeviceLocationSlot: return "slot"; + case MTLDeviceLocationExternal: return "external"; + case MTLDeviceLocationUnspecified: return "unspecified"; + } + return "unknown"; +} + +void AppendSupportedGpuFamilies( + id device, std::vector& families) { + if (@available(macOS 10.15, *)) { + for (int family = 1; family <= 10; ++family) { + if ([device supportsFamily:static_cast(1000 + family)]) { + families.push_back("apple" + std::to_string(family)); + } + } + if ([device supportsFamily:static_cast(2002)]) { + families.push_back("mac2"); + } + for (int family = 1; family <= 3; ++family) { + if ([device supportsFamily:static_cast(3000 + family)]) { + families.push_back("common" + std::to_string(family)); + } + } + for (int family = 3; family <= 4; ++family) { + if ([device supportsFamily:static_cast(4998 + family)]) { + families.push_back("metal" + std::to_string(family)); + } + } + } +} + +void AppendCounterSetNames( + id device, std::vector& counterSets) { + if (@available(macOS 10.15, *)) { + NSArray>* sets = [device counterSets]; + for (id set in sets) { + const std::string name = NSStringToString([set name]); + if (!name.empty()) counterSets.push_back(name); + } + } +} + +DeviceSample DeviceToSample(id device, int id) { + DeviceSample sample{}; + sample.device_id = id; + sample.name = NSStringToString([device name]); + sample.uuid = RegistryUuid(device); + sample.vendor = "Apple"; + + auto& capabilities = sample.telemetry_capabilities; + capabilities.available = { + "process_allocated_mib", "recommended_max_working_set_mib"}; + capabilities.unavailable = { + "gpu_util", "mem_util", "temp_c", "power_mw", "used_mib", + "total_mib", "clock_sm", "fan_speed_pct", "temp_mem_c", + "temp_junction_c", "voltage_mv", "energy_uj", "clock_mem", + "pcie_bw_bps", "ecc_corrected", "ecc_uncorrected"}; + capabilities.allocation_scope = "current_process"; + capabilities.process_allocated_mib = + static_cast([device currentAllocatedSize]) / kMiB; + capabilities.recommended_max_working_set_mib = + [device recommendedMaxWorkingSetSize] / kMiB; + if (@available(macOS 10.15, *)) { + capabilities.memory_model = [device hasUnifiedMemory] + ? "unified" + : "discrete"; + } + + return sample; +} + +GpuStaticDeviceInfo DeviceToStaticInfo(id device, int id) { + GpuStaticDeviceInfo info{}; + info.id = id; + info.name = NSStringToString([device name]); + info.uuid = RegistryUuid(device); + info.vendor = "Apple"; + info.architecture = "Metal"; + + auto& metal = info.metal; + metal.available = true; + metal.registry_id = RegistryId(device); + if (@available(macOS 14.0, *)) { + metal.architecture_name = + NSStringToString([[device architecture] name]); + if (!metal.architecture_name.empty()) { + info.architecture = metal.architecture_name; + } + } + metal.low_power = [device isLowPower]; + metal.headless = [device isHeadless]; + metal.removable = [device isRemovable]; + metal.recommended_max_working_set_bytes = + [device recommendedMaxWorkingSetSize]; + metal.max_buffer_length_bytes = [device maxBufferLength]; + + const MTLSize threads = [device maxThreadsPerThreadgroup]; + metal.max_threads_per_threadgroup = { + static_cast(threads.width), + static_cast(threads.height), + static_cast(threads.depth)}; + + if (@available(macOS 10.15, *)) { + metal.unified_memory = [device hasUnifiedMemory]; + metal.location = DeviceLocationName([device location]); + metal.location_number = [device locationNumber]; + metal.max_transfer_rate_bps = [device maxTransferRate]; + } + AppendSupportedGpuFamilies(device, metal.gpu_families); + AppendCounterSetNames(device, metal.counter_sets); + return info; +} + +} // namespace + +MetalCollector::MetalCollector() : available_(IsAvailable(nullptr)) {} +MetalCollector::~MetalCollector() = default; + +std::vector MetalCollector::sampleAll() { + std::vector samples; + @autoreleasepool { + NSArray>* devices = MTLCopyAllDevices(); + if (devices != nil && [devices count] > 0) { + samples.reserve([devices count]); + int idx = 0; + for (id device in devices) { + if (device != nil) samples.push_back(DeviceToSample(device, idx++)); + } + [devices release]; + return samples; + } + [devices release]; + + id defaultDevice = MTLCreateSystemDefaultDevice(); + if (defaultDevice != nil) { + samples.push_back(DeviceToSample(defaultDevice, 0)); + [(id)defaultDevice release]; + } + } + return samples; +} + +std::vector MetalCollector::sampleStaticInfo() { + std::vector devicesInfo; + @autoreleasepool { + NSArray>* devices = MTLCopyAllDevices(); + if (devices != nil && [devices count] > 0) { + devicesInfo.reserve([devices count]); + int idx = 0; + for (id device in devices) { + if (device != nil) { + devicesInfo.push_back(DeviceToStaticInfo(device, idx++)); + } + } + [devices release]; + return devicesInfo; + } + [devices release]; + + id defaultDevice = MTLCreateSystemDefaultDevice(); + if (defaultDevice != nil) { + devicesInfo.push_back(DeviceToStaticInfo(defaultDevice, 0)); + [(id)defaultDevice release]; + } + } + return devicesInfo; +} + +bool MetalCollector::IsAvailable(std::string* reason) { + @autoreleasepool { + NSArray>* devices = MTLCopyAllDevices(); + if (devices != nil && [devices count] > 0) { + [devices release]; + return true; + } + [devices release]; + + id defaultDevice = MTLCreateSystemDefaultDevice(); + if (defaultDevice != nil) { + [(id)defaultDevice release]; + return true; + } + } + + if (reason) *reason = "Metal framework is present but no Metal device was found."; + return false; +} + +} // namespace gpufl::metal diff --git a/include/gpufl/core/backend_factory.cpp b/include/gpufl/core/backend_factory.cpp index bd673bb..571874d 100644 --- a/include/gpufl/core/backend_factory.cpp +++ b/include/gpufl/core/backend_factory.cpp @@ -12,6 +12,10 @@ #include "gpufl/backends/amd/rocm_collector.hpp" #endif +#if GPUFL_ENABLE_METAL && GPUFL_HAS_METAL +#include "gpufl/backends/metal/metal_collector.hpp" +#endif + namespace gpufl { #if GPUFL_HAS_CUDA @@ -70,6 +74,22 @@ BackendCollectors CreateBackendCollectors(const BackendKind backend, #endif }; + auto tryMetalUnified = [&]() -> std::shared_ptr { +#if GPUFL_ENABLE_METAL && GPUFL_HAS_METAL + std::string reason; + if (!gpufl::metal::MetalCollector::IsAvailable(&reason)) { + setReason("Metal backend unavailable: " + reason); + return nullptr; + } + return std::make_shared(); +#else + setReason( + "Metal backend not available (GPUFL_ENABLE_METAL=OFF or Metal " + "framework not found)."); + return nullptr; +#endif + }; + switch (backend) { case BackendKind::None: break; @@ -78,6 +98,8 @@ BackendCollectors CreateBackendCollectors(const BackendKind backend, out.telemetry_collector = tryNvml(); if (!out.telemetry_collector) { setReason("Requested backend=nvidia but NVML is unavailable."); + } else { + out.telemetry_backend = "nvidia"; } out.static_info_collector = tryNvidiaStatic(); break; @@ -90,6 +112,24 @@ BackendCollectors CreateBackendCollectors(const BackendKind backend, if (out.unified_collector->canSampleTelemetry()) { out.telemetry_collector = std::static_pointer_cast< ISystemCollector>(out.unified_collector); + out.telemetry_backend = "amd"; + } + if (out.unified_collector->canSampleStaticInfo()) { + out.static_info_collector = std::static_pointer_cast< + IGpuStaticInfoCollector>(out.unified_collector); + } + } + break; + + case BackendKind::Metal: + out.unified_collector = tryMetalUnified(); + if (!out.unified_collector) { + setReason("Requested backend=metal but Metal is unavailable."); + } else { + if (out.unified_collector->canSampleTelemetry()) { + out.telemetry_collector = std::static_pointer_cast< + ISystemCollector>(out.unified_collector); + out.telemetry_backend = "metal"; } if (out.unified_collector->canSampleStaticInfo()) { out.static_info_collector = std::static_pointer_cast< @@ -102,6 +142,7 @@ BackendCollectors CreateBackendCollectors(const BackendKind backend, default: out.telemetry_collector = tryNvml(); if (out.telemetry_collector) { + out.telemetry_backend = "nvidia"; out.static_info_collector = tryNvidiaStatic(); break; } @@ -111,6 +152,21 @@ BackendCollectors CreateBackendCollectors(const BackendKind backend, if (out.unified_collector->canSampleTelemetry()) { out.telemetry_collector = std::static_pointer_cast< ISystemCollector>(out.unified_collector); + out.telemetry_backend = "amd"; + } + if (out.unified_collector->canSampleStaticInfo()) { + out.static_info_collector = std::static_pointer_cast< + IGpuStaticInfoCollector>(out.unified_collector); + } + break; + } + + out.unified_collector = tryMetalUnified(); + if (out.unified_collector) { + if (out.unified_collector->canSampleTelemetry()) { + out.telemetry_collector = std::static_pointer_cast< + ISystemCollector>(out.unified_collector); + out.telemetry_backend = "metal"; } if (out.unified_collector->canSampleStaticInfo()) { out.static_info_collector = std::static_pointer_cast< @@ -121,7 +177,7 @@ BackendCollectors CreateBackendCollectors(const BackendKind backend, if (!out.telemetry_collector) { setReason( - "No GPU backend available (NVML/ROCm not compiled in or " + "No GPU backend available (NVML/ROCm/Metal not compiled in or " "not available)."); } out.static_info_collector = tryNvidiaStatic(); diff --git a/include/gpufl/core/backend_interfaces.hpp b/include/gpufl/core/backend_interfaces.hpp index fb4a55a..f80fdf2 100644 --- a/include/gpufl/core/backend_interfaces.hpp +++ b/include/gpufl/core/backend_interfaces.hpp @@ -26,6 +26,7 @@ struct BackendCollectors { std::shared_ptr unified_collector; std::shared_ptr> telemetry_collector; std::shared_ptr static_info_collector; + std::string telemetry_backend; }; } // namespace gpufl diff --git a/include/gpufl/core/client_startup.cpp b/include/gpufl/core/client_startup.cpp index 1ff9ce4..9366e98 100644 --- a/include/gpufl/core/client_startup.cpp +++ b/include/gpufl/core/client_startup.cpp @@ -211,6 +211,7 @@ void ClientStartup::configureCollectors(Runtime& active_runtime) const { active_runtime.collector = std::move(collectors.telemetry_collector); active_runtime.static_info_collector = std::move(collectors.static_info_collector); + active_runtime.telemetry_backend = std::move(collectors.telemetry_backend); if (!active_runtime.collector) { GFL_LOG_ERROR("Failed to initialize GPU backend: ", backend_reason); } @@ -236,7 +237,14 @@ void ClientStartup::emitInitialEvent(Runtime& active_runtime, } event.host = active_runtime.host_collector->sample(); event.session_kind = ProfilingEngineSessionKind(state_->monitor_options.profiling_engine); - event.profiling_engine = Monitor::ResolvedProfilingEngineWireName(); + event.telemetry_backend = active_runtime.telemetry_backend; + if (state_->monitor_options.profiling_engine == ProfilingEngine::Monitor) { + event.profiling_engine = ProfilingEngineWireNameForBackend( + state_->monitor_options.profiling_engine, + active_runtime.telemetry_backend); + } else { + event.profiling_engine = Monitor::ResolvedProfilingEngineWireName(); + } event.run_id = segment.run_id; event.segment_index = segment.segment_index; if (segment.run_part) { diff --git a/include/gpufl/core/env_vars.hpp b/include/gpufl/core/env_vars.hpp index 2fd6d05..4cc468f 100644 --- a/include/gpufl/core/env_vars.hpp +++ b/include/gpufl/core/env_vars.hpp @@ -231,6 +231,7 @@ constexpr const char* kSassDeferScopeFlush = "GPUFL_SASS_DEFER_SCOPE_FLU constexpr const char* kMonitorApp = "GPUFL_MONITOR_APP"; constexpr const char* kMonitorLogDir = "GPUFL_MONITOR_LOG_DIR"; constexpr const char* kMonitorIntervalMs = "GPUFL_MONITOR_INTERVAL_MS"; +constexpr const char* kMonitorBackend = "GPUFL_MONITOR_BACKEND"; // ── External / platform names GPUFlight reads or sets ─────────────────────── // Not our knobs (CUDA-driver / dynamic-loader contracts) but referenced from diff --git a/include/gpufl/core/events/lifecycle_events.hpp b/include/gpufl/core/events/lifecycle_events.hpp index 02f4e27..c429c28 100644 --- a/include/gpufl/core/events/lifecycle_events.hpp +++ b/include/gpufl/core/events/lifecycle_events.hpp @@ -27,10 +27,10 @@ struct InitEvent { // sessions ran with an engine?". // profiling_engine : vendor-namespaced detail like // "nvidia.pc_sampling" / "nvidia.sass_metrics" - // / "nvidia.none" (the latter is what - // ProfilingEngine::Monitor - telemetry only - - // emits). Stored verbatim by the backend. - // The "nvidia.none" string lets the backend + // / "nvidia.none" / "amd.none" / "metal.none". + // The *.none forms identify telemetry-only sessions. + // Stored verbatim by the backend. + // The explicit none string lets the backend // distinguish "user explicitly disabled // profiling" from "pre-V40 client that omitted // the field" - both used to collapse to NULL. @@ -39,6 +39,9 @@ struct InitEvent { // lives next to the InitEvent build site (gpufl.cpp). std::string session_kind; std::string profiling_engine; + // Telemetry provider selected after auto-detection: nvidia, amd, or metal. + // Additive and omitted when no GPU telemetry provider was initialized. + std::string telemetry_backend; // Multi-pass profiling grouping (P1 of the multi-pass workstream). // A single "analysis" = N separately-launched passes (one CUPTI engine // each, isolated to dodge the SASS/kernel-activity deadlock + cross- diff --git a/include/gpufl/core/events/sample_types.hpp b/include/gpufl/core/events/sample_types.hpp index ab7de22..937bc02 100644 --- a/include/gpufl/core/events/sample_types.hpp +++ b/include/gpufl/core/events/sample_types.hpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace gpufl { struct HostSample { @@ -12,6 +13,24 @@ struct HostSample { }; struct GpuStaticDeviceInfo { + struct MetalProperties { + bool available = false; + std::string registry_id; + std::string architecture_name; + bool low_power = false; + bool headless = false; + bool removable = false; + bool unified_memory = false; + std::string location; + uint64_t location_number = 0; + uint64_t recommended_max_working_set_bytes = 0; + uint64_t max_transfer_rate_bps = 0; + uint64_t max_buffer_length_bytes = 0; + std::array max_threads_per_threadgroup{}; + std::vector gpu_families; + std::vector counter_sets; + }; + int id = 0; std::string name; std::string uuid; @@ -49,9 +68,22 @@ struct GpuStaticDeviceInfo { bool memory_pools_supported = false; bool cluster_launch = false; bool tensor_map_access_supported = false; + + // Public MTLDevice inventory. Serialized as an additive nested object only + // for Metal devices so existing NVIDIA/AMD job_start shapes stay stable. + MetalProperties metal; }; struct DeviceSample { + struct TelemetryCapabilities { + std::vector available; + std::vector unavailable; + std::string memory_model; + std::string allocation_scope; + uint64_t process_allocated_mib = 0; + uint64_t recommended_max_working_set_mib = 0; + }; + int device_id = 0; std::string name; std::string uuid; @@ -87,6 +119,11 @@ struct DeviceSample { unsigned long long pcie_rx_bps; // Host -> Device (Upload) unsigned long long pcie_tx_bps; // Device -> Host (Download) + + // Optional semantics for backends that cannot populate the fixed NVML-like + // metric columns. This lets readers distinguish unavailable values from a + // real measurement of zero without changing the batch column contract. + TelemetryCapabilities telemetry_capabilities; }; } // namespace gpufl diff --git a/include/gpufl/core/model/lifecycle_model.cpp b/include/gpufl/core/model/lifecycle_model.cpp index f4d1239..1fd2476 100644 --- a/include/gpufl/core/model/lifecycle_model.cpp +++ b/include/gpufl/core/model/lifecycle_model.cpp @@ -34,6 +34,10 @@ std::string InitEventModel::buildJson() const { << staticDevicesToJsonForVendor(e_.gpu_static_device_infos, "AMD"); oss << ",\"session_kind\":\"" << jsonEscape(e_.session_kind) << "\""; + if (!e_.telemetry_backend.empty()) { + oss << ",\"telemetry_backend\":\"" << jsonEscape(e_.telemetry_backend) + << "\""; + } if (!e_.profiling_engine.empty()) { oss << ",\"profiling_engine\":\"" << jsonEscape(e_.profiling_engine) << "\""; } diff --git a/include/gpufl/core/model/model_utils.hpp b/include/gpufl/core/model/model_utils.hpp index 912342a..74f80db 100644 --- a/include/gpufl/core/model/model_utils.hpp +++ b/include/gpufl/core/model/model_utils.hpp @@ -24,6 +24,16 @@ inline std::string hostToJson(const HostSample& h) { return oss.str(); } +inline void appendStringArray(std::ostringstream& oss, + const std::vector& values) { + oss << '['; + for (size_t i = 0; i < values.size(); ++i) { + if (i != 0) oss << ','; + oss << '"' << jsonEscape(values[i]) << '"'; + } + oss << ']'; +} + inline std::string devicesToJson(const std::vector& devs) { std::ostringstream oss; oss << "["; @@ -49,12 +59,74 @@ inline std::string devicesToJson(const std::vector& devs) { << ",\"throttle_pwr\":" << (d.throttle_power ? 1 : 0) << ",\"throttle_therm\":" << (d.throttle_thermal ? 1 : 0) << ",\"pcie_rx_bw_bps\":" << d.pcie_rx_bps - << ",\"pcie_tx_bw_bps\":" << d.pcie_tx_bps << "}"; + << ",\"pcie_tx_bw_bps\":" << d.pcie_tx_bps; + const auto& capabilities = d.telemetry_capabilities; + if (!capabilities.available.empty() || + !capabilities.unavailable.empty() || + !capabilities.memory_model.empty() || + !capabilities.allocation_scope.empty()) { + oss << ",\"telemetry_capabilities\":{\"available\":"; + appendStringArray(oss, capabilities.available); + oss << ",\"unavailable\":"; + appendStringArray(oss, capabilities.unavailable); + oss << ",\"memory_model\":\"" + << jsonEscape(capabilities.memory_model) << "\"" + << ",\"allocation_scope\":\"" + << jsonEscape(capabilities.allocation_scope) << "\"" + << ",\"process_allocated_mib\":" + << capabilities.process_allocated_mib + << ",\"recommended_max_working_set_mib\":" + << capabilities.recommended_max_working_set_mib << '}'; + } + oss << '}'; } oss << "]"; return oss.str(); } +inline void appendStaticDeviceJson(std::ostringstream& oss, + const GpuStaticDeviceInfo& d) { + oss << "{\"id\":" << d.id << ",\"name\":\"" << jsonEscape(d.name) << "\"" + << ",\"uuid\":\"" << jsonEscape(d.uuid) << "\"" + << ",\"vendor\":\"" << jsonEscape(d.vendor) << "\"" + << ",\"architecture\":\"" << jsonEscape(d.architecture) << "\"" + << ",\"compute_major\":" << d.compute_major + << ",\"compute_minor\":" << d.compute_minor + << ",\"l2_cache_size_bytes\":" << d.l2_cache_size + << ",\"shared_mem_per_block_bytes\":" << d.shared_mem_per_block + << ",\"regs_per_block\":" << d.regs_per_block + << ",\"multi_processor_count\":" << d.multi_processor_count + << ",\"warp_size\":" << d.warp_size; + + if (d.metal.available) { + oss << ",\"metal\":{\"registry_id\":\"" + << jsonEscape(d.metal.registry_id) << "\"" + << ",\"architecture_name\":\"" + << jsonEscape(d.metal.architecture_name) << "\"" + << ",\"low_power\":" << (d.metal.low_power ? "true" : "false") + << ",\"headless\":" << (d.metal.headless ? "true" : "false") + << ",\"removable\":" << (d.metal.removable ? "true" : "false") + << ",\"unified_memory\":" + << (d.metal.unified_memory ? "true" : "false") << ",\"location\":\"" + << jsonEscape(d.metal.location) << "\"" + << ",\"location_number\":" << d.metal.location_number + << ",\"recommended_max_working_set_bytes\":" + << d.metal.recommended_max_working_set_bytes + << ",\"max_transfer_rate_bps\":" << d.metal.max_transfer_rate_bps + << ",\"max_buffer_length_bytes\":" + << d.metal.max_buffer_length_bytes + << ",\"max_threads_per_threadgroup\":[" + << d.metal.max_threads_per_threadgroup[0] << ',' + << d.metal.max_threads_per_threadgroup[1] << ',' + << d.metal.max_threads_per_threadgroup[2] << "],\"gpu_families\":"; + appendStringArray(oss, d.metal.gpu_families); + oss << ",\"counter_sets\":"; + appendStringArray(oss, d.metal.counter_sets); + oss << '}'; + } + oss << '}'; +} + inline std::string staticDevicesToJson( const std::vector& devs) { std::ostringstream oss; @@ -63,18 +135,7 @@ inline std::string staticDevicesToJson( for (const auto& d : devs) { if (!first) oss << ","; first = false; - oss << "{\"id\":" << d.id - << ",\"name\":\"" << jsonEscape(d.name) << "\"" - << ",\"uuid\":\"" << jsonEscape(d.uuid) << "\"" - << ",\"vendor\":\"" << jsonEscape(d.vendor) << "\"" - << ",\"architecture\":\"" << jsonEscape(d.architecture) << "\"" - << ",\"compute_major\":" << d.compute_major - << ",\"compute_minor\":" << d.compute_minor - << ",\"l2_cache_size_bytes\":" << d.l2_cache_size - << ",\"shared_mem_per_block_bytes\":" << d.shared_mem_per_block - << ",\"regs_per_block\":" << d.regs_per_block - << ",\"multi_processor_count\":" << d.multi_processor_count - << ",\"warp_size\":" << d.warp_size << "}"; + appendStaticDeviceJson(oss, d); } oss << "]"; return oss.str(); @@ -89,18 +150,7 @@ inline std::string staticDevicesToJsonForVendor( if (d.vendor != vendor) continue; if (!first) oss << ","; first = false; - oss << "{\"id\":" << d.id - << ",\"name\":\"" << jsonEscape(d.name) << "\"" - << ",\"uuid\":\"" << jsonEscape(d.uuid) << "\"" - << ",\"vendor\":\"" << jsonEscape(d.vendor) << "\"" - << ",\"architecture\":\"" << jsonEscape(d.architecture) << "\"" - << ",\"compute_major\":" << d.compute_major - << ",\"compute_minor\":" << d.compute_minor - << ",\"l2_cache_size_bytes\":" << d.l2_cache_size - << ",\"shared_mem_per_block_bytes\":" << d.shared_mem_per_block - << ",\"regs_per_block\":" << d.regs_per_block - << ",\"multi_processor_count\":" << d.multi_processor_count - << ",\"warp_size\":" << d.warp_size << "}"; + appendStaticDeviceJson(oss, d); } oss << "]"; return oss.str(); diff --git a/include/gpufl/core/monitor.hpp b/include/gpufl/core/monitor.hpp index 5016177..6f332fb 100644 --- a/include/gpufl/core/monitor.hpp +++ b/include/gpufl/core/monitor.hpp @@ -41,6 +41,7 @@ enum class MonitorBackendKind { Nvidia, Amd, None, + Metal, }; /** @@ -122,6 +123,14 @@ inline const char* ProfilingEngineWireName(const ProfilingEngine engine) { return "nvidia.unknown"; } +inline std::string ProfilingEngineWireNameForBackend( + const ProfilingEngine engine, const std::string& telemetry_backend) { + if (engine == ProfilingEngine::Monitor && !telemetry_backend.empty()) { + return telemetry_backend + ".none"; + } + return ProfilingEngineWireName(engine); +} + inline const char* ProfilingEngineSessionKind(const ProfilingEngine engine) { switch (engine) { case ProfilingEngine::Monitor: diff --git a/include/gpufl/core/monitor_adapter.cpp b/include/gpufl/core/monitor_adapter.cpp index a44e892..e453829 100644 --- a/include/gpufl/core/monitor_adapter.cpp +++ b/include/gpufl/core/monitor_adapter.cpp @@ -37,6 +37,8 @@ std::unique_ptr CreateMonitorAdapter(const MonitorOptions& opts #endif case MonitorBackendKind::None: return nullptr; + case MonitorBackendKind::Metal: + return nullptr; case MonitorBackendKind::Auto: default: #if GPUFL_ENABLE_NVIDIA && GPUFL_HAS_CUDA diff --git a/include/gpufl/core/monitor_configuration.cpp b/include/gpufl/core/monitor_configuration.cpp index 1bafe4a..f09dc28 100644 --- a/include/gpufl/core/monitor_configuration.cpp +++ b/include/gpufl/core/monitor_configuration.cpp @@ -17,6 +17,8 @@ MonitorBackendKind toMonitorBackendKind(const BackendKind backend) { return MonitorBackendKind::Amd; case BackendKind::None: return MonitorBackendKind::None; + case BackendKind::Metal: + return MonitorBackendKind::Metal; case BackendKind::Auto: default: return MonitorBackendKind::Auto; diff --git a/include/gpufl/core/runtime.hpp b/include/gpufl/core/runtime.hpp index 702fdde..2b9ecb4 100644 --- a/include/gpufl/core/runtime.hpp +++ b/include/gpufl/core/runtime.hpp @@ -51,6 +51,7 @@ struct Runtime { std::shared_ptr> collector; std::unique_ptr host_collector; std::shared_ptr static_info_collector; + std::string telemetry_backend; // background system sampling std::atomic system_sampling{false}; diff --git a/include/gpufl/core/source_capture_policy.cpp b/include/gpufl/core/source_capture_policy.cpp index da94341..542e970 100644 --- a/include/gpufl/core/source_capture_policy.cpp +++ b/include/gpufl/core/source_capture_policy.cpp @@ -60,6 +60,23 @@ std::vector existingCanonicalRoots( return result; } +std::vector existingLexicalRoots( + const std::vector& roots) { + std::vector result; + for (const auto& value : roots) { + if (value.empty()) continue; + std::error_code ec; + fs::path lexical = fs::absolute(fs::path(value), ec).lexically_normal(); + if (ec) continue; + const fs::path canonical = fs::weakly_canonical(lexical, ec); + if (ec || !fs::is_directory(canonical, ec) || ec) continue; + if (std::find(result.begin(), result.end(), lexical) == result.end()) { + result.push_back(std::move(lexical)); + } + } + return result; +} + void appendEnvironmentRoot(std::vector& roots, const char* key) { const char* raw = std::getenv(key); if (!raw || !*raw) return; @@ -186,6 +203,7 @@ void SourceCapturePolicy::configure( const bool enabled, const SourceCaptureSettings& settings) { enabled_ = enabled; settings_ = settings; + approved_lexical_roots_ = existingLexicalRoots(settings.approved_roots); approved_roots_ = existingCanonicalRoots(settings.approved_roots); manifest_ = {}; manifest_.enabled = enabled_; @@ -197,6 +215,7 @@ void SourceCapturePolicy::configure( void SourceCapturePolicy::reset() { enabled_ = false; settings_ = {}; + approved_lexical_roots_.clear(); approved_roots_.clear(); manifest_ = {}; manifest_dirty_ = false; @@ -260,20 +279,18 @@ SourceCaptureResult SourceCapturePolicy::capture( const fs::path canonical = fs::weakly_canonical(absolute, ec); if (ec) return reject(SourceCaptureDisposition::InvalidPath); - std::size_t lexical_root = approved_roots_.size(); std::size_t canonical_root = approved_roots_.size(); for (std::size_t i = 0; i < approved_roots_.size(); ++i) { - if (lexical_root == approved_roots_.size() && - isWithin(absolute, approved_roots_[i])) { - lexical_root = i; - } if (canonical_root == approved_roots_.size() && isWithin(canonical, approved_roots_[i])) { canonical_root = i; } } if (canonical_root == approved_roots_.size()) { - return reject(lexical_root != approved_roots_.size() + const bool lexically_approved = std::any_of( + approved_lexical_roots_.begin(), approved_lexical_roots_.end(), + [&](const fs::path& root) { return isWithin(absolute, root); }); + return reject(lexically_approved ? SourceCaptureDisposition::SymlinkEscape : SourceCaptureDisposition::OutsideApprovedRoots); } diff --git a/include/gpufl/core/source_capture_policy.hpp b/include/gpufl/core/source_capture_policy.hpp index 00feaf2..ee15273 100644 --- a/include/gpufl/core/source_capture_policy.hpp +++ b/include/gpufl/core/source_capture_policy.hpp @@ -110,6 +110,7 @@ class SourceCapturePolicy { bool enabled_ = false; SourceCaptureSettings settings_; + std::vector approved_lexical_roots_; std::vector approved_roots_; SourceCaptureManifest manifest_; bool manifest_dirty_ = false; diff --git a/include/gpufl/gpufl.hpp b/include/gpufl/gpufl.hpp index 875fd40..ac37f0e 100644 --- a/include/gpufl/gpufl.hpp +++ b/include/gpufl/gpufl.hpp @@ -10,7 +10,7 @@ #include "gpufl/core/monitor.hpp" namespace gpufl { -enum class BackendKind { Auto, Nvidia, Amd, None }; +enum class BackendKind { Auto, Nvidia, Amd, None, Metal }; struct InitOptions { std::string app_name = "gpufl"; diff --git a/include/gpufl/upload/upload_logs.cpp b/include/gpufl/upload/upload_logs.cpp index a401446..6a9f569 100644 --- a/include/gpufl/upload/upload_logs.cpp +++ b/include/gpufl/upload/upload_logs.cpp @@ -581,8 +581,10 @@ std::unique_ptr makeClient(const UrlParts& url, if (url.port > 0) scheme_host_port += ":" + std::to_string(url.port); try { auto cli = std::make_unique(scheme_host_port); - cli->set_connection_timeout(0, opts.connect_timeout_ms * 1000); - cli->set_read_timeout(0, opts.read_timeout_ms * 1000); + cli->set_connection_timeout(opts.connect_timeout_ms / 1000, + (opts.connect_timeout_ms % 1000) * 1000); + cli->set_read_timeout(opts.read_timeout_ms / 1000, + (opts.read_timeout_ms % 1000) * 1000); cli->set_keep_alive(true); return cli; } catch (const std::exception& e) { diff --git a/python/bindings.cpp b/python/bindings.cpp index caf5daa..7c000b6 100644 --- a/python/bindings.cpp +++ b/python/bindings.cpp @@ -62,6 +62,7 @@ PYBIND11_MODULE(_gpufl_client, m) { .value("Nvidia", gpufl::BackendKind::Nvidia) .value("Amd", gpufl::BackendKind::Amd) .value("None", gpufl::BackendKind::None) + .value("Metal", gpufl::BackendKind::Metal) .export_values(); backendKindEnum.attr("None_") = backendKindEnum.attr("__members__")[py::str("None")]; diff --git a/python/gpufl/__init__.py b/python/gpufl/__init__.py index ed9010a..700e016 100644 --- a/python/gpufl/__init__.py +++ b/python/gpufl/__init__.py @@ -238,6 +238,7 @@ class BackendKind: Auto = "Auto" Nvidia = "Nvidia" Amd = "Amd" + Metal = "Metal" None_ = "None" class ProfilingEngine: diff --git a/scripts/run-monitor-macos.sh b/scripts/run-monitor-macos.sh new file mode 100755 index 0000000..701e71e --- /dev/null +++ b/scripts/run-monitor-macos.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BUILD_DIR="${GPUFL_MONITOR_BUILD_DIR:-$ROOT_DIR/build-macos-monitor}" + +"$ROOT_DIR/build-macos.sh" --monitor --build-dir "$BUILD_DIR" + +export GPUFL_MONITOR_BACKEND="${GPUFL_MONITOR_BACKEND:-metal}" +export GPUFL_MONITOR_APP="${GPUFL_MONITOR_APP:-gpufl-monitor-macos}" +export GPUFL_MONITOR_LOG_DIR="${GPUFL_MONITOR_LOG_DIR:-$ROOT_DIR/gpufl-monitor-macos/session}" +export GPUFL_MONITOR_INTERVAL_MS="${GPUFL_MONITOR_INTERVAL_MS:-5000}" + +mkdir -p "$GPUFL_MONITOR_LOG_DIR" + +exec "$BUILD_DIR/daemon/monitor/gpufl-monitor" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ae77592..85a6ff4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -95,6 +95,12 @@ if(GPUFL_ENABLE_AMD AND (GPUFL_HAS_ROCM_SMI OR GPUFL_HAS_HIP)) ) endif() +if(APPLE AND GPUFL_ENABLE_METAL AND GPUFL_HAS_METAL) + list(APPEND GPUFL_TEST_SOURCES + backends/metal/test_metal_collector.mm + ) +endif() + add_executable(gpufl_tests ${GPUFL_TEST_SOURCES}) # Standalone target for the Linux injection readiness regression. It diff --git a/tests/backends/metal/test_metal_collector.mm b/tests/backends/metal/test_metal_collector.mm new file mode 100644 index 0000000..d6efcd2 --- /dev/null +++ b/tests/backends/metal/test_metal_collector.mm @@ -0,0 +1,44 @@ +#include + +#include + +#include "gpufl/backends/metal/metal_collector.hpp" + +namespace { + +TEST(MetalCollector, EmitsPublicInventoryAndExplicitTelemetrySemantics) { + std::string reason; + ASSERT_TRUE(gpufl::metal::MetalCollector::IsAvailable(&reason)) << reason; + + gpufl::metal::MetalCollector collector; + const auto samples = collector.sampleAll(); + const auto static_info = collector.sampleStaticInfo(); + + ASSERT_FALSE(samples.empty()); + ASSERT_EQ(samples.size(), static_info.size()); + + const auto& sample = samples.front(); + EXPECT_EQ(sample.vendor, "Apple"); + EXPECT_FALSE(sample.name.empty()); + EXPECT_EQ(sample.used_mib, 0u); + EXPECT_EQ(sample.free_mib, 0u); + EXPECT_EQ(sample.total_mib, 0u); + EXPECT_EQ(sample.telemetry_capabilities.allocation_scope, + "current_process"); + EXPECT_FALSE(sample.telemetry_capabilities.memory_model.empty()); + EXPECT_GT( + sample.telemetry_capabilities.recommended_max_working_set_mib, 0u); + EXPECT_FALSE(sample.telemetry_capabilities.unavailable.empty()); + + const auto& info = static_info.front(); + EXPECT_EQ(info.vendor, "Apple"); + EXPECT_TRUE(info.metal.available); + EXPECT_FALSE(info.metal.registry_id.empty()); + EXPECT_FALSE(info.architecture.empty()); + EXPECT_GT(info.metal.recommended_max_working_set_bytes, 0u); + EXPECT_GT(info.metal.max_buffer_length_bytes, 0u); + EXPECT_GT(info.metal.max_threads_per_threadgroup[0], 0u); + EXPECT_FALSE(info.metal.gpu_families.empty()); +} + +} // namespace diff --git a/tests/core/test_monitor_configuration.cpp b/tests/core/test_monitor_configuration.cpp index 38d59e3..429dc70 100644 --- a/tests/core/test_monitor_configuration.cpp +++ b/tests/core/test_monitor_configuration.cpp @@ -119,6 +119,15 @@ TEST_F(MonitorConfigurationTest, EnvironmentOverridesTakePrecedence) { EXPECT_TRUE(actual.pm_sampling_scope_only); } +TEST_F(MonitorConfigurationTest, MapsMetalBackendToMetalMonitorBackend) { + gpufl::InitOptions options; + options.backend = gpufl::BackendKind::Metal; + + const auto actual = gpufl::detail::buildMonitorOptions(options); + + EXPECT_EQ(actual.backend_kind, gpufl::MonitorBackendKind::Metal); +} + TEST_F(MonitorConfigurationTest, InvalidOverridesPreserveConfiguredValues) { setEnv(gpufl::env::kProfilingEngine, "not-an-engine"); setEnv(gpufl::env::kPcSamplingPeriod, "32"); diff --git a/tests/core/test_wire_contract.cpp b/tests/core/test_wire_contract.cpp index 3c81df6..6e8e71e 100644 --- a/tests/core/test_wire_contract.cpp +++ b/tests/core/test_wire_contract.cpp @@ -129,6 +129,82 @@ TEST(WireContract, JobStartEmitsNvidiaNoneSentinel) { << "explicit-None sessions must emit profiling_engine: nvidia.none: " << json; } +// Backend selection and Metal capability metadata are additive fields. +TEST(WireContract, MonitorEngineUsesResolvedTelemetryBackend) { + EXPECT_EQ(gpufl::ProfilingEngineWireNameForBackend( + gpufl::ProfilingEngine::Monitor, "metal"), + "metal.none"); + EXPECT_EQ(gpufl::ProfilingEngineWireNameForBackend( + gpufl::ProfilingEngine::Monitor, "amd"), + "amd.none"); + EXPECT_EQ(gpufl::ProfilingEngineWireNameForBackend( + gpufl::ProfilingEngine::Monitor, "nvidia"), + "nvidia.none"); +} + +TEST(WireContract, JobStartEmitsMetalMetadataAndMetricAvailability) { + gpufl::InitEvent e; + e.pid = 1; + e.app = "metal_monitor"; + e.session_id = "sess-metal"; + e.ts_ns = 1; + e.session_kind = "monitor"; + e.profiling_engine = "metal.none"; + e.telemetry_backend = "metal"; + + gpufl::DeviceSample sample{}; + sample.device_id = 0; + sample.name = "Apple Test GPU"; + sample.uuid = "metal-registry-0x1"; + sample.vendor = "Apple"; + sample.telemetry_capabilities.available = { + "process_allocated_mib", "recommended_max_working_set_mib"}; + sample.telemetry_capabilities.unavailable = {"gpu_util", "temp_c", + "power_mw"}; + sample.telemetry_capabilities.memory_model = "unified"; + sample.telemetry_capabilities.allocation_scope = "current_process"; + sample.telemetry_capabilities.process_allocated_mib = 128; + sample.telemetry_capabilities.recommended_max_working_set_mib = 16384; + e.devices.push_back(sample); + + gpufl::GpuStaticDeviceInfo info{}; + info.name = "Apple Test GPU"; + info.uuid = sample.uuid; + info.vendor = "Apple"; + info.architecture = "apple8"; + info.metal.available = true; + info.metal.registry_id = "0x1"; + info.metal.architecture_name = "apple8"; + info.metal.unified_memory = true; + info.metal.location = "built_in"; + info.metal.recommended_max_working_set_bytes = 17179869184ULL; + info.metal.max_buffer_length_bytes = 4294967296ULL; + info.metal.max_threads_per_threadgroup = {1024, 1024, 1024}; + info.metal.gpu_families = {"apple8", "common3", "metal3"}; + info.metal.counter_sets = {"timestamp", "stage_utilization"}; + e.gpu_static_device_infos.push_back(info); + + const std::string json = gpufl::model::InitEventModel(e).buildJson(); + + EXPECT_TRUE(JsonContains(json, "\"telemetry_backend\":\"metal\"")); + EXPECT_TRUE(JsonContains(json, "\"profiling_engine\":\"metal.none\"")); + EXPECT_TRUE(JsonContains(json, + "\"available\":[\"process_allocated_mib\"," + "\"recommended_max_working_set_mib\"]")); + EXPECT_TRUE(JsonContains(json, "\"memory_model\":\"unified\"")); + EXPECT_TRUE(JsonContains(json, "\"allocation_scope\":\"current_process\"")); + EXPECT_TRUE(JsonContains(json, "\"process_allocated_mib\":128")); + EXPECT_TRUE(JsonContains(json, "\"used_mib\":0")); + EXPECT_TRUE(JsonContains(json, "\"total_mib\":0")); + EXPECT_TRUE(JsonContains(json, "\"metal\":{\"registry_id\":\"0x1\"")); + EXPECT_TRUE(JsonContains(json, "\"architecture_name\":\"apple8\"")); + EXPECT_TRUE(JsonContains(json, "\"unified_memory\":true")); + EXPECT_TRUE(JsonContains( + json, "\"gpu_families\":[\"apple8\",\"common3\",\"metal3\"]")); + EXPECT_TRUE(JsonContains(json, "\"cuda_static_devices\":[]")); + EXPECT_TRUE(JsonContains(json, "\"rocm_static_devices\":[]")); +} + // ── job_start multi-pass grouping (P1) ───────────────────────────────────── // // analysis_id / pass_index / pass_count are emitted together, and ONLY when