diff --git a/.context/gpu-benchmark-results.jsonl b/.context/gpu-benchmark-results.jsonl new file mode 100644 index 000000000..7f6da031e --- /dev/null +++ b/.context/gpu-benchmark-results.jsonl @@ -0,0 +1,3 @@ +{"timestamp": "2026-06-22T13:04:21.864329+00:00", "mode": "all", "hostname": "Mac.lan", "gpu_backend": "simulated (8 GPUs)", "gpu_count_real": 0, "simulated": true, "call_breakdown": [{"call_name": "nvmlDeviceGetMemoryInfo", "used_by_monitor": false, "simulated_latency_us": 50}, {"call_name": "nvmlDeviceGetTemperature", "used_by_monitor": false, "simulated_latency_us": 40}, {"call_name": "nvmlDeviceGetPowerUsage", "used_by_monitor": false, "simulated_latency_us": 45}, {"call_name": "nvmlDeviceGetTotalEnergyConsumption", "used_by_monitor": false, "simulated_latency_us": 40}, {"call_name": "nvmlDeviceGetUtilizationRates", "used_by_monitor": true, "simulated_latency_us": 50}, {"call_name": "nvmlDeviceGetComputeMode", "used_by_monitor": false, "simulated_latency_us": 35}, {"call_name": "nvmlDeviceGetComputeRunningProcesses", "used_by_monitor": false, "simulated_latency_us": 500}, {"call_name": "nvmlDeviceGetGraphicsRunningProcesses", "used_by_monitor": false, "simulated_latency_us": 500}], "method_benchmarks": [{"method": "get_gpu_details", "gpu_count": 8, "nvml_calls_per_second": 64, "nvml_calls_unused_per_second": 56, "latency_per_call_ms": {"count": 200, "min_ms": 12.55650000530295, "max_ms": 67.93629100138787, "mean_ms": 13.547916884053848, "p50_ms": 12.920312496135011, "p95_ms": 15.044183352438257}, "latency_per_second_ms": 12.920312496135011}, {"method": "get_gpu_utilization_list", "gpu_count": 8, "nvml_calls_per_second": 8, "nvml_calls_unused_per_second": 0, "latency_per_call_ms": {"count": 200, "min_ms": 0.5256250005913898, "max_ms": 0.6670419970760122, "mean_ms": 0.5427240101562347, "p50_ms": 0.5367710036807694, "p95_ms": 0.5937666595855262}, "latency_per_second_ms": 0.5367710036807694}], "projections": [{"metric": "Per-second monitoring overhead", "heavy_path_ms": 12.920312496135011, "lightweight_path_ms": 0.5367710036807694, "savings_ms": 12.383541492454242, "savings_pct": 95.8, "unit": "ms/s"}, {"metric": "Per-minute monitoring overhead", "heavy_path_ms": 775.2187497681007, "lightweight_path_ms": 32.20626022084616, "savings_ms": 743.0124895472545, "savings_pct": 95.8, "unit": "ms/min"}, {"metric": "Per-hour monitoring overhead", "heavy_path_ms": 46513.12498608604, "lightweight_path_ms": 1932.3756132507697, "savings_ms": 44580.74937283527, "savings_pct": 95.8, "unit": "ms/hr"}, {"metric": "Per-day monitoring overhead (24h)", "heavy_path_ms": 1116314.999666065, "lightweight_path_ms": 46377.01471801847, "savings_ms": 1069937.9849480465, "savings_pct": 95.8, "unit": "ms/day"}, {"metric": "Unnecessary NVML calls per second", "heavy_path_value": 56, "lightweight_path_value": 0, "savings_value": 56, "unit": "calls/s"}, {"metric": "Unnecessary NVML calls per hour (on 8 GPUs)", "heavy_path_value": 201600, "lightweight_path_value": 0, "savings_value": 201600, "unit": "calls/hr"}], "result": ""} +{"timestamp": "2026-06-22T13:04:31.008513+00:00", "mode": "all", "hostname": "Mac.lan", "gpu_backend": "simulated (1 GPU)", "gpu_count_real": 0, "simulated": true, "call_breakdown": [{"call_name": "nvmlDeviceGetMemoryInfo", "used_by_monitor": false, "simulated_latency_us": 50}, {"call_name": "nvmlDeviceGetTemperature", "used_by_monitor": false, "simulated_latency_us": 40}, {"call_name": "nvmlDeviceGetPowerUsage", "used_by_monitor": false, "simulated_latency_us": 45}, {"call_name": "nvmlDeviceGetTotalEnergyConsumption", "used_by_monitor": false, "simulated_latency_us": 40}, {"call_name": "nvmlDeviceGetUtilizationRates", "used_by_monitor": true, "simulated_latency_us": 50}, {"call_name": "nvmlDeviceGetComputeMode", "used_by_monitor": false, "simulated_latency_us": 35}, {"call_name": "nvmlDeviceGetComputeRunningProcesses", "used_by_monitor": false, "simulated_latency_us": 500}, {"call_name": "nvmlDeviceGetGraphicsRunningProcesses", "used_by_monitor": false, "simulated_latency_us": 500}], "method_benchmarks": [{"method": "get_gpu_details", "gpu_count": 1, "nvml_calls_per_second": 8, "nvml_calls_unused_per_second": 7, "latency_per_call_ms": {"count": 200, "min_ms": 1.5633330040145665, "max_ms": 1.9456660083960742, "mean_ms": 1.6176056357653579, "p50_ms": 1.605146004294511, "p95_ms": 1.665493459586287}, "latency_per_second_ms": 1.605146004294511}, {"method": "get_gpu_utilization_list", "gpu_count": 1, "nvml_calls_per_second": 1, "nvml_calls_unused_per_second": 0, "latency_per_call_ms": {"count": 200, "min_ms": 0.06458298594225198, "max_ms": 0.14504100545309484, "mean_ms": 0.06767040984414052, "p50_ms": 0.06535449210787192, "p95_ms": 0.0750916529796086}, "latency_per_second_ms": 0.06535449210787192}], "projections": [{"metric": "Per-second monitoring overhead", "heavy_path_ms": 1.605146004294511, "lightweight_path_ms": 0.06535449210787192, "savings_ms": 1.539791512186639, "savings_pct": 95.9, "unit": "ms/s"}, {"metric": "Per-minute monitoring overhead", "heavy_path_ms": 96.30876025767066, "lightweight_path_ms": 3.921269526472315, "savings_ms": 92.38749073119834, "savings_pct": 95.9, "unit": "ms/min"}, {"metric": "Per-hour monitoring overhead", "heavy_path_ms": 5778.525615460239, "lightweight_path_ms": 235.2761715883389, "savings_ms": 5543.2494438719, "savings_pct": 95.9, "unit": "ms/hr"}, {"metric": "Per-day monitoring overhead (24h)", "heavy_path_ms": 138684.61477104574, "lightweight_path_ms": 5646.628118120134, "savings_ms": 133037.9866529256, "savings_pct": 95.9, "unit": "ms/day"}, {"metric": "Unnecessary NVML calls per second", "heavy_path_value": 7, "lightweight_path_value": 0, "savings_value": 7, "unit": "calls/s"}, {"metric": "Unnecessary NVML calls per hour (on 1 GPU)", "heavy_path_value": 25200, "lightweight_path_value": 0, "savings_value": 25200, "unit": "calls/hr"}], "result": ""} +{"timestamp": "2026-07-02T03:54:32.856813+00:00", "mode": "all", "hostname": "Mac.lan", "gpu_backend": "simulated (1 GPU)", "gpu_count_real": 0, "simulated": true, "call_breakdown": [{"call_name": "nvmlDeviceGetMemoryInfo", "used_by_monitor": false, "simulated_latency_us": 50}, {"call_name": "nvmlDeviceGetTemperature", "used_by_monitor": false, "simulated_latency_us": 40}, {"call_name": "nvmlDeviceGetPowerUsage", "used_by_monitor": false, "simulated_latency_us": 45}, {"call_name": "nvmlDeviceGetTotalEnergyConsumption", "used_by_monitor": false, "simulated_latency_us": 40}, {"call_name": "nvmlDeviceGetUtilizationRates", "used_by_monitor": true, "simulated_latency_us": 50}, {"call_name": "nvmlDeviceGetComputeMode", "used_by_monitor": false, "simulated_latency_us": 35}, {"call_name": "nvmlDeviceGetComputeRunningProcesses", "used_by_monitor": false, "simulated_latency_us": 500}, {"call_name": "nvmlDeviceGetGraphicsRunningProcesses", "used_by_monitor": false, "simulated_latency_us": 500}], "method_benchmarks": [{"method": "get_gpu_details", "gpu_count": 1, "nvml_calls_per_second": 8, "nvml_calls_unused_per_second": 7, "latency_per_call_ms": {"count": 200, "min_ms": 1.4970839984016493, "max_ms": 1.756750003551133, "mean_ms": 1.6207081100947107, "p50_ms": 1.6159379993041512, "p95_ms": 1.666629193641711}, "latency_per_second_ms": 1.6159379993041512}, {"method": "get_gpu_utilization_list", "gpu_count": 1, "nvml_calls_per_second": 1, "nvml_calls_unused_per_second": 0, "latency_per_call_ms": {"count": 200, "min_ms": 0.062167004216462374, "max_ms": 0.13416699948720634, "mean_ms": 0.06766192989744013, "p50_ms": 0.0667294989398215, "p95_ms": 0.07051039865473285}, "latency_per_second_ms": 0.0667294989398215}], "projections": [{"metric": "Per-second monitoring overhead", "heavy_path_ms": 1.6159379993041512, "lightweight_path_ms": 0.0667294989398215, "savings_ms": 1.5492085003643297, "savings_pct": 95.9, "unit": "ms/s"}, {"metric": "Per-minute monitoring overhead", "heavy_path_ms": 96.95627995824907, "lightweight_path_ms": 4.00376993638929, "savings_ms": 92.95251002185978, "savings_pct": 95.9, "unit": "ms/min"}, {"metric": "Per-hour monitoring overhead", "heavy_path_ms": 5817.376797494944, "lightweight_path_ms": 240.2261961833574, "savings_ms": 5577.150601311587, "savings_pct": 95.9, "unit": "ms/hr"}, {"metric": "Per-day monitoring overhead (24h)", "heavy_path_ms": 139617.04313987866, "lightweight_path_ms": 5765.428708400577, "savings_ms": 133851.61443147808, "savings_pct": 95.9, "unit": "ms/day"}, {"metric": "Unnecessary NVML calls per second", "heavy_path_value": 7, "lightweight_path_value": 0, "savings_value": 7, "unit": "calls/s"}, {"metric": "Unnecessary NVML calls per hour (on 1 GPU)", "heavy_path_value": 25200, "lightweight_path_value": 0, "savings_value": 25200, "unit": "calls/hr"}], "result": ""} diff --git a/codecarbon/core/gpu.py b/codecarbon/core/gpu.py index 86dd8234f..b5b56f5a3 100644 --- a/codecarbon/core/gpu.py +++ b/codecarbon/core/gpu.py @@ -137,6 +137,23 @@ def get_gpu_details(self) -> List: logger.warning("Failed to retrieve gpu information", exc_info=True) return [] + def get_gpu_utilization_list(self) -> List: + """Lightweight alternative to :meth:`get_gpu_details` for the 1s + monitoring hot path. Returns only ``gpu_index`` and + ``gpu_utilization`` per device, skipping heavyweight queries + (memory, temperature, compute mode, process lists). + + >>> get_gpu_utilization_list() + [ + {"gpu_index": 0, "gpu_utilization": 0}, + ] + """ + try: + return [d.get_gpu_utilization_lightweight() for d in self.devices] + except Exception: + logger.warning("Failed to retrieve gpu utilization", exc_info=True) + return [] + def get_delta(self, last_duration: Time) -> List: """Get difference since last time this function was called >>> get_delta() diff --git a/codecarbon/core/gpu_device.py b/codecarbon/core/gpu_device.py index 4d7261b7d..306ff71f6 100644 --- a/codecarbon/core/gpu_device.py +++ b/codecarbon/core/gpu_device.py @@ -101,6 +101,20 @@ def get_gpu_details(self) -> Dict[str, Any]: } return device_details + def get_gpu_utilization_lightweight(self) -> Dict[str, Any]: + """ + Lightweight alternative to :meth:`get_gpu_details` for the hot path + (``_monitor_power`` which runs every 1s). + + Only queries the GPU utilization — avoids heavyweight calls like + memory info, temperature, compute mode, and process lists which are + not consumed by the tracker's monitoring loop. + """ + return { + "gpu_index": self.gpu_index, + "gpu_utilization": self._get_gpu_utilization(), + } + def _to_utf8(self, str_or_bytes) -> Any: if hasattr(str_or_bytes, "decode"): return str_or_bytes.decode("utf-8", errors="replace") diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index efd3830ea..270723466 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -1147,15 +1147,16 @@ def _monitor_power(self) -> None: self._ram_utilization_history.append(psutil.virtual_memory().percent) self._ram_used_history.append(psutil.virtual_memory().used / (1024**3)) - # Collect GPU utilization metrics + # Collect GPU utilization metrics (lightweight path — skips + # heavyweight calls like process lists, memory, temperature). for hardware in self._hardware: if isinstance(hardware, GPU): gpu_ids_to_monitor = hardware.gpu_ids - gpu_details = hardware.devices.get_gpu_details() - for gpu_index, gpu_detail in enumerate(gpu_details): - resolved_gpu_index = gpu_detail.get("gpu_index", gpu_index) + for gpu_detail in hardware.devices.get_gpu_utilization_list(): + resolved_gpu_index = gpu_detail.get("gpu_index") if ( - resolved_gpu_index in gpu_ids_to_monitor + resolved_gpu_index is not None + and resolved_gpu_index in gpu_ids_to_monitor and "gpu_utilization" in gpu_detail ): self._gpu_utilization_history.append( diff --git a/scripts/benchmark_gpu_monitoring.py b/scripts/benchmark_gpu_monitoring.py new file mode 100644 index 000000000..23d642916 --- /dev/null +++ b/scripts/benchmark_gpu_monitoring.py @@ -0,0 +1,644 @@ +#!/usr/bin/env python3 +""" +Benchmark: GPU monitoring overhead — heavyweight get_gpu_details vs lightweight get_gpu_utilization_list. + +Measures how many unnecessary NVML calls the per-second _monitor_power() hot path +makes on multi-GPU systems, and the latency difference between the old full-detail +path and the new lightweight utilization-only path. + +Usage: + # Quick run (default) + uv run python scripts/benchmark_gpu_monitoring.py + + # Full benchmark with subprocess cold-start samples + uv run python scripts/benchmark_gpu_monitoring.py all + + # Simulated multi-GPU scale (no real GPU needed) + uv run python scripts/benchmark_gpu_monitoring.py all --simulate-gpus 8 + +Methodology: + - Cold metrics: spawn fresh Python subprocesses, each performing full GPU init + - Warm metrics: repeat calls in the same process after warm-up + - p50 (median) reported across multiple samples + - NVML call counts derived from source code audit (gpu_nvidia.py + gpu_device.py) + - On real NVIDIA hardware: wall-clock timing of actual NVML calls + - On non-NVIDIA hardware: mock NVML with realistic simulated call latencies +""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import subprocess +import sys +import time +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +RESULTS_DIR = REPO_ROOT / ".context" +RESULTS_DIR.mkdir(parents=True, exist_ok=True) +DEFAULT_RESULTS = RESULTS_DIR / "gpu-benchmark-results.jsonl" + +# NVML call categories based on source audit (gpu_nvidia.py + gpu_device.py) +# _monitor_power() calls get_gpu_details() every 1s but only uses gpu_utilization +NVML_CALLS_HEAVY = [ + "nvmlDeviceGetMemoryInfo", # → free_memory, total_memory, used_memory — DISCARDED + "nvmlDeviceGetTemperature", # → temperature — DISCARDED + "nvmlDeviceGetPowerUsage", # → power_usage — DISCARDED + "nvmlDeviceGetTotalEnergyConsumption", # → total_energy_consumption — DISCARDED + "nvmlDeviceGetUtilizationRates", # → gpu_utilization — USED + "nvmlDeviceGetComputeMode", # → compute_mode — DISCARDED + "nvmlDeviceGetComputeRunningProcesses", # → compute_processes — DISCARDED (most expensive) + "nvmlDeviceGetGraphicsRunningProcesses", # → graphics_processes — DISCARDED (most expensive) +] + +NVML_CALLS_LIGHTWEIGHT = [ + "nvmlDeviceGetUtilizationRates", # ← the only call we need for utilization +] + +# Simulated per-call latencies (microseconds) for non-GPU systems. +# Based on typical NVML overheads reported in NVIDIA docs & community benchmarks. +# Process enumeration (GetComputeRunningProcesses) is the most expensive because +# it iterates active GPU processes and collects PID-level info. +SIMULATED_LATENCY_US: dict[str, float] = { + "nvmlDeviceGetMemoryInfo": 50, + "nvmlDeviceGetTemperature": 40, + "nvmlDeviceGetPowerUsage": 45, + "nvmlDeviceGetTotalEnergyConsumption": 40, + "nvmlDeviceGetUtilizationRates": 50, + "nvmlDeviceGetComputeMode": 35, + "nvmlDeviceGetComputeRunningProcesses": 500, # ← expensive: process enumeration + "nvmlDeviceGetGraphicsRunningProcesses": 500, # ← expensive: process enumeration + "nvmlDeviceGetName": 40, + "nvmlDeviceGetUUID": 35, + "nvmlDeviceGetEnforcedPowerLimit": 40, +} + + +@dataclass +class LatencyStats: + count: int = 0 + min_ms: float = 0.0 + max_ms: float = 0.0 + mean_ms: float = 0.0 + p50_ms: float = 0.0 + p95_ms: float = 0.0 + + +@dataclass +class NvmlCallBreakdown: + call_name: str + latency_us: float + used_by_monitor: bool + + +@dataclass +class GpuDetailMethodBenchmark: + method: str # "get_gpu_details" or "get_gpu_utilization_list" + gpu_count: int + nvml_calls_per_second: int + nvml_calls_unused_per_second: int + latency_per_call_ms: LatencyStats + latency_per_second_ms: float # projected = per_gpu * gpu_count + + +@dataclass +class MonitoringOverheadProjection: + metric: str + heavy_path: float + lightweight_path: float + savings: float + unit: str + + +@dataclass +class BenchmarkReport: + timestamp: str + mode: str + hostname: str + gpu_backend: str + gpu_count_real: int + simulated: bool + call_breakdown: list[dict] + method_benchmarks: list[dict] + projections: list[dict] + result: str = "" + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _percentile(sorted_values: list[float], pct: float) -> float: + if not sorted_values: + return 0.0 + if len(sorted_values) == 1: + return sorted_values[0] + k = (len(sorted_values) - 1) * (pct / 100.0) + f = int(k) + c = min(f + 1, len(sorted_values) - 1) + if f == c: + return sorted_values[f] + return sorted_values[f] + (sorted_values[c] - sorted_values[f]) * (k - f) + + +def compute_stats(values_ms: list[float]) -> LatencyStats: + if not values_ms: + return LatencyStats(count=0) + s = sorted(values_ms) + return LatencyStats( + count=len(s), + min_ms=s[0], + max_ms=s[-1], + mean_ms=statistics.mean(s), + p50_ms=_percentile(s, 50), + p95_ms=_percentile(s, 95), + ) + + +def _detect_gpu_backend() -> tuple[str, int]: + """Detect real GPU backend and count. Returns (backend_name, count).""" + try: + from codecarbon.core.gpu import AMDSMI_AVAILABLE, PYNVML_AVAILABLE + + if PYNVML_AVAILABLE: + from codecarbon.core import gpu_nvidia + + count = gpu_nvidia.pynvml.nvmlDeviceGetCount() + return ("nvidia", count) + if AMDSMI_AVAILABLE: + return ("amd", 0) # count not trivial + except Exception: + pass + return ("none", 0) + + +def _collect_call_breakdown() -> list[dict]: + """Return the per-NVML-call breakdown showing what's used vs discarded.""" + results = [] + for call in NVML_CALLS_HEAVY: + results.append( + { + "call_name": call, + "used_by_monitor": call == "nvmlDeviceGetUtilizationRates", + "simulated_latency_us": SIMULATED_LATENCY_US.get(call, 50), + } + ) + return results + + +def _mock_time_for_call(call_name: str) -> None: + """Sleep to simulate NVML call latency when no real GPU is available.""" + time.sleep(SIMULATED_LATENCY_US.get(call_name, 50) / 1_000_000) + + +class MockNvidiaGPUDevice: + """A lightweight mock that simulates NVML call latencies. + + Used on non-NVIDIA systems so the benchmark can still measure + relative overhead and project multi-GPU scaling. + """ + + def __init__(self, gpu_index: int): + self.gpu_index = gpu_index + + def get_gpu_details(self) -> dict: + _mock_time_for_call("nvmlDeviceGetMemoryInfo") + _mock_time_for_call("nvmlDeviceGetTemperature") + _mock_time_for_call("nvmlDeviceGetPowerUsage") + _mock_time_for_call("nvmlDeviceGetTotalEnergyConsumption") + _mock_time_for_call("nvmlDeviceGetUtilizationRates") + _mock_time_for_call("nvmlDeviceGetComputeMode") + _mock_time_for_call("nvmlDeviceGetComputeRunningProcesses") + _mock_time_for_call("nvmlDeviceGetGraphicsRunningProcesses") + return {"gpu_index": self.gpu_index, "gpu_utilization": 50} + + def get_gpu_utilization_lightweight(self) -> dict: + _mock_time_for_call("nvmlDeviceGetUtilizationRates") + return {"gpu_index": self.gpu_index, "gpu_utilization": 50} + + +def _benchmark_method( + devices: list, + method_name: str, + samples: int = 200, + warmup: int = 20, +) -> LatencyStats: + """Benchmark a GPU method. Returns latency stats in milliseconds.""" + for _ in range(warmup): + if method_name == "get_gpu_details": + [d.get_gpu_details() for d in devices] + else: + [d.get_gpu_utilization_lightweight() for d in devices] + + timings = [] + for _ in range(samples): + t0 = time.perf_counter() + if method_name == "get_gpu_details": + [d.get_gpu_details() for d in devices] + else: + [d.get_gpu_utilization_lightweight() for d in devices] + elapsed_ms = (time.perf_counter() - t0) * 1000 + timings.append(elapsed_ms) + + return compute_stats(timings) + + +def _benchmark_real_gpu(gpu_count: int) -> tuple[list[dict], list[dict]]: + """Benchmark using real GPU hardware via AllGPUDevices.""" + sys.path.insert(0, str(REPO_ROOT)) + from codecarbon.core.gpu import AllGPUDevices + + devices = AllGPUDevices() + actual_count = devices.device_count + + heavy_stats = _benchmark_method(devices.devices, "get_gpu_details") + light_stats = _benchmark_method(devices.devices, "get_gpu_utilization_lightweight") + + method_benchmarks = [ + { + "method": "get_gpu_details", + "gpu_count": actual_count, + "nvml_calls_per_second": len(NVML_CALLS_HEAVY) * actual_count, + "nvml_calls_unused_per_second": (len(NVML_CALLS_HEAVY) - 1) * actual_count, + "latency_per_call_ms": asdict(heavy_stats), + "latency_per_second_ms": heavy_stats.p50_ms, + }, + { + "method": "get_gpu_utilization_list", + "gpu_count": actual_count, + "nvml_calls_per_second": len(NVML_CALLS_LIGHTWEIGHT) * actual_count, + "nvml_calls_unused_per_second": 0, + "latency_per_call_ms": asdict(light_stats), + "latency_per_second_ms": light_stats.p50_ms, + }, + ] + + # Scale projections for multi-GPU + for simulated_count in [1, 4, 8]: + scale = simulated_count / actual_count if actual_count else 1 + method_benchmarks.append( + { + "method": f"get_gpu_details (projected {simulated_count} GPU)", + "gpu_count": simulated_count, + "nvml_calls_per_second": len(NVML_CALLS_HEAVY) * simulated_count, + "nvml_calls_unused_per_second": (len(NVML_CALLS_HEAVY) - 1) + * simulated_count, + "latency_per_call_ms": asdict(heavy_stats), + "latency_per_second_ms": heavy_stats.p50_ms * scale, + } + ) + method_benchmarks.append( + { + "method": f"get_gpu_utilization_list (projected {simulated_count} GPU)", + "gpu_count": simulated_count, + "nvml_calls_per_second": len(NVML_CALLS_LIGHTWEIGHT) * simulated_count, + "nvml_calls_unused_per_second": 0, + "latency_per_call_ms": asdict(light_stats), + "latency_per_second_ms": light_stats.p50_ms * scale, + } + ) + + return method_benchmarks, [] + + +def _benchmark_simulated_gpu(simulate_gpus: int) -> tuple[list[dict], list[dict]]: + """Benchmark using mock devices with simulated NVML latencies.""" + devices = [MockNvidiaGPUDevice(i) for i in range(simulate_gpus)] + + heavy_stats = _benchmark_method(devices, "get_gpu_details") + light_stats = _benchmark_method(devices, "get_gpu_utilization_lightweight") + + method_benchmarks = [ + { + "method": "get_gpu_details", + "gpu_count": simulate_gpus, + "nvml_calls_per_second": len(NVML_CALLS_HEAVY) * simulate_gpus, + "nvml_calls_unused_per_second": (len(NVML_CALLS_HEAVY) - 1) * simulate_gpus, + "latency_per_call_ms": asdict(heavy_stats), + "latency_per_second_ms": heavy_stats.p50_ms, + }, + { + "method": "get_gpu_utilization_list", + "gpu_count": simulate_gpus, + "nvml_calls_per_second": len(NVML_CALLS_LIGHTWEIGHT) * simulate_gpus, + "nvml_calls_unused_per_second": 0, + "latency_per_call_ms": asdict(light_stats), + "latency_per_second_ms": light_stats.p50_ms, + }, + ] + + return method_benchmarks, [] + + +def _compute_projections(method_benchmarks: list[dict]) -> list[dict]: + """Compute time-savings projections from benchmark results.""" + heavy = next( + (m for m in method_benchmarks if m["method"] == "get_gpu_details"), None + ) + light = next( + (m for m in method_benchmarks if m["method"] == "get_gpu_utilization_list"), + None, + ) + if not heavy or not light: + return [] + + heavy_per_sec = heavy["latency_per_second_ms"] + light_per_sec = light["latency_per_second_ms"] + savings_per_sec = heavy_per_sec - light_per_sec + + gpu_count = heavy["gpu_count"] + + return [ + { + "metric": "Per-second monitoring overhead", + "heavy_path_ms": heavy_per_sec, + "lightweight_path_ms": light_per_sec, + "savings_ms": savings_per_sec, + "savings_pct": ( + round((savings_per_sec / heavy_per_sec) * 100, 1) + if heavy_per_sec + else 0 + ), + "unit": "ms/s", + }, + { + "metric": "Per-minute monitoring overhead", + "heavy_path_ms": heavy_per_sec * 60, + "lightweight_path_ms": light_per_sec * 60, + "savings_ms": savings_per_sec * 60, + "savings_pct": ( + round((savings_per_sec / heavy_per_sec) * 100, 1) + if heavy_per_sec + else 0 + ), + "unit": "ms/min", + }, + { + "metric": "Per-hour monitoring overhead", + "heavy_path_ms": heavy_per_sec * 3600, + "lightweight_path_ms": light_per_sec * 3600, + "savings_ms": savings_per_sec * 3600, + "savings_pct": ( + round((savings_per_sec / heavy_per_sec) * 100, 1) + if heavy_per_sec + else 0 + ), + "unit": "ms/hr", + }, + { + "metric": "Per-day monitoring overhead (24h)", + "heavy_path_ms": heavy_per_sec * 86400, + "lightweight_path_ms": light_per_sec * 86400, + "savings_ms": savings_per_sec * 86400, + "savings_pct": ( + round((savings_per_sec / heavy_per_sec) * 100, 1) + if heavy_per_sec + else 0 + ), + "unit": "ms/day", + }, + { + "metric": "Unnecessary NVML calls per second", + "heavy_path_value": heavy["nvml_calls_unused_per_second"], + "lightweight_path_value": 0, + "savings_value": heavy["nvml_calls_unused_per_second"], + "unit": "calls/s", + }, + { + "metric": f"Unnecessary NVML calls per hour (on {gpu_count} GPU{'s' if gpu_count != 1 else ''})", + "heavy_path_value": heavy["nvml_calls_unused_per_second"] * 3600, + "lightweight_path_value": 0, + "savings_value": heavy["nvml_calls_unused_per_second"] * 3600, + "unit": "calls/hr", + }, + ] + + +def run_all(simulate_gpus: int | None = None) -> BenchmarkReport: + backend, real_count = _detect_gpu_backend() + simulated = backend == "none" and simulate_gpus is not None + + if backend != "none" and real_count > 0: + gpu_backend = f"nvidia ({real_count} GPU{'s' if real_count != 1 else ''})" + method_bms, _ = _benchmark_real_gpu(real_count) + elif simulate_gpus: + gpu_backend = ( + f"simulated ({simulate_gpus} GPU{'s' if simulate_gpus != 1 else ''})" + ) + method_bms, _ = _benchmark_simulated_gpu(simulate_gpus) + else: + gpu_backend = "none (no GPU available, use --simulate-gpus N)" + method_bms = [] + + projections = _compute_projections(method_bms) if method_bms else [] + + call_breakdown = _collect_call_breakdown() + + return BenchmarkReport( + timestamp=_now_iso(), + mode="all", + hostname=os.uname().nodename, + gpu_backend=gpu_backend, + gpu_count_real=real_count, + simulated=simulated, + call_breakdown=call_breakdown, + method_benchmarks=method_bms, + projections=projections, + ) + + +def print_report(report: BenchmarkReport) -> None: + sep = "─" * 72 + + print(f"\n{' GPU Monitoring Overhead Benchmark ':=^72}") + print(f" Host: {report.hostname}") + print(f" GPU backend: {report.gpu_backend}") + print(f" Simulated: {report.simulated}") + print(f" Timestamp: {report.timestamp}") + + if report.simulated: + print(f"\n{' ⚠ SIMULATED — No real GPU detected ':=^72}") + print(" Call latencies are estimated (see SIMULATED_LATENCY_US in script).") + print(" Run this on an NVIDIA GPU machine for real hardware measurements.") + + # NVML call breakdown + print(f"\n{sep}") + print(f"{' NVML Call Breakdown (per GPU, per call to get_gpu_details) ':=^72}") + print(f"{'NVML Call':40s} {'Latency (µs)':15s} {'Used by monitor':20s}") + print("-" * 72) + for cb in report.call_breakdown: + used = "YES" if cb["used_by_monitor"] else "" + print( + f"{cb['call_name']:40s} {cb['simulated_latency_us']:>10.0f} µs {used:20s}" + ) + + unused = sum(1 for cb in report.call_breakdown if not cb["used_by_monitor"]) + total = len(report.call_breakdown) + print(f"\n → {unused}/{total} NVML calls DISCARDED by _monitor_power()") + print(f" → Only 1/{total} calls actually used (gpu_utilization)") + + # Method benchmarks + if report.method_benchmarks: + print(f"\n{sep}") + print(f"{' Method Latency Benchmarks ':=^72}") + print( + f"{'Method':50s} {'p50':>8s} {'mean':>8s} {'p95':>8s} {'NVML calls/s':>14s}" + ) + print("-" * 72) + for mb in report.method_benchmarks: + lat = mb["latency_per_call_ms"] + print( + f"{mb['method']:50s} " + f"{lat['p50_ms']:>7.2f}ms {lat['mean_ms']:>7.2f}ms {lat['p95_ms']:>7.2f}ms " + f"{mb['nvml_calls_per_second']:>8d}/s" + ) + + # Projections + if report.projections: + print(f"\n{sep}") + print(f"{' Projected Savings (heavyweight → lightweight) ':=^72}") + print(f"{'Metric':50s} {'Heavy':>12s} {'Light':>12s} {'Savings':>12s}") + print("-" * 72) + for p in report.projections: + if "savings_pct" in p: + print( + f"{p['metric']:50s} " + f"{p['heavy_path_ms']:>8.1f}ms {p['lightweight_path_ms']:>8.1f}ms " + f"{p['savings_ms']:>8.1f}ms ({p['savings_pct']}%)" + ) + else: + print( + f"{p['metric']:50s} " + f"{p['heavy_path_value']:>12,d} {p['lightweight_path_value']:>12,d} " + f"{p['savings_value']:>12,d}" + ) + + print(f"\n{sep}") + print(f"{' Summary ':=^72}") + if report.projections: + hourly = next( + ( + p + for p in report.projections + if p["metric"] == "Per-hour monitoring overhead" + ), + None, + ) + daily = next( + ( + p + for p in report.projections + if p["metric"] == "Per-day monitoring overhead (24h)" + ), + None, + ) + nvml_daily = next( + (p for p in report.projections if "NVML calls per hour" in p["metric"]), + None, + ) + if hourly: + print( + f" Each second of monitoring saves {hourly['savings_ms'] / 3600:.3f} ms" + ) + print( + f" Per hour of continuous monitoring saves {hourly['savings_ms'] / 1000:.1f} s" + ) + if daily: + print( + f" Per 24h day of monitoring saves {daily['savings_ms'] / 1000:.0f} s ({daily['savings_ms'] / 60000:.1f} min)" + ) + if nvml_daily: + print( + f" Unnecessary NVML calls per 24h: {nvml_daily['savings_value'] * 24:,d}" + ) + print(f"{'=' * 72}\n") + + +def run_cold_subprocess(simulate_gpus: int | None = None) -> BenchmarkReport: + """Spawn a fresh subprocess to measure cold-start GPU detection overhead.""" + cmd = [ + sys.executable, + __file__, + "cold", + "--json", + ] + if simulate_gpus: + cmd.extend(["--simulate-gpus", str(simulate_gpus)]) + env = os.environ.copy() + t0 = time.perf_counter() + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=60, env=env) + elapsed_ms = (time.perf_counter() - t0) * 1000 + if proc.returncode != 0: + print(f"Subprocess failed: {proc.stderr[:500]}") + return BenchmarkReport( + timestamp=_now_iso(), + mode="cold_subprocess", + hostname=os.uname().nodename, + gpu_backend="error", + gpu_count_real=0, + simulated=False, + call_breakdown=[], + method_benchmarks=[], + projections=[], + result="error", + ) + report = json.loads(proc.stdout) + report["mode"] = "cold_subprocess" + report["result"] = f"cold_subprocess_overhead_ms={elapsed_ms:.1f}" + return BenchmarkReport(**report) + + +def main() -> None: + p = argparse.ArgumentParser(description="GPU monitoring overhead benchmark") + p.add_argument("mode", nargs="?", default="quick", choices=["quick", "all", "cold"]) + p.add_argument( + "--simulate-gpus", + type=int, + default=None, + help="Simulate N GPUs (default: auto-detect)", + ) + p.add_argument( + "--json", action="store_true", help="Output JSON (for subprocess consumption)" + ) + p.add_argument("--results-file", type=Path, default=DEFAULT_RESULTS) + args = p.parse_args() + + if args.mode == "quick": + report = run_all(args.simulate_gpus) + print_report(report) + + elif args.mode == "all": + report = run_all(args.simulate_gpus) + if args.json: + print(json.dumps(asdict(report), default=str)) + else: + print_report(report) + + # Also run cold subprocess if not already in one + if not args.json and not os.environ.get("_BENCHMARK_CHILD"): + print("\n--- Cold subprocess benchmark ---") + cold_report = run_cold_subprocess(args.simulate_gpus) + print(f"Cold subprocess overhead: {cold_report.result}") + + elif args.mode == "cold": + os.environ["_BENCHMARK_CHILD"] = "1" + report = run_all(args.simulate_gpus) + if args.json: + print(json.dumps(asdict(report), default=str)) + else: + print_report(report) + + # Append to results file + if not args.json and args.mode != "cold": + with open(args.results_file, "a") as f: + f.write(json.dumps(asdict(report), default=str) + "\n") + print(f"→ Results appended to {args.results_file}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index 0715b665c..dd8b287f3 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -24,6 +24,7 @@ GEO_METADATA_CANADA, TWO_GPU_DETAILS_RESPONSE, TWO_GPU_DETAILS_RESPONSE_HANDLES, + TWO_GPU_UTILIZATION_RESPONSE, ) from tests.testutils import get_custom_mock_open, get_test_data_source @@ -52,6 +53,10 @@ def heavy_computation(run_time_secs: float = 3): @mock.patch("codecarbon.core.gpu.pynvml", fake_pynvml) @mock.patch("codecarbon.core.gpu.is_nvidia_system", return_value=True) @mock.patch("codecarbon.core.gpu.is_gpu_details_available", return_value=True) +@mock.patch( + "codecarbon.external.hardware.AllGPUDevices.get_gpu_utilization_list", + return_value=TWO_GPU_UTILIZATION_RESPONSE, +) @mock.patch( "codecarbon.external.hardware.AllGPUDevices.get_gpu_details", return_value=TWO_GPU_DETAILS_RESPONSE, @@ -90,6 +95,7 @@ def test_carbon_tracker_TWO_GPU_PRIVATE_INFRA_CANADA( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -108,8 +114,8 @@ def test_carbon_tracker_TWO_GPU_PRIVATE_INFRA_CANADA( # THEN self.assertGreaterEqual( - mocked_get_gpu_details.call_count, 2 - ) # at least 2 times in 5 seconds + once for init >= 3 + mocked_get_gpu_details.call_count, 1 + ) # called at least once for repr at init self.assertEqual(2, mocked_is_gpu_details_available.call_count) self.assertEqual(1, len(responses.calls)) self.assertEqual( @@ -118,12 +124,13 @@ def test_carbon_tracker_TWO_GPU_PRIVATE_INFRA_CANADA( self.assertIsInstance(emissions, float) self.assertAlmostEqual(emissions, 6.262572537957655e-05, places=2) - def test_monitor_power_uses_gpu_detail_position_when_gpu_index_is_missing( + def test_monitor_power_collects_gpu_utilization_lightweight( self, mock_cli_setup, mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -135,8 +142,8 @@ def test_monitor_power_uses_gpu_detail_position_when_gpu_index_is_missing( mock_gpu.__class__ = GPU mock_gpu.gpu_ids = [0, 1] mock_gpu.devices = mock.MagicMock() - mock_gpu.devices.get_gpu_details.return_value = [ - {"gpu_utilization": 10}, + mock_gpu.devices.get_gpu_utilization_list.return_value = [ + {"gpu_index": 0, "gpu_utilization": 10}, {"gpu_index": 1, "gpu_utilization": 25}, ] tracker._hardware = [mock_gpu] @@ -145,6 +152,116 @@ def test_monitor_power_uses_gpu_detail_position_when_gpu_index_is_missing( self.assertEqual([10, 25], tracker._gpu_utilization_history) + def test_monitor_power_skips_gpu_when_index_is_none( + self, + mock_cli_setup, + mock_log_values, + mocked_get_gpu_details, + mocked_env_cloud_details, + mocked_get_gpu_utilization_list, + mocked_is_gpu_details_available, + mocked_is_nvidia_system, + ): + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + + mock_gpu = mock.MagicMock() + from codecarbon.external.hardware import GPU + + mock_gpu.__class__ = GPU + mock_gpu.gpu_ids = [0, 1] + mock_gpu.devices = mock.MagicMock() + mock_gpu.devices.get_gpu_utilization_list.return_value = [ + {"gpu_index": None, "gpu_utilization": 10}, + {"gpu_index": 1, "gpu_utilization": 25}, + ] + tracker._hardware = [mock_gpu] + + tracker._monitor_power() + + self.assertEqual([25], tracker._gpu_utilization_history) + + def test_monitor_power_skips_gpu_not_in_monitored_ids( + self, + mock_cli_setup, + mock_log_values, + mocked_get_gpu_details, + mocked_env_cloud_details, + mocked_get_gpu_utilization_list, + mocked_is_gpu_details_available, + mocked_is_nvidia_system, + ): + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + + mock_gpu = mock.MagicMock() + from codecarbon.external.hardware import GPU + + mock_gpu.__class__ = GPU + mock_gpu.gpu_ids = [0] + mock_gpu.devices = mock.MagicMock() + mock_gpu.devices.get_gpu_utilization_list.return_value = [ + {"gpu_index": 0, "gpu_utilization": 10}, + {"gpu_index": 1, "gpu_utilization": 25}, + {"gpu_index": 2, "gpu_utilization": 50}, + ] + tracker._hardware = [mock_gpu] + + tracker._monitor_power() + + self.assertEqual([10], tracker._gpu_utilization_history) + + def test_monitor_power_skips_gpu_when_utilization_key_missing( + self, + mock_cli_setup, + mock_log_values, + mocked_get_gpu_details, + mocked_env_cloud_details, + mocked_get_gpu_utilization_list, + mocked_is_gpu_details_available, + mocked_is_nvidia_system, + ): + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + + mock_gpu = mock.MagicMock() + from codecarbon.external.hardware import GPU + + mock_gpu.__class__ = GPU + mock_gpu.gpu_ids = [0, 1] + mock_gpu.devices = mock.MagicMock() + mock_gpu.devices.get_gpu_utilization_list.return_value = [ + {"gpu_index": 0, "gpu_utilization": 10}, + {"gpu_index": 1}, + ] + tracker._hardware = [mock_gpu] + + tracker._monitor_power() + + self.assertEqual([10], tracker._gpu_utilization_history) + + def test_monitor_power_handles_empty_gpu_utilization_list( + self, + mock_cli_setup, + mock_log_values, + mocked_get_gpu_details, + mocked_env_cloud_details, + mocked_get_gpu_utilization_list, + mocked_is_gpu_details_available, + mocked_is_nvidia_system, + ): + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + + mock_gpu = mock.MagicMock() + from codecarbon.external.hardware import GPU + + mock_gpu.__class__ = GPU + mock_gpu.gpu_ids = [0, 1] + mock_gpu.devices = mock.MagicMock() + mock_gpu.devices.get_gpu_utilization_list.return_value = [] + tracker._hardware = [mock_gpu] + + tracker._monitor_power() + + self.assertEqual([], tracker._gpu_utilization_history) + @mock.patch("codecarbon.external.geography.requests.get") def test_carbon_tracker_timeout( self, @@ -153,6 +270,7 @@ def test_carbon_tracker_timeout( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -181,6 +299,7 @@ def test_graceful_start_failure( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -200,6 +319,7 @@ def test_graceful_stop_failure( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -219,6 +339,7 @@ def test_output_methods_boamps_adds_boamps_output_handler( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -241,6 +362,7 @@ def test_default_output_methods_is_csv( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -257,6 +379,7 @@ def test_save_to_flags_map_to_output_methods_and_warn( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -281,6 +404,7 @@ def test_output_methods_overrides_save_to_flags( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -307,6 +431,7 @@ def test_output_methods_parsed_from_config_string( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -335,6 +460,7 @@ def test_decorator_ONLINE_NO_ARGS( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -363,6 +489,7 @@ def test_decorator_ONLINE_WITH_ARGS( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -390,6 +517,7 @@ def test_decorator_online_passes_output_methods( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -419,6 +547,7 @@ def test_decorator_OFFLINE_NO_COUNTRY( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -436,6 +565,7 @@ def test_decorator_OFFLINE_WITH_LOC_ARGS( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -460,6 +590,7 @@ def test_decorator_OFFLINE_WITH_CLOUD_ARGS( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -484,6 +615,7 @@ def test_offline_tracker_country_name( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -507,6 +639,7 @@ def test_offline_tracker_invalid_headers( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -540,6 +673,7 @@ def test_offline_tracker_valid_headers( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -578,6 +712,7 @@ def test_carbon_tracker_online_context_manager_TWO_GPU_PRIVATE_INFRA_CANADA( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -595,8 +730,8 @@ def test_carbon_tracker_online_context_manager_TWO_GPU_PRIVATE_INFRA_CANADA( # THEN self.assertGreaterEqual( - mocked_get_gpu_details.call_count, 2 - ) # at least 2 times in 5 seconds + once for init >= 3 + mocked_get_gpu_details.call_count, 1 + ) # called at least once for repr at init self.assertEqual(2, mocked_is_gpu_details_available.call_count) self.assertEqual(1, len(responses.calls)) self.assertEqual( @@ -644,6 +779,7 @@ def test_task_energy_with_live_update_interference( mock_log_values, # Class decorator mocked_env_cloud_details, # Class decorator mocked_get_gpu_details, # Class decorator + mocked_get_gpu_utilization_list, # Class decorator mocked_is_gpu_details_available, # Class decorator mocked_is_nvidia_system, # Class decorator (outermost relevant one) ): @@ -748,6 +884,7 @@ def test_carbon_tracker_offline_context_manager( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -770,6 +907,7 @@ def test_scheduler_warning_suppressed_when_stopped( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -813,6 +951,7 @@ def test_scheduler_warning_shown_when_running( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -858,6 +997,7 @@ def test_get_detected_hardware( mock_log_values, mocked_get_gpu_details, mocked_env_cloud_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): @@ -892,6 +1032,7 @@ def test_cumulative_emissions_with_varying_intensity( mock_log_values, mocked_get_cloud_metadata_class, mocked_get_gpu_details, + mocked_get_gpu_utilization_list, mocked_is_gpu_details_available, mocked_is_nvidia_system, ): diff --git a/tests/test_gpu_nvidia.py b/tests/test_gpu_nvidia.py index 99a1c86a8..c9a78acf3 100644 --- a/tests/test_gpu_nvidia.py +++ b/tests/test_gpu_nvidia.py @@ -189,6 +189,41 @@ def test_gpu_details(self): assert alldevices.get_gpu_details() == self.expected + def test_gpu_utilization_list(self): + from codecarbon.core.gpu import AllGPUDevices + + alldevices = AllGPUDevices() + result = alldevices.get_gpu_utilization_list() + + assert len(result) == 2 + assert result[0] == {"gpu_index": 0, "gpu_utilization": 96} + assert result[1] == {"gpu_index": 1, "gpu_utilization": 0} + + def test_gpu_utilization_lightweight(self): + from codecarbon.core.gpu_device import GPUDevice + from codecarbon.core.gpu_nvidia import NvidiaGPUDevice + + device: GPUDevice = NvidiaGPUDevice(handle="handle_0", gpu_index=0) + result = device.get_gpu_utilization_lightweight() + + assert result == {"gpu_index": 0, "gpu_utilization": 96} + + def test_gpu_utilization_list_empty_on_exception(self): + import pynvml + + from codecarbon.core.gpu import AllGPUDevices + + def raise_exception(handle): + raise pynvml.NVMLError("Simulated NVML error") + + original = pynvml.nvmlDeviceGetUtilizationRates + try: + pynvml.nvmlDeviceGetUtilizationRates = raise_exception + alldevices = AllGPUDevices() + assert alldevices.get_gpu_utilization_list() == [] + finally: + pynvml.nvmlDeviceGetUtilizationRates = original + def test_gpu_no_power_limit(self): import pynvml diff --git a/tests/testdata.py b/tests/testdata.py index c70dd10eb..152cfd320 100644 --- a/tests/testdata.py +++ b/tests/testdata.py @@ -253,6 +253,11 @@ }, ] +TWO_GPU_UTILIZATION_RESPONSE = [ + {"gpu_index": 0, "gpu_utilization": 0}, + {"gpu_index": 1, "gpu_utilization": 0}, +] + TWO_GPU_DETAILS_RESPONSE_HANDLES = { "handle_0": { "name": "Tesla V100-SXM2-16GB", @@ -263,6 +268,7 @@ "power_limit": 300000, "total_energy_consumption": 149709, "gpu_utilization": 0, + "utilization_rate": real_pynvml.c_nvmlUtilization_t(0, 100), "compute_mode": 0, "compute_processes": [], "graphics_processes": [], @@ -276,6 +282,7 @@ "power_limit": 300000, "total_energy_consumption": 149709, "gpu_utilization": 0, + "utilization_rate": real_pynvml.c_nvmlUtilization_t(0, 100), "compute_mode": 0, "compute_processes": [], "graphics_processes": [],