Keep topology discovery CUDA-context agnostic

docs/topology-and-configuration.md

@@ -8,6 +8,7 @@ Hardware topology discovery, NUMA-aware memory placement, and system configurati
 - [Hardware Topology Discovery](#hardware-topology-discovery)
   - [What Gets Detected](#what-gets-detected)
   - [How Detection Works](#how-detection-works)
+  - [CUDA Context Independence](#cuda-context-independence)
   - [GPU-to-NUMA Affinity](#gpu-to-numa-affinity)
   - [PCIe Path Types](#pcie-path-types)
 - [Configuring cuCascade](#configuring-cucascade)
@@ -105,6 +106,13 @@ throws rather than sizing a host space from device memory.
 5. **Network devices** -- scans `/sys/class/infiniband/` for NICs with NUMA info, with configurable verification (see [Network Device Verification](#network-device-verification))
 6. **Storage devices** -- scans `/sys/block/` for NVMe and SATA devices with NUMA info
 
+### CUDA Context Independence
+
+Topology discovery is independent of CUDA runtime and context state. `discover()` may be called
+before or after CUDA initialization, but it must never initialize CUDA, create or require a CUDA
+context, or otherwise alter CUDA process state. This is a public API invariant, not merely an
+implementation detail.
+
 ### Network Device Verification
 
 The `discover()` method accepts a `NetworkDeviceVerification` parameter that controls how strictly network devices are validated before inclusion. This prevents assigning non-functional devices to `UCX_NET_DEVICES` (which would cause connection failures).

include/cucascade/memory/topology_discovery.hpp

@@ -192,6 +192,10 @@ class topology_discovery {
    * This method performs the actual discovery of GPUs, NUMA nodes, CPU affinity,
    * and network devices. It must be called before `get_topology()`.
    *
+   * @note CUDA-context invariant: this method is CUDA runtime and context agnostic.
+   * It must not initialize CUDA, create a CUDA context, require an active CUDA context,
+   * or otherwise alter CUDA process state.
+   *
    * @param net_verification Controls how strictly network devices are validated.
    * @return true if discovery was successful, false otherwise.
    */

src/memory/topology_discovery.cpp

@@ -5,9 +5,6 @@
 
 #include <cucascade/memory/topology_discovery.hpp>
 
-#include <rmm/cuda_device.hpp>
-#include <rmm/detail/runtime_capabilities.hpp>
-
 #include <dlfcn.h>
 #include <ifaddrs.h>
 #include <nvml.h>
@@ -63,23 +60,18 @@ void report_nvml_error(nvmlReturn_t result, std::string const& context)
 /**
  * @brief Query whether a CUDA device supports hardware-accelerated decompression.
  *
- * Delegates to `rmm::detail::hwdecompress::is_supported()`, which checks the CUDA
- * driver version. RMM's capability queries are scoped to the current device, so the
- * call is wrapped in an `rmm::cuda_set_device_raii`. Best-effort: any failure while
- * setting the device or probing yields false.
+ * Hardware decompression requires CUDA driver 12.8 or newer. Use NVML rather than
+ * the CUDA runtime so topology discovery neither initializes CUDA nor observes the
+ * process's CUDA_VISIBLE_DEVICES setting.
  *
- * @param cuda_ordinal CUDA device ordinal (matches the runtime device index used
- * elsewhere in discovery under the same CUDA_VISIBLE_DEVICES ordering).
  * @return true iff the hardware decompression engine is available.
  */
-bool query_hw_decompression(unsigned int cuda_ordinal)
+bool query_hw_decompression()
 {
-  try {
-    rmm::cuda_set_device_raii set_device{rmm::cuda_device_id{static_cast<int>(cuda_ordinal)}};
-    return rmm::detail::hwdecompress::is_supported();
-  } catch (...) {
-    return false;
-  }
+  constexpr int min_hw_decompression_cuda_version = 12080;
+  int driver_version                              = 0;
+  return nvmlSystemGetCudaDriverVersion(&driver_version) == NVML_SUCCESS &&
+         driver_version >= min_hw_decompression_cuda_version;
 }
 
 /**
@@ -991,13 +983,14 @@ bool topology_discovery::discover(NetworkDeviceVerification net_verification)
 
   auto visible_indices =
     resolve_visible_gpu_indices(nvml_gpus, nvml_index_by_pci, nvml_index_by_uuid);
-  topology.num_gpus = static_cast<unsigned int>(visible_indices.size());
+  auto const hw_decompression_available = query_hw_decompression();
+  topology.num_gpus                     = static_cast<unsigned int>(visible_indices.size());
   for (size_t visible_idx = 0; visible_idx < visible_indices.size(); ++visible_idx) {
     size_t nvml_idx = visible_indices[visible_idx];
     if (nvml_idx >= nvml_gpus.size()) { continue; }
     auto gpu                       = nvml_gpus[nvml_idx];
     gpu.id                         = static_cast<unsigned int>(visible_idx);
-    gpu.hw_decompression_available = query_hw_decompression(gpu.id);
+    gpu.hw_decompression_available = hw_decompression_available;
     topology.gpus.push_back(std::move(gpu));
   }
 

test/CMakeLists.txt

@@ -132,8 +132,10 @@ target_include_directories(
                                              ${CMAKE_CURRENT_SOURCE_DIR}/utils)
 
 # Link dependencies
-target_link_libraries(cucascade_topology_discovery_tests
-                      PRIVATE cucascade_topology_discovery Catch2::Catch2)
+target_link_libraries(
+  cucascade_topology_discovery_tests
+  PRIVATE cucascade_topology_discovery Catch2::Catch2 CUDA::cuda_driver
+          CUDA::cudart CUDA::nvml_static)
 
 # Register tests with CTest
 add_test(NAME cucascade_topology_discovery_tests

test/memory/test_topology_discovery.cpp

@@ -31,7 +31,13 @@
 
 #include <cucascade/memory/topology_discovery.hpp>
 
+#include <cuda.h>
+#include <cuda_runtime_api.h>
+
 #include <catch2/catch_all.hpp>
+#include <nvml.h>
+#include <sys/wait.h>
+#include <unistd.h>
 
 #include <algorithm>
 #include <cstdlib>
@@ -290,3 +296,50 @@ TEST_CASE("Topology Discovery rejects overflow CUDA_VISIBLE_DEVICES", "[hw_topol
   REQUIRE_THROWS_WITH(discovery.discover(),
                       "Invalid numeric CUDA_VISIBLE_DEVICES entry: 999999999999999999999999999999");
 }
+
+TEST_CASE("Topology Discovery does not initialize CUDA while visibility is widened",
+          "[hw_topology][cuda-context]")
+{
+  auto const child = fork();
+  REQUIRE(child >= 0);
+
+  if (child == 0) {
+    // NVML is used only to make this test meaningful on a multi-GPU host. It
+    // does not create a CUDA context or initialize the CUDA runtime.
+    if (nvmlInit_v2() != NVML_SUCCESS) { _exit(77); }
+    unsigned int device_count = 0;
+    auto const count_status   = nvmlDeviceGetCount_v2(&device_count);
+    nvmlShutdown();
+    if (count_status != NVML_SUCCESS || device_count < 2) { _exit(77); }
+
+    // This mirrors callers that temporarily widen visibility to collect
+    // physical topology, then restore the process's assigned GPU. If
+    // discover() initializes the CUDA runtime while CVD is unset, the runtime
+    // permanently sees all GPUs and the final cudaGetDeviceCount() is wrong.
+    setenv("CUDA_VISIBLE_DEVICES", "1", 1);
+    unsetenv("CUDA_VISIBLE_DEVICES");
+    topology_discovery discovery;
+    bool const discovered = discovery.discover();
+    setenv("CUDA_VISIBLE_DEVICES", "1", 1);
+    if (!discovered) { _exit(1); }
+
+    CUcontext context = nullptr;
+    if (cuInit(0) != CUDA_SUCCESS || cuCtxGetCurrent(&context) != CUDA_SUCCESS ||
+        context != nullptr) {
+      _exit(1);
+    }
+
+    int visible_count = 0;
+    if (cudaGetDeviceCount(&visible_count) != cudaSuccess || visible_count != 1) { _exit(1); }
+    _exit(0);
+  }
+
+  int status = 0;
+  REQUIRE(waitpid(child, &status, 0) == child);
+  if (WIFEXITED(status) && WEXITSTATUS(status) == 77) {
+    SUCCEED("Skipped: requires NVML and at least two visible GPUs");
+    return;
+  }
+  REQUIRE(WIFEXITED(status));
+  REQUIRE(WEXITSTATUS(status) == 0);
+}