Follow gpu deterministic test setup

Author: ChSonnabend

Common/ML/CMakeLists.txt

@@ -52,6 +52,6 @@ if(BUILD_TESTING)
                           "ORT_CUDA_BUILD=${ONNXRUNTIME_INFERENCE_TEST_CUDA}"
                           "ORT_MIGRAPHX_BUILD=${ONNXRUNTIME_INFERENCE_TEST_MIGRAPHX}"
                           "ORT_TENSORRT_BUILD=${ONNXRUNTIME_INFERENCE_TEST_TENSORRT}"
-                        LABELS ml onnxruntime
+                        LABELS ml onnxruntime gpu
                         TIMEOUT 300)
 endif()

Common/ML/test/onnxruntime-inference/run-onnxruntime-all-eps.sh

@@ -11,6 +11,9 @@ fi
 if [[ -n ${ONNXRUNTIME_ROOT:-} && -f $ONNXRUNTIME_ROOT/etc/ort-init.sh ]]; then
   source "$ONNXRUNTIME_ROOT/etc/ort-init.sh"
 fi
+if [[ -n ${GPU_SYSTEM_ROOT:-} && -f $GPU_SYSTEM_ROOT/etc/gpu-features-available.sh ]]; then
+  source "$GPU_SYSTEM_ROOT/etc/gpu-features-available.sh"
+fi
 
 has_cuda_device() {
   if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then
@@ -28,25 +31,49 @@ has_rocm_device() {
 
 if [[ -n ${ONNXRUNTIME_INFERENCE_TEST_PROVIDERS:-} ]]; then
   IFS=', ' read -r -a PROVIDERS <<< "$ONNXRUNTIME_INFERENCE_TEST_PROVIDERS"
+  for PROVIDER in "${PROVIDERS[@]}"; do
+    PROVIDER=${PROVIDER,,}
+    if [[ $PROVIDER == "cuda" && ${O2_GPU_CUDA_AVAILABLE:-0} == 1 ]] && ! has_cuda_device; then
+      echo "onnxruntime-inference-test: CUDA is available but no CUDA device was detected" >&2
+      exit 1
+    fi
+    if [[ $PROVIDER == "tensorrt" && ${O2_GPU_CUDA_AVAILABLE:-0} == 1 ]] && ! has_cuda_device; then
+      echo "onnxruntime-inference-test: TensorRT is available but no CUDA device was detected" >&2
+      exit 1
+    fi
+    if [[ $PROVIDER == "migraphx" && ${O2_GPU_ROCM_AVAILABLE:-0} == 1 ]] && ! has_rocm_device; then
+      echo "onnxruntime-inference-test: ROCm is available but no ROCm device was detected" >&2
+      exit 1
+    fi
+  done
 else
   PROVIDERS=(cpu)
   if [[ ${ORT_MIGRAPHX_BUILD:-0} == 1 ]]; then
     if has_rocm_device; then
       PROVIDERS+=(migraphx)
+    elif [[ ${O2_GPU_ROCM_AVAILABLE:-0} == 1 ]]; then
+      echo "onnxruntime-inference-test: ROCm is available but no ROCm device was detected" >&2
+      exit 1
     else
       echo "onnxruntime-inference-test: skipping migraphx, no ROCm device detected"
     fi
   fi
   if [[ ${ORT_CUDA_BUILD:-0} == 1 ]]; then
     if has_cuda_device; then
       PROVIDERS+=(cuda)
+    elif [[ ${O2_GPU_CUDA_AVAILABLE:-0} == 1 ]]; then
+      echo "onnxruntime-inference-test: CUDA is available but no CUDA device was detected" >&2
+      exit 1
     else
       echo "onnxruntime-inference-test: skipping cuda, no CUDA device detected"
     fi
   fi
   if [[ ${ORT_TENSORRT_BUILD:-0} == 1 ]]; then
     if has_cuda_device; then
       PROVIDERS+=(tensorrt)
+    elif [[ ${O2_GPU_CUDA_AVAILABLE:-0} == 1 ]]; then
+      echo "onnxruntime-inference-test: TensorRT is available but no CUDA device was detected" >&2
+      exit 1
     else
       echo "onnxruntime-inference-test: skipping tensorrt, no CUDA device detected"
     fi