test: sdk upgrade tests

Author: pawelpaszki

images/tests/Dockerfile

@@ -35,12 +35,20 @@ RUN poetry install --no-root --with test && rm -rf $POETRY_CACHE_DIR
 # Runtime stage
 FROM python:3.12-slim
 
-# Install system dependencies for runtime
+# Install system dependencies for runtime including Chrome for Selenium tests
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     curl \
     ca-certificates \
-    && rm -rf /var/lib/apt/lists/*
+    wget \
+    gnupg \
+    && wget -q -O /tmp/google-chrome-key.pub https://dl-ssl.google.com/linux/linux_signing_key.pub \
+    && gpg --dearmor -o /usr/share/keyrings/google-chrome-keyring.gpg /tmp/google-chrome-key.pub \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome-keyring.gpg] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+    google-chrome-stable \
+    && rm -rf /var/lib/apt/lists/* /tmp/google-chrome-key.pub
 
 # Install OpenShift CLI (oc)
 RUN curl -L https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/linux/oc.tar.gz | \

images/tests/run-tests.sh

@@ -55,19 +55,23 @@ cleanup_on_exit() {
                     echo "WARNING: TEST_USER_USERNAME not found, cannot cleanup RBAC"
                 fi
 
-                # Set Kueue Component to Removed State
-                echo "Setting Kueue component to Removed state..."
-                DSC_NAME=$(get_dsc_name 2>/dev/null || echo "")
-
-                if [ -n "$DSC_NAME" ] && [[ ! "$DSC_NAME" =~ ^ERROR ]]; then
-                    set_kueue_management_state "Removed" "$DSC_NAME" 2>/dev/null || {
-                        echo "WARNING: Failed to set Kueue to Removed state"
-                    }
-                    wait_for_dsc_ready 600 2>/dev/null || {
-                        echo "WARNING: DataScienceCluster did not reach Ready state after setting Kueue to Removed"
-                    }
+                # Set Kueue Component to Removed State (only if we changed it)
+                if [ "${INITIAL_KUEUE_STATE:-}" = "Unmanaged" ]; then
+                    echo "Kueue was already Unmanaged at start, skipping state change in cleanup"
                 else
-                    echo "WARNING: Failed to get DataScienceCluster name, skipping Kueue cleanup"
+                    echo "Setting Kueue component to Removed state..."
+                    DSC_NAME=$(get_dsc_name 2>/dev/null || echo "")
+
+                    if [ -n "$DSC_NAME" ] && [[ ! "$DSC_NAME" =~ ^ERROR ]]; then
+                        set_kueue_management_state "Removed" "$DSC_NAME" 2>/dev/null || {
+                            echo "WARNING: Failed to set Kueue to Removed state"
+                        }
+                        wait_for_dsc_ready 600 2>/dev/null || {
+                            echo "WARNING: DataScienceCluster did not reach Ready state after setting Kueue to Removed"
+                        }
+                    else
+                        echo "WARNING: Failed to get DataScienceCluster name, skipping Kueue cleanup"
+                    fi
                 fi
             else
                 echo "WARNING: Failed to login with OCP_ADMIN_USER for cleanup"
@@ -186,6 +190,28 @@ wait_for_dsc_ready() {
     return 1
 }
 
+# Get Kueue component management state
+# Arguments: cluster_name
+get_kueue_management_state() {
+    local cluster_name=$1
+
+    if [ -z "$cluster_name" ]; then
+        echo "ERROR: Invalid arguments for get_kueue_management_state"
+        return 1
+    fi
+
+    local state
+    state=$(oc get DataScienceCluster "$cluster_name" -o jsonpath='{.spec.components.kueue.managementState}' 2>/dev/null)
+
+    if [ -z "$state" ]; then
+        echo "ERROR: Failed to get Kueue management state"
+        return 1
+    fi
+
+    echo "$state"
+    return 0
+}
+
 # Set Kueue component management state
 # Arguments: state (Unmanaged or Removed), cluster_name
 set_kueue_management_state() {
@@ -296,22 +322,38 @@ echo "Successfully logged in with OCP_ADMIN_USER (verified: $CURRENT_USER)"
 # ============================================================================
 # Set Kueue Component to Unmanaged State
 # ============================================================================
-echo "Setting Kueue component to Unmanaged state..."
+echo "Checking current Kueue component state..."
 DSC_NAME=$(get_dsc_name) || {
     echo "ERROR: Failed to get DataScienceCluster name"
     exit 1
 }
 
-set_kueue_management_state "Unmanaged" "$DSC_NAME" || {
-    echo "ERROR: Failed to set Kueue to Unmanaged state"
+# Get and store the initial Kueue management state
+INITIAL_KUEUE_STATE=$(get_kueue_management_state "$DSC_NAME") || {
+    echo "ERROR: Failed to get initial Kueue management state"
     exit 1
 }
+echo "Initial Kueue management state: $INITIAL_KUEUE_STATE"
 
-# Wait for DataScienceCluster to be Ready after setting Kueue to Unmanaged
-wait_for_dsc_ready 600 || {
-    echo "ERROR: DataScienceCluster did not reach Ready state after setting Kueue to Unmanaged"
-    exit 1
-}
+# Export it so cleanup function can access it
+export INITIAL_KUEUE_STATE
+
+# Only set to Unmanaged if it's not already Unmanaged
+if [ "$INITIAL_KUEUE_STATE" = "Unmanaged" ]; then
+    echo "Kueue is already in Unmanaged state, skipping state change"
+else
+    echo "Setting Kueue component to Unmanaged state..."
+    set_kueue_management_state "Unmanaged" "$DSC_NAME" || {
+        echo "ERROR: Failed to set Kueue to Unmanaged state"
+        exit 1
+    }
+
+    # Wait for DataScienceCluster to be Ready after setting Kueue to Unmanaged
+    wait_for_dsc_ready 600 || {
+        echo "ERROR: DataScienceCluster did not reach Ready state after setting Kueue to Unmanaged"
+        exit 1
+    }
+fi
 
 # ============================================================================
 # Apply RBAC Policies
@@ -387,6 +429,20 @@ cp "${TEMP_KUBECONFIG}" ~/.kube/config || {
 
 echo "Successfully logged in with TEST_USER"
 
+# ============================================================================
+# Get RHOAI Dashboard URL for UI Tests
+# ============================================================================
+echo "Retrieving RHOAI Dashboard URL..."
+ODH_DASHBOARD_URL=$(oc get consolelink rhodslink -o jsonpath='{.spec.href}' 2>/dev/null)
+
+if [ -z "$ODH_DASHBOARD_URL" ]; then
+    echo "WARNING: Failed to retrieve Dashboard URL from consolelink rhodslink"
+    echo "         UI tests will be skipped or may fail"
+else
+    echo "Dashboard URL: $ODH_DASHBOARD_URL"
+    export ODH_DASHBOARD_URL
+fi
+
 # ============================================================================
 # Run Tests
 # ============================================================================

poetry.lock

@@ -66,6 +66,8 @@ coverage = "7.6.4"
 pytest-mock = "3.11.1"
 pytest-timeout = "2.3.1"
 jupyterlab = "4.3.1"
+selenium = "4.27.1"
+webdriver-manager = "4.0.2"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -83,7 +85,8 @@ markers = [
     "smoke: Smoke tests - quick validation tests",
     "tier1: Tier1 tests - standard test suite",
     "pre_upgrade: Tests to run before upgrade",
-    "post_upgrade: Tests to run after upgrade"
+    "post_upgrade: Tests to run after upgrade",
+    "ui: UI tests requiring browser automation"
 ]
 addopts = "--timeout=900 --ignore=src/codeflare_sdk/vendored"
 testpaths = ["src/codeflare_sdk"]

pyproject.toml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="codeflare-sdk" errors="0" failures="0" skipped="0" tests="2" time="201.043" timestamp="2025-11-27T10:03:00.294945" hostname="73450509f8a1"><testcase classname="tests.upgrade.01_raycluster_sdk_upgrade_test.TestMnistJobSubmit" name="test_mnist_job_submission" time="27.377" /><testcase classname="tests.upgrade.02_dashboard_ui_upgrade_test.TestDistributedWorkloadsUIPostUpgrade" name="test_verify_cluster_persists_after_upgrade" time="169.695" /></testsuite></testsuites>