diff --git a/controllers/object_controls.go b/controllers/object_controls.go index f7ea8abf8..10fde0307 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -3675,10 +3675,15 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy // set up subscription entitlements for RHEL(using K8s with a non-CRIO runtime) and SLES if (osID == "rhel" && n.openshift == "" && n.runtime != gpuv1.CRIO) || osID == "sles" || osID == "sl-micro" { - n.logger.Info("Mounting subscriptions into the driver container", "OS", osID) - pathToVolumeSource, err := n.getSubscriptionPathsToVolumeSources() - if err != nil { - return fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %v", err) + pathToVolumeSource := MountPathToVolumeSource{} + if config.Driver.RepoConfig != nil && config.Driver.RepoConfig.ConfigMapName != "" && osID == "rhel" { + n.logger.Info("Skipping host subscription mounts because repoConfig is enabled", "OS", osID) + } else { + n.logger.Info("Mounting subscriptions into the driver container", "OS", osID) + pathToVolumeSource, err = n.getSubscriptionPathsToVolumeSources() + if err != nil { + return fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %v", err) + } } // sort host path volumes to ensure ordering is preserved when adding to pod spec diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index b1547a750..46ec7c96e 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -18,6 +18,7 @@ package controllers import ( "path/filepath" + "strings" "testing" kata_v1alpha1 "github.com/NVIDIA/k8s-kata-manager/api/v1alpha1/config" @@ -4223,6 +4224,115 @@ func TestTransformDriverWithAdditionalConfig(t *testing.T) { } } +func TestTransformDriverSubscriptionMounts(t *testing.T) { + repoConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo-config", + Namespace: "test-ns", + }, + Data: map[string]string{ + "redhat.repo": "[test-repo]", + }, + } + mockClient := fake.NewFakeClient(repoConfigMap) + + testCases := []struct { + description string + osRelease string + osTag string + repoConfigEnabled bool + expectSubscriptionMounts bool + expectedSubscriptionHostMap map[string]corev1.HostPathType + }{ + { + description: "rhel with repo config skips host subscription mounts", + osRelease: "rhel", + osTag: "rhel8.10", + repoConfigEnabled: true, + expectSubscriptionMounts: false, + }, + { + description: "rhel without repo config mounts host subscription paths", + osRelease: "rhel", + osTag: "rhel8.10", + expectSubscriptionMounts: true, + expectedSubscriptionHostMap: map[string]corev1.HostPathType{ + "/etc/pki/entitlement": corev1.HostPathDirectory, + "/etc/yum.repos.d/redhat.repo": corev1.HostPathFile, + "/etc/rhsm": corev1.HostPathDirectory, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + ds := NewDaemonset().WithContainer(corev1.Container{Name: "nvidia-driver-ctr"}). + WithInitContainer(corev1.Container{Name: "k8s-driver-manager"}) + cpSpec := &gpuv1.ClusterPolicySpec{ + Driver: gpuv1.DriverSpec{ + Repository: "nvcr.io/nvidia", + Image: "driver", + ImagePullPolicy: "IfNotPresent", + Version: "580.126.16", + Manager: gpuv1.DriverManagerSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "k8s-driver-manager", + ImagePullPolicy: "IfNotPresent", + Version: "v0.8.0", + }, + }, + } + if tc.repoConfigEnabled { + cpSpec.Driver.RepoConfig = &gpuv1.DriverRepoConfigSpec{ConfigMapName: "test-repo-config"} + } + + err := TransformDriver(ds.DaemonSet, cpSpec, ClusterPolicyController{ + client: mockClient, + runtime: gpuv1.Containerd, + operatorNamespace: "test-ns", + logger: ctrl.Log.WithName("test"), + gpuNodeOSRelease: tc.osRelease, + gpuNodeOSTag: tc.osTag, + }) + require.NoError(t, err) + + driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr") + require.NotNil(t, driverContainer) + assertSubscriptionHostPathVolumesForTransform(t, ds.Spec.Template.Spec.Volumes, tc.expectedSubscriptionHostMap) + assert.Equal(t, tc.expectSubscriptionMounts, hasSubscriptionVolumeMountForTransform(driverContainer.VolumeMounts)) + }) + } +} + +func hasSubscriptionVolumeMountForTransform(volumeMounts []corev1.VolumeMount) bool { + for _, volumeMount := range volumeMounts { + if strings.HasPrefix(volumeMount.Name, "subscription-config-") { + return true + } + } + return false +} + +func assertSubscriptionHostPathVolumesForTransform(t *testing.T, volumes []corev1.Volume, expected map[string]corev1.HostPathType) { + t.Helper() + + if expected == nil { + expected = map[string]corev1.HostPathType{} + } + + actual := map[string]corev1.HostPathType{} + for _, volume := range volumes { + if !strings.HasPrefix(volume.Name, "subscription-config-") { + continue + } + require.NotNil(t, volume.HostPath) + require.NotNil(t, volume.HostPath.Type) + actual[volume.HostPath.Path] = *volume.HostPath.Type + } + + assert.Equal(t, expected, actual) +} + // baseDriverDaemonSetSpec returns a minimal DaemonSetSpec representative of // the post-transformation driver DaemonSet in the ClusterPolicy path. // Only fields relevant to extractDriverInstallConfig extraction are diff --git a/internal/state/driver_test.go b/internal/state/driver_test.go index 8d9eaa354..02b2736f2 100644 --- a/internal/state/driver_test.go +++ b/internal/state/driver_test.go @@ -18,6 +18,7 @@ package state import ( "bytes" + "context" "os" "path/filepath" "strings" @@ -34,8 +35,10 @@ import ( apitypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" + "github.com/NVIDIA/gpu-operator/internal/consts" "github.com/NVIDIA/gpu-operator/internal/render" ) @@ -44,6 +47,27 @@ const ( manifestResultDir = "./testdata/golden" ) +type testClusterInfo struct { + runtime string + openshiftVersion string +} + +func (i testClusterInfo) GetContainerRuntime() (string, error) { + return i.runtime, nil +} + +func (i testClusterInfo) GetOpenshiftVersion() (string, error) { + return i.openshiftVersion, nil +} + +func (i testClusterInfo) GetOpenshiftDriverToolkitImages() map[string]string { + return nil +} + +func (i testClusterInfo) GetOpenshiftProxySpec() (*configv1.ProxySpec, error) { + return nil, nil +} + func getYAMLString(objs []*unstructured.Unstructured) (string, error) { s := json.NewSerializerWithOptions(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme, json.SerializerOptions{Yaml: true, Pretty: false, Strict: false}) @@ -60,6 +84,35 @@ func getYAMLString(objs []*unstructured.Unstructured) (string, error) { return sb.String(), nil } +func hasSubscriptionVolumeMount(volumeMounts []corev1.VolumeMount) bool { + for _, volumeMount := range volumeMounts { + if strings.HasPrefix(volumeMount.Name, "subscription-config-") { + return true + } + } + return false +} + +func assertSubscriptionHostPathVolumes(t *testing.T, volumes []corev1.Volume, expected map[string]corev1.HostPathType) { + t.Helper() + + if expected == nil { + expected = map[string]corev1.HostPathType{} + } + + actual := map[string]corev1.HostPathType{} + for _, volume := range volumes { + if !strings.HasPrefix(volume.Name, "subscription-config-") { + continue + } + require.NotNil(t, volume.HostPath) + require.NotNil(t, volume.HostPath.Type) + actual[volume.HostPath.Path] = *volume.HostPath.Type + } + + assert.Equal(t, expected, actual) +} + func TestDriverRenderMinimal(t *testing.T) { // Construct a sample driver state manager const ( @@ -440,6 +493,69 @@ func TestDriverAdditionalConfigs(t *testing.T) { require.Equal(t, string(o), actual) } +func TestDriverAdditionalConfigsSubscriptionMounts(t *testing.T) { + repoConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo-config", + Namespace: "test-ns", + }, + Data: map[string]string{ + "redhat.repo": "[test-repo]", + }, + } + + testCases := []struct { + description string + osRelease string + repoConfigEnabled bool + expectSubscriptionMounts bool + expectedSubscriptionHostMap map[string]corev1.HostPathType + }{ + { + description: "rhel with repo config skips host subscription mounts", + osRelease: "rhel", + repoConfigEnabled: true, + expectSubscriptionMounts: false, + }, + { + description: "rhel without repo config mounts host subscription paths", + osRelease: "rhel", + expectSubscriptionMounts: true, + expectedSubscriptionHostMap: map[string]corev1.HostPathType{ + "/etc/pki/entitlement": corev1.HostPathDirectory, + "/etc/yum.repos.d/redhat.repo": corev1.HostPathFile, + "/etc/rhsm": corev1.HostPathDirectory, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + stateDriver := &stateDriver{ + stateSkel: stateSkel{ + client: fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(repoConfigMap).Build(), + namespace: "test-ns", + }, + } + driver := &nvidiav1alpha1.NVIDIADriver{} + if tc.repoConfigEnabled { + driver.Spec.RepoConfig = &nvidiav1alpha1.DriverRepoConfigSpec{Name: "test-repo-config"} + } + + configs, err := stateDriver.getDriverAdditionalConfigs( + context.Background(), + driver, + testClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: tc.osRelease, osVersion: tc.osRelease}, + ) + require.NoError(t, err) + + assertSubscriptionHostPathVolumes(t, configs.Volumes, tc.expectedSubscriptionHostMap) + assert.Equal(t, tc.expectSubscriptionMounts, hasSubscriptionVolumeMount(configs.VolumeMounts)) + }) + } +} + func TestDriverOpenshiftDriverToolkit(t *testing.T) { const ( testName = "driver-openshift-drivertoolkit" diff --git a/internal/state/driver_volumes.go b/internal/state/driver_volumes.go index a1f59c94c..a8e1ff2c7 100644 --- a/internal/state/driver_volumes.go +++ b/internal/state/driver_volumes.go @@ -182,10 +182,19 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph // set up subscription entitlements for RHEL(using K8s with a non-CRIO runtime) and SLES if (pool.osRelease == "rhel" && openshiftVersion == "" && runtime != consts.CRIO) || pool.osRelease == "sles" || pool.osRelease == "sl-micro" { - logger.Info("Mounting subscriptions into the driver container", "OS", pool.osVersion) - pathToVolumeSource, err := getSubscriptionPathsToVolumeSources(pool.osRelease) - if err != nil { - return nil, fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %v", err) + pathToVolumeSource := MountPathToVolumeSource{} + // Custom repo ConfigMap supplies yum repos in offline/air-gapped installs. Skip + // mounting host RHSM paths (/etc/pki/entitlement, redhat.repo, /etc/rhsm): they may + // be missing or not directories on minimal nodes, and are not needed when packages + // come only from the mounted repo ConfigMap. + if cr.Spec.IsRepoConfigEnabled() && pool.osRelease == "rhel" { + logger.Info("Skipping host subscription mounts because repoConfig is enabled", "OS", pool.osVersion) + } else { + logger.Info("Mounting subscriptions into the driver container", "OS", pool.osVersion) + pathToVolumeSource, err = getSubscriptionPathsToVolumeSources(pool.osRelease) + if err != nil { + return nil, fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %v", err) + } } // sort host path volumes to ensure ordering is preserved when adding to pod spec