diff --git a/controllers/object_controls.go b/controllers/object_controls.go index 9181d5b9f..805ebfedb 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -1489,21 +1489,19 @@ func transformForRuntime(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, socketVol) } - if config.CDI.IsNRIPluginEnabled() { - // setup mounts for the runtime NRI socket file - nriSocketFile := getContainerEnv(container, "NRI_SOCKET") - if nriSocketFile == "" { - nriSocketFile = DefaultRuntimeNRISocketFile - } - setContainerEnv(container, "NRI_SOCKET", DefaultRuntimeNRISocketTargetDir+path.Base(nriSocketFile)) + // setup mounts for the runtime NRI socket file + nriSocketFile := getContainerEnv(container, "NRI_SOCKET") + if nriSocketFile == "" { + nriSocketFile = DefaultRuntimeNRISocketFile + } + setContainerEnv(container, "NRI_SOCKET", DefaultRuntimeNRISocketTargetDir+path.Base(nriSocketFile)) - nriVolMountSocketName := "nri-socket" - nriVolMountSocket := corev1.VolumeMount{Name: nriVolMountSocketName, MountPath: DefaultRuntimeNRISocketTargetDir} - container.VolumeMounts = append(container.VolumeMounts, nriVolMountSocket) + nriVolMountSocketName := "nri-socket" + nriVolMountSocket := corev1.VolumeMount{Name: nriVolMountSocketName, MountPath: DefaultRuntimeNRISocketTargetDir} + container.VolumeMounts = append(container.VolumeMounts, nriVolMountSocket) - nriSocketVol := corev1.Volume{Name: nriVolMountSocketName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: path.Dir(nriSocketFile), Type: ptr.To(corev1.HostPathDirectoryOrCreate)}}} - obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, nriSocketVol) - } + nriSocketVol := corev1.Volume{Name: nriVolMountSocketName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: path.Dir(nriSocketFile), Type: ptr.To(corev1.HostPathDirectoryOrCreate)}}} + obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, nriSocketVol) return nil } @@ -2771,7 +2769,7 @@ func setRuntimeClassName(podSpec *corev1.PodSpec, config *gpuv1.ClusterPolicySpe return } runtimeClassName := getRuntimeClassName(config) - podSpec.RuntimeClassName = &runtimeClassName + podSpec.RuntimeClassName = new(runtimeClassName) } func setContainerProbe(container *corev1.Container, probe *gpuv1.ContainerProbeSpec, probeType ContainerProbe) { diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 30310c1b2..0d25d5534 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -402,6 +402,7 @@ func TestTransformForRuntime(t *testing.T) { WithHostPathVolume("containerd-config", filepath.Dir(DefaultContainerdConfigFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-drop-in-config", "/etc/containerd/conf.d", ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-socket", filepath.Dir(DefaultContainerdSocketFile), nil). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithContainer(corev1.Container{ Name: "test-ctr", Env: []corev1.EnvVar{ @@ -413,11 +414,13 @@ func TestTransformForRuntime(t *testing.T) { {Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/containerd/conf.d/99-nvidia.toml"}, {Name: "RUNTIME_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultContainerdSocketFile))}, {Name: "CONTAINERD_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultContainerdSocketFile))}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "containerd-config", MountPath: DefaultRuntimeConfigTargetDir}, {Name: "containerd-drop-in-config", MountPath: "/runtime/config-dir.d/"}, {Name: "containerd-socket", MountPath: DefaultRuntimeSocketTargetDir}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }), }, @@ -435,6 +438,7 @@ func TestTransformForRuntime(t *testing.T) { WithHostPathVolume("containerd-config", filepath.Dir(DefaultContainerdConfigFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-drop-in-config", "/etc/containerd/conf.d", ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-socket", filepath.Dir(DefaultContainerdSocketFile), nil). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithContainer(corev1.Container{ Name: "test-ctr", Env: []corev1.EnvVar{ @@ -447,11 +451,13 @@ func TestTransformForRuntime(t *testing.T) { {Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/containerd/conf.d/99-nvidia.toml"}, {Name: "RUNTIME_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultContainerdSocketFile))}, {Name: "CONTAINERD_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultContainerdSocketFile))}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "containerd-config", MountPath: DefaultRuntimeConfigTargetDir}, {Name: "containerd-drop-in-config", MountPath: "/runtime/config-dir.d/"}, {Name: "containerd-socket", MountPath: DefaultRuntimeSocketTargetDir}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }), }, @@ -469,6 +475,7 @@ func TestTransformForRuntime(t *testing.T) { WithHostPathVolume("containerd-config", filepath.Dir(DefaultContainerdConfigFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-drop-in-config", "/etc/containerd/conf.d", ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-socket", filepath.Dir(DefaultContainerdSocketFile), nil). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithContainer(corev1.Container{ Name: "test-ctr", Env: []corev1.EnvVar{ @@ -481,11 +488,13 @@ func TestTransformForRuntime(t *testing.T) { {Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/containerd/conf.d/99-nvidia.toml"}, {Name: "RUNTIME_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultContainerdSocketFile))}, {Name: "CONTAINERD_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultContainerdSocketFile))}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "containerd-config", MountPath: DefaultRuntimeConfigTargetDir}, {Name: "containerd-drop-in-config", MountPath: "/runtime/config-dir.d/"}, {Name: "containerd-socket", MountPath: DefaultRuntimeSocketTargetDir}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }), }, @@ -496,6 +505,7 @@ func TestTransformForRuntime(t *testing.T) { expectedOutput: NewDaemonset(). WithHostPathVolume("crio-config", "/etc/crio", ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("crio-drop-in-config", "/etc/crio/crio.conf.d", ptr.To(corev1.HostPathDirectoryOrCreate)). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithContainer(corev1.Container{ Name: "test-ctr", Env: []corev1.EnvVar{ @@ -504,10 +514,12 @@ func TestTransformForRuntime(t *testing.T) { {Name: "CRIO_CONFIG", Value: "/runtime/config-dir/config.toml"}, {Name: "RUNTIME_DROP_IN_CONFIG", Value: "/runtime/config-dir.d/99-nvidia.conf"}, {Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/crio/crio.conf.d/99-nvidia.conf"}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "crio-config", MountPath: DefaultRuntimeConfigTargetDir}, {Name: "crio-drop-in-config", MountPath: "/runtime/config-dir.d/"}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }), }, @@ -520,6 +532,7 @@ func TestTransformForRuntime(t *testing.T) { expectedOutput: NewDaemonset(). WithHostPathVolume("containerd-config", filepath.Dir(DefaultContainerdConfigFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-socket", filepath.Dir(DefaultContainerdSocketFile), nil). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithContainer(corev1.Container{ Name: "nvidia-kata-manager", Env: []corev1.EnvVar{ @@ -529,10 +542,12 @@ func TestTransformForRuntime(t *testing.T) { {Name: "CONTAINERD_CONFIG", Value: filepath.Join(DefaultRuntimeConfigTargetDir, filepath.Base(DefaultContainerdConfigFile))}, {Name: "RUNTIME_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultContainerdSocketFile))}, {Name: "CONTAINERD_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultContainerdSocketFile))}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "containerd-config", MountPath: DefaultRuntimeConfigTargetDir}, {Name: "containerd-socket", MountPath: DefaultRuntimeSocketTargetDir}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }), }, @@ -543,6 +558,7 @@ func TestTransformForRuntime(t *testing.T) { expectedOutput: NewDaemonset(). WithHostPathVolume("docker-config", filepath.Dir(DefaultDockerConfigFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("docker-socket", filepath.Dir(DefaultDockerSocketFile), nil). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithContainer(corev1.Container{ Name: "test-ctr", Env: []corev1.EnvVar{ @@ -551,10 +567,12 @@ func TestTransformForRuntime(t *testing.T) { {Name: "DOCKER_CONFIG", Value: filepath.Join(DefaultRuntimeConfigTargetDir, filepath.Base(DefaultDockerConfigFile))}, {Name: "RUNTIME_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultDockerSocketFile))}, {Name: "DOCKER_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultDockerSocketFile))}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "docker-config", MountPath: DefaultRuntimeConfigTargetDir}, {Name: "docker-socket", MountPath: DefaultRuntimeSocketTargetDir}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }), }, @@ -910,16 +928,19 @@ func TestTransformToolkit(t *testing.T) { {Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/containerd/conf.d/99-nvidia.toml"}, {Name: "RUNTIME_SOCKET", Value: "/runtime/sock-dir/containerd.sock"}, {Name: "CONTAINERD_SOCKET", Value: "/runtime/sock-dir/containerd.sock"}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "containerd-config", MountPath: "/runtime/config-dir/"}, {Name: "containerd-drop-in-config", MountPath: "/runtime/config-dir.d/"}, {Name: "containerd-socket", MountPath: "/runtime/sock-dir/"}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }). WithHostPathVolume("containerd-config", "/etc/containerd", ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-drop-in-config", "/etc/containerd/conf.d", ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-socket", "/run/containerd", nil). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithPullSecret("pull-secret"), }, { @@ -989,16 +1010,19 @@ func TestTransformToolkit(t *testing.T) { {Name: "RUNTIME_DROP_IN_CONFIG", Value: "/runtime/config-dir.d/99-nvidia.toml"}, {Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/containerd/conf.d/99-nvidia.toml"}, {Name: "RUNTIME_SOCKET", Value: "/runtime/sock-dir/containerd.sock"}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "containerd-config", MountPath: "/runtime/config-dir/"}, {Name: "containerd-drop-in-config", MountPath: "/runtime/config-dir.d/"}, {Name: "containerd-socket", MountPath: "/runtime/sock-dir/"}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }). WithHostPathVolume("containerd-config", "/var/lib/rancher/k3s/agent/etc/containerd", ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-drop-in-config", "/etc/containerd/conf.d", ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-socket", "/run/k3s/containerd", nil). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)). WithPullSecret("pull-secret"), }, { @@ -1028,14 +1052,17 @@ func TestTransformToolkit(t *testing.T) { {Name: "CRIO_CONFIG", Value: "/runtime/config-dir/config.toml"}, {Name: "RUNTIME_DROP_IN_CONFIG", Value: "/runtime/config-dir.d/99-nvidia.conf"}, {Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/crio/crio.conf.d/99-nvidia.conf"}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "crio-config", MountPath: DefaultRuntimeConfigTargetDir}, {Name: "crio-drop-in-config", MountPath: "/runtime/config-dir.d/"}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }). WithHostPathVolume("crio-config", "/etc/crio", ptr.To(corev1.HostPathDirectoryOrCreate)). - WithHostPathVolume("crio-drop-in-config", "/etc/crio/crio.conf.d", ptr.To(corev1.HostPathDirectoryOrCreate)), + WithHostPathVolume("crio-drop-in-config", "/etc/crio/crio.conf.d", ptr.To(corev1.HostPathDirectoryOrCreate)). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)), }, { description: "transform nvidia-container-toolkit-ctr container, cri-o runtime, cdi disabled", @@ -1064,14 +1091,17 @@ func TestTransformToolkit(t *testing.T) { {Name: "CRIO_CONFIG", Value: "/runtime/config-dir/config.toml"}, {Name: "RUNTIME_DROP_IN_CONFIG", Value: "/runtime/config-dir.d/99-nvidia.conf"}, {Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/crio/crio.conf.d/99-nvidia.conf"}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "crio-config", MountPath: DefaultRuntimeConfigTargetDir}, {Name: "crio-drop-in-config", MountPath: "/runtime/config-dir.d/"}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }). WithHostPathVolume("crio-config", "/etc/crio", ptr.To(corev1.HostPathDirectoryOrCreate)). - WithHostPathVolume("crio-drop-in-config", "/etc/crio/crio.conf.d", ptr.To(corev1.HostPathDirectoryOrCreate)), + WithHostPathVolume("crio-drop-in-config", "/etc/crio/crio.conf.d", ptr.To(corev1.HostPathDirectoryOrCreate)). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)), }, } @@ -2016,13 +2046,19 @@ func TestTransformKataManager(t *testing.T) { {Name: "CONTAINERD_CONFIG", Value: "/runtime/config-dir/config.toml"}, {Name: "RUNTIME_SOCKET", Value: "/runtime/sock-dir/containerd.sock"}, {Name: "CONTAINERD_SOCKET", Value: "/runtime/sock-dir/containerd.sock"}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "kata-artifacts", MountPath: "/var/lib/kata"}, {Name: "containerd-config", MountPath: "/runtime/config-dir/"}, {Name: "containerd-socket", MountPath: "/runtime/sock-dir/"}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, - }).WithPullSecret("pull-secret").WithPodAnnotations(map[string]string{"nvidia.com/kata-manager.last-applied-hash": "1929911998"}).WithHostPathVolume("kata-artifacts", "/var/lib/kata", ptr.To(corev1.HostPathDirectoryOrCreate)).WithHostPathVolume("containerd-config", "/etc/containerd", ptr.To(corev1.HostPathDirectoryOrCreate)).WithHostPathVolume("containerd-socket", "/run/containerd", nil), + }).WithPullSecret("pull-secret").WithPodAnnotations(map[string]string{"nvidia.com/kata-manager.last-applied-hash": "1929911998"}). + WithHostPathVolume("kata-artifacts", "/var/lib/kata", ptr.To(corev1.HostPathDirectoryOrCreate)). + WithHostPathVolume("containerd-config", "/etc/containerd", ptr.To(corev1.HostPathDirectoryOrCreate)). + WithHostPathVolume("containerd-socket", "/run/containerd", nil). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)), }, { description: "transform kata manager with custom container runtime socket", @@ -2068,17 +2104,20 @@ func TestTransformKataManager(t *testing.T) { {Name: "RUNTIME", Value: "containerd"}, {Name: "RUNTIME_CONFIG", Value: "/runtime/config-dir/config.toml"}, {Name: "RUNTIME_SOCKET", Value: "/runtime/sock-dir/containerd.sock"}, + {Name: "NRI_SOCKET", Value: "/runtime/nri-sock-dir/nri.sock"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "kata-artifacts", MountPath: "/var/lib/kata"}, {Name: "containerd-config", MountPath: "/runtime/config-dir/"}, {Name: "containerd-socket", MountPath: "/runtime/sock-dir/"}, + {Name: "nri-socket", MountPath: "/runtime/nri-sock-dir/"}, }, }).WithPullSecret("pull-secret"). WithPodAnnotations(map[string]string{"nvidia.com/kata-manager.last-applied-hash": "1929911998"}). WithHostPathVolume("kata-artifacts", "/var/lib/kata", ptr.To(corev1.HostPathDirectoryOrCreate)). WithHostPathVolume("containerd-config", "/var/lib/rancher/k3s/agent/etc/containerd", ptr.To(corev1.HostPathDirectoryOrCreate)). - WithHostPathVolume("containerd-socket", "/run/k3s/containerd", nil), + WithHostPathVolume("containerd-socket", "/run/k3s/containerd", nil). + WithHostPathVolume("nri-socket", filepath.Dir(DefaultRuntimeNRISocketFile), ptr.To(corev1.HostPathDirectoryOrCreate)), }, }