Merge pull request #115891 from bart0sh/PR103-CRI-add-CDI-devices

DRA: Pass CDI devices with a new CRI field

pkg/kubelet/cm/container_manager_linux.go

@@ -661,7 +661,11 @@ func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Containe
 		if err != nil {
 			return nil, err
 		}
+		// NOTE: Passing CDI device names as annotations is a temporary solution
+		// It will be removed after all runtimes are updated
+		// to get CDI device names from the ContainerConfig.CDIDevices field
 		opts.Annotations = append(opts.Annotations, resOpts.Annotations...)
+		opts.CDIDevices = append(opts.CDIDevices, resOpts.CDIDevices...)
 	}
 	// Allocate should already be called during predicateAdmitHandler.Admit(),
 	// just try to fetch device runtime information from cached state here

pkg/kubelet/cm/dra/manager.go

@@ -126,6 +126,9 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error {
 
 				klog.V(3).InfoS("NodePrepareResource succeeded", "response", response)
 
+				// NOTE: Passing CDI device names as annotations is a temporary solution
+				// It will be removed after all runtimes are updated
+				// to get CDI device names from the ContainerConfig.CDIDevices field
 				annotations, err := generateCDIAnnotations(resourceClaim.UID, driverName, response.CdiDevices)
 				if err != nil {
 					return fmt.Errorf("failed to generate container annotations, err: %+v", err)
@@ -163,6 +166,7 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error {
 // This information is used by the caller to update a container config.
 func (m *ManagerImpl) GetResources(pod *v1.Pod, container *v1.Container) (*ContainerInfo, error) {
 	annotations := []kubecontainer.Annotation{}
+	cdiDevices := []kubecontainer.CDIDevice{}
 
 	for i, podResourceClaim := range pod.Spec.ResourceClaims {
 		claimName := resourceclaim.Name(pod, &pod.Spec.ResourceClaims[i])
@@ -179,10 +183,13 @@ func (m *ManagerImpl) GetResources(pod *v1.Pod, container *v1.Container) (*Conta
 
 			klog.V(3).InfoS("add resource annotations", "claim", claimName, "annotations", claimInfo.annotations)
 			annotations = append(annotations, claimInfo.annotations...)
+			for _, cdiDevice := range claimInfo.cdiDevices {
+				cdiDevices = append(cdiDevices, kubecontainer.CDIDevice{Name: cdiDevice})
+			}
 		}
 	}
 
-	return &ContainerInfo{Annotations: annotations}, nil
+	return &ContainerInfo{Annotations: annotations, CDIDevices: cdiDevices}, nil
 }
 
 // UnprepareResources calls a plugin's NodeUnprepareResource API for each resource claim owned by a pod.

pkg/kubelet/cm/dra/types.go

@@ -44,4 +44,6 @@ type Manager interface {
 type ContainerInfo struct {
 	// The Annotations for the container
 	Annotations []kubecontainer.Annotation
+	// CDI Devices for the container
+	CDIDevices []kubecontainer.CDIDevice
 }

pkg/kubelet/container/runtime.go

@@ -463,6 +463,12 @@ type DeviceInfo struct {
 	Permissions string
 }
 
+// CDIDevice contains information about CDI device
+type CDIDevice struct {
+	// Name is a fully qualified device name
+	Name string
+}
+
 // RunContainerOptions specify the options which are necessary for running containers
 type RunContainerOptions struct {
 	// The environment variables list.
@@ -471,6 +477,8 @@ type RunContainerOptions struct {
 	Mounts []Mount
 	// The host devices mapped into the containers.
 	Devices []DeviceInfo
+	// The CDI devices for the container
+	CDIDevices []CDIDevice
 	// The annotations for the container
 	// These annotations are generated by other components (i.e.,
 	// not users). Currently, only device plugins populate the annotations.

pkg/kubelet/kuberuntime/kuberuntime_container.go

@@ -335,6 +335,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(ctx context.Context,
 		Labels:      newContainerLabels(container, pod),
 		Annotations: newContainerAnnotations(container, pod, restartCount, opts),
 		Devices:     makeDevices(opts),
+		CDIDevices:  makeCDIDevices(opts),
 		Mounts:      m.makeMounts(opts, container),
 		LogPath:     containerLogsPath,
 		Stdin:       container.Stdin,
@@ -390,6 +391,19 @@ func makeDevices(opts *kubecontainer.RunContainerOptions) []*runtimeapi.Device {
 	return devices
 }
 
+// makeCDIDevices generates container CDIDevices for kubelet runtime v1.
+func makeCDIDevices(opts *kubecontainer.RunContainerOptions) []*runtimeapi.CDIDevice {
+	devices := make([]*runtimeapi.CDIDevice, len(opts.CDIDevices))
+
+	for i, device := range opts.CDIDevices {
+		devices[i] = &runtimeapi.CDIDevice{
+			Name: device.Name,
+		}
+	}
+
+	return devices
+}
+
 // makeMounts generates container volume mounts for kubelet runtime v1.
 func (m *kubeGenericRuntimeManager) makeMounts(opts *kubecontainer.RunContainerOptions, container *v1.Container) []*runtimeapi.Mount {
 	volumeMounts := []*runtimeapi.Mount{}

pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go

@@ -71,6 +71,7 @@ func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerInde
 		Tty:         container.TTY,
 		Linux:       l,
 		Envs:        envs,
+		CDIDevices:  makeCDIDevices(opts),
 	}
 	return expectedConfig
 }

staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto

@@ -1017,6 +1017,15 @@ message Device {
     string permissions = 3;
 }
 
+// CDIDevice specifies a CDI device information.
+message CDIDevice {
+    // Fully qualified CDI device name
+    // for example: vendor.com/gpu=gpudevice1
+    // see more details in the CDI specification:
+    // https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md
+    string name = 1;
+}
+
 // ContainerConfig holds all the required and optional fields for creating a
 // container.
 message ContainerConfig {
@@ -1074,6 +1083,9 @@ message ContainerConfig {
     LinuxContainerConfig linux = 15;
     // Configuration specific to Windows containers.
     WindowsContainerConfig windows = 16;
+
+    // CDI devices for the container.
+    repeated CDIDevice CDI_devices = 17;
 }
 
 message CreateContainerRequest {