From 5a8689507037e4e2c98c1af9837dbd1420f6188a Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Mon, 20 Feb 2023 01:02:46 +0200 Subject: [PATCH] DRA: pass CDI devices through CRI CDIDevice field --- pkg/kubelet/cm/container_manager_linux.go | 4 ++++ pkg/kubelet/cm/dra/manager.go | 9 ++++++++- pkg/kubelet/cm/dra/types.go | 2 ++ pkg/kubelet/container/runtime.go | 8 ++++++++ pkg/kubelet/kuberuntime/kuberuntime_container.go | 14 ++++++++++++++ .../kuberuntime_container_linux_test.go | 1 + 6 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index ff3b197aeb0..7d735e8bfd7 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -661,7 +661,11 @@ func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Containe if err != nil { return nil, err } + // NOTE: Passing CDI device names as annotations is a temporary solution + // It will be removed after all runtimes are updated + // to get CDI device names from the ContainerConfig.CDIDevices field opts.Annotations = append(opts.Annotations, resOpts.Annotations...) + opts.CDIDevices = append(opts.CDIDevices, resOpts.CDIDevices...) } // Allocate should already be called during predicateAdmitHandler.Admit(), // just try to fetch device runtime information from cached state here diff --git a/pkg/kubelet/cm/dra/manager.go b/pkg/kubelet/cm/dra/manager.go index 95eb9687c05..2513af1b498 100644 --- a/pkg/kubelet/cm/dra/manager.go +++ b/pkg/kubelet/cm/dra/manager.go @@ -126,6 +126,9 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { klog.V(3).InfoS("NodePrepareResource succeeded", "response", response) + // NOTE: Passing CDI device names as annotations is a temporary solution + // It will be removed after all runtimes are updated + // to get CDI device names from the ContainerConfig.CDIDevices field annotations, err := generateCDIAnnotations(resourceClaim.UID, driverName, response.CdiDevices) if err != nil { return fmt.Errorf("failed to generate container annotations, err: %+v", err) @@ -163,6 +166,7 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { // This information is used by the caller to update a container config. func (m *ManagerImpl) GetResources(pod *v1.Pod, container *v1.Container) (*ContainerInfo, error) { annotations := []kubecontainer.Annotation{} + cdiDevices := []kubecontainer.CDIDevice{} for i, podResourceClaim := range pod.Spec.ResourceClaims { claimName := resourceclaim.Name(pod, &pod.Spec.ResourceClaims[i]) @@ -179,10 +183,13 @@ func (m *ManagerImpl) GetResources(pod *v1.Pod, container *v1.Container) (*Conta klog.V(3).InfoS("add resource annotations", "claim", claimName, "annotations", claimInfo.annotations) annotations = append(annotations, claimInfo.annotations...) + for _, cdiDevice := range claimInfo.cdiDevices { + cdiDevices = append(cdiDevices, kubecontainer.CDIDevice{Name: cdiDevice}) + } } } - return &ContainerInfo{Annotations: annotations}, nil + return &ContainerInfo{Annotations: annotations, CDIDevices: cdiDevices}, nil } // UnprepareResources calls a plugin's NodeUnprepareResource API for each resource claim owned by a pod. diff --git a/pkg/kubelet/cm/dra/types.go b/pkg/kubelet/cm/dra/types.go index 6b52ad8ef5c..7e559a06356 100644 --- a/pkg/kubelet/cm/dra/types.go +++ b/pkg/kubelet/cm/dra/types.go @@ -44,4 +44,6 @@ type Manager interface { type ContainerInfo struct { // The Annotations for the container Annotations []kubecontainer.Annotation + // CDI Devices for the container + CDIDevices []kubecontainer.CDIDevice } diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index 5f54799c7ed..bbc322b1975 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -463,6 +463,12 @@ type DeviceInfo struct { Permissions string } +// CDIDevice contains information about CDI device +type CDIDevice struct { + // Name is a fully qualified device name + Name string +} + // RunContainerOptions specify the options which are necessary for running containers type RunContainerOptions struct { // The environment variables list. @@ -471,6 +477,8 @@ type RunContainerOptions struct { Mounts []Mount // The host devices mapped into the containers. Devices []DeviceInfo + // The CDI devices for the container + CDIDevices []CDIDevice // The annotations for the container // These annotations are generated by other components (i.e., // not users). Currently, only device plugins populate the annotations. diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 745d70fc7ed..881af3f2190 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -335,6 +335,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(ctx context.Context, Labels: newContainerLabels(container, pod), Annotations: newContainerAnnotations(container, pod, restartCount, opts), Devices: makeDevices(opts), + CDIDevices: makeCDIDevices(opts), Mounts: m.makeMounts(opts, container), LogPath: containerLogsPath, Stdin: container.Stdin, @@ -390,6 +391,19 @@ func makeDevices(opts *kubecontainer.RunContainerOptions) []*runtimeapi.Device { return devices } +// makeCDIDevices generates container CDIDevices for kubelet runtime v1. +func makeCDIDevices(opts *kubecontainer.RunContainerOptions) []*runtimeapi.CDIDevice { + devices := make([]*runtimeapi.CDIDevice, len(opts.CDIDevices)) + + for i, device := range opts.CDIDevices { + devices[i] = &runtimeapi.CDIDevice{ + Name: device.Name, + } + } + + return devices +} + // makeMounts generates container volume mounts for kubelet runtime v1. func (m *kubeGenericRuntimeManager) makeMounts(opts *kubecontainer.RunContainerOptions, container *v1.Container) []*runtimeapi.Mount { volumeMounts := []*runtimeapi.Mount{} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go index 2297cffcdf9..82b3ab16ff0 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go @@ -71,6 +71,7 @@ func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerInde Tty: container.TTY, Linux: l, Envs: envs, + CDIDevices: makeCDIDevices(opts), } return expectedConfig }