From f0e3c32fe560ece3aece6403e6303a9d966f8c8e Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 6 Jul 2023 14:33:01 +0200 Subject: [PATCH 1/4] Move CDI annotation code to utils package Signed-off-by: Evan Lezar --- pkg/kubelet/cm/dra/claiminfo.go | 3 +- pkg/kubelet/cm/{dra => util/cdi}/cdi.go | 6 +-- pkg/kubelet/cm/util/cdi/cdi_test.go | 52 +++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) rename pkg/kubelet/cm/{dra => util/cdi}/cdi.go (98%) create mode 100644 pkg/kubelet/cm/util/cdi/cdi_test.go diff --git a/pkg/kubelet/cm/dra/claiminfo.go b/pkg/kubelet/cm/dra/claiminfo.go index d1711a0771d..ae4c12a67b6 100644 --- a/pkg/kubelet/cm/dra/claiminfo.go +++ b/pkg/kubelet/cm/dra/claiminfo.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/kubelet/cm/dra/state" + "k8s.io/kubernetes/pkg/kubelet/cm/util/cdi" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -57,7 +58,7 @@ func (info *ClaimInfo) addCDIDevices(pluginName string, cdiDevices []string) err // NOTE: Passing CDI device names as annotations is a temporary solution // It will be removed after all runtimes are updated // to get CDI device names from the ContainerConfig.CDIDevices field - annotations, err := generateCDIAnnotations(info.ClaimUID, info.DriverName, cdiDevices) + annotations, err := cdi.GenerateAnnotations(info.ClaimUID, info.DriverName, cdiDevices) if err != nil { return fmt.Errorf("failed to generate container annotations, err: %+v", err) } diff --git a/pkg/kubelet/cm/dra/cdi.go b/pkg/kubelet/cm/util/cdi/cdi.go similarity index 98% rename from pkg/kubelet/cm/dra/cdi.go rename to pkg/kubelet/cm/util/cdi/cdi.go index b6118337817..6cd9a1b8127 100644 --- a/pkg/kubelet/cm/dra/cdi.go +++ b/pkg/kubelet/cm/util/cdi/cdi.go @@ -23,7 +23,7 @@ limitations under the License. // Long term it would be good to avoid this duplication: // https://github.com/container-orchestrated-devices/container-device-interface/issues/97 -package dra +package cdi import ( "errors" @@ -39,8 +39,8 @@ const ( annotationPrefix = "cdi.k8s.io/" ) -// generate container annotations using CDI UpdateAnnotations API. -func generateCDIAnnotations( +// GenerateAnnotations generate container annotations using CDI UpdateAnnotations API. +func GenerateAnnotations( claimUID types.UID, driverName string, cdiDevices []string, diff --git a/pkg/kubelet/cm/util/cdi/cdi_test.go b/pkg/kubelet/cm/util/cdi/cdi_test.go new file mode 100644 index 00000000000..ce84daadc0e --- /dev/null +++ b/pkg/kubelet/cm/util/cdi/cdi_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cdi + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/kubernetes/pkg/kubelet/container" +) + +func TestGenerateAnnotations(t *testing.T) { + testCases := []struct { + description string + deviceIDs []string + expecteError error + expectedAnnotations []container.Annotation + }{ + { + description: "no devices", + deviceIDs: []string{}, + }, + { + description: "one device", + deviceIDs: []string{"vendor.com/class=device1"}, + expectedAnnotations: []container.Annotation{{Name: "cdi.k8s.io/test-driver-name_test-claim-uid", Value: "vendor.com/class=device1"}}, + }, + } + + as := assert.New(t) + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + annotations, err := GenerateAnnotations("test-claim-uid", "test-driver-name", tc.deviceIDs) + as.ErrorIs(err, tc.expecteError) + as.Equal(tc.expectedAnnotations, annotations) + }) + } +} From db2a1edbdd63d0313f6a3a3de383a4232c493705 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 7 Jul 2023 17:38:48 +0200 Subject: [PATCH 2/4] Generate empty cdi annotations Signed-off-by: Evan Lezar --- pkg/kubelet/cm/util/cdi/cdi.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/kubelet/cm/util/cdi/cdi.go b/pkg/kubelet/cm/util/cdi/cdi.go index 6cd9a1b8127..b228b5a66d8 100644 --- a/pkg/kubelet/cm/util/cdi/cdi.go +++ b/pkg/kubelet/cm/util/cdi/cdi.go @@ -45,6 +45,9 @@ func GenerateAnnotations( driverName string, cdiDevices []string, ) ([]kubecontainer.Annotation, error) { + if len(cdiDevices) == 0 { + return nil, nil + } annotations, err := updateAnnotations(map[string]string{}, driverName, string(claimUID), cdiDevices) if err != nil { return nil, fmt.Errorf("can't generate CDI annotations: %+v", err) From cd14e97ea80fba6bb09a85187ed685bd298af246 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 31 May 2023 14:57:25 +0200 Subject: [PATCH 3/4] Add a builder for ContainerAllocateResponse objects This chagne introduces a helper to construct ContainerAllocateResponse instances. Test cases are updated to use a new constructor accepting functional options allowing the response contents to be set based on the test requirements. This can then be extended to also test additional fields in the device plugin API such as annotations which are not currently covered or new fields. Signed-off-by: Evan Lezar --- pkg/kubelet/cm/devicemanager/manager_test.go | 117 ++++++++++++++---- .../cm/devicemanager/pod_devices_test.go | 6 +- 2 files changed, 98 insertions(+), 25 deletions(-) diff --git a/pkg/kubelet/cm/devicemanager/manager_test.go b/pkg/kubelet/cm/devicemanager/manager_test.go index 56881f9e25a..f50e82f69fd 100644 --- a/pkg/kubelet/cm/devicemanager/manager_test.go +++ b/pkg/kubelet/cm/devicemanager/manager_test.go @@ -571,16 +571,58 @@ func constructDevices(devices []string) checkpoint.DevicesPerNUMA { return ret } -func constructAllocResp(devices, mounts, envs map[string]string) *pluginapi.ContainerAllocateResponse { +// containerAllocateResponseBuilder is a helper to build a ContainerAllocateResponse +type containerAllocateResponseBuilder struct { + devices map[string]string + mounts map[string]string + envs map[string]string +} + +// containerAllocateResponseBuilderOption defines a functional option for a containerAllocateResponseBuilder +type containerAllocateResponseBuilderOption func(*containerAllocateResponseBuilder) + +// withDevices sets the devices for the containerAllocateResponseBuilder +func withDevices(devices map[string]string) containerAllocateResponseBuilderOption { + return func(b *containerAllocateResponseBuilder) { + b.devices = devices + } +} + +// withMounts sets the mounts for the containerAllocateResponseBuilder +func withMounts(mounts map[string]string) containerAllocateResponseBuilderOption { + return func(b *containerAllocateResponseBuilder) { + b.mounts = mounts + } +} + +// withEnvs sets the envs for the containerAllocateResponseBuilder +func withEnvs(envs map[string]string) containerAllocateResponseBuilderOption { + return func(b *containerAllocateResponseBuilder) { + b.envs = envs + } +} + +// newContainerAllocateResponse creates a ContainerAllocateResponse with the given options. +func newContainerAllocateResponse(opts ...containerAllocateResponseBuilderOption) *pluginapi.ContainerAllocateResponse { + b := &containerAllocateResponseBuilder{} + for _, opt := range opts { + opt(b) + } + + return b.Build() +} + +// Build uses the configured builder to create a ContainerAllocateResponse. +func (b *containerAllocateResponseBuilder) Build() *pluginapi.ContainerAllocateResponse { resp := &pluginapi.ContainerAllocateResponse{} - for k, v := range devices { + for k, v := range b.devices { resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{ HostPath: k, ContainerPath: v, Permissions: "mrw", }) } - for k, v := range mounts { + for k, v := range b.mounts { resp.Mounts = append(resp.Mounts, &pluginapi.Mount{ ContainerPath: k, HostPath: v, @@ -588,7 +630,7 @@ func constructAllocResp(devices, mounts, envs map[string]string) *pluginapi.Cont }) } resp.Envs = make(map[string]string) - for k, v := range envs { + for k, v := range b.envs { resp.Envs[k] = v } return resp @@ -615,25 +657,40 @@ func TestCheckpoint(t *testing.T) { testManager.podDevices.insert("pod1", "con1", resourceName1, constructDevices([]string{"dev1", "dev2"}), - constructAllocResp(map[string]string{"/dev/r1dev1": "/dev/r1dev1", "/dev/r1dev2": "/dev/r1dev2"}, - map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{})) + newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r1dev1": "/dev/r1dev1", "/dev/r1dev2": "/dev/r1dev2"}), + withMounts(map[string]string{"/home/r1lib1": "/usr/r1lib1"}), + ), + ) testManager.podDevices.insert("pod1", "con1", resourceName2, constructDevices([]string{"dev1", "dev2"}), - constructAllocResp(map[string]string{"/dev/r2dev1": "/dev/r2dev1", "/dev/r2dev2": "/dev/r2dev2"}, - map[string]string{"/home/r2lib1": "/usr/r2lib1"}, - map[string]string{"r2devices": "dev1 dev2"})) + newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r2dev1": "/dev/r2dev1", "/dev/r2dev2": "/dev/r2dev2"}), + withMounts(map[string]string{"/home/r2lib1": "/usr/r2lib1"}), + withEnvs(map[string]string{"r2devices": "dev1 dev2"}), + ), + ) testManager.podDevices.insert("pod1", "con2", resourceName1, constructDevices([]string{"dev3"}), - constructAllocResp(map[string]string{"/dev/r1dev3": "/dev/r1dev3"}, - map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{})) + newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r1dev3": "/dev/r1dev3"}), + withMounts(map[string]string{"/home/r1lib1": "/usr/r1lib1"}), + ), + ) testManager.podDevices.insert("pod2", "con1", resourceName1, constructDevices([]string{"dev4"}), - constructAllocResp(map[string]string{"/dev/r1dev4": "/dev/r1dev4"}, - map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{})) + newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r1dev4": "/dev/r1dev4"}), + withMounts(map[string]string{"/home/r1lib1": "/usr/r1lib1"}), + ), + ) testManager.podDevices.insert("pod3", "con3", resourceName3, checkpoint.DevicesPerNUMA{nodeWithoutTopology: []string{"dev5"}}, - constructAllocResp(map[string]string{"/dev/r1dev5": "/dev/r1dev5"}, - map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{})) + newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r3dev5": "/dev/r3dev5"}), + withMounts(map[string]string{"/home/r3lib1": "/usr/r3lib1"}), + ), + ) testManager.healthyDevices[resourceName1] = sets.NewString() testManager.healthyDevices[resourceName1].Insert("dev1") @@ -1012,17 +1069,26 @@ func TestPodContainerDeviceToAllocate(t *testing.T) { testManager.podDevices.insert("pod1", "con1", resourceName1, constructDevices([]string{"dev1", "dev2"}), - constructAllocResp(map[string]string{"/dev/r2dev1": "/dev/r2dev1", "/dev/r2dev2": "/dev/r2dev2"}, - map[string]string{"/home/r2lib1": "/usr/r2lib1"}, - map[string]string{"r2devices": "dev1 dev2"})) + newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r2dev1": "/dev/r2dev1", "/dev/r2dev2": "/dev/r2dev2"}), + withMounts(map[string]string{"/home/r2lib1": "/usr/r2lib1"}), + withEnvs(map[string]string{"r2devices": "dev1 dev2"}), + ), + ) testManager.podDevices.insert("pod2", "con2", resourceName2, checkpoint.DevicesPerNUMA{nodeWithoutTopology: []string{"dev5"}}, - constructAllocResp(map[string]string{"/dev/r1dev5": "/dev/r1dev5"}, - map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{})) + newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r1dev5": "/dev/r1dev5"}), + withMounts(map[string]string{"/home/r1lib1": "/usr/r1lib1"}), + ), + ) testManager.podDevices.insert("pod3", "con3", resourceName3, checkpoint.DevicesPerNUMA{nodeWithoutTopology: []string{"dev5"}}, - constructAllocResp(map[string]string{"/dev/r1dev5": "/dev/r1dev5"}, - map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{})) + newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r1dev5": "/dev/r1dev5"}), + withMounts(map[string]string{"/home/r1lib1": "/usr/r1lib1"}), + ), + ) // no healthy devices for resourceName1 and devices corresponding to // resource2 are intentionally omitted to simulate that the resource @@ -1402,8 +1468,11 @@ func TestResetExtendedResource(t *testing.T) { extendedResourceName := "domain.com/resource" testManager.podDevices.insert("pod", "con", extendedResourceName, constructDevices([]string{"dev1"}), - constructAllocResp(map[string]string{"/dev/dev1": "/dev/dev1"}, - map[string]string{"/home/lib1": "/usr/lib1"}, map[string]string{})) + newContainerAllocateResponse( + withDevices(map[string]string{"/dev/dev1": "/dev/dev1"}), + withMounts(map[string]string{"/home/lib1": "/usr/lib1"}), + ), + ) testManager.healthyDevices[extendedResourceName] = sets.NewString() testManager.healthyDevices[extendedResourceName].Insert("dev1") diff --git a/pkg/kubelet/cm/devicemanager/pod_devices_test.go b/pkg/kubelet/cm/devicemanager/pod_devices_test.go index f21fee36816..15fc07a8dab 100644 --- a/pkg/kubelet/cm/devicemanager/pod_devices_test.go +++ b/pkg/kubelet/cm/devicemanager/pod_devices_test.go @@ -36,7 +36,11 @@ func TestGetContainerDevices(t *testing.T) { podDevices.insert(podID, contID, resourceName1, devices, - constructAllocResp(map[string]string{"/dev/r1dev1": "/dev/r1dev1", "/dev/r1dev2": "/dev/r1dev2"}, map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{})) + newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r1dev1": "/dev/r1dev1", "/dev/r1dev2": "/dev/r1dev2"}), + withMounts(map[string]string{"/home/r1lib1": "/usr/r1lib1"}), + ), + ) resContDevices := podDevices.getContainerDevices(podID, contID) contDevices, ok := resContDevices[resourceName1] From b57c7e2fe4bb466ff1614aa9df7cc164e90b24b6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 16 May 2023 15:12:32 +0200 Subject: [PATCH 4/4] Add CDI devices to device plugin API This change adds CDI device IDs to the ContainerAllocateResponse in the device plugin API. This allows a device plugin to specify CDI devices by their unique fully-qualified CDI device names using the related field in the CRI specification. Signed-off-by: Evan Lezar --- pkg/features/kube_features.go | 9 + pkg/kubelet/cm/container_manager_linux.go | 1 + pkg/kubelet/cm/devicemanager/manager_test.go | 25 +- pkg/kubelet/cm/devicemanager/pod_devices.go | 75 ++++ .../cm/devicemanager/pod_devices_test.go | 139 ++++++ pkg/kubelet/cm/devicemanager/types.go | 2 + .../pkg/apis/deviceplugin/v1beta1/api.pb.go | 405 ++++++++++++++---- .../pkg/apis/deviceplugin/v1beta1/api.proto | 11 + 8 files changed, 590 insertions(+), 77 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index cb2f976cb2b..b79fa2d5c63 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -221,6 +221,13 @@ const ( // (e.g. in a Deployment), which is the historical default. DefaultHostNetworkHostPortsInPodTemplates featuregate.Feature = "DefaultHostNetworkHostPortsInPodTemplates" + // owner: @elezar + // kep: http://kep.k8s.io/4009 + // alpha: v1.28 + // + // Add support for CDI Device IDs in the Device Plugin API. + DevicePluginCDIDevices featuregate.Feature = "DevicePluginCDIDevices" + // owner: @andrewsykim // alpha: v1.22 // @@ -908,6 +915,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS DisableKubeletCloudCredentialProviders: {Default: false, PreRelease: featuregate.Alpha}, + DevicePluginCDIDevices: {Default: false, PreRelease: featuregate.Alpha}, + DownwardAPIHugePages: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in v1.29 DynamicResourceAllocation: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 02cb34ddcdc..c4a19371566 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -673,6 +673,7 @@ func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Containe opts.Mounts = append(opts.Mounts, devOpts.Mounts...) opts.Envs = append(opts.Envs, devOpts.Envs...) opts.Annotations = append(opts.Annotations, devOpts.Annotations...) + opts.CDIDevices = append(opts.CDIDevices, devOpts.CDIDevices...) return opts, nil } diff --git a/pkg/kubelet/cm/devicemanager/manager_test.go b/pkg/kubelet/cm/devicemanager/manager_test.go index f50e82f69fd..a46722e26a1 100644 --- a/pkg/kubelet/cm/devicemanager/manager_test.go +++ b/pkg/kubelet/cm/devicemanager/manager_test.go @@ -573,9 +573,10 @@ func constructDevices(devices []string) checkpoint.DevicesPerNUMA { // containerAllocateResponseBuilder is a helper to build a ContainerAllocateResponse type containerAllocateResponseBuilder struct { - devices map[string]string - mounts map[string]string - envs map[string]string + devices map[string]string + mounts map[string]string + envs map[string]string + cdiDevices []string } // containerAllocateResponseBuilderOption defines a functional option for a containerAllocateResponseBuilder @@ -602,6 +603,13 @@ func withEnvs(envs map[string]string) containerAllocateResponseBuilderOption { } } +// withCDIDevices sets the cdiDevices for the containerAllocateResponseBuilder +func withCDIDevices(cdiDevices ...string) containerAllocateResponseBuilderOption { + return func(b *containerAllocateResponseBuilder) { + b.cdiDevices = cdiDevices + } +} + // newContainerAllocateResponse creates a ContainerAllocateResponse with the given options. func newContainerAllocateResponse(opts ...containerAllocateResponseBuilderOption) *pluginapi.ContainerAllocateResponse { b := &containerAllocateResponseBuilder{} @@ -633,6 +641,16 @@ func (b *containerAllocateResponseBuilder) Build() *pluginapi.ContainerAllocateR for k, v := range b.envs { resp.Envs[k] = v } + + var cdiDevices []*pluginapi.CDIDevice + for _, dev := range b.cdiDevices { + cdiDevice := pluginapi.CDIDevice{ + Name: dev, + } + cdiDevices = append(cdiDevices, &cdiDevice) + } + resp.CDIDevices = cdiDevices + return resp } @@ -660,6 +678,7 @@ func TestCheckpoint(t *testing.T) { newContainerAllocateResponse( withDevices(map[string]string{"/dev/r1dev1": "/dev/r1dev1", "/dev/r1dev2": "/dev/r1dev2"}), withMounts(map[string]string{"/home/r1lib1": "/usr/r1lib1"}), + withCDIDevices("domain1.com/resource1=dev1", "domain1.com/resource1=dev2"), ), ) testManager.podDevices.insert("pod1", "con1", resourceName2, diff --git a/pkg/kubelet/cm/devicemanager/pod_devices.go b/pkg/kubelet/cm/devicemanager/pod_devices.go index 7a12e8de813..fe4eb65e405 100644 --- a/pkg/kubelet/cm/devicemanager/pod_devices.go +++ b/pkg/kubelet/cm/devicemanager/pod_devices.go @@ -21,9 +21,13 @@ import ( "k8s.io/klog/v2" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" + "k8s.io/kubernetes/pkg/kubelet/cm/util/cdi" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -244,6 +248,8 @@ func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *Devi mountsMap := make(map[string]string) envsMap := make(map[string]string) annotationsMap := make(map[string]string) + // Keep track of all CDI devices requested for the container. + allCDIDevices := sets.New[string]() // Loops through AllocationResponses of all cached device resources. for _, devices := range resources { resp := devices.allocResp @@ -252,6 +258,7 @@ func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *Devi // Mount points // Device files // Container annotations + // CDI device IDs // These artifacts are per resource per container. // Updates RunContainerOptions.Envs. for k, v := range resp.Envs { @@ -321,10 +328,78 @@ func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *Devi annotationsMap[k] = v opts.Annotations = append(opts.Annotations, kubecontainer.Annotation{Name: k, Value: v}) } + + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DevicePluginCDIDevices) { + // Updates for CDI devices. + cdiDevices := getCDIDeviceInfo(resp, allCDIDevices) + opts.CDIDevices = append(opts.CDIDevices, cdiDevices...) + } } + + // Although the CDI devices are expected to be empty when this feature is disabled, we still + // guard this with a feature gate to avoid any potential issues. + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DevicePluginCDIDevices) { + // We construct a resource ID from the pod UID and container name. + // This ID has no semantic meaning, and is only used to ensure that the generated CDI annotation key is unique + // for a given container. Since this is only called once per pod-container combination, this should be the case. + resourceID := podUID + "-" + contName + cdiAnnotations := getCDIAnnotations(resourceID, allCDIDevices, annotationsMap) + opts.Annotations = append(opts.Annotations, cdiAnnotations...) + } + return opts } +// getCDIAnnotations returns the cdi annotations for a given container. +// This creates a CDI annotation with a key of the form: devicemanager_{{resourceID}}. +// The value of the annotation is a comma separated list of sorted CDI device IDs. +// If the annotation key is already defined in the provided annotations map, then the existing value is used. +func getCDIAnnotations(resourceID string, cdiDevices sets.Set[string], annotationsMap map[string]string) []kubecontainer.Annotation { + // We sort the CDI devices to ensure that the annotation value is deterministic. + sortedCDIDevices := sets.List[string](cdiDevices) + annotations, err := cdi.GenerateAnnotations(types.UID(resourceID), "devicemanager", sortedCDIDevices) + if err != nil { + klog.ErrorS(err, "Failed to create CDI annotations") + return nil + } + + var cdiAnnotations []kubecontainer.Annotation + for _, annotation := range annotations { + if e, ok := annotationsMap[annotation.Name]; ok { + klog.V(4).InfoS("Skip existing annotation", "annotationKey", annotation.Name, "annotationValue", annotation.Value) + if e != annotation.Value { + klog.ErrorS(nil, "Annotation has conflicting setting", "annotationKey", annotation.Name, "expected", e, "got", annotation.Value) + } + continue + } + klog.V(4).InfoS("Add annotation", "annotationKey", annotation.Name, "annotationValue", annotation.Value) + annotationsMap[annotation.Name] = annotation.Value + cdiAnnotations = append(cdiAnnotations, kubecontainer.Annotation{Name: annotation.Name, Value: annotation.Value}) + } + + return cdiAnnotations +} + +// getCDIDeviceInfo returns CDI devices from an allocate response +func getCDIDeviceInfo(resp *pluginapi.ContainerAllocateResponse, knownCDIDevices sets.Set[string]) []kubecontainer.CDIDevice { + var cdiDevices []kubecontainer.CDIDevice + for _, cdiDevice := range resp.CDIDevices { + if knownCDIDevices.Has(cdiDevice.Name) { + klog.V(4).InfoS("Skip existing CDI Device", "name", cdiDevice.Name) + continue + } + klog.V(4).InfoS("Add CDI device", "name", cdiDevice.Name) + knownCDIDevices.Insert(cdiDevice.Name) + + device := kubecontainer.CDIDevice{ + Name: cdiDevice.Name, + } + cdiDevices = append(cdiDevices, device) + } + + return cdiDevices +} + // getContainerDevices returns the devices assigned to the provided container for all ResourceNames func (pdev *podDevices) getContainerDevices(podUID, contName string) ResourceDeviceInstances { pdev.RLock() diff --git a/pkg/kubelet/cm/devicemanager/pod_devices_test.go b/pkg/kubelet/cm/devicemanager/pod_devices_test.go index 15fc07a8dab..70d5f6e6dea 100644 --- a/pkg/kubelet/cm/devicemanager/pod_devices_test.go +++ b/pkg/kubelet/cm/devicemanager/pod_devices_test.go @@ -20,11 +20,16 @@ import ( "encoding/json" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) func TestGetContainerDevices(t *testing.T) { @@ -153,3 +158,137 @@ func expectResourceDeviceInstances(t *testing.T, resp ResourceDeviceInstances, e t.Errorf("expected %q got %q", expected, got) } } + +func TestDeviceRunContainerOptions(t *testing.T) { + const ( + podUID = "pod" + containerName = "container" + resource1 = "example1.com/resource1" + resource2 = "example2.com/resource2" + ) + testCases := []struct { + description string + gate bool + responsesPerResource map[string]*pluginapi.ContainerAllocateResponse + expected *DeviceRunContainerOptions + }{ + { + description: "empty response", + gate: false, + responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{ + resource1: newContainerAllocateResponse(), + }, + expected: &DeviceRunContainerOptions{}, + }, + { + description: "cdi devices are ingored when feature gate is disabled", + gate: false, + responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{ + resource1: newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r1": "/dev/r1"}), + withMounts(map[string]string{"/home/lib1": "/home/lib1"}), + withEnvs(map[string]string{"ENV1": "VALUE1"}), + withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"), + ), + }, + expected: &DeviceRunContainerOptions{ + Devices: []kubecontainer.DeviceInfo{ + {PathOnHost: "/dev/r1", PathInContainer: "/dev/r1", Permissions: "mrw"}, + }, + Mounts: []kubecontainer.Mount{ + {Name: "/home/lib1", HostPath: "/home/lib1", ContainerPath: "/home/lib1", ReadOnly: true}, + }, + Envs: []kubecontainer.EnvVar{ + {Name: "ENV1", Value: "VALUE1"}, + }, + }, + }, + { + description: "cdi devices are handled when feature gate is enabled", + gate: true, + responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{ + resource1: newContainerAllocateResponse( + withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"), + ), + }, + expected: &DeviceRunContainerOptions{ + Annotations: []kubecontainer.Annotation{ + {Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2"}, + }, + CDIDevices: []kubecontainer.CDIDevice{ + {Name: "vendor1.com/class1=device1"}, + {Name: "vendor2.com/class2=device2"}, + }, + }, + }, + { + description: "cdi devices from multiple resources are handled when feature gate is enabled", + gate: true, + responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{ + resource1: newContainerAllocateResponse( + withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"), + ), + resource2: newContainerAllocateResponse( + withCDIDevices("vendor3.com/class3=device3", "vendor4.com/class4=device4"), + ), + }, + expected: &DeviceRunContainerOptions{ + Annotations: []kubecontainer.Annotation{ + {Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2,vendor3.com/class3=device3,vendor4.com/class4=device4"}, + }, + CDIDevices: []kubecontainer.CDIDevice{ + {Name: "vendor1.com/class1=device1"}, + {Name: "vendor2.com/class2=device2"}, + {Name: "vendor3.com/class3=device3"}, + {Name: "vendor4.com/class4=device4"}, + }, + }, + }, + { + description: "duplicate cdi devices are skipped", + gate: true, + responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{ + resource1: newContainerAllocateResponse( + withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"), + ), + resource2: newContainerAllocateResponse( + withCDIDevices("vendor2.com/class2=device2", "vendor3.com/class3=device3"), + ), + }, + expected: &DeviceRunContainerOptions{ + Annotations: []kubecontainer.Annotation{ + {Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2,vendor3.com/class3=device3"}, + }, + CDIDevices: []kubecontainer.CDIDevice{ + {Name: "vendor1.com/class1=device1"}, + {Name: "vendor2.com/class2=device2"}, + {Name: "vendor3.com/class3=device3"}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + as := assert.New(t) + + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DevicePluginCDIDevices, tc.gate)() + podDevices := newPodDevices() + for resourceName, response := range tc.responsesPerResource { + podDevices.insert("pod", "container", resourceName, + nil, + response, + ) + } + opts := podDevices.deviceRunContainerOptions(podUID, containerName) + + // The exact ordering of the options depends on the order of the resources in the map. + // We therefore use `ElementsMatch` instead of `Equal` on the member slices. + as.ElementsMatch(tc.expected.Annotations, opts.Annotations) + as.ElementsMatch(tc.expected.CDIDevices, opts.CDIDevices) + as.ElementsMatch(tc.expected.Devices, opts.Devices) + as.ElementsMatch(tc.expected.Envs, opts.Envs) + as.ElementsMatch(tc.expected.Mounts, opts.Mounts) + }) + } +} diff --git a/pkg/kubelet/cm/devicemanager/types.go b/pkg/kubelet/cm/devicemanager/types.go index d508e8c9969..4b63944a656 100644 --- a/pkg/kubelet/cm/devicemanager/types.go +++ b/pkg/kubelet/cm/devicemanager/types.go @@ -91,6 +91,8 @@ type DeviceRunContainerOptions struct { Devices []kubecontainer.DeviceInfo // The Annotations for the container Annotations []kubecontainer.Annotation + // CDI Devices for the container + CDIDevices []kubecontainer.CDIDevice } // TODO: evaluate whether we need this error definition. diff --git a/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.pb.go b/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.pb.go index a24d42881f3..e15ca7ccf99 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.pb.go +++ b/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.pb.go @@ -814,6 +814,56 @@ func (m *ContainerAllocateRequest) GetDevicesIDs() []string { return nil } +// CDIDevice specifies a CDI device information. +type CDIDevice struct { + // Fully qualified CDI device name + // for example: vendor.com/gpu=gpudevice1 + // see more details in the CDI specification: + // https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CDIDevice) Reset() { *m = CDIDevice{} } +func (*CDIDevice) ProtoMessage() {} +func (*CDIDevice) Descriptor() ([]byte, []int) { + return fileDescriptor_00212fb1f9d3bf1c, []int{15} +} +func (m *CDIDevice) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CDIDevice) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CDIDevice.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CDIDevice) XXX_Merge(src proto.Message) { + xxx_messageInfo_CDIDevice.Merge(m, src) +} +func (m *CDIDevice) XXX_Size() int { + return m.Size() +} +func (m *CDIDevice) XXX_DiscardUnknown() { + xxx_messageInfo_CDIDevice.DiscardUnknown(m) +} + +var xxx_messageInfo_CDIDevice proto.InternalMessageInfo + +func (m *CDIDevice) GetName() string { + if m != nil { + return m.Name + } + return "" +} + // AllocateResponse includes the artifacts that needs to be injected into // a container for accessing 'deviceIDs' that were mentioned as part of // 'AllocateRequest'. @@ -831,7 +881,7 @@ type AllocateResponse struct { func (m *AllocateResponse) Reset() { *m = AllocateResponse{} } func (*AllocateResponse) ProtoMessage() {} func (*AllocateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{15} + return fileDescriptor_00212fb1f9d3bf1c, []int{16} } func (m *AllocateResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -875,15 +925,17 @@ type ContainerAllocateResponse struct { // Devices for the container. Devices []*DeviceSpec `protobuf:"bytes,3,rep,name=devices,proto3" json:"devices,omitempty"` // Container annotations to pass to the container runtime - Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_sizecache int32 `json:"-"` + Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // CDI devices for the container. + CDIDevices []*CDIDevice `protobuf:"bytes,5,rep,name=cdi_devices,json=cdiDevices,proto3" json:"cdi_devices,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ContainerAllocateResponse) Reset() { *m = ContainerAllocateResponse{} } func (*ContainerAllocateResponse) ProtoMessage() {} func (*ContainerAllocateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{16} + return fileDescriptor_00212fb1f9d3bf1c, []int{17} } func (m *ContainerAllocateResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -940,6 +992,13 @@ func (m *ContainerAllocateResponse) GetAnnotations() map[string]string { return nil } +func (m *ContainerAllocateResponse) GetCDIDevices() []*CDIDevice { + if m != nil { + return m.CDIDevices + } + return nil +} + // Mount specifies a host volume to mount into a container. // where device library or tools are installed on host and container type Mount struct { @@ -956,7 +1015,7 @@ type Mount struct { func (m *Mount) Reset() { *m = Mount{} } func (*Mount) ProtoMessage() {} func (*Mount) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{17} + return fileDescriptor_00212fb1f9d3bf1c, []int{18} } func (m *Mount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1024,7 +1083,7 @@ type DeviceSpec struct { func (m *DeviceSpec) Reset() { *m = DeviceSpec{} } func (*DeviceSpec) ProtoMessage() {} func (*DeviceSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{18} + return fileDescriptor_00212fb1f9d3bf1c, []int{19} } func (m *DeviceSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1090,6 +1149,7 @@ func init() { proto.RegisterType((*ContainerPreferredAllocationResponse)(nil), "v1beta1.ContainerPreferredAllocationResponse") proto.RegisterType((*AllocateRequest)(nil), "v1beta1.AllocateRequest") proto.RegisterType((*ContainerAllocateRequest)(nil), "v1beta1.ContainerAllocateRequest") + proto.RegisterType((*CDIDevice)(nil), "v1beta1.CDIDevice") proto.RegisterType((*AllocateResponse)(nil), "v1beta1.AllocateResponse") proto.RegisterType((*ContainerAllocateResponse)(nil), "v1beta1.ContainerAllocateResponse") proto.RegisterMapType((map[string]string)(nil), "v1beta1.ContainerAllocateResponse.AnnotationsEntry") @@ -1101,73 +1161,75 @@ func init() { func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) } var fileDescriptor_00212fb1f9d3bf1c = []byte{ - // 1044 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0x1b, 0x45, - 0x14, 0xce, 0xc6, 0x4d, 0x62, 0x1f, 0xa7, 0xf9, 0x99, 0x84, 0xc8, 0xd9, 0x14, 0x37, 0x9d, 0x14, - 0x1a, 0xa4, 0xc4, 0x26, 0x2e, 0x6a, 0x2b, 0x2e, 0x10, 0x2e, 0x0e, 0x60, 0x42, 0xd3, 0x68, 0x43, - 0x85, 0x04, 0x08, 0x6b, 0xbc, 0x3b, 0xb1, 0x57, 0x59, 0xcf, 0x2c, 0x3b, 0x63, 0x4b, 0xae, 0x84, - 0xc4, 0x05, 0x0f, 0xd0, 0x77, 0x80, 0x57, 0xe0, 0x1d, 0x7a, 0xc9, 0x25, 0x57, 0x88, 0x9a, 0x17, - 0x41, 0x9e, 0xd9, 0x3f, 0x6d, 0x36, 0x56, 0x2a, 0x71, 0xe7, 0x39, 0xe7, 0x7c, 0xe7, 0xe7, 0x3b, - 0x67, 0xcf, 0x31, 0x94, 0x88, 0xef, 0xd6, 0xfc, 0x80, 0x4b, 0x8e, 0x96, 0x46, 0x47, 0x5d, 0x2a, - 0xc9, 0x91, 0x79, 0xd8, 0x73, 0x65, 0x7f, 0xd8, 0xad, 0xd9, 0x7c, 0x50, 0xef, 0xf1, 0x1e, 0xaf, - 0x2b, 0x7d, 0x77, 0x78, 0xa1, 0x5e, 0xea, 0xa1, 0x7e, 0x69, 0x1c, 0x7e, 0x65, 0xc0, 0x46, 0x8b, - 0x8e, 0x5c, 0x9b, 0x9e, 0x79, 0xc3, 0x9e, 0xcb, 0x9e, 0xfb, 0xd2, 0xe5, 0x4c, 0xa0, 0x03, 0x40, - 0x7e, 0x40, 0x3b, 0x42, 0x92, 0x40, 0x76, 0x02, 0xfa, 0xd3, 0xd0, 0x0d, 0xa8, 0x53, 0x31, 0x76, - 0x8d, 0xfd, 0xa2, 0xb5, 0xe6, 0x07, 0xf4, 0x7c, 0xaa, 0xb0, 0x42, 0x39, 0x3a, 0x01, 0xdc, 0xa3, - 0xb2, 0xe3, 0x07, 0xf4, 0x82, 0x06, 0x01, 0x75, 0x3a, 0xc4, 0xf3, 0xb8, 0x4d, 0xa6, 0xae, 0x3a, - 0x64, 0x44, 0x5c, 0x8f, 0x74, 0x3d, 0x5a, 0x99, 0x57, 0xe8, 0xbb, 0x3d, 0x2a, 0xcf, 0x22, 0xc3, - 0x66, 0x6c, 0xd7, 0x8c, 0xcc, 0xf0, 0xef, 0x06, 0xac, 0x5a, 0xb4, 0xe7, 0x0a, 0x49, 0x83, 0x69, - 0x04, 0x2a, 0x24, 0xaa, 0xc0, 0xd2, 0x88, 0x06, 0xc2, 0xe5, 0x4c, 0xe5, 0x50, 0xb2, 0xa2, 0x27, - 0x32, 0xa1, 0x48, 0x99, 0xe3, 0x73, 0x97, 0x49, 0x15, 0xa0, 0x64, 0xc5, 0x6f, 0xb4, 0x07, 0xb7, - 0x03, 0x2a, 0xf8, 0x30, 0xb0, 0x69, 0x87, 0x91, 0x01, 0xad, 0x14, 0x94, 0xc1, 0x72, 0x24, 0x3c, - 0x25, 0x03, 0x8a, 0x1e, 0xc1, 0x12, 0xd7, 0x45, 0x57, 0x6e, 0xed, 0x1a, 0xfb, 0xe5, 0xc6, 0x9d, - 0x5a, 0xc8, 0x65, 0x2d, 0x87, 0x18, 0x2b, 0x32, 0xc6, 0x4b, 0xb0, 0x70, 0x3c, 0xf0, 0xe5, 0x18, - 0x37, 0x61, 0xf3, 0x6b, 0x57, 0xc8, 0x26, 0x73, 0xbe, 0x25, 0xd2, 0xee, 0x5b, 0x54, 0xf8, 0x9c, - 0x09, 0x8a, 0x3e, 0x80, 0x25, 0x47, 0x39, 0x10, 0x15, 0x63, 0xb7, 0xb0, 0x5f, 0x6e, 0xac, 0x66, - 0x1c, 0x5b, 0x91, 0x1e, 0x3f, 0x86, 0xe5, 0x6f, 0xb8, 0xcf, 0x3d, 0xde, 0x1b, 0xb7, 0xd9, 0x05, - 0x47, 0x0f, 0x60, 0x81, 0x71, 0x27, 0x06, 0xae, 0xc7, 0xc0, 0xd3, 0x17, 0xcf, 0x9a, 0xa7, 0xdc, - 0xa1, 0x96, 0xd6, 0x63, 0x13, 0x8a, 0x91, 0x08, 0xad, 0xc0, 0x7c, 0xbb, 0xa5, 0xe8, 0x29, 0x58, - 0xf3, 0xed, 0x16, 0xb6, 0x61, 0x51, 0xc7, 0x49, 0x69, 0x4a, 0x53, 0x0d, 0xda, 0x82, 0xc5, 0x3e, - 0x25, 0x9e, 0xec, 0x87, 0x8c, 0x85, 0x2f, 0x74, 0x04, 0x45, 0x19, 0xa6, 0xa1, 0xa8, 0x2a, 0x37, - 0xde, 0x89, 0x23, 0xa7, 0xf3, 0xb3, 0x62, 0x33, 0x7c, 0x02, 0x95, 0xb3, 0x70, 0x1a, 0x3e, 0xe3, - 0x4c, 0x12, 0x97, 0x25, 0x4d, 0xab, 0x43, 0x39, 0x2c, 0xb0, 0xe3, 0x3a, 0xba, 0x96, 0xd2, 0xd3, - 0x95, 0xc9, 0xdf, 0x77, 0x41, 0xe7, 0x25, 0xda, 0x2d, 0x61, 0x41, 0x68, 0xd2, 0x76, 0x04, 0xde, - 0x81, 0xed, 0x1c, 0x67, 0x9a, 0x4e, 0x3c, 0x06, 0x33, 0x67, 0x6c, 0xa2, 0x58, 0xdf, 0x03, 0xb2, - 0x23, 0x88, 0x9a, 0x57, 0x2a, 0x64, 0x44, 0xdf, 0x41, 0x5c, 0x44, 0xec, 0xf5, 0x7a, 0x4f, 0xd6, - 0xba, 0x9d, 0xa9, 0x43, 0xe0, 0x3f, 0x0c, 0xd8, 0xbb, 0x01, 0x14, 0xd5, 0x61, 0x23, 0x9e, 0xf6, - 0x8e, 0xae, 0xab, 0xdd, 0x0a, 0x0b, 0xb7, 0x50, 0xac, 0x6a, 0x45, 0x1a, 0xf4, 0x11, 0x6c, 0x0d, - 0x86, 0x42, 0x76, 0x5c, 0x66, 0x7b, 0x43, 0x27, 0x8d, 0x99, 0x57, 0x98, 0xcd, 0xa9, 0xb6, 0xad, - 0x95, 0x09, 0xea, 0x01, 0xac, 0xa6, 0xbe, 0x2f, 0xe1, 0xbe, 0xd4, 0x83, 0xbd, 0x60, 0xad, 0x24, - 0xe2, 0x73, 0xf7, 0x25, 0xc5, 0x3f, 0xc3, 0x4e, 0x6e, 0xb6, 0xe1, 0x80, 0xfe, 0x08, 0x1b, 0x69, - 0xce, 0xb4, 0x34, 0x22, 0xed, 0xf0, 0x86, 0xa4, 0x69, 0x94, 0x85, 0xec, 0x6c, 0xc3, 0x04, 0x6e, - 0xc1, 0xfd, 0x9b, 0x60, 0xd1, 0x1d, 0x28, 0x65, 0xc9, 0x4a, 0x04, 0xd8, 0x86, 0xd5, 0x10, 0x43, - 0x23, 0x9e, 0xcf, 0x66, 0x34, 0xfb, 0xde, 0xd5, 0xbc, 0x33, 0xf0, 0xbc, 0x0e, 0x9f, 0x40, 0xe5, - 0x3a, 0xf3, 0xb7, 0x1f, 0xe3, 0x1e, 0xac, 0x25, 0x3e, 0xc2, 0x1a, 0xcf, 0x67, 0x71, 0x8d, 0x67, - 0xe5, 0x3c, 0x83, 0xe0, 0x5f, 0x0b, 0xb0, 0x7d, 0x2d, 0x02, 0x7d, 0x0a, 0xb7, 0x28, 0x1b, 0xcd, - 0xf8, 0x08, 0xb2, 0x88, 0xda, 0x31, 0x1b, 0x89, 0x63, 0x26, 0x83, 0xb1, 0xa5, 0x90, 0xe8, 0x7d, - 0x58, 0x1c, 0xf0, 0x21, 0x93, 0x7a, 0x1c, 0xcb, 0x8d, 0x95, 0xd8, 0xc7, 0xb3, 0xa9, 0xd8, 0x0a, - 0xb5, 0xe8, 0x30, 0xd9, 0x74, 0x05, 0x65, 0xb8, 0x91, 0xd9, 0x74, 0xe7, 0x3e, 0xb5, 0xe3, 0x6d, - 0x87, 0x5e, 0x40, 0x99, 0x30, 0xc6, 0x25, 0x89, 0xb6, 0xee, 0x14, 0xf2, 0xf0, 0x06, 0xf9, 0x35, - 0x13, 0x94, 0x4e, 0x33, 0xed, 0xc7, 0x7c, 0x0c, 0xa5, 0xb8, 0x00, 0xb4, 0x06, 0x85, 0x4b, 0x3a, - 0x0e, 0x77, 0xde, 0xf4, 0x27, 0xda, 0x84, 0x85, 0x11, 0xf1, 0x86, 0x34, 0xdc, 0x79, 0xfa, 0xf1, - 0xf1, 0xfc, 0x13, 0xc3, 0xfc, 0x04, 0xd6, 0xb2, 0x9e, 0xdf, 0x06, 0x8f, 0xfb, 0xb0, 0xa0, 0xf8, - 0x40, 0xef, 0xc1, 0x4a, 0xd2, 0x64, 0x9f, 0xc8, 0x7e, 0x88, 0xbf, 0x1d, 0x4b, 0xcf, 0x88, 0xec, - 0xa3, 0x1d, 0x28, 0xf5, 0xb9, 0x90, 0xda, 0x22, 0xbc, 0x59, 0x53, 0x41, 0xa4, 0x0c, 0x28, 0x71, - 0x3a, 0x9c, 0x79, 0x7a, 0x09, 0x17, 0xad, 0xe2, 0x54, 0xf0, 0x9c, 0x79, 0x63, 0x1c, 0x00, 0x24, - 0x84, 0xfe, 0x2f, 0xe1, 0x76, 0xa1, 0xec, 0xd3, 0x60, 0xe0, 0x0a, 0xa1, 0x7a, 0xa1, 0x0f, 0x64, - 0x5a, 0xd4, 0xf8, 0x1c, 0x96, 0xf5, 0x35, 0x0e, 0x14, 0x3f, 0xe8, 0x11, 0x14, 0xa3, 0xeb, 0x8c, - 0x2a, 0x71, 0xd3, 0x32, 0x07, 0xdb, 0x4c, 0x46, 0x45, 0x1f, 0xc9, 0xb9, 0xc6, 0x6f, 0x05, 0x58, - 0x4e, 0x1f, 0x54, 0xf4, 0x25, 0x6c, 0x7d, 0x41, 0x65, 0xde, 0x9f, 0x8f, 0x0c, 0xd8, 0x9c, 0x79, - 0x91, 0xf1, 0x1c, 0x6a, 0xc2, 0x72, 0xfa, 0x02, 0x5f, 0xc1, 0xbf, 0x1b, 0xbf, 0xf3, 0x0e, 0x35, - 0x9e, 0xfb, 0xd0, 0x40, 0x54, 0x25, 0x93, 0xb3, 0xa5, 0xd0, 0x5e, 0x0c, 0xbe, 0x7e, 0xf3, 0x9b, - 0xf7, 0x67, 0x1b, 0x45, 0x81, 0x50, 0x13, 0x8a, 0xd1, 0x54, 0xa7, 0xc8, 0xcb, 0x6c, 0x1c, 0x73, - 0x3b, 0x47, 0x13, 0xbb, 0xf8, 0x01, 0xd6, 0xaf, 0x1c, 0x49, 0x74, 0x2f, 0x1d, 0x3f, 0xf7, 0x1a, - 0x9b, 0x78, 0x96, 0x49, 0xe4, 0xfd, 0xe9, 0x57, 0xaf, 0xdf, 0x54, 0x8d, 0xbf, 0xde, 0x54, 0xe7, - 0x7e, 0x99, 0x54, 0x8d, 0xd7, 0x93, 0xaa, 0xf1, 0xe7, 0xa4, 0x6a, 0xfc, 0x33, 0xa9, 0x1a, 0xaf, - 0xfe, 0xad, 0xce, 0x7d, 0x77, 0x70, 0xf9, 0x44, 0xd4, 0x5c, 0x5e, 0xbf, 0x1c, 0x76, 0xa9, 0x47, - 0x65, 0xdd, 0xbf, 0xec, 0xd5, 0x89, 0xef, 0x8a, 0xba, 0xfe, 0xb4, 0x7d, 0xd5, 0x97, 0x7a, 0x18, - 0xa7, 0xbb, 0xa8, 0xfe, 0x62, 0x3e, 0xfc, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xe2, 0xf2, 0x09, 0x79, - 0xa7, 0x0a, 0x00, 0x00, + // 1088 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xef, 0x6e, 0x1b, 0x45, + 0x10, 0xcf, 0xc5, 0x71, 0x62, 0x8f, 0xd3, 0xfc, 0xd9, 0x84, 0xc8, 0x71, 0x8a, 0x93, 0x6e, 0x0a, + 0x0d, 0x52, 0x62, 0x13, 0x17, 0xb5, 0x15, 0x1f, 0x10, 0x6e, 0x1c, 0xc0, 0x84, 0xa6, 0xd1, 0x85, + 0x0a, 0x09, 0x10, 0xa7, 0xf3, 0xdd, 0xc6, 0x3e, 0xe5, 0xbc, 0x7b, 0xdc, 0xae, 0x2d, 0xb9, 0x12, + 0x12, 0x8f, 0xd0, 0x77, 0x80, 0x57, 0xe0, 0x1d, 0xca, 0x37, 0x3e, 0xf2, 0x09, 0xd1, 0xf0, 0x22, + 0xc8, 0xbb, 0xb7, 0x77, 0xa7, 0x8b, 0x63, 0xa5, 0x52, 0xbf, 0xdd, 0xce, 0xcc, 0x6f, 0x66, 0xf6, + 0x37, 0xb3, 0x33, 0x07, 0x45, 0x3b, 0xf0, 0x6a, 0x41, 0xc8, 0x04, 0x43, 0x0b, 0xc3, 0xc3, 0x0e, + 0x11, 0xf6, 0x61, 0xe5, 0xa0, 0xeb, 0x89, 0xde, 0xa0, 0x53, 0x73, 0x58, 0xbf, 0xde, 0x65, 0x5d, + 0x56, 0x97, 0xfa, 0xce, 0xe0, 0x42, 0x9e, 0xe4, 0x41, 0x7e, 0x29, 0x1c, 0x7e, 0x65, 0xc0, 0x5a, + 0x8b, 0x0c, 0x3d, 0x87, 0x9c, 0xf9, 0x83, 0xae, 0x47, 0x9f, 0x07, 0xc2, 0x63, 0x94, 0xa3, 0x7d, + 0x40, 0x41, 0x48, 0x2c, 0x2e, 0xec, 0x50, 0x58, 0x21, 0xf9, 0x79, 0xe0, 0x85, 0xc4, 0x2d, 0x1b, + 0x3b, 0xc6, 0x5e, 0xc1, 0x5c, 0x09, 0x42, 0x72, 0x3e, 0x56, 0x98, 0x91, 0x1c, 0x9d, 0x00, 0xee, + 0x12, 0x61, 0x05, 0x21, 0xb9, 0x20, 0x61, 0x48, 0x5c, 0xcb, 0xf6, 0x7d, 0xe6, 0xd8, 0x63, 0x57, + 0x96, 0x3d, 0xb4, 0x3d, 0xdf, 0xee, 0xf8, 0xa4, 0x3c, 0x2b, 0xd1, 0xdb, 0x5d, 0x22, 0xce, 0xb4, + 0x61, 0x33, 0xb6, 0x6b, 0x6a, 0x33, 0xfc, 0xbb, 0x01, 0xcb, 0x26, 0xe9, 0x7a, 0x5c, 0x90, 0x70, + 0x1c, 0x81, 0x70, 0x81, 0xca, 0xb0, 0x30, 0x24, 0x21, 0xf7, 0x18, 0x95, 0x39, 0x14, 0x4d, 0x7d, + 0x44, 0x15, 0x28, 0x10, 0xea, 0x06, 0xcc, 0xa3, 0x42, 0x06, 0x28, 0x9a, 0xf1, 0x19, 0xed, 0xc2, + 0x9d, 0x90, 0x70, 0x36, 0x08, 0x1d, 0x62, 0x51, 0xbb, 0x4f, 0xca, 0x39, 0x69, 0xb0, 0xa8, 0x85, + 0xa7, 0x76, 0x9f, 0xa0, 0x47, 0xb0, 0xc0, 0xd4, 0xa5, 0xcb, 0x73, 0x3b, 0xc6, 0x5e, 0xa9, 0x71, + 0xb7, 0x16, 0x71, 0x59, 0x9b, 0x40, 0x8c, 0xa9, 0x8d, 0xf1, 0x02, 0xe4, 0x8f, 0xfb, 0x81, 0x18, + 0xe1, 0x26, 0xac, 0x7f, 0xe3, 0x71, 0xd1, 0xa4, 0xee, 0x77, 0xb6, 0x70, 0x7a, 0x26, 0xe1, 0x01, + 0xa3, 0x9c, 0xa0, 0x8f, 0x60, 0xc1, 0x95, 0x0e, 0x78, 0xd9, 0xd8, 0xc9, 0xed, 0x95, 0x1a, 0xcb, + 0x19, 0xc7, 0xa6, 0xd6, 0xe3, 0xc7, 0xb0, 0xf8, 0x2d, 0x0b, 0x98, 0xcf, 0xba, 0xa3, 0x36, 0xbd, + 0x60, 0xe8, 0x01, 0xe4, 0x29, 0x73, 0x63, 0xe0, 0x6a, 0x0c, 0x3c, 0x7d, 0xf1, 0xac, 0x79, 0xca, + 0x5c, 0x62, 0x2a, 0x3d, 0xae, 0x40, 0x41, 0x8b, 0xd0, 0x12, 0xcc, 0xb6, 0x5b, 0x92, 0x9e, 0x9c, + 0x39, 0xdb, 0x6e, 0x61, 0x07, 0xe6, 0x55, 0x9c, 0x94, 0xa6, 0x38, 0xd6, 0xa0, 0x0d, 0x98, 0xef, + 0x11, 0xdb, 0x17, 0xbd, 0x88, 0xb1, 0xe8, 0x84, 0x0e, 0xa1, 0x20, 0xa2, 0x34, 0x24, 0x55, 0xa5, + 0xc6, 0x7b, 0x71, 0xe4, 0x74, 0x7e, 0x66, 0x6c, 0x86, 0x4f, 0xa0, 0x7c, 0x16, 0x75, 0xc3, 0x11, + 0xa3, 0xc2, 0xf6, 0x68, 0x52, 0xb4, 0x3a, 0x94, 0xa2, 0x0b, 0x5a, 0x9e, 0xab, 0xee, 0x52, 0x7c, + 0xba, 0x74, 0xf5, 0xcf, 0x36, 0xa8, 0xbc, 0x78, 0xbb, 0xc5, 0x4d, 0x88, 0x4c, 0xda, 0x2e, 0xc7, + 0x5b, 0xb0, 0x39, 0xc1, 0x99, 0xa2, 0x13, 0x8f, 0xa0, 0x32, 0xa1, 0x6d, 0x74, 0xac, 0x1f, 0x00, + 0x39, 0x1a, 0x22, 0xfb, 0x95, 0x70, 0xa1, 0xe9, 0xdb, 0x8f, 0x2f, 0x11, 0x7b, 0xbd, 0xd9, 0x93, + 0xb9, 0xea, 0x64, 0xee, 0xc1, 0xf1, 0x1f, 0x06, 0xec, 0xde, 0x02, 0x8a, 0xea, 0xb0, 0x16, 0x77, + 0xbb, 0xa5, 0xee, 0xd5, 0x6e, 0x45, 0x17, 0x37, 0x51, 0xac, 0x6a, 0x69, 0x0d, 0xfa, 0x04, 0x36, + 0xfa, 0x03, 0x2e, 0x2c, 0x8f, 0x3a, 0xfe, 0xc0, 0x4d, 0x63, 0x66, 0x25, 0x66, 0x7d, 0xac, 0x6d, + 0x2b, 0x65, 0x82, 0x7a, 0x00, 0xcb, 0xa9, 0xf7, 0xc5, 0xbd, 0x97, 0xaa, 0xb1, 0xf3, 0xe6, 0x52, + 0x22, 0x3e, 0xf7, 0x5e, 0x12, 0xfc, 0x0b, 0x6c, 0x4d, 0xcc, 0x36, 0x6a, 0xd0, 0x9f, 0x60, 0x2d, + 0xcd, 0x99, 0x92, 0x6a, 0xd2, 0x0e, 0x6e, 0x49, 0x9a, 0x42, 0x99, 0xc8, 0xc9, 0x16, 0x8c, 0xe3, + 0x16, 0xdc, 0xbf, 0x0d, 0x16, 0xdd, 0x85, 0x62, 0x96, 0xac, 0x44, 0x80, 0x1d, 0x58, 0x8e, 0x30, + 0x44, 0xf3, 0x7c, 0x36, 0xa5, 0xd8, 0xf7, 0xae, 0xe7, 0x9d, 0x81, 0x4f, 0xaa, 0xf0, 0x09, 0x94, + 0x6f, 0x32, 0x7f, 0xfb, 0x36, 0xde, 0x86, 0xe2, 0x51, 0xab, 0x1d, 0xbd, 0x3d, 0x04, 0x73, 0x72, + 0xf4, 0xa8, 0xd7, 0x27, 0xbf, 0x71, 0x17, 0x56, 0x92, 0x20, 0x11, 0x09, 0xe7, 0xd3, 0x8a, 0x81, + 0xa7, 0x5d, 0x6a, 0x4a, 0x05, 0xfe, 0xcc, 0xc1, 0xe6, 0x8d, 0x08, 0xf4, 0x39, 0xcc, 0x11, 0x3a, + 0x9c, 0xf2, 0x4a, 0xb2, 0x88, 0xda, 0x31, 0x1d, 0xf2, 0x63, 0x2a, 0xc2, 0x91, 0x29, 0x91, 0xe8, + 0x43, 0x98, 0xef, 0xb3, 0x01, 0x15, 0xaa, 0x5f, 0x4b, 0x8d, 0xa5, 0xd8, 0xc7, 0xb3, 0xb1, 0xd8, + 0x8c, 0xb4, 0xe8, 0x20, 0x19, 0x85, 0x39, 0x69, 0xb8, 0x96, 0x19, 0x85, 0xe7, 0x01, 0x71, 0xe2, + 0x71, 0x88, 0x5e, 0x40, 0xc9, 0xa6, 0x94, 0x09, 0x5b, 0x8f, 0xe5, 0x31, 0xe4, 0xe1, 0x2d, 0xf2, + 0x6b, 0x26, 0x28, 0x95, 0x66, 0xda, 0x0f, 0x3a, 0x82, 0x92, 0xe3, 0x7a, 0x96, 0xce, 0x24, 0x2f, + 0xdd, 0xa2, 0xc4, 0xad, 0xae, 0x99, 0x2a, 0x6e, 0x7c, 0xe4, 0x26, 0x38, 0xae, 0x17, 0x7d, 0x57, + 0x1e, 0x43, 0x31, 0x66, 0x01, 0xad, 0x40, 0xee, 0x92, 0x8c, 0xa2, 0xda, 0x8e, 0x3f, 0xd1, 0x3a, + 0xe4, 0x87, 0xb6, 0x3f, 0x20, 0xd1, 0x64, 0x55, 0x87, 0x4f, 0x67, 0x9f, 0x18, 0x95, 0xcf, 0x60, + 0x25, 0x9b, 0xde, 0xdb, 0xe0, 0x71, 0x0f, 0xf2, 0x92, 0x54, 0xf4, 0x01, 0x2c, 0x25, 0x9d, 0x12, + 0xd8, 0xa2, 0x17, 0xe1, 0xef, 0xc4, 0xd2, 0x33, 0x5b, 0xf4, 0xd0, 0x16, 0x14, 0x7b, 0x8c, 0x0b, + 0x65, 0x11, 0x6d, 0xc6, 0xb1, 0x40, 0x2b, 0x43, 0x62, 0xbb, 0x16, 0xa3, 0xbe, 0x1a, 0xf5, 0x05, + 0xb3, 0x30, 0x16, 0x3c, 0xa7, 0xfe, 0x08, 0x87, 0x00, 0x49, 0x55, 0xde, 0x49, 0xb8, 0x1d, 0x28, + 0x05, 0x24, 0xec, 0x7b, 0x9c, 0xcb, 0x82, 0xaa, 0x35, 0x9c, 0x16, 0x35, 0xbe, 0x80, 0x45, 0xb5, + 0xf3, 0x43, 0xc9, 0x0f, 0x7a, 0x04, 0x05, 0xfd, 0x0f, 0x80, 0xca, 0x71, 0x89, 0x32, 0xbf, 0x05, + 0x95, 0xa4, 0xdf, 0xd4, 0x2a, 0x9e, 0x69, 0xfc, 0x96, 0x83, 0xc5, 0xf4, 0xda, 0x46, 0x5f, 0xc1, + 0xc6, 0x97, 0x44, 0x4c, 0xfa, 0xc5, 0xc9, 0x80, 0x2b, 0x53, 0xf7, 0x3e, 0x9e, 0x41, 0x4d, 0x58, + 0x4c, 0xef, 0xf9, 0x6b, 0xf8, 0xf7, 0xe3, 0xf3, 0xa4, 0xdf, 0x01, 0x3c, 0xf3, 0xb1, 0x81, 0x88, + 0x4c, 0x66, 0xc2, 0x2c, 0x44, 0xbb, 0x31, 0xf8, 0xe6, 0xfd, 0x52, 0xb9, 0x3f, 0xdd, 0x48, 0x07, + 0x42, 0x4d, 0x28, 0xe8, 0xa7, 0x91, 0x22, 0x2f, 0x33, 0xd7, 0x2a, 0x9b, 0x13, 0x34, 0xb1, 0x8b, + 0x1f, 0x61, 0xf5, 0xda, 0x2a, 0x46, 0xf7, 0xd2, 0xf1, 0x27, 0xee, 0xfc, 0x0a, 0x9e, 0x66, 0xa2, + 0xbd, 0x3f, 0xfd, 0xfa, 0xf5, 0x9b, 0xaa, 0xf1, 0xf7, 0x9b, 0xea, 0xcc, 0xaf, 0x57, 0x55, 0xe3, + 0xf5, 0x55, 0xd5, 0xf8, 0xeb, 0xaa, 0x6a, 0xfc, 0x7b, 0x55, 0x35, 0x5e, 0xfd, 0x57, 0x9d, 0xf9, + 0x7e, 0xff, 0xf2, 0x09, 0xaf, 0x79, 0xac, 0x7e, 0x39, 0xe8, 0x10, 0x9f, 0x88, 0x7a, 0x70, 0xd9, + 0xad, 0xdb, 0x81, 0xc7, 0xeb, 0xea, 0xe5, 0x06, 0xb2, 0x2e, 0xf5, 0x28, 0x4e, 0x67, 0x5e, 0xfe, + 0xc8, 0x3e, 0xfc, 0x3f, 0x00, 0x00, 0xff, 0xff, 0xe0, 0x26, 0x52, 0x96, 0x0d, 0x0b, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2075,6 +2137,36 @@ func (m *ContainerAllocateRequest) MarshalToSizedBuffer(dAtA []byte) (int, error return len(dAtA) - i, nil } +func (m *CDIDevice) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CDIDevice) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CDIDevice) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintApi(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *AllocateResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2132,6 +2224,20 @@ func (m *ContainerAllocateResponse) MarshalToSizedBuffer(dAtA []byte) (int, erro _ = i var l int _ = l + if len(m.CDIDevices) > 0 { + for iNdEx := len(m.CDIDevices) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.CDIDevices[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintApi(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } + } if len(m.Annotations) > 0 { for k := range m.Annotations { v := m.Annotations[k] @@ -2538,6 +2644,19 @@ func (m *ContainerAllocateRequest) Size() (n int) { return n } +func (m *CDIDevice) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + return n +} + func (m *AllocateResponse) Size() (n int) { if m == nil { return 0 @@ -2587,6 +2706,12 @@ func (m *ContainerAllocateResponse) Size() (n int) { n += mapEntrySize + 1 + sovApi(uint64(mapEntrySize)) } } + if len(m.CDIDevices) > 0 { + for _, e := range m.CDIDevices { + l = e.Size() + n += 1 + l + sovApi(uint64(l)) + } + } return n } @@ -2818,6 +2943,16 @@ func (this *ContainerAllocateRequest) String() string { }, "") return s } +func (this *CDIDevice) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&CDIDevice{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `}`, + }, "") + return s +} func (this *AllocateResponse) String() string { if this == nil { return "nil" @@ -2847,6 +2982,11 @@ func (this *ContainerAllocateResponse) String() string { repeatedStringForDevices += strings.Replace(f.String(), "DeviceSpec", "DeviceSpec", 1) + "," } repeatedStringForDevices += "}" + repeatedStringForCDIDevices := "[]*CDIDevice{" + for _, f := range this.CDIDevices { + repeatedStringForCDIDevices += strings.Replace(f.String(), "CDIDevice", "CDIDevice", 1) + "," + } + repeatedStringForCDIDevices += "}" keysForEnvs := make([]string, 0, len(this.Envs)) for k := range this.Envs { keysForEnvs = append(keysForEnvs, k) @@ -2872,6 +3012,7 @@ func (this *ContainerAllocateResponse) String() string { `Mounts:` + repeatedStringForMounts + `,`, `Devices:` + repeatedStringForDevices + `,`, `Annotations:` + mapStringForAnnotations + `,`, + `CDIDevices:` + repeatedStringForCDIDevices + `,`, `}`, }, "") return s @@ -4298,6 +4439,88 @@ func (m *ContainerAllocateRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *CDIDevice) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CDIDevice: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CDIDevice: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *AllocateResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -4733,6 +4956,40 @@ func (m *ContainerAllocateResponse) Unmarshal(dAtA []byte) error { } m.Annotations[mapkey] = mapvalue iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CDIDevices", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CDIDevices = append(m.CDIDevices, &CDIDevice{}) + if err := m.CDIDevices[len(m.CDIDevices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipApi(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.proto b/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.proto index f44e7cebd2d..97019bd5b93 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.proto +++ b/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.proto @@ -164,6 +164,15 @@ message ContainerAllocateRequest { repeated string devices_ids = 1 [(gogoproto.customname) = "DevicesIDs"]; } +// CDIDevice specifies a CDI device information. +message CDIDevice { + // Fully qualified CDI device name + // for example: vendor.com/gpu=gpudevice1 + // see more details in the CDI specification: + // https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md + string name = 1; +} + // AllocateResponse includes the artifacts that needs to be injected into // a container for accessing 'deviceIDs' that were mentioned as part of // 'AllocateRequest'. @@ -185,6 +194,8 @@ message ContainerAllocateResponse { repeated DeviceSpec devices = 3; // Container annotations to pass to the container runtime map annotations = 4; + // CDI devices for the container. + repeated CDIDevice cdi_devices = 5 [(gogoproto.customname) = "CDIDevices"]; } // Mount specifies a host volume to mount into a container.