From b57c7e2fe4bb466ff1614aa9df7cc164e90b24b6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 16 May 2023 15:12:32 +0200 Subject: [PATCH] Add CDI devices to device plugin API This change adds CDI device IDs to the ContainerAllocateResponse in the device plugin API. This allows a device plugin to specify CDI devices by their unique fully-qualified CDI device names using the related field in the CRI specification. Signed-off-by: Evan Lezar --- pkg/features/kube_features.go | 9 + pkg/kubelet/cm/container_manager_linux.go | 1 + pkg/kubelet/cm/devicemanager/manager_test.go | 25 +- pkg/kubelet/cm/devicemanager/pod_devices.go | 75 ++++ .../cm/devicemanager/pod_devices_test.go | 139 ++++++ pkg/kubelet/cm/devicemanager/types.go | 2 + .../pkg/apis/deviceplugin/v1beta1/api.pb.go | 405 ++++++++++++++---- .../pkg/apis/deviceplugin/v1beta1/api.proto | 11 + 8 files changed, 590 insertions(+), 77 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index cb2f976cb2b..b79fa2d5c63 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -221,6 +221,13 @@ const ( // (e.g. in a Deployment), which is the historical default. DefaultHostNetworkHostPortsInPodTemplates featuregate.Feature = "DefaultHostNetworkHostPortsInPodTemplates" + // owner: @elezar + // kep: http://kep.k8s.io/4009 + // alpha: v1.28 + // + // Add support for CDI Device IDs in the Device Plugin API. + DevicePluginCDIDevices featuregate.Feature = "DevicePluginCDIDevices" + // owner: @andrewsykim // alpha: v1.22 // @@ -908,6 +915,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS DisableKubeletCloudCredentialProviders: {Default: false, PreRelease: featuregate.Alpha}, + DevicePluginCDIDevices: {Default: false, PreRelease: featuregate.Alpha}, + DownwardAPIHugePages: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in v1.29 DynamicResourceAllocation: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 02cb34ddcdc..c4a19371566 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -673,6 +673,7 @@ func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Containe opts.Mounts = append(opts.Mounts, devOpts.Mounts...) opts.Envs = append(opts.Envs, devOpts.Envs...) opts.Annotations = append(opts.Annotations, devOpts.Annotations...) + opts.CDIDevices = append(opts.CDIDevices, devOpts.CDIDevices...) return opts, nil } diff --git a/pkg/kubelet/cm/devicemanager/manager_test.go b/pkg/kubelet/cm/devicemanager/manager_test.go index f50e82f69fd..a46722e26a1 100644 --- a/pkg/kubelet/cm/devicemanager/manager_test.go +++ b/pkg/kubelet/cm/devicemanager/manager_test.go @@ -573,9 +573,10 @@ func constructDevices(devices []string) checkpoint.DevicesPerNUMA { // containerAllocateResponseBuilder is a helper to build a ContainerAllocateResponse type containerAllocateResponseBuilder struct { - devices map[string]string - mounts map[string]string - envs map[string]string + devices map[string]string + mounts map[string]string + envs map[string]string + cdiDevices []string } // containerAllocateResponseBuilderOption defines a functional option for a containerAllocateResponseBuilder @@ -602,6 +603,13 @@ func withEnvs(envs map[string]string) containerAllocateResponseBuilderOption { } } +// withCDIDevices sets the cdiDevices for the containerAllocateResponseBuilder +func withCDIDevices(cdiDevices ...string) containerAllocateResponseBuilderOption { + return func(b *containerAllocateResponseBuilder) { + b.cdiDevices = cdiDevices + } +} + // newContainerAllocateResponse creates a ContainerAllocateResponse with the given options. func newContainerAllocateResponse(opts ...containerAllocateResponseBuilderOption) *pluginapi.ContainerAllocateResponse { b := &containerAllocateResponseBuilder{} @@ -633,6 +641,16 @@ func (b *containerAllocateResponseBuilder) Build() *pluginapi.ContainerAllocateR for k, v := range b.envs { resp.Envs[k] = v } + + var cdiDevices []*pluginapi.CDIDevice + for _, dev := range b.cdiDevices { + cdiDevice := pluginapi.CDIDevice{ + Name: dev, + } + cdiDevices = append(cdiDevices, &cdiDevice) + } + resp.CDIDevices = cdiDevices + return resp } @@ -660,6 +678,7 @@ func TestCheckpoint(t *testing.T) { newContainerAllocateResponse( withDevices(map[string]string{"/dev/r1dev1": "/dev/r1dev1", "/dev/r1dev2": "/dev/r1dev2"}), withMounts(map[string]string{"/home/r1lib1": "/usr/r1lib1"}), + withCDIDevices("domain1.com/resource1=dev1", "domain1.com/resource1=dev2"), ), ) testManager.podDevices.insert("pod1", "con1", resourceName2, diff --git a/pkg/kubelet/cm/devicemanager/pod_devices.go b/pkg/kubelet/cm/devicemanager/pod_devices.go index 7a12e8de813..fe4eb65e405 100644 --- a/pkg/kubelet/cm/devicemanager/pod_devices.go +++ b/pkg/kubelet/cm/devicemanager/pod_devices.go @@ -21,9 +21,13 @@ import ( "k8s.io/klog/v2" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" + "k8s.io/kubernetes/pkg/kubelet/cm/util/cdi" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -244,6 +248,8 @@ func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *Devi mountsMap := make(map[string]string) envsMap := make(map[string]string) annotationsMap := make(map[string]string) + // Keep track of all CDI devices requested for the container. + allCDIDevices := sets.New[string]() // Loops through AllocationResponses of all cached device resources. for _, devices := range resources { resp := devices.allocResp @@ -252,6 +258,7 @@ func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *Devi // Mount points // Device files // Container annotations + // CDI device IDs // These artifacts are per resource per container. // Updates RunContainerOptions.Envs. for k, v := range resp.Envs { @@ -321,10 +328,78 @@ func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *Devi annotationsMap[k] = v opts.Annotations = append(opts.Annotations, kubecontainer.Annotation{Name: k, Value: v}) } + + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DevicePluginCDIDevices) { + // Updates for CDI devices. + cdiDevices := getCDIDeviceInfo(resp, allCDIDevices) + opts.CDIDevices = append(opts.CDIDevices, cdiDevices...) + } } + + // Although the CDI devices are expected to be empty when this feature is disabled, we still + // guard this with a feature gate to avoid any potential issues. + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DevicePluginCDIDevices) { + // We construct a resource ID from the pod UID and container name. + // This ID has no semantic meaning, and is only used to ensure that the generated CDI annotation key is unique + // for a given container. Since this is only called once per pod-container combination, this should be the case. + resourceID := podUID + "-" + contName + cdiAnnotations := getCDIAnnotations(resourceID, allCDIDevices, annotationsMap) + opts.Annotations = append(opts.Annotations, cdiAnnotations...) + } + return opts } +// getCDIAnnotations returns the cdi annotations for a given container. +// This creates a CDI annotation with a key of the form: devicemanager_{{resourceID}}. +// The value of the annotation is a comma separated list of sorted CDI device IDs. +// If the annotation key is already defined in the provided annotations map, then the existing value is used. +func getCDIAnnotations(resourceID string, cdiDevices sets.Set[string], annotationsMap map[string]string) []kubecontainer.Annotation { + // We sort the CDI devices to ensure that the annotation value is deterministic. + sortedCDIDevices := sets.List[string](cdiDevices) + annotations, err := cdi.GenerateAnnotations(types.UID(resourceID), "devicemanager", sortedCDIDevices) + if err != nil { + klog.ErrorS(err, "Failed to create CDI annotations") + return nil + } + + var cdiAnnotations []kubecontainer.Annotation + for _, annotation := range annotations { + if e, ok := annotationsMap[annotation.Name]; ok { + klog.V(4).InfoS("Skip existing annotation", "annotationKey", annotation.Name, "annotationValue", annotation.Value) + if e != annotation.Value { + klog.ErrorS(nil, "Annotation has conflicting setting", "annotationKey", annotation.Name, "expected", e, "got", annotation.Value) + } + continue + } + klog.V(4).InfoS("Add annotation", "annotationKey", annotation.Name, "annotationValue", annotation.Value) + annotationsMap[annotation.Name] = annotation.Value + cdiAnnotations = append(cdiAnnotations, kubecontainer.Annotation{Name: annotation.Name, Value: annotation.Value}) + } + + return cdiAnnotations +} + +// getCDIDeviceInfo returns CDI devices from an allocate response +func getCDIDeviceInfo(resp *pluginapi.ContainerAllocateResponse, knownCDIDevices sets.Set[string]) []kubecontainer.CDIDevice { + var cdiDevices []kubecontainer.CDIDevice + for _, cdiDevice := range resp.CDIDevices { + if knownCDIDevices.Has(cdiDevice.Name) { + klog.V(4).InfoS("Skip existing CDI Device", "name", cdiDevice.Name) + continue + } + klog.V(4).InfoS("Add CDI device", "name", cdiDevice.Name) + knownCDIDevices.Insert(cdiDevice.Name) + + device := kubecontainer.CDIDevice{ + Name: cdiDevice.Name, + } + cdiDevices = append(cdiDevices, device) + } + + return cdiDevices +} + // getContainerDevices returns the devices assigned to the provided container for all ResourceNames func (pdev *podDevices) getContainerDevices(podUID, contName string) ResourceDeviceInstances { pdev.RLock() diff --git a/pkg/kubelet/cm/devicemanager/pod_devices_test.go b/pkg/kubelet/cm/devicemanager/pod_devices_test.go index 15fc07a8dab..70d5f6e6dea 100644 --- a/pkg/kubelet/cm/devicemanager/pod_devices_test.go +++ b/pkg/kubelet/cm/devicemanager/pod_devices_test.go @@ -20,11 +20,16 @@ import ( "encoding/json" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) func TestGetContainerDevices(t *testing.T) { @@ -153,3 +158,137 @@ func expectResourceDeviceInstances(t *testing.T, resp ResourceDeviceInstances, e t.Errorf("expected %q got %q", expected, got) } } + +func TestDeviceRunContainerOptions(t *testing.T) { + const ( + podUID = "pod" + containerName = "container" + resource1 = "example1.com/resource1" + resource2 = "example2.com/resource2" + ) + testCases := []struct { + description string + gate bool + responsesPerResource map[string]*pluginapi.ContainerAllocateResponse + expected *DeviceRunContainerOptions + }{ + { + description: "empty response", + gate: false, + responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{ + resource1: newContainerAllocateResponse(), + }, + expected: &DeviceRunContainerOptions{}, + }, + { + description: "cdi devices are ingored when feature gate is disabled", + gate: false, + responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{ + resource1: newContainerAllocateResponse( + withDevices(map[string]string{"/dev/r1": "/dev/r1"}), + withMounts(map[string]string{"/home/lib1": "/home/lib1"}), + withEnvs(map[string]string{"ENV1": "VALUE1"}), + withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"), + ), + }, + expected: &DeviceRunContainerOptions{ + Devices: []kubecontainer.DeviceInfo{ + {PathOnHost: "/dev/r1", PathInContainer: "/dev/r1", Permissions: "mrw"}, + }, + Mounts: []kubecontainer.Mount{ + {Name: "/home/lib1", HostPath: "/home/lib1", ContainerPath: "/home/lib1", ReadOnly: true}, + }, + Envs: []kubecontainer.EnvVar{ + {Name: "ENV1", Value: "VALUE1"}, + }, + }, + }, + { + description: "cdi devices are handled when feature gate is enabled", + gate: true, + responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{ + resource1: newContainerAllocateResponse( + withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"), + ), + }, + expected: &DeviceRunContainerOptions{ + Annotations: []kubecontainer.Annotation{ + {Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2"}, + }, + CDIDevices: []kubecontainer.CDIDevice{ + {Name: "vendor1.com/class1=device1"}, + {Name: "vendor2.com/class2=device2"}, + }, + }, + }, + { + description: "cdi devices from multiple resources are handled when feature gate is enabled", + gate: true, + responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{ + resource1: newContainerAllocateResponse( + withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"), + ), + resource2: newContainerAllocateResponse( + withCDIDevices("vendor3.com/class3=device3", "vendor4.com/class4=device4"), + ), + }, + expected: &DeviceRunContainerOptions{ + Annotations: []kubecontainer.Annotation{ + {Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2,vendor3.com/class3=device3,vendor4.com/class4=device4"}, + }, + CDIDevices: []kubecontainer.CDIDevice{ + {Name: "vendor1.com/class1=device1"}, + {Name: "vendor2.com/class2=device2"}, + {Name: "vendor3.com/class3=device3"}, + {Name: "vendor4.com/class4=device4"}, + }, + }, + }, + { + description: "duplicate cdi devices are skipped", + gate: true, + responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{ + resource1: newContainerAllocateResponse( + withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"), + ), + resource2: newContainerAllocateResponse( + withCDIDevices("vendor2.com/class2=device2", "vendor3.com/class3=device3"), + ), + }, + expected: &DeviceRunContainerOptions{ + Annotations: []kubecontainer.Annotation{ + {Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2,vendor3.com/class3=device3"}, + }, + CDIDevices: []kubecontainer.CDIDevice{ + {Name: "vendor1.com/class1=device1"}, + {Name: "vendor2.com/class2=device2"}, + {Name: "vendor3.com/class3=device3"}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + as := assert.New(t) + + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DevicePluginCDIDevices, tc.gate)() + podDevices := newPodDevices() + for resourceName, response := range tc.responsesPerResource { + podDevices.insert("pod", "container", resourceName, + nil, + response, + ) + } + opts := podDevices.deviceRunContainerOptions(podUID, containerName) + + // The exact ordering of the options depends on the order of the resources in the map. + // We therefore use `ElementsMatch` instead of `Equal` on the member slices. + as.ElementsMatch(tc.expected.Annotations, opts.Annotations) + as.ElementsMatch(tc.expected.CDIDevices, opts.CDIDevices) + as.ElementsMatch(tc.expected.Devices, opts.Devices) + as.ElementsMatch(tc.expected.Envs, opts.Envs) + as.ElementsMatch(tc.expected.Mounts, opts.Mounts) + }) + } +} diff --git a/pkg/kubelet/cm/devicemanager/types.go b/pkg/kubelet/cm/devicemanager/types.go index d508e8c9969..4b63944a656 100644 --- a/pkg/kubelet/cm/devicemanager/types.go +++ b/pkg/kubelet/cm/devicemanager/types.go @@ -91,6 +91,8 @@ type DeviceRunContainerOptions struct { Devices []kubecontainer.DeviceInfo // The Annotations for the container Annotations []kubecontainer.Annotation + // CDI Devices for the container + CDIDevices []kubecontainer.CDIDevice } // TODO: evaluate whether we need this error definition. diff --git a/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.pb.go b/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.pb.go index a24d42881f3..e15ca7ccf99 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.pb.go +++ b/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.pb.go @@ -814,6 +814,56 @@ func (m *ContainerAllocateRequest) GetDevicesIDs() []string { return nil } +// CDIDevice specifies a CDI device information. +type CDIDevice struct { + // Fully qualified CDI device name + // for example: vendor.com/gpu=gpudevice1 + // see more details in the CDI specification: + // https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CDIDevice) Reset() { *m = CDIDevice{} } +func (*CDIDevice) ProtoMessage() {} +func (*CDIDevice) Descriptor() ([]byte, []int) { + return fileDescriptor_00212fb1f9d3bf1c, []int{15} +} +func (m *CDIDevice) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CDIDevice) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CDIDevice.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CDIDevice) XXX_Merge(src proto.Message) { + xxx_messageInfo_CDIDevice.Merge(m, src) +} +func (m *CDIDevice) XXX_Size() int { + return m.Size() +} +func (m *CDIDevice) XXX_DiscardUnknown() { + xxx_messageInfo_CDIDevice.DiscardUnknown(m) +} + +var xxx_messageInfo_CDIDevice proto.InternalMessageInfo + +func (m *CDIDevice) GetName() string { + if m != nil { + return m.Name + } + return "" +} + // AllocateResponse includes the artifacts that needs to be injected into // a container for accessing 'deviceIDs' that were mentioned as part of // 'AllocateRequest'. @@ -831,7 +881,7 @@ type AllocateResponse struct { func (m *AllocateResponse) Reset() { *m = AllocateResponse{} } func (*AllocateResponse) ProtoMessage() {} func (*AllocateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{15} + return fileDescriptor_00212fb1f9d3bf1c, []int{16} } func (m *AllocateResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -875,15 +925,17 @@ type ContainerAllocateResponse struct { // Devices for the container. Devices []*DeviceSpec `protobuf:"bytes,3,rep,name=devices,proto3" json:"devices,omitempty"` // Container annotations to pass to the container runtime - Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_sizecache int32 `json:"-"` + Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // CDI devices for the container. + CDIDevices []*CDIDevice `protobuf:"bytes,5,rep,name=cdi_devices,json=cdiDevices,proto3" json:"cdi_devices,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ContainerAllocateResponse) Reset() { *m = ContainerAllocateResponse{} } func (*ContainerAllocateResponse) ProtoMessage() {} func (*ContainerAllocateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{16} + return fileDescriptor_00212fb1f9d3bf1c, []int{17} } func (m *ContainerAllocateResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -940,6 +992,13 @@ func (m *ContainerAllocateResponse) GetAnnotations() map[string]string { return nil } +func (m *ContainerAllocateResponse) GetCDIDevices() []*CDIDevice { + if m != nil { + return m.CDIDevices + } + return nil +} + // Mount specifies a host volume to mount into a container. // where device library or tools are installed on host and container type Mount struct { @@ -956,7 +1015,7 @@ type Mount struct { func (m *Mount) Reset() { *m = Mount{} } func (*Mount) ProtoMessage() {} func (*Mount) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{17} + return fileDescriptor_00212fb1f9d3bf1c, []int{18} } func (m *Mount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1024,7 +1083,7 @@ type DeviceSpec struct { func (m *DeviceSpec) Reset() { *m = DeviceSpec{} } func (*DeviceSpec) ProtoMessage() {} func (*DeviceSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{18} + return fileDescriptor_00212fb1f9d3bf1c, []int{19} } func (m *DeviceSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1090,6 +1149,7 @@ func init() { proto.RegisterType((*ContainerPreferredAllocationResponse)(nil), "v1beta1.ContainerPreferredAllocationResponse") proto.RegisterType((*AllocateRequest)(nil), "v1beta1.AllocateRequest") proto.RegisterType((*ContainerAllocateRequest)(nil), "v1beta1.ContainerAllocateRequest") + proto.RegisterType((*CDIDevice)(nil), "v1beta1.CDIDevice") proto.RegisterType((*AllocateResponse)(nil), "v1beta1.AllocateResponse") proto.RegisterType((*ContainerAllocateResponse)(nil), "v1beta1.ContainerAllocateResponse") proto.RegisterMapType((map[string]string)(nil), "v1beta1.ContainerAllocateResponse.AnnotationsEntry") @@ -1101,73 +1161,75 @@ func init() { func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) } var fileDescriptor_00212fb1f9d3bf1c = []byte{ - // 1044 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0x1b, 0x45, - 0x14, 0xce, 0xc6, 0x4d, 0x62, 0x1f, 0xa7, 0xf9, 0x99, 0x84, 0xc8, 0xd9, 0x14, 0x37, 0x9d, 0x14, - 0x1a, 0xa4, 0xc4, 0x26, 0x2e, 0x6a, 0x2b, 0x2e, 0x10, 0x2e, 0x0e, 0x60, 0x42, 0xd3, 0x68, 0x43, - 0x85, 0x04, 0x08, 0x6b, 0xbc, 0x3b, 0xb1, 0x57, 0x59, 0xcf, 0x2c, 0x3b, 0x63, 0x4b, 0xae, 0x84, - 0xc4, 0x05, 0x0f, 0xd0, 0x77, 0x80, 0x57, 0xe0, 0x1d, 0x7a, 0xc9, 0x25, 0x57, 0x88, 0x9a, 0x17, - 0x41, 0x9e, 0xd9, 0x3f, 0x6d, 0x36, 0x56, 0x2a, 0x71, 0xe7, 0x39, 0xe7, 0x7c, 0xe7, 0xe7, 0x3b, - 0x67, 0xcf, 0x31, 0x94, 0x88, 0xef, 0xd6, 0xfc, 0x80, 0x4b, 0x8e, 0x96, 0x46, 0x47, 0x5d, 0x2a, - 0xc9, 0x91, 0x79, 0xd8, 0x73, 0x65, 0x7f, 0xd8, 0xad, 0xd9, 0x7c, 0x50, 0xef, 0xf1, 0x1e, 0xaf, - 0x2b, 0x7d, 0x77, 0x78, 0xa1, 0x5e, 0xea, 0xa1, 0x7e, 0x69, 0x1c, 0x7e, 0x65, 0xc0, 0x46, 0x8b, - 0x8e, 0x5c, 0x9b, 0x9e, 0x79, 0xc3, 0x9e, 0xcb, 0x9e, 0xfb, 0xd2, 0xe5, 0x4c, 0xa0, 0x03, 0x40, - 0x7e, 0x40, 0x3b, 0x42, 0x92, 0x40, 0x76, 0x02, 0xfa, 0xd3, 0xd0, 0x0d, 0xa8, 0x53, 0x31, 0x76, - 0x8d, 0xfd, 0xa2, 0xb5, 0xe6, 0x07, 0xf4, 0x7c, 0xaa, 0xb0, 0x42, 0x39, 0x3a, 0x01, 0xdc, 0xa3, - 0xb2, 0xe3, 0x07, 0xf4, 0x82, 0x06, 0x01, 0x75, 0x3a, 0xc4, 0xf3, 0xb8, 0x4d, 0xa6, 0xae, 0x3a, - 0x64, 0x44, 0x5c, 0x8f, 0x74, 0x3d, 0x5a, 0x99, 0x57, 0xe8, 0xbb, 0x3d, 0x2a, 0xcf, 0x22, 0xc3, - 0x66, 0x6c, 0xd7, 0x8c, 0xcc, 0xf0, 0xef, 0x06, 0xac, 0x5a, 0xb4, 0xe7, 0x0a, 0x49, 0x83, 0x69, - 0x04, 0x2a, 0x24, 0xaa, 0xc0, 0xd2, 0x88, 0x06, 0xc2, 0xe5, 0x4c, 0xe5, 0x50, 0xb2, 0xa2, 0x27, - 0x32, 0xa1, 0x48, 0x99, 0xe3, 0x73, 0x97, 0x49, 0x15, 0xa0, 0x64, 0xc5, 0x6f, 0xb4, 0x07, 0xb7, - 0x03, 0x2a, 0xf8, 0x30, 0xb0, 0x69, 0x87, 0x91, 0x01, 0xad, 0x14, 0x94, 0xc1, 0x72, 0x24, 0x3c, - 0x25, 0x03, 0x8a, 0x1e, 0xc1, 0x12, 0xd7, 0x45, 0x57, 0x6e, 0xed, 0x1a, 0xfb, 0xe5, 0xc6, 0x9d, - 0x5a, 0xc8, 0x65, 0x2d, 0x87, 0x18, 0x2b, 0x32, 0xc6, 0x4b, 0xb0, 0x70, 0x3c, 0xf0, 0xe5, 0x18, - 0x37, 0x61, 0xf3, 0x6b, 0x57, 0xc8, 0x26, 0x73, 0xbe, 0x25, 0xd2, 0xee, 0x5b, 0x54, 0xf8, 0x9c, - 0x09, 0x8a, 0x3e, 0x80, 0x25, 0x47, 0x39, 0x10, 0x15, 0x63, 0xb7, 0xb0, 0x5f, 0x6e, 0xac, 0x66, - 0x1c, 0x5b, 0x91, 0x1e, 0x3f, 0x86, 0xe5, 0x6f, 0xb8, 0xcf, 0x3d, 0xde, 0x1b, 0xb7, 0xd9, 0x05, - 0x47, 0x0f, 0x60, 0x81, 0x71, 0x27, 0x06, 0xae, 0xc7, 0xc0, 0xd3, 0x17, 0xcf, 0x9a, 0xa7, 0xdc, - 0xa1, 0x96, 0xd6, 0x63, 0x13, 0x8a, 0x91, 0x08, 0xad, 0xc0, 0x7c, 0xbb, 0xa5, 0xe8, 0x29, 0x58, - 0xf3, 0xed, 0x16, 0xb6, 0x61, 0x51, 0xc7, 0x49, 0x69, 0x4a, 0x53, 0x0d, 0xda, 0x82, 0xc5, 0x3e, - 0x25, 0x9e, 0xec, 0x87, 0x8c, 0x85, 0x2f, 0x74, 0x04, 0x45, 0x19, 0xa6, 0xa1, 0xa8, 0x2a, 0x37, - 0xde, 0x89, 0x23, 0xa7, 0xf3, 0xb3, 0x62, 0x33, 0x7c, 0x02, 0x95, 0xb3, 0x70, 0x1a, 0x3e, 0xe3, - 0x4c, 0x12, 0x97, 0x25, 0x4d, 0xab, 0x43, 0x39, 0x2c, 0xb0, 0xe3, 0x3a, 0xba, 0x96, 0xd2, 0xd3, - 0x95, 0xc9, 0xdf, 0x77, 0x41, 0xe7, 0x25, 0xda, 0x2d, 0x61, 0x41, 0x68, 0xd2, 0x76, 0x04, 0xde, - 0x81, 0xed, 0x1c, 0x67, 0x9a, 0x4e, 0x3c, 0x06, 0x33, 0x67, 0x6c, 0xa2, 0x58, 0xdf, 0x03, 0xb2, - 0x23, 0x88, 0x9a, 0x57, 0x2a, 0x64, 0x44, 0xdf, 0x41, 0x5c, 0x44, 0xec, 0xf5, 0x7a, 0x4f, 0xd6, - 0xba, 0x9d, 0xa9, 0x43, 0xe0, 0x3f, 0x0c, 0xd8, 0xbb, 0x01, 0x14, 0xd5, 0x61, 0x23, 0x9e, 0xf6, - 0x8e, 0xae, 0xab, 0xdd, 0x0a, 0x0b, 0xb7, 0x50, 0xac, 0x6a, 0x45, 0x1a, 0xf4, 0x11, 0x6c, 0x0d, - 0x86, 0x42, 0x76, 0x5c, 0x66, 0x7b, 0x43, 0x27, 0x8d, 0x99, 0x57, 0x98, 0xcd, 0xa9, 0xb6, 0xad, - 0x95, 0x09, 0xea, 0x01, 0xac, 0xa6, 0xbe, 0x2f, 0xe1, 0xbe, 0xd4, 0x83, 0xbd, 0x60, 0xad, 0x24, - 0xe2, 0x73, 0xf7, 0x25, 0xc5, 0x3f, 0xc3, 0x4e, 0x6e, 0xb6, 0xe1, 0x80, 0xfe, 0x08, 0x1b, 0x69, - 0xce, 0xb4, 0x34, 0x22, 0xed, 0xf0, 0x86, 0xa4, 0x69, 0x94, 0x85, 0xec, 0x6c, 0xc3, 0x04, 0x6e, - 0xc1, 0xfd, 0x9b, 0x60, 0xd1, 0x1d, 0x28, 0x65, 0xc9, 0x4a, 0x04, 0xd8, 0x86, 0xd5, 0x10, 0x43, - 0x23, 0x9e, 0xcf, 0x66, 0x34, 0xfb, 0xde, 0xd5, 0xbc, 0x33, 0xf0, 0xbc, 0x0e, 0x9f, 0x40, 0xe5, - 0x3a, 0xf3, 0xb7, 0x1f, 0xe3, 0x1e, 0xac, 0x25, 0x3e, 0xc2, 0x1a, 0xcf, 0x67, 0x71, 0x8d, 0x67, - 0xe5, 0x3c, 0x83, 0xe0, 0x5f, 0x0b, 0xb0, 0x7d, 0x2d, 0x02, 0x7d, 0x0a, 0xb7, 0x28, 0x1b, 0xcd, - 0xf8, 0x08, 0xb2, 0x88, 0xda, 0x31, 0x1b, 0x89, 0x63, 0x26, 0x83, 0xb1, 0xa5, 0x90, 0xe8, 0x7d, - 0x58, 0x1c, 0xf0, 0x21, 0x93, 0x7a, 0x1c, 0xcb, 0x8d, 0x95, 0xd8, 0xc7, 0xb3, 0xa9, 0xd8, 0x0a, - 0xb5, 0xe8, 0x30, 0xd9, 0x74, 0x05, 0x65, 0xb8, 0x91, 0xd9, 0x74, 0xe7, 0x3e, 0xb5, 0xe3, 0x6d, - 0x87, 0x5e, 0x40, 0x99, 0x30, 0xc6, 0x25, 0x89, 0xb6, 0xee, 0x14, 0xf2, 0xf0, 0x06, 0xf9, 0x35, - 0x13, 0x94, 0x4e, 0x33, 0xed, 0xc7, 0x7c, 0x0c, 0xa5, 0xb8, 0x00, 0xb4, 0x06, 0x85, 0x4b, 0x3a, - 0x0e, 0x77, 0xde, 0xf4, 0x27, 0xda, 0x84, 0x85, 0x11, 0xf1, 0x86, 0x34, 0xdc, 0x79, 0xfa, 0xf1, - 0xf1, 0xfc, 0x13, 0xc3, 0xfc, 0x04, 0xd6, 0xb2, 0x9e, 0xdf, 0x06, 0x8f, 0xfb, 0xb0, 0xa0, 0xf8, - 0x40, 0xef, 0xc1, 0x4a, 0xd2, 0x64, 0x9f, 0xc8, 0x7e, 0x88, 0xbf, 0x1d, 0x4b, 0xcf, 0x88, 0xec, - 0xa3, 0x1d, 0x28, 0xf5, 0xb9, 0x90, 0xda, 0x22, 0xbc, 0x59, 0x53, 0x41, 0xa4, 0x0c, 0x28, 0x71, - 0x3a, 0x9c, 0x79, 0x7a, 0x09, 0x17, 0xad, 0xe2, 0x54, 0xf0, 0x9c, 0x79, 0x63, 0x1c, 0x00, 0x24, - 0x84, 0xfe, 0x2f, 0xe1, 0x76, 0xa1, 0xec, 0xd3, 0x60, 0xe0, 0x0a, 0xa1, 0x7a, 0xa1, 0x0f, 0x64, - 0x5a, 0xd4, 0xf8, 0x1c, 0x96, 0xf5, 0x35, 0x0e, 0x14, 0x3f, 0xe8, 0x11, 0x14, 0xa3, 0xeb, 0x8c, - 0x2a, 0x71, 0xd3, 0x32, 0x07, 0xdb, 0x4c, 0x46, 0x45, 0x1f, 0xc9, 0xb9, 0xc6, 0x6f, 0x05, 0x58, - 0x4e, 0x1f, 0x54, 0xf4, 0x25, 0x6c, 0x7d, 0x41, 0x65, 0xde, 0x9f, 0x8f, 0x0c, 0xd8, 0x9c, 0x79, - 0x91, 0xf1, 0x1c, 0x6a, 0xc2, 0x72, 0xfa, 0x02, 0x5f, 0xc1, 0xbf, 0x1b, 0xbf, 0xf3, 0x0e, 0x35, - 0x9e, 0xfb, 0xd0, 0x40, 0x54, 0x25, 0x93, 0xb3, 0xa5, 0xd0, 0x5e, 0x0c, 0xbe, 0x7e, 0xf3, 0x9b, - 0xf7, 0x67, 0x1b, 0x45, 0x81, 0x50, 0x13, 0x8a, 0xd1, 0x54, 0xa7, 0xc8, 0xcb, 0x6c, 0x1c, 0x73, - 0x3b, 0x47, 0x13, 0xbb, 0xf8, 0x01, 0xd6, 0xaf, 0x1c, 0x49, 0x74, 0x2f, 0x1d, 0x3f, 0xf7, 0x1a, - 0x9b, 0x78, 0x96, 0x49, 0xe4, 0xfd, 0xe9, 0x57, 0xaf, 0xdf, 0x54, 0x8d, 0xbf, 0xde, 0x54, 0xe7, - 0x7e, 0x99, 0x54, 0x8d, 0xd7, 0x93, 0xaa, 0xf1, 0xe7, 0xa4, 0x6a, 0xfc, 0x33, 0xa9, 0x1a, 0xaf, - 0xfe, 0xad, 0xce, 0x7d, 0x77, 0x70, 0xf9, 0x44, 0xd4, 0x5c, 0x5e, 0xbf, 0x1c, 0x76, 0xa9, 0x47, - 0x65, 0xdd, 0xbf, 0xec, 0xd5, 0x89, 0xef, 0x8a, 0xba, 0xfe, 0xb4, 0x7d, 0xd5, 0x97, 0x7a, 0x18, - 0xa7, 0xbb, 0xa8, 0xfe, 0x62, 0x3e, 0xfc, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xe2, 0xf2, 0x09, 0x79, - 0xa7, 0x0a, 0x00, 0x00, + // 1088 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xef, 0x6e, 0x1b, 0x45, + 0x10, 0xcf, 0xc5, 0x71, 0x62, 0x8f, 0xd3, 0xfc, 0xd9, 0x84, 0xc8, 0x71, 0x8a, 0x93, 0x6e, 0x0a, + 0x0d, 0x52, 0x62, 0x13, 0x17, 0xb5, 0x15, 0x1f, 0x10, 0x6e, 0x1c, 0xc0, 0x84, 0xa6, 0xd1, 0x85, + 0x0a, 0x09, 0x10, 0xa7, 0xf3, 0xdd, 0xc6, 0x3e, 0xe5, 0xbc, 0x7b, 0xdc, 0xae, 0x2d, 0xb9, 0x12, + 0x12, 0x8f, 0xd0, 0x77, 0x80, 0x57, 0xe0, 0x1d, 0xca, 0x37, 0x3e, 0xf2, 0x09, 0xd1, 0xf0, 0x22, + 0xc8, 0xbb, 0xb7, 0x77, 0xa7, 0x8b, 0x63, 0xa5, 0x52, 0xbf, 0xdd, 0xce, 0xcc, 0x6f, 0x66, 0xf6, + 0x37, 0xb3, 0x33, 0x07, 0x45, 0x3b, 0xf0, 0x6a, 0x41, 0xc8, 0x04, 0x43, 0x0b, 0xc3, 0xc3, 0x0e, + 0x11, 0xf6, 0x61, 0xe5, 0xa0, 0xeb, 0x89, 0xde, 0xa0, 0x53, 0x73, 0x58, 0xbf, 0xde, 0x65, 0x5d, + 0x56, 0x97, 0xfa, 0xce, 0xe0, 0x42, 0x9e, 0xe4, 0x41, 0x7e, 0x29, 0x1c, 0x7e, 0x65, 0xc0, 0x5a, + 0x8b, 0x0c, 0x3d, 0x87, 0x9c, 0xf9, 0x83, 0xae, 0x47, 0x9f, 0x07, 0xc2, 0x63, 0x94, 0xa3, 0x7d, + 0x40, 0x41, 0x48, 0x2c, 0x2e, 0xec, 0x50, 0x58, 0x21, 0xf9, 0x79, 0xe0, 0x85, 0xc4, 0x2d, 0x1b, + 0x3b, 0xc6, 0x5e, 0xc1, 0x5c, 0x09, 0x42, 0x72, 0x3e, 0x56, 0x98, 0x91, 0x1c, 0x9d, 0x00, 0xee, + 0x12, 0x61, 0x05, 0x21, 0xb9, 0x20, 0x61, 0x48, 0x5c, 0xcb, 0xf6, 0x7d, 0xe6, 0xd8, 0x63, 0x57, + 0x96, 0x3d, 0xb4, 0x3d, 0xdf, 0xee, 0xf8, 0xa4, 0x3c, 0x2b, 0xd1, 0xdb, 0x5d, 0x22, 0xce, 0xb4, + 0x61, 0x33, 0xb6, 0x6b, 0x6a, 0x33, 0xfc, 0xbb, 0x01, 0xcb, 0x26, 0xe9, 0x7a, 0x5c, 0x90, 0x70, + 0x1c, 0x81, 0x70, 0x81, 0xca, 0xb0, 0x30, 0x24, 0x21, 0xf7, 0x18, 0x95, 0x39, 0x14, 0x4d, 0x7d, + 0x44, 0x15, 0x28, 0x10, 0xea, 0x06, 0xcc, 0xa3, 0x42, 0x06, 0x28, 0x9a, 0xf1, 0x19, 0xed, 0xc2, + 0x9d, 0x90, 0x70, 0x36, 0x08, 0x1d, 0x62, 0x51, 0xbb, 0x4f, 0xca, 0x39, 0x69, 0xb0, 0xa8, 0x85, + 0xa7, 0x76, 0x9f, 0xa0, 0x47, 0xb0, 0xc0, 0xd4, 0xa5, 0xcb, 0x73, 0x3b, 0xc6, 0x5e, 0xa9, 0x71, + 0xb7, 0x16, 0x71, 0x59, 0x9b, 0x40, 0x8c, 0xa9, 0x8d, 0xf1, 0x02, 0xe4, 0x8f, 0xfb, 0x81, 0x18, + 0xe1, 0x26, 0xac, 0x7f, 0xe3, 0x71, 0xd1, 0xa4, 0xee, 0x77, 0xb6, 0x70, 0x7a, 0x26, 0xe1, 0x01, + 0xa3, 0x9c, 0xa0, 0x8f, 0x60, 0xc1, 0x95, 0x0e, 0x78, 0xd9, 0xd8, 0xc9, 0xed, 0x95, 0x1a, 0xcb, + 0x19, 0xc7, 0xa6, 0xd6, 0xe3, 0xc7, 0xb0, 0xf8, 0x2d, 0x0b, 0x98, 0xcf, 0xba, 0xa3, 0x36, 0xbd, + 0x60, 0xe8, 0x01, 0xe4, 0x29, 0x73, 0x63, 0xe0, 0x6a, 0x0c, 0x3c, 0x7d, 0xf1, 0xac, 0x79, 0xca, + 0x5c, 0x62, 0x2a, 0x3d, 0xae, 0x40, 0x41, 0x8b, 0xd0, 0x12, 0xcc, 0xb6, 0x5b, 0x92, 0x9e, 0x9c, + 0x39, 0xdb, 0x6e, 0x61, 0x07, 0xe6, 0x55, 0x9c, 0x94, 0xa6, 0x38, 0xd6, 0xa0, 0x0d, 0x98, 0xef, + 0x11, 0xdb, 0x17, 0xbd, 0x88, 0xb1, 0xe8, 0x84, 0x0e, 0xa1, 0x20, 0xa2, 0x34, 0x24, 0x55, 0xa5, + 0xc6, 0x7b, 0x71, 0xe4, 0x74, 0x7e, 0x66, 0x6c, 0x86, 0x4f, 0xa0, 0x7c, 0x16, 0x75, 0xc3, 0x11, + 0xa3, 0xc2, 0xf6, 0x68, 0x52, 0xb4, 0x3a, 0x94, 0xa2, 0x0b, 0x5a, 0x9e, 0xab, 0xee, 0x52, 0x7c, + 0xba, 0x74, 0xf5, 0xcf, 0x36, 0xa8, 0xbc, 0x78, 0xbb, 0xc5, 0x4d, 0x88, 0x4c, 0xda, 0x2e, 0xc7, + 0x5b, 0xb0, 0x39, 0xc1, 0x99, 0xa2, 0x13, 0x8f, 0xa0, 0x32, 0xa1, 0x6d, 0x74, 0xac, 0x1f, 0x00, + 0x39, 0x1a, 0x22, 0xfb, 0x95, 0x70, 0xa1, 0xe9, 0xdb, 0x8f, 0x2f, 0x11, 0x7b, 0xbd, 0xd9, 0x93, + 0xb9, 0xea, 0x64, 0xee, 0xc1, 0xf1, 0x1f, 0x06, 0xec, 0xde, 0x02, 0x8a, 0xea, 0xb0, 0x16, 0x77, + 0xbb, 0xa5, 0xee, 0xd5, 0x6e, 0x45, 0x17, 0x37, 0x51, 0xac, 0x6a, 0x69, 0x0d, 0xfa, 0x04, 0x36, + 0xfa, 0x03, 0x2e, 0x2c, 0x8f, 0x3a, 0xfe, 0xc0, 0x4d, 0x63, 0x66, 0x25, 0x66, 0x7d, 0xac, 0x6d, + 0x2b, 0x65, 0x82, 0x7a, 0x00, 0xcb, 0xa9, 0xf7, 0xc5, 0xbd, 0x97, 0xaa, 0xb1, 0xf3, 0xe6, 0x52, + 0x22, 0x3e, 0xf7, 0x5e, 0x12, 0xfc, 0x0b, 0x6c, 0x4d, 0xcc, 0x36, 0x6a, 0xd0, 0x9f, 0x60, 0x2d, + 0xcd, 0x99, 0x92, 0x6a, 0xd2, 0x0e, 0x6e, 0x49, 0x9a, 0x42, 0x99, 0xc8, 0xc9, 0x16, 0x8c, 0xe3, + 0x16, 0xdc, 0xbf, 0x0d, 0x16, 0xdd, 0x85, 0x62, 0x96, 0xac, 0x44, 0x80, 0x1d, 0x58, 0x8e, 0x30, + 0x44, 0xf3, 0x7c, 0x36, 0xa5, 0xd8, 0xf7, 0xae, 0xe7, 0x9d, 0x81, 0x4f, 0xaa, 0xf0, 0x09, 0x94, + 0x6f, 0x32, 0x7f, 0xfb, 0x36, 0xde, 0x86, 0xe2, 0x51, 0xab, 0x1d, 0xbd, 0x3d, 0x04, 0x73, 0x72, + 0xf4, 0xa8, 0xd7, 0x27, 0xbf, 0x71, 0x17, 0x56, 0x92, 0x20, 0x11, 0x09, 0xe7, 0xd3, 0x8a, 0x81, + 0xa7, 0x5d, 0x6a, 0x4a, 0x05, 0xfe, 0xcc, 0xc1, 0xe6, 0x8d, 0x08, 0xf4, 0x39, 0xcc, 0x11, 0x3a, + 0x9c, 0xf2, 0x4a, 0xb2, 0x88, 0xda, 0x31, 0x1d, 0xf2, 0x63, 0x2a, 0xc2, 0x91, 0x29, 0x91, 0xe8, + 0x43, 0x98, 0xef, 0xb3, 0x01, 0x15, 0xaa, 0x5f, 0x4b, 0x8d, 0xa5, 0xd8, 0xc7, 0xb3, 0xb1, 0xd8, + 0x8c, 0xb4, 0xe8, 0x20, 0x19, 0x85, 0x39, 0x69, 0xb8, 0x96, 0x19, 0x85, 0xe7, 0x01, 0x71, 0xe2, + 0x71, 0x88, 0x5e, 0x40, 0xc9, 0xa6, 0x94, 0x09, 0x5b, 0x8f, 0xe5, 0x31, 0xe4, 0xe1, 0x2d, 0xf2, + 0x6b, 0x26, 0x28, 0x95, 0x66, 0xda, 0x0f, 0x3a, 0x82, 0x92, 0xe3, 0x7a, 0x96, 0xce, 0x24, 0x2f, + 0xdd, 0xa2, 0xc4, 0xad, 0xae, 0x99, 0x2a, 0x6e, 0x7c, 0xe4, 0x26, 0x38, 0xae, 0x17, 0x7d, 0x57, + 0x1e, 0x43, 0x31, 0x66, 0x01, 0xad, 0x40, 0xee, 0x92, 0x8c, 0xa2, 0xda, 0x8e, 0x3f, 0xd1, 0x3a, + 0xe4, 0x87, 0xb6, 0x3f, 0x20, 0xd1, 0x64, 0x55, 0x87, 0x4f, 0x67, 0x9f, 0x18, 0x95, 0xcf, 0x60, + 0x25, 0x9b, 0xde, 0xdb, 0xe0, 0x71, 0x0f, 0xf2, 0x92, 0x54, 0xf4, 0x01, 0x2c, 0x25, 0x9d, 0x12, + 0xd8, 0xa2, 0x17, 0xe1, 0xef, 0xc4, 0xd2, 0x33, 0x5b, 0xf4, 0xd0, 0x16, 0x14, 0x7b, 0x8c, 0x0b, + 0x65, 0x11, 0x6d, 0xc6, 0xb1, 0x40, 0x2b, 0x43, 0x62, 0xbb, 0x16, 0xa3, 0xbe, 0x1a, 0xf5, 0x05, + 0xb3, 0x30, 0x16, 0x3c, 0xa7, 0xfe, 0x08, 0x87, 0x00, 0x49, 0x55, 0xde, 0x49, 0xb8, 0x1d, 0x28, + 0x05, 0x24, 0xec, 0x7b, 0x9c, 0xcb, 0x82, 0xaa, 0x35, 0x9c, 0x16, 0x35, 0xbe, 0x80, 0x45, 0xb5, + 0xf3, 0x43, 0xc9, 0x0f, 0x7a, 0x04, 0x05, 0xfd, 0x0f, 0x80, 0xca, 0x71, 0x89, 0x32, 0xbf, 0x05, + 0x95, 0xa4, 0xdf, 0xd4, 0x2a, 0x9e, 0x69, 0xfc, 0x96, 0x83, 0xc5, 0xf4, 0xda, 0x46, 0x5f, 0xc1, + 0xc6, 0x97, 0x44, 0x4c, 0xfa, 0xc5, 0xc9, 0x80, 0x2b, 0x53, 0xf7, 0x3e, 0x9e, 0x41, 0x4d, 0x58, + 0x4c, 0xef, 0xf9, 0x6b, 0xf8, 0xf7, 0xe3, 0xf3, 0xa4, 0xdf, 0x01, 0x3c, 0xf3, 0xb1, 0x81, 0x88, + 0x4c, 0x66, 0xc2, 0x2c, 0x44, 0xbb, 0x31, 0xf8, 0xe6, 0xfd, 0x52, 0xb9, 0x3f, 0xdd, 0x48, 0x07, + 0x42, 0x4d, 0x28, 0xe8, 0xa7, 0x91, 0x22, 0x2f, 0x33, 0xd7, 0x2a, 0x9b, 0x13, 0x34, 0xb1, 0x8b, + 0x1f, 0x61, 0xf5, 0xda, 0x2a, 0x46, 0xf7, 0xd2, 0xf1, 0x27, 0xee, 0xfc, 0x0a, 0x9e, 0x66, 0xa2, + 0xbd, 0x3f, 0xfd, 0xfa, 0xf5, 0x9b, 0xaa, 0xf1, 0xf7, 0x9b, 0xea, 0xcc, 0xaf, 0x57, 0x55, 0xe3, + 0xf5, 0x55, 0xd5, 0xf8, 0xeb, 0xaa, 0x6a, 0xfc, 0x7b, 0x55, 0x35, 0x5e, 0xfd, 0x57, 0x9d, 0xf9, + 0x7e, 0xff, 0xf2, 0x09, 0xaf, 0x79, 0xac, 0x7e, 0x39, 0xe8, 0x10, 0x9f, 0x88, 0x7a, 0x70, 0xd9, + 0xad, 0xdb, 0x81, 0xc7, 0xeb, 0xea, 0xe5, 0x06, 0xb2, 0x2e, 0xf5, 0x28, 0x4e, 0x67, 0x5e, 0xfe, + 0xc8, 0x3e, 0xfc, 0x3f, 0x00, 0x00, 0xff, 0xff, 0xe0, 0x26, 0x52, 0x96, 0x0d, 0x0b, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2075,6 +2137,36 @@ func (m *ContainerAllocateRequest) MarshalToSizedBuffer(dAtA []byte) (int, error return len(dAtA) - i, nil } +func (m *CDIDevice) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CDIDevice) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CDIDevice) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintApi(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *AllocateResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2132,6 +2224,20 @@ func (m *ContainerAllocateResponse) MarshalToSizedBuffer(dAtA []byte) (int, erro _ = i var l int _ = l + if len(m.CDIDevices) > 0 { + for iNdEx := len(m.CDIDevices) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.CDIDevices[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintApi(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } + } if len(m.Annotations) > 0 { for k := range m.Annotations { v := m.Annotations[k] @@ -2538,6 +2644,19 @@ func (m *ContainerAllocateRequest) Size() (n int) { return n } +func (m *CDIDevice) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + return n +} + func (m *AllocateResponse) Size() (n int) { if m == nil { return 0 @@ -2587,6 +2706,12 @@ func (m *ContainerAllocateResponse) Size() (n int) { n += mapEntrySize + 1 + sovApi(uint64(mapEntrySize)) } } + if len(m.CDIDevices) > 0 { + for _, e := range m.CDIDevices { + l = e.Size() + n += 1 + l + sovApi(uint64(l)) + } + } return n } @@ -2818,6 +2943,16 @@ func (this *ContainerAllocateRequest) String() string { }, "") return s } +func (this *CDIDevice) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&CDIDevice{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `}`, + }, "") + return s +} func (this *AllocateResponse) String() string { if this == nil { return "nil" @@ -2847,6 +2982,11 @@ func (this *ContainerAllocateResponse) String() string { repeatedStringForDevices += strings.Replace(f.String(), "DeviceSpec", "DeviceSpec", 1) + "," } repeatedStringForDevices += "}" + repeatedStringForCDIDevices := "[]*CDIDevice{" + for _, f := range this.CDIDevices { + repeatedStringForCDIDevices += strings.Replace(f.String(), "CDIDevice", "CDIDevice", 1) + "," + } + repeatedStringForCDIDevices += "}" keysForEnvs := make([]string, 0, len(this.Envs)) for k := range this.Envs { keysForEnvs = append(keysForEnvs, k) @@ -2872,6 +3012,7 @@ func (this *ContainerAllocateResponse) String() string { `Mounts:` + repeatedStringForMounts + `,`, `Devices:` + repeatedStringForDevices + `,`, `Annotations:` + mapStringForAnnotations + `,`, + `CDIDevices:` + repeatedStringForCDIDevices + `,`, `}`, }, "") return s @@ -4298,6 +4439,88 @@ func (m *ContainerAllocateRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *CDIDevice) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CDIDevice: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CDIDevice: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *AllocateResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -4733,6 +4956,40 @@ func (m *ContainerAllocateResponse) Unmarshal(dAtA []byte) error { } m.Annotations[mapkey] = mapvalue iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CDIDevices", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CDIDevices = append(m.CDIDevices, &CDIDevice{}) + if err := m.CDIDevices[len(m.CDIDevices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipApi(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.proto b/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.proto index f44e7cebd2d..97019bd5b93 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.proto +++ b/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.proto @@ -164,6 +164,15 @@ message ContainerAllocateRequest { repeated string devices_ids = 1 [(gogoproto.customname) = "DevicesIDs"]; } +// CDIDevice specifies a CDI device information. +message CDIDevice { + // Fully qualified CDI device name + // for example: vendor.com/gpu=gpudevice1 + // see more details in the CDI specification: + // https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md + string name = 1; +} + // AllocateResponse includes the artifacts that needs to be injected into // a container for accessing 'deviceIDs' that were mentioned as part of // 'AllocateRequest'. @@ -185,6 +194,8 @@ message ContainerAllocateResponse { repeated DeviceSpec devices = 3; // Container annotations to pass to the container runtime map annotations = 4; + // CDI devices for the container. + repeated CDIDevice cdi_devices = 5 [(gogoproto.customname) = "CDIDevices"]; } // Mount specifies a host volume to mount into a container.