From 9f54dccc9224d7e695be605ac41697f5d9ebb73f Mon Sep 17 00:00:00 2001 From: Alexey Perevalov Date: Tue, 10 Nov 2020 16:09:32 +0300 Subject: [PATCH 1/5] Change GetDevices interface This change is necessary for supporting Topology in the ContainerDevices. Signed-off-by: Alexey Perevalov --- pkg/kubelet/apis/podresources/server_v1.go | 13 +------------ pkg/kubelet/apis/podresources/server_v1_test.go | 11 +++++------ pkg/kubelet/apis/podresources/server_v1alpha1.go | 16 +++++++++++++++- .../apis/podresources/server_v1alpha1_test.go | 15 ++++++++------- pkg/kubelet/apis/podresources/types.go | 4 ++-- pkg/kubelet/cm/BUILD | 2 +- pkg/kubelet/cm/container_manager.go | 2 +- pkg/kubelet/cm/container_manager_linux.go | 2 +- pkg/kubelet/cm/container_manager_stub.go | 2 +- pkg/kubelet/cm/container_manager_windows.go | 2 +- pkg/kubelet/cm/devicemanager/BUILD | 2 +- pkg/kubelet/cm/devicemanager/manager.go | 2 +- pkg/kubelet/cm/devicemanager/manager_stub.go | 2 +- pkg/kubelet/cm/devicemanager/pod_devices.go | 2 +- pkg/kubelet/cm/devicemanager/types.go | 2 +- 15 files changed, 41 insertions(+), 38 deletions(-) diff --git a/pkg/kubelet/apis/podresources/server_v1.go b/pkg/kubelet/apis/podresources/server_v1.go index 20a6abb8ec5..fe4666a969c 100644 --- a/pkg/kubelet/apis/podresources/server_v1.go +++ b/pkg/kubelet/apis/podresources/server_v1.go @@ -22,7 +22,6 @@ import ( "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubelet/pkg/apis/podresources/v1" - "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" ) // podResourcesServerV1alpha1 implements PodResourcesListerServer @@ -40,16 +39,6 @@ func NewV1PodResourcesServer(podsProvider PodsProvider, devicesProvider DevicesP } } -func alphaDevicesToV1(alphaDevs []*v1alpha1.ContainerDevices) []*v1.ContainerDevices { - var devs []*v1.ContainerDevices - for _, alphaDev := range alphaDevs { - dev := v1.ContainerDevices(*alphaDev) - devs = append(devs, &dev) - } - - return devs -} - // List returns information about the resources assigned to pods on the node func (p *v1PodResourcesServer) List(ctx context.Context, req *v1.ListPodResourcesRequest) (*v1.ListPodResourcesResponse, error) { metrics.PodResourcesEndpointRequestsTotalCount.WithLabelValues("v1").Inc() @@ -68,7 +57,7 @@ func (p *v1PodResourcesServer) List(ctx context.Context, req *v1.ListPodResource for j, container := range pod.Spec.Containers { pRes.Containers[j] = &v1.ContainerResources{ Name: container.Name, - Devices: alphaDevicesToV1(p.devicesProvider.GetDevices(string(pod.UID), container.Name)), + Devices: p.devicesProvider.GetDevices(string(pod.UID), container.Name), } } podResources[i] = &pRes diff --git a/pkg/kubelet/apis/podresources/server_v1_test.go b/pkg/kubelet/apis/podresources/server_v1_test.go index c0d1dfba68b..40ace68f351 100644 --- a/pkg/kubelet/apis/podresources/server_v1_test.go +++ b/pkg/kubelet/apis/podresources/server_v1_test.go @@ -24,7 +24,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" - "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" ) func TestListPodResourcesV1(t *testing.T) { @@ -33,7 +32,7 @@ func TestListPodResourcesV1(t *testing.T) { podUID := types.UID("pod-uid") containerName := "container-name" - devs := []*v1alpha1.ContainerDevices{ + devs := []*podresourcesapi.ContainerDevices{ { ResourceName: "resource", DeviceIds: []string{"dev0", "dev1"}, @@ -43,13 +42,13 @@ func TestListPodResourcesV1(t *testing.T) { for _, tc := range []struct { desc string pods []*v1.Pod - devices []*v1alpha1.ContainerDevices + devices []*podresourcesapi.ContainerDevices expectedResponse *podresourcesapi.ListPodResourcesResponse }{ { desc: "no pods", pods: []*v1.Pod{}, - devices: []*v1alpha1.ContainerDevices{}, + devices: []*podresourcesapi.ContainerDevices{}, expectedResponse: &podresourcesapi.ListPodResourcesResponse{}, }, { @@ -70,7 +69,7 @@ func TestListPodResourcesV1(t *testing.T) { }, }, }, - devices: []*v1alpha1.ContainerDevices{}, + devices: []*podresourcesapi.ContainerDevices{}, expectedResponse: &podresourcesapi.ListPodResourcesResponse{ PodResources: []*podresourcesapi.PodResources{ { @@ -113,7 +112,7 @@ func TestListPodResourcesV1(t *testing.T) { Containers: []*podresourcesapi.ContainerResources{ { Name: containerName, - Devices: alphaDevicesToV1(devs), + Devices: devs, }, }, }, diff --git a/pkg/kubelet/apis/podresources/server_v1alpha1.go b/pkg/kubelet/apis/podresources/server_v1alpha1.go index 4d4cb7ca5cb..baacd2722fb 100644 --- a/pkg/kubelet/apis/podresources/server_v1alpha1.go +++ b/pkg/kubelet/apis/podresources/server_v1alpha1.go @@ -21,6 +21,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/metrics" + "k8s.io/kubelet/pkg/apis/podresources/v1" "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" ) @@ -39,6 +40,19 @@ func NewV1alpha1PodResourcesServer(podsProvider PodsProvider, devicesProvider De } } +func v1DevicesToAlphaV1(alphaDevs []*v1.ContainerDevices) []*v1alpha1.ContainerDevices { + var devs []*v1alpha1.ContainerDevices + for _, alphaDev := range alphaDevs { + dev := v1alpha1.ContainerDevices{ + ResourceName: alphaDev.ResourceName, + DeviceIds: alphaDev.DeviceIds, + } + devs = append(devs, &dev) + } + + return devs +} + // List returns information about the resources assigned to pods on the node func (p *v1alpha1PodResourcesServer) List(ctx context.Context, req *v1alpha1.ListPodResourcesRequest) (*v1alpha1.ListPodResourcesResponse, error) { metrics.PodResourcesEndpointRequestsTotalCount.WithLabelValues("v1alpha1").Inc() @@ -56,7 +70,7 @@ func (p *v1alpha1PodResourcesServer) List(ctx context.Context, req *v1alpha1.Lis for j, container := range pod.Spec.Containers { pRes.Containers[j] = &v1alpha1.ContainerResources{ Name: container.Name, - Devices: p.devicesProvider.GetDevices(string(pod.UID), container.Name), + Devices: v1DevicesToAlphaV1(p.devicesProvider.GetDevices(string(pod.UID), container.Name)), } } podResources[i] = &pRes diff --git a/pkg/kubelet/apis/podresources/server_v1alpha1_test.go b/pkg/kubelet/apis/podresources/server_v1alpha1_test.go index 302d8c95fa0..b87ac10fa8c 100644 --- a/pkg/kubelet/apis/podresources/server_v1alpha1_test.go +++ b/pkg/kubelet/apis/podresources/server_v1alpha1_test.go @@ -25,6 +25,7 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + podresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1" "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" ) @@ -37,9 +38,9 @@ func (m *mockProvider) GetPods() []*v1.Pod { return args.Get(0).([]*v1.Pod) } -func (m *mockProvider) GetDevices(podUID, containerName string) []*v1alpha1.ContainerDevices { +func (m *mockProvider) GetDevices(podUID, containerName string) []*podresourcesv1.ContainerDevices { args := m.Called(podUID, containerName) - return args.Get(0).([]*v1alpha1.ContainerDevices) + return args.Get(0).([]*podresourcesv1.ContainerDevices) } func (m *mockProvider) UpdateAllocatedDevices() { @@ -52,7 +53,7 @@ func TestListPodResourcesV1alpha1(t *testing.T) { podUID := types.UID("pod-uid") containerName := "container-name" - devs := []*v1alpha1.ContainerDevices{ + devs := []*podresourcesv1.ContainerDevices{ { ResourceName: "resource", DeviceIds: []string{"dev0", "dev1"}, @@ -62,13 +63,13 @@ func TestListPodResourcesV1alpha1(t *testing.T) { for _, tc := range []struct { desc string pods []*v1.Pod - devices []*v1alpha1.ContainerDevices + devices []*podresourcesv1.ContainerDevices expectedResponse *v1alpha1.ListPodResourcesResponse }{ { desc: "no pods", pods: []*v1.Pod{}, - devices: []*v1alpha1.ContainerDevices{}, + devices: []*podresourcesv1.ContainerDevices{}, expectedResponse: &v1alpha1.ListPodResourcesResponse{}, }, { @@ -89,7 +90,7 @@ func TestListPodResourcesV1alpha1(t *testing.T) { }, }, }, - devices: []*v1alpha1.ContainerDevices{}, + devices: []*podresourcesv1.ContainerDevices{}, expectedResponse: &v1alpha1.ListPodResourcesResponse{ PodResources: []*v1alpha1.PodResources{ { @@ -132,7 +133,7 @@ func TestListPodResourcesV1alpha1(t *testing.T) { Containers: []*v1alpha1.ContainerResources{ { Name: containerName, - Devices: devs, + Devices: v1DevicesToAlphaV1(devs), }, }, }, diff --git a/pkg/kubelet/apis/podresources/types.go b/pkg/kubelet/apis/podresources/types.go index 7f330185686..266c0c8af70 100644 --- a/pkg/kubelet/apis/podresources/types.go +++ b/pkg/kubelet/apis/podresources/types.go @@ -18,12 +18,12 @@ package podresources import ( "k8s.io/api/core/v1" - "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" ) // DevicesProvider knows how to provide the devices used by the given container type DevicesProvider interface { - GetDevices(podUID, containerName string) []*v1alpha1.ContainerDevices + GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices UpdateAllocatedDevices() } diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index 107878444be..6935ba92d09 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -42,7 +42,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis:go_default_library", - "//staging/src/k8s.io/kubelet/pkg/apis/podresources/v1alpha1:go_default_library", + "//staging/src/k8s.io/kubelet/pkg/apis/podresources/v1:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:aix": [ diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 83c150cbec1..603a150a26a 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -23,7 +23,7 @@ import ( // TODO: Migrate kubelet to either use its own internal objects or client library. v1 "k8s.io/api/core/v1" internalapi "k8s.io/cri-api/pkg/apis" - podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index aaf6b88f573..f0c0e33cf1d 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -47,7 +47,7 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" internalapi "k8s.io/cri-api/pkg/apis" - podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm/containermap" diff --git a/pkg/kubelet/cm/container_manager_stub.go b/pkg/kubelet/cm/container_manager_stub.go index 03387f2c479..6d8fa08dcab 100644 --- a/pkg/kubelet/cm/container_manager_stub.go +++ b/pkg/kubelet/cm/container_manager_stub.go @@ -22,7 +22,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" internalapi "k8s.io/cri-api/pkg/apis" - podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/config" diff --git a/pkg/kubelet/cm/container_manager_windows.go b/pkg/kubelet/cm/container_manager_windows.go index 47a17a0f837..3bc0f00e2a6 100644 --- a/pkg/kubelet/cm/container_manager_windows.go +++ b/pkg/kubelet/cm/container_manager_windows.go @@ -32,7 +32,7 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" internalapi "k8s.io/cri-api/pkg/apis" - podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" diff --git a/pkg/kubelet/cm/devicemanager/BUILD b/pkg/kubelet/cm/devicemanager/BUILD index 92e254992ea..a2a58b0b363 100644 --- a/pkg/kubelet/cm/devicemanager/BUILD +++ b/pkg/kubelet/cm/devicemanager/BUILD @@ -36,7 +36,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1:go_default_library", "//staging/src/k8s.io/kubelet/pkg/apis/pluginregistration/v1:go_default_library", - "//staging/src/k8s.io/kubelet/pkg/apis/podresources/v1alpha1:go_default_library", + "//staging/src/k8s.io/kubelet/pkg/apis/podresources/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go index 0b9afa98a9f..59ecf7f2a60 100644 --- a/pkg/kubelet/cm/devicemanager/manager.go +++ b/pkg/kubelet/cm/devicemanager/manager.go @@ -36,7 +36,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" - podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go index 626f4c39791..ebffa5c138e 100644 --- a/pkg/kubelet/cm/devicemanager/manager_stub.go +++ b/pkg/kubelet/cm/devicemanager/manager_stub.go @@ -18,7 +18,7 @@ package devicemanager import ( v1 "k8s.io/api/core/v1" - podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/lifecycle" diff --git a/pkg/kubelet/cm/devicemanager/pod_devices.go b/pkg/kubelet/cm/devicemanager/pod_devices.go index fef1e7785a2..f8625c8a06c 100644 --- a/pkg/kubelet/cm/devicemanager/pod_devices.go +++ b/pkg/kubelet/cm/devicemanager/pod_devices.go @@ -21,7 +21,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" - podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) diff --git a/pkg/kubelet/cm/devicemanager/types.go b/pkg/kubelet/cm/devicemanager/types.go index 4559da0ce55..771b5099c3e 100644 --- a/pkg/kubelet/cm/devicemanager/types.go +++ b/pkg/kubelet/cm/devicemanager/types.go @@ -20,7 +20,7 @@ import ( "time" v1 "k8s.io/api/core/v1" - podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" From ce921c039b933d4ec9ccb2c3db0aea3b9c56283a Mon Sep 17 00:00:00 2001 From: Alexey Perevalov Date: Thu, 22 Oct 2020 11:47:34 +0300 Subject: [PATCH 2/5] Add TopologyInfo and cpu_ids into podresources Signed-off-by: Alexey Perevalov --- .../kubelet/pkg/apis/podresources/v1/api.proto | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.proto b/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.proto index ef4c7f96d6a..cd8dbeaf957 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.proto +++ b/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.proto @@ -39,10 +39,22 @@ message PodResources { message ContainerResources { string name = 1; repeated ContainerDevices devices = 2; + repeated int64 cpu_ids = 3; } // ContainerDevices contains information about the devices assigned to a container message ContainerDevices { string resource_name = 1; repeated string device_ids = 2; + TopologyInfo topology = 3; +} + +// Topology describes hardware topology of the resource +message TopologyInfo { + repeated NUMANode nodes = 1; +} + +// NUMA representation of NUMA node +message NUMANode { + int64 ID = 1; } From db0a515be0cfc3fe10ff6b48212dd56ea3bb50c2 Mon Sep 17 00:00:00 2001 From: Alexey Perevalov Date: Wed, 28 Oct 2020 12:43:13 +0300 Subject: [PATCH 3/5] Generate podresources API for TopologyInfo and cpu_ids Signed-off-by: Alexey Perevalov --- .../pkg/apis/podresources/v1/api.pb.go | 601 +++++++++++++++++- 1 file changed, 574 insertions(+), 27 deletions(-) diff --git a/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.pb.go b/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.pb.go index 8bca22bd2a9..961027e556e 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.pb.go +++ b/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.pb.go @@ -195,6 +195,7 @@ func (m *PodResources) GetContainers() []*ContainerResources { type ContainerResources struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Devices []*ContainerDevices `protobuf:"bytes,2,rep,name=devices,proto3" json:"devices,omitempty"` + CpuIds []int64 `protobuf:"varint,3,rep,packed,name=cpu_ids,json=cpuIds,proto3" json:"cpu_ids,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_sizecache int32 `json:"-"` } @@ -245,12 +246,20 @@ func (m *ContainerResources) GetDevices() []*ContainerDevices { return nil } +func (m *ContainerResources) GetCpuIds() []int64 { + if m != nil { + return m.CpuIds + } + return nil +} + // ContainerDevices contains information about the devices assigned to a container type ContainerDevices struct { - ResourceName string `protobuf:"bytes,1,opt,name=resource_name,json=resourceName,proto3" json:"resource_name,omitempty"` - DeviceIds []string `protobuf:"bytes,2,rep,name=device_ids,json=deviceIds,proto3" json:"device_ids,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_sizecache int32 `json:"-"` + ResourceName string `protobuf:"bytes,1,opt,name=resource_name,json=resourceName,proto3" json:"resource_name,omitempty"` + DeviceIds []string `protobuf:"bytes,2,rep,name=device_ids,json=deviceIds,proto3" json:"device_ids,omitempty"` + Topology *TopologyInfo `protobuf:"bytes,3,opt,name=topology,proto3" json:"topology,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ContainerDevices) Reset() { *m = ContainerDevices{} } @@ -299,40 +308,146 @@ func (m *ContainerDevices) GetDeviceIds() []string { return nil } +func (m *ContainerDevices) GetTopology() *TopologyInfo { + if m != nil { + return m.Topology + } + return nil +} + +// Topology describes hardware topology of the resource +type TopologyInfo struct { + Nodes []*NUMANode `protobuf:"bytes,1,rep,name=nodes,proto3" json:"nodes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TopologyInfo) Reset() { *m = TopologyInfo{} } +func (*TopologyInfo) ProtoMessage() {} +func (*TopologyInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_00212fb1f9d3bf1c, []int{5} +} +func (m *TopologyInfo) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *TopologyInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_TopologyInfo.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *TopologyInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_TopologyInfo.Merge(m, src) +} +func (m *TopologyInfo) XXX_Size() int { + return m.Size() +} +func (m *TopologyInfo) XXX_DiscardUnknown() { + xxx_messageInfo_TopologyInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_TopologyInfo proto.InternalMessageInfo + +func (m *TopologyInfo) GetNodes() []*NUMANode { + if m != nil { + return m.Nodes + } + return nil +} + +// NUMA representation of NUMA node +type NUMANode struct { + ID int64 `protobuf:"varint,1,opt,name=ID,json=iD,proto3" json:"ID,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NUMANode) Reset() { *m = NUMANode{} } +func (*NUMANode) ProtoMessage() {} +func (*NUMANode) Descriptor() ([]byte, []int) { + return fileDescriptor_00212fb1f9d3bf1c, []int{6} +} +func (m *NUMANode) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *NUMANode) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_NUMANode.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *NUMANode) XXX_Merge(src proto.Message) { + xxx_messageInfo_NUMANode.Merge(m, src) +} +func (m *NUMANode) XXX_Size() int { + return m.Size() +} +func (m *NUMANode) XXX_DiscardUnknown() { + xxx_messageInfo_NUMANode.DiscardUnknown(m) +} + +var xxx_messageInfo_NUMANode proto.InternalMessageInfo + +func (m *NUMANode) GetID() int64 { + if m != nil { + return m.ID + } + return 0 +} + func init() { proto.RegisterType((*ListPodResourcesRequest)(nil), "v1.ListPodResourcesRequest") proto.RegisterType((*ListPodResourcesResponse)(nil), "v1.ListPodResourcesResponse") proto.RegisterType((*PodResources)(nil), "v1.PodResources") proto.RegisterType((*ContainerResources)(nil), "v1.ContainerResources") proto.RegisterType((*ContainerDevices)(nil), "v1.ContainerDevices") + proto.RegisterType((*TopologyInfo)(nil), "v1.TopologyInfo") + proto.RegisterType((*NUMANode)(nil), "v1.NUMANode") } func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) } var fileDescriptor_00212fb1f9d3bf1c = []byte{ - // 339 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x92, 0xc1, 0x4e, 0xf2, 0x40, - 0x10, 0xc7, 0x59, 0x20, 0xdf, 0x97, 0x8e, 0x90, 0x90, 0x8d, 0xd1, 0x8a, 0xb8, 0x21, 0xeb, 0x85, - 0x8b, 0x25, 0x60, 0xf4, 0x01, 0xc4, 0x8b, 0x89, 0x31, 0xda, 0x83, 0xd1, 0x13, 0x29, 0xed, 0x8a, - 0x7b, 0xa0, 0xbb, 0x76, 0x5b, 0xe2, 0xd1, 0x47, 0xf0, 0xb1, 0x38, 0x7a, 0xf4, 0x28, 0xf5, 0x45, - 0xcc, 0xee, 0xa6, 0xa1, 0x08, 0x9e, 0x3a, 0xf3, 0xff, 0xcd, 0xcc, 0xbf, 0x9d, 0x29, 0x38, 0x81, - 0xe4, 0x9e, 0x4c, 0x44, 0x2a, 0x70, 0x75, 0x3e, 0x68, 0x9f, 0x4c, 0x79, 0xfa, 0x9c, 0x4d, 0xbc, - 0x50, 0xcc, 0xfa, 0x53, 0x31, 0x15, 0x7d, 0x83, 0x26, 0xd9, 0x93, 0xc9, 0x4c, 0x62, 0x22, 0xdb, - 0x42, 0x0f, 0x60, 0xff, 0x9a, 0xab, 0xf4, 0x56, 0x44, 0x3e, 0x53, 0x22, 0x4b, 0x42, 0xa6, 0x7c, - 0xf6, 0x92, 0x31, 0x95, 0xd2, 0x3b, 0x70, 0x37, 0x91, 0x92, 0x22, 0x56, 0x0c, 0x9f, 0x41, 0x53, - 0x8a, 0x68, 0x9c, 0x14, 0xc0, 0x45, 0xdd, 0x5a, 0x6f, 0x67, 0xd8, 0xf2, 0xe6, 0x03, 0x6f, 0xad, - 0xa1, 0x21, 0x4b, 0x19, 0x7d, 0x85, 0x46, 0x99, 0x62, 0x0c, 0xf5, 0x38, 0x98, 0x31, 0x17, 0x75, - 0x51, 0xcf, 0xf1, 0x4d, 0x8c, 0x3b, 0xe0, 0xe8, 0xa7, 0x92, 0x41, 0xc8, 0xdc, 0xaa, 0x01, 0x2b, - 0x01, 0x9f, 0x03, 0x84, 0x22, 0x4e, 0x03, 0x1e, 0xb3, 0x44, 0xb9, 0x35, 0xe3, 0xba, 0xa7, 0x5d, - 0x47, 0x85, 0xba, 0xf2, 0x2e, 0x55, 0xd2, 0x07, 0xc0, 0x9b, 0x15, 0x5b, 0xfd, 0x3d, 0xf8, 0x1f, - 0xb1, 0x39, 0xd7, 0x1f, 0x55, 0x35, 0xe3, 0x77, 0xd7, 0xc6, 0x5f, 0x5a, 0xe6, 0x17, 0x45, 0xf4, - 0x1e, 0x5a, 0xbf, 0x21, 0x3e, 0x86, 0x66, 0xb1, 0x9a, 0x71, 0xc9, 0xa0, 0x51, 0x88, 0x37, 0xda, - 0xe8, 0x08, 0xc0, 0xce, 0x18, 0xf3, 0xc8, 0x7a, 0x39, 0xbe, 0x63, 0x95, 0xab, 0x48, 0x0d, 0x1f, - 0x01, 0x97, 0x77, 0xa5, 0x4f, 0xc1, 0x12, 0x3c, 0x82, 0xba, 0x8e, 0xf0, 0xa1, 0x7e, 0xa9, 0x3f, - 0x2e, 0xd7, 0xee, 0x6c, 0x87, 0xf6, 0x76, 0xb4, 0x72, 0xd1, 0x59, 0x2c, 0x09, 0xfa, 0x5c, 0x92, - 0xca, 0x5b, 0x4e, 0xd0, 0x22, 0x27, 0xe8, 0x23, 0x27, 0xe8, 0x2b, 0x27, 0xe8, 0xfd, 0x9b, 0x54, - 0x26, 0xff, 0xcc, 0x9f, 0x71, 0xfa, 0x13, 0x00, 0x00, 0xff, 0xff, 0x1c, 0x28, 0xb2, 0xaa, 0x59, - 0x02, 0x00, 0x00, + // 424 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0xc1, 0x6e, 0xd3, 0x40, + 0x10, 0xcd, 0xda, 0xa5, 0xad, 0x07, 0x17, 0x55, 0x2b, 0x44, 0x4d, 0x08, 0x56, 0xb4, 0x5c, 0x7a, + 0x00, 0x57, 0x0d, 0x82, 0x3b, 0x34, 0x17, 0x4b, 0x10, 0xc1, 0x0a, 0x0e, 0x9c, 0x22, 0xc7, 0xbb, + 0x35, 0x96, 0xa8, 0x67, 0xeb, 0xb5, 0x23, 0xb8, 0x71, 0xe0, 0x03, 0xf8, 0xac, 0x1e, 0x39, 0x72, + 0xa4, 0xe6, 0x47, 0xd0, 0xae, 0x71, 0xe3, 0x90, 0x70, 0xf2, 0xcc, 0x7b, 0x33, 0xef, 0x8d, 0x77, + 0x06, 0xbc, 0x44, 0xe5, 0x91, 0x2a, 0xb1, 0x42, 0xea, 0x2c, 0x4f, 0x87, 0x4f, 0xb2, 0xbc, 0xfa, + 0x58, 0x2f, 0xa2, 0x14, 0x2f, 0x4e, 0x32, 0xcc, 0xf0, 0xc4, 0x52, 0x8b, 0xfa, 0xdc, 0x66, 0x36, + 0xb1, 0x51, 0xdb, 0xc2, 0xee, 0xc3, 0xd1, 0xab, 0x5c, 0x57, 0x6f, 0x50, 0x70, 0xa9, 0xb1, 0x2e, + 0x53, 0xa9, 0xb9, 0xbc, 0xac, 0xa5, 0xae, 0xd8, 0x5b, 0x08, 0x36, 0x29, 0xad, 0xb0, 0xd0, 0x92, + 0x3e, 0x83, 0x03, 0x85, 0x62, 0x5e, 0x76, 0x44, 0x40, 0xc6, 0xee, 0xf1, 0xed, 0xc9, 0x61, 0xb4, + 0x3c, 0x8d, 0xd6, 0x1a, 0x7c, 0xd5, 0xcb, 0xd8, 0x67, 0xf0, 0xfb, 0x2c, 0xa5, 0xb0, 0x53, 0x24, + 0x17, 0x32, 0x20, 0x63, 0x72, 0xec, 0x71, 0x1b, 0xd3, 0x11, 0x78, 0xe6, 0xab, 0x55, 0x92, 0xca, + 0xc0, 0xb1, 0xc4, 0x0a, 0xa0, 0xcf, 0x01, 0x52, 0x2c, 0xaa, 0x24, 0x2f, 0x64, 0xa9, 0x03, 0xd7, + 0xba, 0xde, 0x33, 0xae, 0x67, 0x1d, 0xba, 0xf2, 0xee, 0x55, 0xb2, 0x4b, 0xa0, 0x9b, 0x15, 0x5b, + 0xfd, 0x23, 0xd8, 0x13, 0x72, 0x99, 0x9b, 0x9f, 0x72, 0xac, 0xfc, 0xdd, 0x35, 0xf9, 0x69, 0xcb, + 0xf1, 0xae, 0x88, 0x1e, 0xc1, 0x5e, 0xaa, 0xea, 0x79, 0x2e, 0xda, 0x71, 0x5c, 0xbe, 0x9b, 0xaa, + 0x3a, 0x16, 0x9a, 0x7d, 0x23, 0x70, 0xf8, 0x6f, 0x1b, 0x7d, 0x04, 0x07, 0xdd, 0xa3, 0xcd, 0x7b, + 0xd6, 0x7e, 0x07, 0xce, 0xcc, 0x08, 0x0f, 0x01, 0x5a, 0x75, 0xab, 0x6a, 0xa6, 0xf0, 0xb8, 0xd7, + 0x22, 0xb1, 0xd0, 0xf4, 0x31, 0xec, 0x57, 0xa8, 0xf0, 0x13, 0x66, 0x5f, 0x02, 0x77, 0x4c, 0xba, + 0x77, 0x7f, 0xf7, 0x17, 0x8b, 0x8b, 0x73, 0xe4, 0x37, 0x15, 0x6c, 0x02, 0x7e, 0x9f, 0xa1, 0x0c, + 0x6e, 0x15, 0x28, 0x6e, 0x56, 0xe6, 0x9b, 0xd6, 0xd9, 0xfb, 0xd7, 0x2f, 0x66, 0x28, 0x24, 0x6f, + 0x29, 0x36, 0x84, 0xfd, 0x0e, 0xa2, 0x77, 0xc0, 0x89, 0xa7, 0x76, 0x4c, 0x97, 0x3b, 0xf9, 0x74, + 0xf2, 0x01, 0x68, 0x7f, 0x87, 0xe6, 0x44, 0x64, 0x49, 0xcf, 0x60, 0xc7, 0x44, 0xf4, 0x81, 0x91, + 0xfb, 0xcf, 0x45, 0x0d, 0x47, 0xdb, 0xc9, 0xf6, 0xa6, 0xd8, 0xe0, 0xe5, 0xe8, 0xea, 0x3a, 0x24, + 0x3f, 0xaf, 0xc3, 0xc1, 0xd7, 0x26, 0x24, 0x57, 0x4d, 0x48, 0x7e, 0x34, 0x21, 0xf9, 0xd5, 0x84, + 0xe4, 0xfb, 0xef, 0x70, 0xb0, 0xd8, 0xb5, 0x17, 0xfb, 0xf4, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, + 0x34, 0x5b, 0xe8, 0xc9, 0xf1, 0x02, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -546,6 +661,25 @@ func (m *ContainerResources) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.CpuIds) > 0 { + dAtA2 := make([]byte, len(m.CpuIds)*10) + var j1 int + for _, num1 := range m.CpuIds { + num := uint64(num1) + for num >= 1<<7 { + dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j1++ + } + dAtA2[j1] = uint8(num) + j1++ + } + i -= j1 + copy(dAtA[i:], dAtA2[:j1]) + i = encodeVarintApi(dAtA, i, uint64(j1)) + i-- + dAtA[i] = 0x1a + } if len(m.Devices) > 0 { for iNdEx := len(m.Devices) - 1; iNdEx >= 0; iNdEx-- { { @@ -590,6 +724,18 @@ func (m *ContainerDevices) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Topology != nil { + { + size, err := m.Topology.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintApi(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } if len(m.DeviceIds) > 0 { for iNdEx := len(m.DeviceIds) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.DeviceIds[iNdEx]) @@ -609,6 +755,71 @@ func (m *ContainerDevices) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *TopologyInfo) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TopologyInfo) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *TopologyInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Nodes) > 0 { + for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Nodes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintApi(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *NUMANode) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NUMANode) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NUMANode) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.ID != 0 { + i = encodeVarintApi(dAtA, i, uint64(m.ID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func encodeVarintApi(dAtA []byte, offset int, v uint64) int { offset -= sovApi(v) base := offset @@ -683,6 +894,13 @@ func (m *ContainerResources) Size() (n int) { n += 1 + l + sovApi(uint64(l)) } } + if len(m.CpuIds) > 0 { + l = 0 + for _, e := range m.CpuIds { + l += sovApi(uint64(e)) + } + n += 1 + sovApi(uint64(l)) + l + } return n } @@ -702,6 +920,37 @@ func (m *ContainerDevices) Size() (n int) { n += 1 + l + sovApi(uint64(l)) } } + if m.Topology != nil { + l = m.Topology.Size() + n += 1 + l + sovApi(uint64(l)) + } + return n +} + +func (m *TopologyInfo) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Nodes) > 0 { + for _, e := range m.Nodes { + l = e.Size() + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + +func (m *NUMANode) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.ID != 0 { + n += 1 + sovApi(uint64(m.ID)) + } return n } @@ -764,6 +1013,7 @@ func (this *ContainerResources) String() string { s := strings.Join([]string{`&ContainerResources{`, `Name:` + fmt.Sprintf("%v", this.Name) + `,`, `Devices:` + repeatedStringForDevices + `,`, + `CpuIds:` + fmt.Sprintf("%v", this.CpuIds) + `,`, `}`, }, "") return s @@ -775,6 +1025,32 @@ func (this *ContainerDevices) String() string { s := strings.Join([]string{`&ContainerDevices{`, `ResourceName:` + fmt.Sprintf("%v", this.ResourceName) + `,`, `DeviceIds:` + fmt.Sprintf("%v", this.DeviceIds) + `,`, + `Topology:` + strings.Replace(this.Topology.String(), "TopologyInfo", "TopologyInfo", 1) + `,`, + `}`, + }, "") + return s +} +func (this *TopologyInfo) String() string { + if this == nil { + return "nil" + } + repeatedStringForNodes := "[]*NUMANode{" + for _, f := range this.Nodes { + repeatedStringForNodes += strings.Replace(f.String(), "NUMANode", "NUMANode", 1) + "," + } + repeatedStringForNodes += "}" + s := strings.Join([]string{`&TopologyInfo{`, + `Nodes:` + repeatedStringForNodes + `,`, + `}`, + }, "") + return s +} +func (this *NUMANode) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&NUMANode{`, + `ID:` + fmt.Sprintf("%v", this.ID) + `,`, `}`, }, "") return s @@ -1173,6 +1449,82 @@ func (m *ContainerResources) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType == 0 { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.CpuIds = append(m.CpuIds, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.CpuIds) == 0 { + m.CpuIds = make([]int64, 0, elementCount) + } + for iNdEx < postIndex { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.CpuIds = append(m.CpuIds, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field CpuIds", wireType) + } default: iNdEx = preIndex skippy, err := skipApi(dAtA[iNdEx:]) @@ -1290,6 +1642,201 @@ func (m *ContainerDevices) Unmarshal(dAtA []byte) error { } m.DeviceIds = append(m.DeviceIds, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topology", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Topology == nil { + m.Topology = &TopologyInfo{} + } + if err := m.Topology.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *TopologyInfo) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TopologyInfo: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TopologyInfo: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Nodes = append(m.Nodes, &NUMANode{}) + if err := m.Nodes[len(m.Nodes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NUMANode) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NUMANode: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NUMANode: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + } + m.ID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ID |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipApi(dAtA[iNdEx:]) From 62326a1846eb18dae2a378648e6d3462b86344dd Mon Sep 17 00:00:00 2001 From: Alexey Perevalov Date: Sun, 25 Oct 2020 23:59:49 +0300 Subject: [PATCH 4/5] Convert podDevices to struct PodDevices will have its own guard Signed-off-by: Alexey Perevalov --- pkg/kubelet/cm/devicemanager/manager.go | 20 ++-- pkg/kubelet/cm/devicemanager/manager_test.go | 16 +-- pkg/kubelet/cm/devicemanager/pod_devices.go | 105 +++++++++++++----- .../cm/devicemanager/topology_hints_test.go | 6 +- 4 files changed, 94 insertions(+), 53 deletions(-) diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go index 59ecf7f2a60..c374e1a7c7a 100644 --- a/pkg/kubelet/cm/devicemanager/manager.go +++ b/pkg/kubelet/cm/devicemanager/manager.go @@ -96,7 +96,7 @@ type ManagerImpl struct { allocatedDevices map[string]sets.String // podDevices contains pod to allocated device mapping. - podDevices podDevices + podDevices *podDevices checkpointManager checkpointmanager.CheckpointManager // List of NUMA Nodes available on the underlying machine @@ -150,7 +150,7 @@ func newManagerImpl(socketPath string, topology []cadvisorapi.Node, topologyAffi healthyDevices: make(map[string]sets.String), unhealthyDevices: make(map[string]sets.String), allocatedDevices: make(map[string]sets.String), - podDevices: make(podDevices), + podDevices: newPodDevices(), numaNodes: numaNodes, topologyAffinityStore: topologyAffinityStore, devicesToReuse: make(PodReusableDevices), @@ -393,11 +393,8 @@ func (m *ManagerImpl) Allocate(pod *v1.Pod, container *v1.Container) error { func (m *ManagerImpl) UpdatePluginResources(node *schedulerframework.NodeInfo, attrs *lifecycle.PodAdmitAttributes) error { pod := attrs.Pod - m.mutex.Lock() - defer m.mutex.Unlock() - // quick return if no pluginResources requested - if _, podRequireDevicePluginResource := m.podDevices[string(pod.UID)]; !podRequireDevicePluginResource { + if !m.podDevices.hasPod(string(pod.UID)) { return nil } @@ -904,9 +901,7 @@ func (m *ManagerImpl) allocateContainerResources(pod *v1.Pod, container *v1.Cont } // Update internal cached podDevices state. - m.mutex.Lock() m.podDevices.insert(podUID, contName, resource, allocDevices, resp.ContainerResponses[0]) - m.mutex.Unlock() } if needsUpdateCheckpoint { @@ -945,8 +940,6 @@ func (m *ManagerImpl) GetDeviceRunContainerOptions(pod *v1.Pod, container *v1.Co return nil, err } } - m.mutex.Lock() - defer m.mutex.Unlock() return m.podDevices.deviceRunContainerOptions(string(pod.UID), container.Name), nil } @@ -1019,6 +1012,9 @@ func (m *ManagerImpl) sanitizeNodeAllocatable(node *schedulerframework.NodeInfo) if allocatableResource.ScalarResources == nil { allocatableResource.ScalarResources = make(map[v1.ResourceName]int64) } + + m.mutex.Lock() + defer m.mutex.Unlock() for resource, devices := range m.allocatedDevices { needed := devices.Len() quant, ok := allocatableResource.ScalarResources[v1.ResourceName(resource)] @@ -1038,6 +1034,8 @@ func (m *ManagerImpl) sanitizeNodeAllocatable(node *schedulerframework.NodeInfo) } func (m *ManagerImpl) isDevicePluginResource(resource string) bool { + m.mutex.Lock() + defer m.mutex.Unlock() _, registeredResource := m.healthyDevices[resource] _, allocatedResource := m.allocatedDevices[resource] // Return true if this is either an active device plugin resource or @@ -1050,8 +1048,6 @@ func (m *ManagerImpl) isDevicePluginResource(resource string) bool { // GetDevices returns the devices used by the specified container func (m *ManagerImpl) GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices { - m.mutex.Lock() - defer m.mutex.Unlock() return m.podDevices.getContainerDevices(podUID, containerName) } diff --git a/pkg/kubelet/cm/devicemanager/manager_test.go b/pkg/kubelet/cm/devicemanager/manager_test.go index 6b40f1f9ef1..ca45ebc87e8 100644 --- a/pkg/kubelet/cm/devicemanager/manager_test.go +++ b/pkg/kubelet/cm/devicemanager/manager_test.go @@ -477,7 +477,7 @@ func TestCheckpoint(t *testing.T) { healthyDevices: make(map[string]sets.String), unhealthyDevices: make(map[string]sets.String), allocatedDevices: make(map[string]sets.String), - podDevices: make(podDevices), + podDevices: newPodDevices(), checkpointManager: ckm, } @@ -516,12 +516,12 @@ func TestCheckpoint(t *testing.T) { err = testManager.writeCheckpoint() as.Nil(err) - testManager.podDevices = make(podDevices) + testManager.podDevices = newPodDevices() err = testManager.readCheckpoint() as.Nil(err) - as.Equal(len(expectedPodDevices), len(testManager.podDevices)) - for podUID, containerDevices := range expectedPodDevices { + as.Equal(expectedPodDevices.size(), testManager.podDevices.size()) + for podUID, containerDevices := range expectedPodDevices.devs { for conName, resources := range containerDevices { for resource := range resources { expDevices := expectedPodDevices.containerDevices(podUID, conName, resource) @@ -615,7 +615,7 @@ func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestReso unhealthyDevices: make(map[string]sets.String), allocatedDevices: make(map[string]sets.String), endpoints: make(map[string]endpointInfo), - podDevices: make(podDevices), + podDevices: newPodDevices(), devicesToReuse: make(PodReusableDevices), topologyAffinityStore: topologymanager.NewFakeManager(), activePods: activePods, @@ -882,10 +882,10 @@ func TestUpdatePluginResources(t *testing.T) { callback: monitorCallback, allocatedDevices: make(map[string]sets.String), healthyDevices: make(map[string]sets.String), - podDevices: make(podDevices), + podDevices: newPodDevices(), checkpointManager: ckm, } - testManager.podDevices[string(pod.UID)] = make(containerDevices) + testManager.podDevices.devs[string(pod.UID)] = make(containerDevices) // require one of resource1 and one of resource2 testManager.allocatedDevices[resourceName1] = sets.NewString() @@ -983,7 +983,7 @@ func TestResetExtendedResource(t *testing.T) { healthyDevices: make(map[string]sets.String), unhealthyDevices: make(map[string]sets.String), allocatedDevices: make(map[string]sets.String), - podDevices: make(podDevices), + podDevices: newPodDevices(), checkpointManager: ckm, } diff --git a/pkg/kubelet/cm/devicemanager/pod_devices.go b/pkg/kubelet/cm/devicemanager/pod_devices.go index f8625c8a06c..d3ffab80da4 100644 --- a/pkg/kubelet/cm/devicemanager/pod_devices.go +++ b/pkg/kubelet/cm/devicemanager/pod_devices.go @@ -17,6 +17,8 @@ limitations under the License. package devicemanager import ( + "sync" + "k8s.io/klog/v2" "k8s.io/apimachinery/pkg/util/sets" @@ -35,45 +37,74 @@ type deviceAllocateInfo struct { type resourceAllocateInfo map[string]deviceAllocateInfo // Keyed by resourceName. type containerDevices map[string]resourceAllocateInfo // Keyed by containerName. -type podDevices map[string]containerDevices // Keyed by podUID. +type podDevices struct { + sync.RWMutex + devs map[string]containerDevices // Keyed by podUID. +} -func (pdev podDevices) pods() sets.String { +// NewPodDevices is a function that returns object of podDevices type with its own guard +// RWMutex and a map where key is a pod UID and value contains +// container devices information of type containerDevices. +func newPodDevices() *podDevices { + return &podDevices{devs: make(map[string]containerDevices)} +} + +func (pdev *podDevices) pods() sets.String { + pdev.RLock() + defer pdev.RUnlock() ret := sets.NewString() - for k := range pdev { + for k := range pdev.devs { ret.Insert(k) } return ret } -func (pdev podDevices) insert(podUID, contName, resource string, devices sets.String, resp *pluginapi.ContainerAllocateResponse) { - if _, podExists := pdev[podUID]; !podExists { - pdev[podUID] = make(containerDevices) +func (pdev *podDevices) size() int { + pdev.RLock() + defer pdev.RUnlock() + return len(pdev.devs) +} + +func (pdev *podDevices) hasPod(podUID string) bool { + _, podExists := pdev.devs[podUID] + return podExists +} + +func (pdev *podDevices) insert(podUID, contName, resource string, devices sets.String, resp *pluginapi.ContainerAllocateResponse) { + pdev.Lock() + defer pdev.Unlock() + if _, podExists := pdev.devs[podUID]; !podExists { + pdev.devs[podUID] = make(containerDevices) } - if _, contExists := pdev[podUID][contName]; !contExists { - pdev[podUID][contName] = make(resourceAllocateInfo) + if _, contExists := pdev.devs[podUID][contName]; !contExists { + pdev.devs[podUID][contName] = make(resourceAllocateInfo) } - pdev[podUID][contName][resource] = deviceAllocateInfo{ + pdev.devs[podUID][contName][resource] = deviceAllocateInfo{ deviceIds: devices, allocResp: resp, } } -func (pdev podDevices) delete(pods []string) { +func (pdev *podDevices) delete(pods []string) { + pdev.Lock() + defer pdev.Unlock() for _, uid := range pods { - delete(pdev, uid) + delete(pdev.devs, uid) } } // Returns list of device Ids allocated to the given container for the given resource. // Returns nil if we don't have cached state for the given . -func (pdev podDevices) containerDevices(podUID, contName, resource string) sets.String { - if _, podExists := pdev[podUID]; !podExists { +func (pdev *podDevices) containerDevices(podUID, contName, resource string) sets.String { + pdev.RLock() + defer pdev.RUnlock() + if _, podExists := pdev.devs[podUID]; !podExists { return nil } - if _, contExists := pdev[podUID][contName]; !contExists { + if _, contExists := pdev.devs[podUID][contName]; !contExists { return nil } - devs, resourceExists := pdev[podUID][contName][resource] + devs, resourceExists := pdev.devs[podUID][contName][resource] if !resourceExists { return nil } @@ -81,8 +112,10 @@ func (pdev podDevices) containerDevices(podUID, contName, resource string) sets. } // Populates allocatedResources with the device resources allocated to the specified . -func (pdev podDevices) addContainerAllocatedResources(podUID, contName string, allocatedResources map[string]sets.String) { - containers, exists := pdev[podUID] +func (pdev *podDevices) addContainerAllocatedResources(podUID, contName string, allocatedResources map[string]sets.String) { + pdev.RLock() + defer pdev.RUnlock() + containers, exists := pdev.devs[podUID] if !exists { return } @@ -96,8 +129,10 @@ func (pdev podDevices) addContainerAllocatedResources(podUID, contName string, a } // Removes the device resources allocated to the specified from allocatedResources. -func (pdev podDevices) removeContainerAllocatedResources(podUID, contName string, allocatedResources map[string]sets.String) { - containers, exists := pdev[podUID] +func (pdev *podDevices) removeContainerAllocatedResources(podUID, contName string, allocatedResources map[string]sets.String) { + pdev.RLock() + defer pdev.RUnlock() + containers, exists := pdev.devs[podUID] if !exists { return } @@ -111,9 +146,11 @@ func (pdev podDevices) removeContainerAllocatedResources(podUID, contName string } // Returns all of devices allocated to the pods being tracked, keyed by resourceName. -func (pdev podDevices) devices() map[string]sets.String { +func (pdev *podDevices) devices() map[string]sets.String { ret := make(map[string]sets.String) - for _, containerDevices := range pdev { + pdev.RLock() + defer pdev.RUnlock() + for _, containerDevices := range pdev.devs { for _, resources := range containerDevices { for resource, devices := range resources { if _, exists := ret[resource]; !exists { @@ -129,9 +166,11 @@ func (pdev podDevices) devices() map[string]sets.String { } // Turns podDevices to checkpointData. -func (pdev podDevices) toCheckpointData() []checkpoint.PodDevicesEntry { +func (pdev *podDevices) toCheckpointData() []checkpoint.PodDevicesEntry { var data []checkpoint.PodDevicesEntry - for podUID, containerDevices := range pdev { + pdev.RLock() + defer pdev.RUnlock() + for podUID, containerDevices := range pdev.devs { for conName, resources := range containerDevices { for resource, devices := range resources { devIds := devices.deviceIds.UnsortedList() @@ -158,7 +197,7 @@ func (pdev podDevices) toCheckpointData() []checkpoint.PodDevicesEntry { } // Populates podDevices from the passed in checkpointData. -func (pdev podDevices) fromCheckpointData(data []checkpoint.PodDevicesEntry) { +func (pdev *podDevices) fromCheckpointData(data []checkpoint.PodDevicesEntry) { for _, entry := range data { klog.V(2).Infof("Get checkpoint entry: %v %v %v %v %v\n", entry.PodUID, entry.ContainerName, entry.ResourceName, entry.DeviceIDs, entry.AllocResp) @@ -177,8 +216,11 @@ func (pdev podDevices) fromCheckpointData(data []checkpoint.PodDevicesEntry) { } // Returns combined container runtime settings to consume the container's allocated devices. -func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *DeviceRunContainerOptions { - containers, exists := pdev[podUID] +func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *DeviceRunContainerOptions { + pdev.RLock() + defer pdev.RUnlock() + + containers, exists := pdev.devs[podUID] if !exists { return nil } @@ -274,15 +316,18 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic } // getContainerDevices returns the devices assigned to the provided container for all ResourceNames -func (pdev podDevices) getContainerDevices(podUID, contName string) []*podresourcesapi.ContainerDevices { - if _, podExists := pdev[podUID]; !podExists { +func (pdev *podDevices) getContainerDevices(podUID, contName string) []*podresourcesapi.ContainerDevices { + pdev.RLock() + defer pdev.RUnlock() + + if _, podExists := pdev.devs[podUID]; !podExists { return nil } - if _, contExists := pdev[podUID][contName]; !contExists { + if _, contExists := pdev.devs[podUID][contName]; !contExists { return nil } cDev := []*podresourcesapi.ContainerDevices{} - for resource, allocateInfo := range pdev[podUID][contName] { + for resource, allocateInfo := range pdev.devs[podUID][contName] { cDev = append(cDev, &podresourcesapi.ContainerDevices{ ResourceName: resource, DeviceIds: allocateInfo.deviceIds.UnsortedList(), diff --git a/pkg/kubelet/cm/devicemanager/topology_hints_test.go b/pkg/kubelet/cm/devicemanager/topology_hints_test.go index 3cd072183e8..5f2db322953 100644 --- a/pkg/kubelet/cm/devicemanager/topology_hints_test.go +++ b/pkg/kubelet/cm/devicemanager/topology_hints_test.go @@ -385,7 +385,7 @@ func TestGetTopologyHints(t *testing.T) { allDevices: make(map[string]map[string]pluginapi.Device), healthyDevices: make(map[string]sets.String), allocatedDevices: make(map[string]sets.String), - podDevices: make(podDevices), + podDevices: newPodDevices(), sourcesReady: &sourcesReadyStub{}, activePods: func() []*v1.Pod { return []*v1.Pod{pod} }, numaNodes: []int{0, 1}, @@ -739,7 +739,7 @@ func TestTopologyAlignedAllocation(t *testing.T) { healthyDevices: make(map[string]sets.String), allocatedDevices: make(map[string]sets.String), endpoints: make(map[string]endpointInfo), - podDevices: make(podDevices), + podDevices: newPodDevices(), sourcesReady: &sourcesReadyStub{}, activePods: func() []*v1.Pod { return []*v1.Pod{} }, topologyAffinityStore: &mockAffinityStore{tc.hint}, @@ -928,7 +928,7 @@ func TestGetPreferredAllocationParameters(t *testing.T) { healthyDevices: make(map[string]sets.String), allocatedDevices: make(map[string]sets.String), endpoints: make(map[string]endpointInfo), - podDevices: make(podDevices), + podDevices: newPodDevices(), sourcesReady: &sourcesReadyStub{}, activePods: func() []*v1.Pod { return []*v1.Pod{} }, topologyAffinityStore: &mockAffinityStore{tc.hint}, From a8b8995ef241e93e9486d475126450f33f24ef4e Mon Sep 17 00:00:00 2001 From: Alexey Perevalov Date: Tue, 14 Jul 2020 18:03:56 +0300 Subject: [PATCH 5/5] Implement TopologyInfo and cpu_ids in podresources It covers deviceplugin & cpumanager. It has drawback, since cpuset and all other structs including cadvisor's keep cpu as int, but for protobuf based interface is better to have fixed int. This patch also introduces additional interface CPUsProvider, while DeviceProvider might have been extended too. Checkpoint not covered by unit test. Signed-off-by: Swati Sehgal Signed-off-by: Alexey Perevalov --- pkg/kubelet/apis/podresources/server_v1.go | 5 +- .../apis/podresources/server_v1_test.go | 12 ++++- .../apis/podresources/server_v1alpha1_test.go | 5 ++ pkg/kubelet/apis/podresources/types.go | 5 ++ pkg/kubelet/cm/container_manager.go | 3 ++ pkg/kubelet/cm/container_manager_linux.go | 4 ++ pkg/kubelet/cm/container_manager_stub.go | 4 ++ pkg/kubelet/cm/container_manager_windows.go | 4 ++ pkg/kubelet/cm/cpumanager/cpu_manager.go | 13 +++++ pkg/kubelet/cm/cpumanager/fake_cpu_manager.go | 5 ++ pkg/kubelet/cm/devicemanager/BUILD | 2 + pkg/kubelet/cm/devicemanager/checkpoint/BUILD | 1 + .../cm/devicemanager/checkpoint/checkpoint.go | 22 ++++++++- pkg/kubelet/cm/devicemanager/manager.go | 11 ++++- pkg/kubelet/cm/devicemanager/manager_test.go | 39 +++++++++------ pkg/kubelet/cm/devicemanager/pod_devices.go | 32 ++++++------- .../cm/devicemanager/pod_devices_test.go | 47 +++++++++++++++++++ .../cm/devicemanager/topology_hints_test.go | 2 +- pkg/kubelet/kubelet.go | 2 +- pkg/kubelet/server/server.go | 4 +- 20 files changed, 183 insertions(+), 39 deletions(-) create mode 100644 pkg/kubelet/cm/devicemanager/pod_devices_test.go diff --git a/pkg/kubelet/apis/podresources/server_v1.go b/pkg/kubelet/apis/podresources/server_v1.go index fe4666a969c..8482f51a19d 100644 --- a/pkg/kubelet/apis/podresources/server_v1.go +++ b/pkg/kubelet/apis/podresources/server_v1.go @@ -28,14 +28,16 @@ import ( type v1PodResourcesServer struct { podsProvider PodsProvider devicesProvider DevicesProvider + cpusProvider CPUsProvider } // NewV1PodResourcesServer returns a PodResourcesListerServer which lists pods provided by the PodsProvider // with device information provided by the DevicesProvider -func NewV1PodResourcesServer(podsProvider PodsProvider, devicesProvider DevicesProvider) v1.PodResourcesListerServer { +func NewV1PodResourcesServer(podsProvider PodsProvider, devicesProvider DevicesProvider, cpusProvider CPUsProvider) v1.PodResourcesListerServer { return &v1PodResourcesServer{ podsProvider: podsProvider, devicesProvider: devicesProvider, + cpusProvider: cpusProvider, } } @@ -58,6 +60,7 @@ func (p *v1PodResourcesServer) List(ctx context.Context, req *v1.ListPodResource pRes.Containers[j] = &v1.ContainerResources{ Name: container.Name, Devices: p.devicesProvider.GetDevices(string(pod.UID), container.Name), + CpuIds: p.cpusProvider.GetCPUs(string(pod.UID), container.Name), } } podResources[i] = &pRes diff --git a/pkg/kubelet/apis/podresources/server_v1_test.go b/pkg/kubelet/apis/podresources/server_v1_test.go index 40ace68f351..c25912ee3ba 100644 --- a/pkg/kubelet/apis/podresources/server_v1_test.go +++ b/pkg/kubelet/apis/podresources/server_v1_test.go @@ -31,24 +31,30 @@ func TestListPodResourcesV1(t *testing.T) { podNamespace := "pod-namespace" podUID := types.UID("pod-uid") containerName := "container-name" + numaID := int64(1) devs := []*podresourcesapi.ContainerDevices{ { ResourceName: "resource", DeviceIds: []string{"dev0", "dev1"}, + Topology: &podresourcesapi.TopologyInfo{Nodes: []*podresourcesapi.NUMANode{{ID: numaID}}}, }, } + cpus := []int64{12, 23, 30} + for _, tc := range []struct { desc string pods []*v1.Pod devices []*podresourcesapi.ContainerDevices + cpus []int64 expectedResponse *podresourcesapi.ListPodResourcesResponse }{ { desc: "no pods", pods: []*v1.Pod{}, devices: []*podresourcesapi.ContainerDevices{}, + cpus: []int64{}, expectedResponse: &podresourcesapi.ListPodResourcesResponse{}, }, { @@ -70,6 +76,7 @@ func TestListPodResourcesV1(t *testing.T) { }, }, devices: []*podresourcesapi.ContainerDevices{}, + cpus: []int64{}, expectedResponse: &podresourcesapi.ListPodResourcesResponse{ PodResources: []*podresourcesapi.PodResources{ { @@ -104,6 +111,7 @@ func TestListPodResourcesV1(t *testing.T) { }, }, devices: devs, + cpus: cpus, expectedResponse: &podresourcesapi.ListPodResourcesResponse{ PodResources: []*podresourcesapi.PodResources{ { @@ -113,6 +121,7 @@ func TestListPodResourcesV1(t *testing.T) { { Name: containerName, Devices: devs, + CpuIds: cpus, }, }, }, @@ -124,8 +133,9 @@ func TestListPodResourcesV1(t *testing.T) { m := new(mockProvider) m.On("GetPods").Return(tc.pods) m.On("GetDevices", string(podUID), containerName).Return(tc.devices) + m.On("GetCPUs", string(podUID), containerName).Return(tc.cpus) m.On("UpdateAllocatedDevices").Return() - server := NewV1PodResourcesServer(m, m) + server := NewV1PodResourcesServer(m, m, m) resp, err := server.List(context.TODO(), &podresourcesapi.ListPodResourcesRequest{}) if err != nil { t.Errorf("want err = %v, got %q", nil, err) diff --git a/pkg/kubelet/apis/podresources/server_v1alpha1_test.go b/pkg/kubelet/apis/podresources/server_v1alpha1_test.go index b87ac10fa8c..5fe6f966e9d 100644 --- a/pkg/kubelet/apis/podresources/server_v1alpha1_test.go +++ b/pkg/kubelet/apis/podresources/server_v1alpha1_test.go @@ -43,6 +43,11 @@ func (m *mockProvider) GetDevices(podUID, containerName string) []*podresourcesv return args.Get(0).([]*podresourcesv1.ContainerDevices) } +func (m *mockProvider) GetCPUs(podUID, containerName string) []int64 { + args := m.Called(podUID, containerName) + return args.Get(0).([]int64) +} + func (m *mockProvider) UpdateAllocatedDevices() { m.Called() } diff --git a/pkg/kubelet/apis/podresources/types.go b/pkg/kubelet/apis/podresources/types.go index 266c0c8af70..433d92c5996 100644 --- a/pkg/kubelet/apis/podresources/types.go +++ b/pkg/kubelet/apis/podresources/types.go @@ -31,3 +31,8 @@ type DevicesProvider interface { type PodsProvider interface { GetPods() []*v1.Pod } + +// CPUsProvider knows how to provide the cpus used by the given container +type CPUsProvider interface { + GetCPUs(podUID, containerName string) []int64 +} diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 603a150a26a..bf8122610a2 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -106,6 +106,9 @@ type ContainerManager interface { // GetDevices returns information about the devices assigned to pods and containers GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices + // GetCPUs returns information about the cpus assigned to pods and containers + GetCPUs(podUID, containerName string) []int64 + // ShouldResetExtendedResourceCapacity returns whether or not the extended resources should be zeroed, // due to node recreation. ShouldResetExtendedResourceCapacity() bool diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index f0c0e33cf1d..1c95b652918 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -1027,6 +1027,10 @@ func (cm *containerManagerImpl) GetDevices(podUID, containerName string) []*podr return cm.deviceManager.GetDevices(podUID, containerName) } +func (cm *containerManagerImpl) GetCPUs(podUID, containerName string) []int64 { + return cm.cpuManager.GetCPUs(podUID, containerName) +} + func (cm *containerManagerImpl) ShouldResetExtendedResourceCapacity() bool { return cm.deviceManager.ShouldResetExtendedResourceCapacity() } diff --git a/pkg/kubelet/cm/container_manager_stub.go b/pkg/kubelet/cm/container_manager_stub.go index 6d8fa08dcab..65eac7aadcf 100644 --- a/pkg/kubelet/cm/container_manager_stub.go +++ b/pkg/kubelet/cm/container_manager_stub.go @@ -125,6 +125,10 @@ func (cm *containerManagerStub) UpdateAllocatedDevices() { return } +func (cm *containerManagerStub) GetCPUs(_, _ string) []int64 { + return nil +} + func NewStubContainerManager() ContainerManager { return &containerManagerStub{shouldResetExtendedResourceCapacity: false} } diff --git a/pkg/kubelet/cm/container_manager_windows.go b/pkg/kubelet/cm/container_manager_windows.go index 3bc0f00e2a6..e6637b4d443 100644 --- a/pkg/kubelet/cm/container_manager_windows.go +++ b/pkg/kubelet/cm/container_manager_windows.go @@ -192,3 +192,7 @@ func (cm *containerManagerImpl) GetAllocateResourcesPodAdmitHandler() lifecycle. func (cm *containerManagerImpl) UpdateAllocatedDevices() { return } + +func (cm *containerManagerImpl) GetCPUs(_, _ string) []int64 { + return nil +} diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go index 1e6cc936d82..5f68c5957cd 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go @@ -77,6 +77,10 @@ type Manager interface { // and is consulted to achieve NUMA aware resource alignment among this // and other resource controllers. GetTopologyHints(*v1.Pod, *v1.Container) map[string][]topologymanager.TopologyHint + + // GetCPUs implements the podresources.CPUsProvider interface to provide allocated + // cpus for the container + GetCPUs(podUID, containerName string) []int64 } type manager struct { @@ -461,3 +465,12 @@ func (m *manager) updateContainerCPUSet(containerID string, cpus cpuset.CPUSet) CpusetCpus: cpus.String(), }) } + +func (m *manager) GetCPUs(podUID, containerName string) []int64 { + cpus := m.state.GetCPUSetOrDefault(string(podUID), containerName) + result := []int64{} + for _, cpu := range cpus.ToSliceNoSort() { + result = append(result, int64(cpu)) + } + return result +} diff --git a/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go b/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go index d8d707173bd..f73d038298f 100644 --- a/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go +++ b/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go @@ -64,6 +64,11 @@ func (m *fakeManager) State() state.Reader { return m.state } +func (m *fakeManager) GetCPUs(podUID, containerName string) []int64 { + klog.Infof("[fake cpumanager] GetCPUs(podUID: %s, containerName: %s)", podUID, containerName) + return nil +} + // NewFakeManager creates empty/fake cpu manager func NewFakeManager() Manager { return &fakeManager{ diff --git a/pkg/kubelet/cm/devicemanager/BUILD b/pkg/kubelet/cm/devicemanager/BUILD index a2a58b0b363..571f8b37389 100644 --- a/pkg/kubelet/cm/devicemanager/BUILD +++ b/pkg/kubelet/cm/devicemanager/BUILD @@ -48,11 +48,13 @@ go_test( srcs = [ "endpoint_test.go", "manager_test.go", + "pod_devices_test.go", "topology_hints_test.go", ], embed = [":go_default_library"], deps = [ "//pkg/kubelet/checkpointmanager:go_default_library", + "//pkg/kubelet/cm/devicemanager/checkpoint:go_default_library", "//pkg/kubelet/cm/topologymanager:go_default_library", "//pkg/kubelet/cm/topologymanager/bitmask:go_default_library", "//pkg/kubelet/config:go_default_library", diff --git a/pkg/kubelet/cm/devicemanager/checkpoint/BUILD b/pkg/kubelet/cm/devicemanager/checkpoint/BUILD index 91506a71f7c..afdace3db4f 100644 --- a/pkg/kubelet/cm/devicemanager/checkpoint/BUILD +++ b/pkg/kubelet/cm/devicemanager/checkpoint/BUILD @@ -8,6 +8,7 @@ go_library( deps = [ "//pkg/kubelet/checkpointmanager:go_default_library", "//pkg/kubelet/checkpointmanager/checksum:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", ], ) diff --git a/pkg/kubelet/cm/devicemanager/checkpoint/checkpoint.go b/pkg/kubelet/cm/devicemanager/checkpoint/checkpoint.go index 441eebd6684..0430fcd1d2b 100644 --- a/pkg/kubelet/cm/devicemanager/checkpoint/checkpoint.go +++ b/pkg/kubelet/cm/devicemanager/checkpoint/checkpoint.go @@ -19,6 +19,7 @@ package checkpoint import ( "encoding/json" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/checksum" ) @@ -29,12 +30,15 @@ type DeviceManagerCheckpoint interface { GetData() ([]PodDevicesEntry, map[string][]string) } +// DevicesPerNUMA represents device ids obtained from device plugin per NUMA node id +type DevicesPerNUMA map[int64][]string + // PodDevicesEntry connects pod information to devices type PodDevicesEntry struct { PodUID string ContainerName string ResourceName string - DeviceIDs []string + DeviceIDs DevicesPerNUMA AllocResp []byte } @@ -52,6 +56,22 @@ type Data struct { Checksum checksum.Checksum } +// NewDevicesPerNUMA is a function that creates DevicesPerNUMA map +func NewDevicesPerNUMA() DevicesPerNUMA { + return make(DevicesPerNUMA) +} + +// Devices is a function that returns all device ids for all NUMA nodes +// and represent it as sets.String +func (dev DevicesPerNUMA) Devices() sets.String { + result := sets.NewString() + + for _, devs := range dev { + result.Insert(devs...) + } + return result +} + // New returns an instance of Checkpoint func New(devEntries []PodDevicesEntry, devices map[string][]string) DeviceManagerCheckpoint { diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go index c374e1a7c7a..626f25da9ef 100644 --- a/pkg/kubelet/cm/devicemanager/manager.go +++ b/pkg/kubelet/cm/devicemanager/manager.go @@ -900,8 +900,17 @@ func (m *ManagerImpl) allocateContainerResources(pod *v1.Pod, container *v1.Cont return fmt.Errorf("no containers return in allocation response %v", resp) } + allocDevicesWithNUMA := checkpoint.NewDevicesPerNUMA() // Update internal cached podDevices state. - m.podDevices.insert(podUID, contName, resource, allocDevices, resp.ContainerResponses[0]) + m.mutex.Lock() + for dev := range allocDevices { + for idx := range m.allDevices[resource][dev].Topology.Nodes { + node := m.allDevices[resource][dev].Topology.Nodes[idx] + allocDevicesWithNUMA[node.ID] = append(allocDevicesWithNUMA[node.ID], dev) + } + } + m.mutex.Unlock() + m.podDevices.insert(podUID, contName, resource, allocDevicesWithNUMA, resp.ContainerResponses[0]) } if needsUpdateCheckpoint { diff --git a/pkg/kubelet/cm/devicemanager/manager_test.go b/pkg/kubelet/cm/devicemanager/manager_test.go index ca45ebc87e8..8f1ec0bd362 100644 --- a/pkg/kubelet/cm/devicemanager/manager_test.go +++ b/pkg/kubelet/cm/devicemanager/manager_test.go @@ -38,6 +38,7 @@ import ( pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" watcherapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" + "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/lifecycle" @@ -432,10 +433,10 @@ func TestUpdateCapacityAllocatable(t *testing.T) { as.True(testManager.isDevicePluginResource(resourceName2)) } -func constructDevices(devices []string) sets.String { - ret := sets.NewString() +func constructDevices(devices []string) checkpoint.DevicesPerNUMA { + ret := checkpoint.DevicesPerNUMA{} for _, dev := range devices { - ret.Insert(dev) + ret[0] = append(ret[0], dev) } return ret } @@ -621,13 +622,11 @@ func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestReso activePods: activePods, sourcesReady: &sourcesReadyStub{}, checkpointManager: ckm, + allDevices: make(map[string]map[string]pluginapi.Device), } for _, res := range testRes { - testManager.healthyDevices[res.resourceName] = sets.NewString() - for _, dev := range res.devs { - testManager.healthyDevices[res.resourceName].Insert(dev) - } + testManager.healthyDevices[res.resourceName] = sets.NewString(res.devs.Devices().UnsortedList()...) if res.resourceName == "domain1.com/resource1" { testManager.endpoints[res.resourceName] = endpointInfo{ e: &MockEndpoint{allocateFunc: allocateStubFunc()}, @@ -657,6 +656,8 @@ func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestReso opts: nil, } } + testManager.allDevices[res.resourceName] = makeDevice(res.devs) + } return testManager, nil } @@ -664,19 +665,19 @@ func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestReso type TestResource struct { resourceName string resourceQuantity resource.Quantity - devs []string + devs checkpoint.DevicesPerNUMA } func TestPodContainerDeviceAllocation(t *testing.T) { res1 := TestResource{ resourceName: "domain1.com/resource1", resourceQuantity: *resource.NewQuantity(int64(2), resource.DecimalSI), - devs: []string{"dev1", "dev2"}, + devs: checkpoint.DevicesPerNUMA{0: []string{"dev1", "dev2"}}, } res2 := TestResource{ resourceName: "domain2.com/resource2", resourceQuantity: *resource.NewQuantity(int64(1), resource.DecimalSI), - devs: []string{"dev3", "dev4"}, + devs: checkpoint.DevicesPerNUMA{0: []string{"dev3", "dev4"}}, } testResources := make([]TestResource, 2) testResources = append(testResources, res1) @@ -767,12 +768,12 @@ func TestInitContainerDeviceAllocation(t *testing.T) { res1 := TestResource{ resourceName: "domain1.com/resource1", resourceQuantity: *resource.NewQuantity(int64(2), resource.DecimalSI), - devs: []string{"dev1", "dev2"}, + devs: checkpoint.DevicesPerNUMA{0: []string{"dev1", "dev2"}}, } res2 := TestResource{ resourceName: "domain2.com/resource2", resourceQuantity: *resource.NewQuantity(int64(1), resource.DecimalSI), - devs: []string{"dev3", "dev4"}, + devs: checkpoint.DevicesPerNUMA{0: []string{"dev3", "dev4"}}, } testResources := make([]TestResource, 2) testResources = append(testResources, res1) @@ -920,7 +921,7 @@ func TestDevicePreStartContainer(t *testing.T) { res1 := TestResource{ resourceName: "domain1.com/resource1", resourceQuantity: *resource.NewQuantity(int64(2), resource.DecimalSI), - devs: []string{"dev1", "dev2"}, + devs: checkpoint.DevicesPerNUMA{0: []string{"dev1", "dev2"}}, } as := require.New(t) podsStub := activePodsStub{ @@ -960,7 +961,7 @@ func TestDevicePreStartContainer(t *testing.T) { as.Contains(initializedDevs, "dev1") as.Contains(initializedDevs, "dev2") - as.Equal(len(initializedDevs), len(res1.devs)) + as.Equal(len(initializedDevs), res1.devs.Devices().Len()) expectedResps, err := allocateStubFunc()([]string{"dev1", "dev2"}) as.Nil(err) @@ -1057,3 +1058,13 @@ func allocateStubFunc() func(devs []string) (*pluginapi.AllocateResponse, error) return resps, nil } } + +func makeDevice(devOnNUMA checkpoint.DevicesPerNUMA) map[string]pluginapi.Device { + res := make(map[string]pluginapi.Device) + for node, devs := range devOnNUMA { + for idx := range devs { + res[devs[idx]] = pluginapi.Device{ID: devs[idx], Topology: &pluginapi.TopologyInfo{Nodes: []*pluginapi.NUMANode{{ID: node}}}} + } + } + return res +} diff --git a/pkg/kubelet/cm/devicemanager/pod_devices.go b/pkg/kubelet/cm/devicemanager/pod_devices.go index d3ffab80da4..6f640666211 100644 --- a/pkg/kubelet/cm/devicemanager/pod_devices.go +++ b/pkg/kubelet/cm/devicemanager/pod_devices.go @@ -30,7 +30,7 @@ import ( type deviceAllocateInfo struct { // deviceIds contains device Ids allocated to this container for the given resourceName. - deviceIds sets.String + deviceIds checkpoint.DevicesPerNUMA // allocResp contains cached rpc AllocateResponse. allocResp *pluginapi.ContainerAllocateResponse } @@ -70,7 +70,7 @@ func (pdev *podDevices) hasPod(podUID string) bool { return podExists } -func (pdev *podDevices) insert(podUID, contName, resource string, devices sets.String, resp *pluginapi.ContainerAllocateResponse) { +func (pdev *podDevices) insert(podUID, contName, resource string, devices checkpoint.DevicesPerNUMA, resp *pluginapi.ContainerAllocateResponse) { pdev.Lock() defer pdev.Unlock() if _, podExists := pdev.devs[podUID]; !podExists { @@ -108,7 +108,7 @@ func (pdev *podDevices) containerDevices(podUID, contName, resource string) sets if !resourceExists { return nil } - return devs.deviceIds + return devs.deviceIds.Devices() } // Populates allocatedResources with the device resources allocated to the specified . @@ -124,7 +124,7 @@ func (pdev *podDevices) addContainerAllocatedResources(podUID, contName string, return } for resource, devices := range resources { - allocatedResources[resource] = allocatedResources[resource].Union(devices.deviceIds) + allocatedResources[resource] = allocatedResources[resource].Union(devices.deviceIds.Devices()) } } @@ -141,7 +141,7 @@ func (pdev *podDevices) removeContainerAllocatedResources(podUID, contName strin return } for resource, devices := range resources { - allocatedResources[resource] = allocatedResources[resource].Difference(devices.deviceIds) + allocatedResources[resource] = allocatedResources[resource].Difference(devices.deviceIds.Devices()) } } @@ -157,7 +157,7 @@ func (pdev *podDevices) devices() map[string]sets.String { ret[resource] = sets.NewString() } if devices.allocResp != nil { - ret[resource] = ret[resource].Union(devices.deviceIds) + ret[resource] = ret[resource].Union(devices.deviceIds.Devices()) } } } @@ -173,7 +173,6 @@ func (pdev *podDevices) toCheckpointData() []checkpoint.PodDevicesEntry { for podUID, containerDevices := range pdev.devs { for conName, resources := range containerDevices { for resource, devices := range resources { - devIds := devices.deviceIds.UnsortedList() if devices.allocResp == nil { klog.Errorf("Can't marshal allocResp for %v %v %v: allocation response is missing", podUID, conName, resource) continue @@ -188,7 +187,7 @@ func (pdev *podDevices) toCheckpointData() []checkpoint.PodDevicesEntry { PodUID: podUID, ContainerName: conName, ResourceName: resource, - DeviceIDs: devIds, + DeviceIDs: devices.deviceIds, AllocResp: allocResp}) } } @@ -201,17 +200,13 @@ func (pdev *podDevices) fromCheckpointData(data []checkpoint.PodDevicesEntry) { for _, entry := range data { klog.V(2).Infof("Get checkpoint entry: %v %v %v %v %v\n", entry.PodUID, entry.ContainerName, entry.ResourceName, entry.DeviceIDs, entry.AllocResp) - devIDs := sets.NewString() - for _, devID := range entry.DeviceIDs { - devIDs.Insert(devID) - } allocResp := &pluginapi.ContainerAllocateResponse{} err := allocResp.Unmarshal(entry.AllocResp) if err != nil { klog.Errorf("Can't unmarshal allocResp for %v %v %v: %v", entry.PodUID, entry.ContainerName, entry.ResourceName, err) continue } - pdev.insert(entry.PodUID, entry.ContainerName, entry.ResourceName, devIDs, allocResp) + pdev.insert(entry.PodUID, entry.ContainerName, entry.ResourceName, entry.DeviceIDs, allocResp) } } @@ -328,10 +323,13 @@ func (pdev *podDevices) getContainerDevices(podUID, contName string) []*podresou } cDev := []*podresourcesapi.ContainerDevices{} for resource, allocateInfo := range pdev.devs[podUID][contName] { - cDev = append(cDev, &podresourcesapi.ContainerDevices{ - ResourceName: resource, - DeviceIds: allocateInfo.deviceIds.UnsortedList(), - }) + for numaid, devlist := range allocateInfo.deviceIds { + cDev = append(cDev, &podresourcesapi.ContainerDevices{ + ResourceName: resource, + DeviceIds: devlist, + Topology: &podresourcesapi.TopologyInfo{Nodes: []*podresourcesapi.NUMANode{{ID: numaid}}}, + }) + } } return cDev } diff --git a/pkg/kubelet/cm/devicemanager/pod_devices_test.go b/pkg/kubelet/cm/devicemanager/pod_devices_test.go new file mode 100644 index 00000000000..b2ff8376a73 --- /dev/null +++ b/pkg/kubelet/cm/devicemanager/pod_devices_test.go @@ -0,0 +1,47 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package devicemanager + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" +) + +func TestGetContainerDevices(t *testing.T) { + podDevices := newPodDevices() + resourceName1 := "domain1.com/resource1" + podID := "pod1" + contID := "con1" + devices := checkpoint.DevicesPerNUMA{0: []string{"dev1"}, 1: []string{"dev1"}} + + podDevices.insert(podID, contID, resourceName1, + devices, + constructAllocResp(map[string]string{"/dev/r1dev1": "/dev/r1dev1", "/dev/r1dev2": "/dev/r1dev2"}, map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{})) + + contDevices := podDevices.getContainerDevices(podID, contID) + require.Equal(t, len(devices), len(contDevices), "Incorrect container devices") + for _, contDev := range contDevices { + for _, node := range contDev.Topology.Nodes { + dev, ok := devices[node.ID] + require.True(t, ok, "NUMA id %v doesn't exist in result", node.ID) + require.Equal(t, contDev.DeviceIds[0], dev[0], "Can't find device %s in result", dev[0]) + } + } +} diff --git a/pkg/kubelet/cm/devicemanager/topology_hints_test.go b/pkg/kubelet/cm/devicemanager/topology_hints_test.go index 5f2db322953..84f000a8cc6 100644 --- a/pkg/kubelet/cm/devicemanager/topology_hints_test.go +++ b/pkg/kubelet/cm/devicemanager/topology_hints_test.go @@ -404,7 +404,7 @@ func TestGetTopologyHints(t *testing.T) { for p := range tc.allocatedDevices { for c := range tc.allocatedDevices[p] { for r, devices := range tc.allocatedDevices[p][c] { - m.podDevices.insert(p, c, r, sets.NewString(devices...), nil) + m.podDevices.insert(p, c, r, constructDevices(devices), nil) m.allocatedDevices[r] = sets.NewString() for _, d := range devices { diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index d6d620790c0..f8f0fc0a0ef 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -2193,7 +2193,7 @@ func (kl *Kubelet) ListenAndServePodResources() { klog.V(2).Infof("Failed to get local endpoint for PodResources endpoint: %v", err) return } - server.ListenAndServePodResources(socket, kl.podManager, kl.containerManager) + server.ListenAndServePodResources(socket, kl.podManager, kl.containerManager, kl.containerManager) } // Delete the eligible dead container instances in a pod. Depending on the configuration, the latest dead containers may be kept around. diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index 23e7c362afc..fb8c7e12e7c 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -179,10 +179,10 @@ func ListenAndServeKubeletReadOnlyServer(host HostInterface, resourceAnalyzer st } // ListenAndServePodResources initializes a gRPC server to serve the PodResources service -func ListenAndServePodResources(socket string, podsProvider podresources.PodsProvider, devicesProvider podresources.DevicesProvider) { +func ListenAndServePodResources(socket string, podsProvider podresources.PodsProvider, devicesProvider podresources.DevicesProvider, cpusProvider podresources.CPUsProvider) { server := grpc.NewServer() podresourcesapiv1alpha1.RegisterPodResourcesListerServer(server, podresources.NewV1alpha1PodResourcesServer(podsProvider, devicesProvider)) - podresourcesapi.RegisterPodResourcesListerServer(server, podresources.NewV1PodResourcesServer(podsProvider, devicesProvider)) + podresourcesapi.RegisterPodResourcesListerServer(server, podresources.NewV1PodResourcesServer(podsProvider, devicesProvider, cpusProvider)) l, err := util.CreateListener(socket) if err != nil { klog.Fatalf("Failed to create listener for podResources endpoint: %v", err)