mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 19:31:44 +00:00
Merge pull request #115847 from moshe010/pod-resource-api-dra-upstream
Extend the PodResources API to include resources allocated by DRA
This commit is contained in:
commit
28fa3cbbf1
@ -439,6 +439,18 @@ const (
|
||||
// Enables the kubelet's pod resources grpc endpoint
|
||||
KubeletPodResources featuregate.Feature = "KubeletPodResources"
|
||||
|
||||
// owner: @moshe010
|
||||
// alpha: v1.27
|
||||
//
|
||||
// Enable POD resources API to return resources allocated by Dynamic Resource Allocation
|
||||
KubeletPodResourcesDynamicResources featuregate.Feature = "KubeletPodResourcesDynamicResources"
|
||||
|
||||
// owner: @moshe010
|
||||
// alpha: v1.27
|
||||
//
|
||||
// Enable POD resources API with Get method
|
||||
KubeletPodResourcesGet featuregate.Feature = "KubeletPodResourcesGet"
|
||||
|
||||
// owner: @fromanirh
|
||||
// alpha: v1.21
|
||||
// beta: v1.23
|
||||
@ -995,6 +1007,10 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
|
||||
|
||||
KubeletPodResources: {Default: true, PreRelease: featuregate.Beta},
|
||||
|
||||
KubeletPodResourcesDynamicResources: {Default: false, PreRelease: featuregate.Alpha},
|
||||
|
||||
KubeletPodResourcesGet: {Default: false, PreRelease: featuregate.Alpha},
|
||||
|
||||
KubeletPodResourcesGetAllocatable: {Default: true, PreRelease: featuregate.Beta},
|
||||
|
||||
KubeletTracing: {Default: false, PreRelease: featuregate.Alpha},
|
||||
|
@ -27,22 +27,24 @@ import (
|
||||
"k8s.io/kubelet/pkg/apis/podresources/v1"
|
||||
)
|
||||
|
||||
// podResourcesServerV1alpha1 implements PodResourcesListerServer
|
||||
// v1PodResourcesServer implements PodResourcesListerServer
|
||||
type v1PodResourcesServer struct {
|
||||
podsProvider PodsProvider
|
||||
devicesProvider DevicesProvider
|
||||
cpusProvider CPUsProvider
|
||||
memoryProvider MemoryProvider
|
||||
podsProvider PodsProvider
|
||||
devicesProvider DevicesProvider
|
||||
cpusProvider CPUsProvider
|
||||
memoryProvider MemoryProvider
|
||||
dynamicResourcesProvider DynamicResourcesProvider
|
||||
}
|
||||
|
||||
// NewV1PodResourcesServer returns a PodResourcesListerServer which lists pods provided by the PodsProvider
|
||||
// with device information provided by the DevicesProvider
|
||||
func NewV1PodResourcesServer(podsProvider PodsProvider, devicesProvider DevicesProvider, cpusProvider CPUsProvider, memoryProvider MemoryProvider) v1.PodResourcesListerServer {
|
||||
func NewV1PodResourcesServer(providers PodResourcesProviders) v1.PodResourcesListerServer {
|
||||
return &v1PodResourcesServer{
|
||||
podsProvider: podsProvider,
|
||||
devicesProvider: devicesProvider,
|
||||
cpusProvider: cpusProvider,
|
||||
memoryProvider: memoryProvider,
|
||||
podsProvider: providers.Pods,
|
||||
devicesProvider: providers.Devices,
|
||||
cpusProvider: providers.Cpus,
|
||||
memoryProvider: providers.Memory,
|
||||
dynamicResourcesProvider: providers.DynamicResources,
|
||||
}
|
||||
}
|
||||
|
||||
@ -69,6 +71,10 @@ func (p *v1PodResourcesServer) List(ctx context.Context, req *v1.ListPodResource
|
||||
CpuIds: p.cpusProvider.GetCPUs(string(pod.UID), container.Name),
|
||||
Memory: p.memoryProvider.GetMemory(string(pod.UID), container.Name),
|
||||
}
|
||||
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.KubeletPodResourcesDynamicResources) {
|
||||
pRes.Containers[j].DynamicResources = p.dynamicResourcesProvider.GetDynamicResources(pod, &container)
|
||||
}
|
||||
|
||||
}
|
||||
podResources[i] = &pRes
|
||||
}
|
||||
@ -85,7 +91,7 @@ func (p *v1PodResourcesServer) GetAllocatableResources(ctx context.Context, req
|
||||
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.KubeletPodResourcesGetAllocatable) {
|
||||
metrics.PodResourcesEndpointErrorsGetAllocatableCount.WithLabelValues("v1").Inc()
|
||||
return nil, fmt.Errorf("Pod Resources API GetAllocatableResources disabled")
|
||||
return nil, fmt.Errorf("PodResources API GetAllocatableResources disabled")
|
||||
}
|
||||
|
||||
return &v1.AllocatableResourcesResponse{
|
||||
@ -94,3 +100,43 @@ func (p *v1PodResourcesServer) GetAllocatableResources(ctx context.Context, req
|
||||
Memory: p.memoryProvider.GetAllocatableMemory(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Get returns information about the resources assigned to a specific pod
|
||||
func (p *v1PodResourcesServer) Get(ctx context.Context, req *v1.GetPodResourcesRequest) (*v1.GetPodResourcesResponse, error) {
|
||||
metrics.PodResourcesEndpointRequestsTotalCount.WithLabelValues("v1").Inc()
|
||||
metrics.PodResourcesEndpointRequestsGetCount.WithLabelValues("v1").Inc()
|
||||
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.KubeletPodResourcesGet) {
|
||||
metrics.PodResourcesEndpointErrorsGetCount.WithLabelValues("v1").Inc()
|
||||
return nil, fmt.Errorf("PodResources API Get method disabled")
|
||||
}
|
||||
|
||||
pod, exist := p.podsProvider.GetPodByName(req.PodNamespace, req.PodName)
|
||||
if !exist {
|
||||
metrics.PodResourcesEndpointErrorsGetCount.WithLabelValues("v1").Inc()
|
||||
return nil, fmt.Errorf("pod %s in namespace %s not found", req.PodName, req.PodNamespace)
|
||||
}
|
||||
|
||||
podResources := &v1.PodResources{
|
||||
Name: pod.Name,
|
||||
Namespace: pod.Namespace,
|
||||
Containers: make([]*v1.ContainerResources, len(pod.Spec.Containers)),
|
||||
}
|
||||
|
||||
for i, container := range pod.Spec.Containers {
|
||||
podResources.Containers[i] = &v1.ContainerResources{
|
||||
Name: container.Name,
|
||||
Devices: p.devicesProvider.GetDevices(string(pod.UID), container.Name),
|
||||
CpuIds: p.cpusProvider.GetCPUs(string(pod.UID), container.Name),
|
||||
Memory: p.memoryProvider.GetMemory(string(pod.UID), container.Name),
|
||||
}
|
||||
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.KubeletPodResourcesDynamicResources) {
|
||||
podResources.Containers[i].DynamicResources = p.dynamicResourcesProvider.GetDynamicResources(pod, &container)
|
||||
}
|
||||
}
|
||||
|
||||
response := &v1.GetPodResourcesResponse{
|
||||
PodResources: podResources,
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
@ -18,6 +18,7 @@ package podresources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
@ -66,12 +67,41 @@ func TestListPodResourcesV1(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
containers := []v1.Container{
|
||||
{
|
||||
Name: containerName,
|
||||
},
|
||||
}
|
||||
pods := []*v1.Pod{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: podName,
|
||||
Namespace: podNamespace,
|
||||
UID: podUID,
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: containers,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pluginCDIDevices := []*podresourcesapi.CDIDevice{{Name: "dra-dev0"}, {Name: "dra-dev1"}}
|
||||
draDevs := []*podresourcesapi.DynamicResource{
|
||||
{
|
||||
ClassName: "resource-class",
|
||||
ClaimName: "claim-name",
|
||||
ClaimNamespace: "default",
|
||||
ClaimResources: []*podresourcesapi.ClaimResource{{CDIDevices: pluginCDIDevices}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
desc string
|
||||
pods []*v1.Pod
|
||||
devices []*podresourcesapi.ContainerDevices
|
||||
cpus []int64
|
||||
memory []*podresourcesapi.ContainerMemory
|
||||
dynamicResources []*podresourcesapi.DynamicResource
|
||||
expectedResponse *podresourcesapi.ListPodResourcesResponse
|
||||
}{
|
||||
{
|
||||
@ -80,29 +110,16 @@ func TestListPodResourcesV1(t *testing.T) {
|
||||
devices: []*podresourcesapi.ContainerDevices{},
|
||||
cpus: []int64{},
|
||||
memory: []*podresourcesapi.ContainerMemory{},
|
||||
dynamicResources: []*podresourcesapi.DynamicResource{},
|
||||
expectedResponse: &podresourcesapi.ListPodResourcesResponse{},
|
||||
},
|
||||
{
|
||||
desc: "pod without devices",
|
||||
pods: []*v1.Pod{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: podName,
|
||||
Namespace: podNamespace,
|
||||
UID: podUID,
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: containerName,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
devices: []*podresourcesapi.ContainerDevices{},
|
||||
cpus: []int64{},
|
||||
memory: []*podresourcesapi.ContainerMemory{},
|
||||
desc: "pod without devices",
|
||||
pods: pods,
|
||||
devices: []*podresourcesapi.ContainerDevices{},
|
||||
cpus: []int64{},
|
||||
memory: []*podresourcesapi.ContainerMemory{},
|
||||
dynamicResources: []*podresourcesapi.DynamicResource{},
|
||||
expectedResponse: &podresourcesapi.ListPodResourcesResponse{
|
||||
PodResources: []*podresourcesapi.PodResources{
|
||||
{
|
||||
@ -110,8 +127,9 @@ func TestListPodResourcesV1(t *testing.T) {
|
||||
Namespace: podNamespace,
|
||||
Containers: []*podresourcesapi.ContainerResources{
|
||||
{
|
||||
Name: containerName,
|
||||
Devices: []*podresourcesapi.ContainerDevices{},
|
||||
Name: containerName,
|
||||
Devices: []*podresourcesapi.ContainerDevices{},
|
||||
DynamicResources: []*podresourcesapi.DynamicResource{},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -119,26 +137,12 @@ func TestListPodResourcesV1(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "pod with devices",
|
||||
pods: []*v1.Pod{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: podName,
|
||||
Namespace: podNamespace,
|
||||
UID: podUID,
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: containerName,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
devices: devs,
|
||||
cpus: cpus,
|
||||
memory: memory,
|
||||
desc: "pod with devices",
|
||||
pods: pods,
|
||||
devices: devs,
|
||||
cpus: cpus,
|
||||
memory: memory,
|
||||
dynamicResources: []*podresourcesapi.DynamicResource{},
|
||||
expectedResponse: &podresourcesapi.ListPodResourcesResponse{
|
||||
PodResources: []*podresourcesapi.PodResources{
|
||||
{
|
||||
@ -146,10 +150,61 @@ func TestListPodResourcesV1(t *testing.T) {
|
||||
Namespace: podNamespace,
|
||||
Containers: []*podresourcesapi.ContainerResources{
|
||||
{
|
||||
Name: containerName,
|
||||
Devices: devs,
|
||||
CpuIds: cpus,
|
||||
Memory: memory,
|
||||
Name: containerName,
|
||||
Devices: devs,
|
||||
CpuIds: cpus,
|
||||
Memory: memory,
|
||||
DynamicResources: []*podresourcesapi.DynamicResource{},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "pod with dynamic resources",
|
||||
pods: pods,
|
||||
devices: []*podresourcesapi.ContainerDevices{},
|
||||
cpus: cpus,
|
||||
memory: memory,
|
||||
dynamicResources: draDevs,
|
||||
expectedResponse: &podresourcesapi.ListPodResourcesResponse{
|
||||
PodResources: []*podresourcesapi.PodResources{
|
||||
{
|
||||
Name: podName,
|
||||
Namespace: podNamespace,
|
||||
Containers: []*podresourcesapi.ContainerResources{
|
||||
{
|
||||
Name: containerName,
|
||||
Devices: []*podresourcesapi.ContainerDevices{},
|
||||
CpuIds: cpus,
|
||||
Memory: memory,
|
||||
DynamicResources: draDevs,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "pod with dynamic resources and devices",
|
||||
pods: pods,
|
||||
devices: devs,
|
||||
cpus: cpus,
|
||||
memory: memory,
|
||||
dynamicResources: draDevs,
|
||||
expectedResponse: &podresourcesapi.ListPodResourcesResponse{
|
||||
PodResources: []*podresourcesapi.PodResources{
|
||||
{
|
||||
Name: podName,
|
||||
Namespace: podNamespace,
|
||||
Containers: []*podresourcesapi.ContainerResources{
|
||||
{
|
||||
Name: containerName,
|
||||
Devices: devs,
|
||||
CpuIds: cpus,
|
||||
Memory: memory,
|
||||
DynamicResources: draDevs,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -158,21 +213,31 @@ func TestListPodResourcesV1(t *testing.T) {
|
||||
},
|
||||
} {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.KubeletPodResourcesDynamicResources, true)()
|
||||
mockDevicesProvider := podresourcetest.NewMockDevicesProvider(mockCtrl)
|
||||
mockPodsProvider := podresourcetest.NewMockPodsProvider(mockCtrl)
|
||||
mockCPUsProvider := podresourcetest.NewMockCPUsProvider(mockCtrl)
|
||||
mockMemoryProvider := podresourcetest.NewMockMemoryProvider(mockCtrl)
|
||||
mockDynamicResourcesProvider := podresourcetest.NewMockDynamicResourcesProvider(mockCtrl)
|
||||
|
||||
mockPodsProvider.EXPECT().GetPods().Return(tc.pods).AnyTimes().AnyTimes()
|
||||
mockDevicesProvider.EXPECT().GetDevices(string(podUID), containerName).Return(tc.devices).AnyTimes()
|
||||
mockCPUsProvider.EXPECT().GetCPUs(string(podUID), containerName).Return(tc.cpus).AnyTimes()
|
||||
mockMemoryProvider.EXPECT().GetMemory(string(podUID), containerName).Return(tc.memory).AnyTimes()
|
||||
mockDynamicResourcesProvider.EXPECT().GetDynamicResources(pods[0], &containers[0]).Return(tc.dynamicResources).AnyTimes()
|
||||
mockDevicesProvider.EXPECT().UpdateAllocatedDevices().Return().AnyTimes()
|
||||
mockCPUsProvider.EXPECT().GetAllocatableCPUs().Return([]int64{}).AnyTimes()
|
||||
mockDevicesProvider.EXPECT().GetAllocatableDevices().Return([]*podresourcesapi.ContainerDevices{}).AnyTimes()
|
||||
mockMemoryProvider.EXPECT().GetAllocatableMemory().Return([]*podresourcesapi.ContainerMemory{}).AnyTimes()
|
||||
|
||||
server := NewV1PodResourcesServer(mockPodsProvider, mockDevicesProvider, mockCPUsProvider, mockMemoryProvider)
|
||||
providers := PodResourcesProviders{
|
||||
Pods: mockPodsProvider,
|
||||
Devices: mockDevicesProvider,
|
||||
Cpus: mockCPUsProvider,
|
||||
Memory: mockMemoryProvider,
|
||||
DynamicResources: mockDynamicResourcesProvider,
|
||||
}
|
||||
server := NewV1PodResourcesServer(providers)
|
||||
resp, err := server.List(context.TODO(), &podresourcesapi.ListPodResourcesRequest{})
|
||||
if err != nil {
|
||||
t.Errorf("want err = %v, got %q", nil, err)
|
||||
@ -459,7 +524,13 @@ func TestAllocatableResources(t *testing.T) {
|
||||
mockCPUsProvider.EXPECT().GetAllocatableCPUs().Return(tc.allCPUs).AnyTimes()
|
||||
mockMemoryProvider.EXPECT().GetAllocatableMemory().Return(tc.allMemory).AnyTimes()
|
||||
|
||||
server := NewV1PodResourcesServer(mockPodsProvider, mockDevicesProvider, mockCPUsProvider, mockMemoryProvider)
|
||||
providers := PodResourcesProviders{
|
||||
Pods: mockPodsProvider,
|
||||
Devices: mockDevicesProvider,
|
||||
Cpus: mockCPUsProvider,
|
||||
Memory: mockMemoryProvider,
|
||||
}
|
||||
server := NewV1PodResourcesServer(providers)
|
||||
|
||||
resp, err := server.GetAllocatableResources(context.TODO(), &podresourcesapi.AllocatableResourcesRequest{})
|
||||
if err != nil {
|
||||
@ -473,6 +544,185 @@ func TestAllocatableResources(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPodResourcesV1(t *testing.T) {
|
||||
podName := "pod-name"
|
||||
podNamespace := "pod-namespace"
|
||||
podUID := types.UID("pod-uid")
|
||||
containerName := "container-name"
|
||||
numaID := int64(1)
|
||||
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
|
||||
devs := []*podresourcesapi.ContainerDevices{
|
||||
{
|
||||
ResourceName: "resource",
|
||||
DeviceIds: []string{"dev0", "dev1"},
|
||||
Topology: &podresourcesapi.TopologyInfo{Nodes: []*podresourcesapi.NUMANode{{ID: numaID}}},
|
||||
},
|
||||
}
|
||||
|
||||
cpus := []int64{12, 23, 30}
|
||||
|
||||
memory := []*podresourcesapi.ContainerMemory{
|
||||
{
|
||||
MemoryType: "memory",
|
||||
Size_: 1073741824,
|
||||
Topology: &podresourcesapi.TopologyInfo{Nodes: []*podresourcesapi.NUMANode{{ID: numaID}}},
|
||||
},
|
||||
{
|
||||
MemoryType: "hugepages-1Gi",
|
||||
Size_: 1073741824,
|
||||
Topology: &podresourcesapi.TopologyInfo{Nodes: []*podresourcesapi.NUMANode{{ID: numaID}}},
|
||||
},
|
||||
}
|
||||
|
||||
containers := []v1.Container{
|
||||
{
|
||||
Name: containerName,
|
||||
},
|
||||
}
|
||||
|
||||
pod := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: podName,
|
||||
Namespace: podNamespace,
|
||||
UID: podUID,
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: containers,
|
||||
},
|
||||
}
|
||||
|
||||
pluginCDIDevices := []*podresourcesapi.CDIDevice{{Name: "dra-dev0"}, {Name: "dra-dev1"}}
|
||||
draDevs := []*podresourcesapi.DynamicResource{
|
||||
{
|
||||
ClassName: "resource-class",
|
||||
ClaimName: "claim-name",
|
||||
ClaimNamespace: "default",
|
||||
ClaimResources: []*podresourcesapi.ClaimResource{{CDIDevices: pluginCDIDevices}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
desc string
|
||||
err error
|
||||
exist bool
|
||||
pod *v1.Pod
|
||||
devices []*podresourcesapi.ContainerDevices
|
||||
cpus []int64
|
||||
memory []*podresourcesapi.ContainerMemory
|
||||
dynamicResources []*podresourcesapi.DynamicResource
|
||||
expectedResponse *podresourcesapi.GetPodResourcesResponse
|
||||
}{
|
||||
{
|
||||
desc: "pod not exist",
|
||||
err: fmt.Errorf("pod %s in namespace %s not found", podName, podNamespace),
|
||||
exist: false,
|
||||
pod: nil,
|
||||
devices: []*podresourcesapi.ContainerDevices{},
|
||||
cpus: []int64{},
|
||||
memory: []*podresourcesapi.ContainerMemory{},
|
||||
dynamicResources: []*podresourcesapi.DynamicResource{},
|
||||
|
||||
expectedResponse: &podresourcesapi.GetPodResourcesResponse{},
|
||||
},
|
||||
{
|
||||
desc: "pod without devices",
|
||||
err: nil,
|
||||
exist: true,
|
||||
pod: pod,
|
||||
devices: []*podresourcesapi.ContainerDevices{},
|
||||
cpus: []int64{},
|
||||
memory: []*podresourcesapi.ContainerMemory{},
|
||||
dynamicResources: []*podresourcesapi.DynamicResource{},
|
||||
expectedResponse: &podresourcesapi.GetPodResourcesResponse{
|
||||
PodResources: &podresourcesapi.PodResources{
|
||||
Name: podName,
|
||||
Namespace: podNamespace,
|
||||
Containers: []*podresourcesapi.ContainerResources{
|
||||
{
|
||||
Name: containerName,
|
||||
Devices: []*podresourcesapi.ContainerDevices{},
|
||||
DynamicResources: []*podresourcesapi.DynamicResource{},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "pod with devices",
|
||||
err: nil,
|
||||
exist: true,
|
||||
pod: pod,
|
||||
devices: devs,
|
||||
cpus: cpus,
|
||||
memory: memory,
|
||||
dynamicResources: draDevs,
|
||||
expectedResponse: &podresourcesapi.GetPodResourcesResponse{
|
||||
PodResources: &podresourcesapi.PodResources{
|
||||
Name: podName,
|
||||
Namespace: podNamespace,
|
||||
Containers: []*podresourcesapi.ContainerResources{
|
||||
{
|
||||
Name: containerName,
|
||||
Devices: devs,
|
||||
CpuIds: cpus,
|
||||
Memory: memory,
|
||||
DynamicResources: draDevs,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.KubeletPodResourcesGet, true)()
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.KubeletPodResourcesDynamicResources, true)()
|
||||
mockDevicesProvider := podresourcetest.NewMockDevicesProvider(mockCtrl)
|
||||
mockPodsProvider := podresourcetest.NewMockPodsProvider(mockCtrl)
|
||||
mockCPUsProvider := podresourcetest.NewMockCPUsProvider(mockCtrl)
|
||||
mockMemoryProvider := podresourcetest.NewMockMemoryProvider(mockCtrl)
|
||||
mockDynamicResourcesProvider := podresourcetest.NewMockDynamicResourcesProvider(mockCtrl)
|
||||
|
||||
mockPodsProvider.EXPECT().GetPodByName(podNamespace, podName).Return(tc.pod, tc.exist).AnyTimes()
|
||||
mockDevicesProvider.EXPECT().GetDevices(string(podUID), containerName).Return(tc.devices).AnyTimes()
|
||||
mockCPUsProvider.EXPECT().GetCPUs(string(podUID), containerName).Return(tc.cpus).AnyTimes()
|
||||
mockMemoryProvider.EXPECT().GetMemory(string(podUID), containerName).Return(tc.memory).AnyTimes()
|
||||
mockDynamicResourcesProvider.EXPECT().GetDynamicResources(pod, &containers[0]).Return(tc.dynamicResources).AnyTimes()
|
||||
mockDevicesProvider.EXPECT().UpdateAllocatedDevices().Return().AnyTimes()
|
||||
mockCPUsProvider.EXPECT().GetAllocatableCPUs().Return([]int64{}).AnyTimes()
|
||||
mockDevicesProvider.EXPECT().GetAllocatableDevices().Return([]*podresourcesapi.ContainerDevices{}).AnyTimes()
|
||||
mockMemoryProvider.EXPECT().GetAllocatableMemory().Return([]*podresourcesapi.ContainerMemory{}).AnyTimes()
|
||||
|
||||
providers := PodResourcesProviders{
|
||||
Pods: mockPodsProvider,
|
||||
Devices: mockDevicesProvider,
|
||||
Cpus: mockCPUsProvider,
|
||||
Memory: mockMemoryProvider,
|
||||
DynamicResources: mockDynamicResourcesProvider,
|
||||
}
|
||||
server := NewV1PodResourcesServer(providers)
|
||||
podReq := &podresourcesapi.GetPodResourcesRequest{PodName: podName, PodNamespace: podNamespace}
|
||||
resp, err := server.Get(context.TODO(), podReq)
|
||||
if err != nil {
|
||||
if err.Error() != tc.err.Error() {
|
||||
t.Errorf("want exit = %v, got %v", tc.err, err)
|
||||
}
|
||||
} else {
|
||||
if err != err {
|
||||
t.Errorf("want exit = %v, got %v", tc.err, err)
|
||||
} else {
|
||||
if !equalGetResponse(tc.expectedResponse, resp) {
|
||||
t.Errorf("want resp = %s, got %s", tc.expectedResponse.String(), resp.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func equalListResponse(respA, respB *podresourcesapi.ListPodResourcesResponse) bool {
|
||||
if len(respA.PodResources) != len(respB.PodResources) {
|
||||
return false
|
||||
@ -503,11 +753,54 @@ func equalListResponse(respA, respB *podresourcesapi.ListPodResourcesResponse) b
|
||||
if !equalContainerDevices(cntA.Devices, cntB.Devices) {
|
||||
return false
|
||||
}
|
||||
|
||||
if !euqalDynamicResources(cntA.DynamicResources, cntB.DynamicResources) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func euqalDynamicResources(draResA, draResB []*podresourcesapi.DynamicResource) bool {
|
||||
if len(draResA) != len(draResB) {
|
||||
return false
|
||||
}
|
||||
|
||||
for idx := 0; idx < len(draResA); idx++ {
|
||||
cntDraResA := draResA[idx]
|
||||
cntDraResB := draResB[idx]
|
||||
|
||||
if cntDraResA.ClassName != cntDraResB.ClassName {
|
||||
return false
|
||||
}
|
||||
if cntDraResA.ClaimName != cntDraResB.ClaimName {
|
||||
return false
|
||||
}
|
||||
if cntDraResA.ClaimNamespace != cntDraResB.ClaimNamespace {
|
||||
return false
|
||||
}
|
||||
if len(cntDraResA.ClaimResources) != len(cntDraResB.ClaimResources) {
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(cntDraResA.ClaimResources); i++ {
|
||||
claimResA := cntDraResA.ClaimResources[i]
|
||||
claimResB := cntDraResB.ClaimResources[i]
|
||||
if len(claimResA.CDIDevices) != len(claimResB.CDIDevices) {
|
||||
return false
|
||||
}
|
||||
for y := 0; y < len(claimResA.CDIDevices); y++ {
|
||||
cdiDeviceA := claimResA.CDIDevices[y]
|
||||
cdiDeviceB := claimResB.CDIDevices[y]
|
||||
if cdiDeviceA.Name != cdiDeviceB.Name {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
func equalContainerDevices(devA, devB []*podresourcesapi.ContainerDevices) bool {
|
||||
if len(devA) != len(devB) {
|
||||
return false
|
||||
@ -569,3 +862,38 @@ func equalAllocatableResourcesResponse(respA, respB *podresourcesapi.Allocatable
|
||||
}
|
||||
return equalContainerDevices(respA.Devices, respB.Devices)
|
||||
}
|
||||
|
||||
func equalGetResponse(ResA, ResB *podresourcesapi.GetPodResourcesResponse) bool {
|
||||
podResA := ResA.PodResources
|
||||
podResB := ResB.PodResources
|
||||
if podResA.Name != podResB.Name {
|
||||
return false
|
||||
}
|
||||
if podResA.Namespace != podResB.Namespace {
|
||||
return false
|
||||
}
|
||||
if len(podResA.Containers) != len(podResB.Containers) {
|
||||
return false
|
||||
}
|
||||
for jdx := 0; jdx < len(podResA.Containers); jdx++ {
|
||||
cntA := podResA.Containers[jdx]
|
||||
cntB := podResB.Containers[jdx]
|
||||
|
||||
if cntA.Name != cntB.Name {
|
||||
return false
|
||||
}
|
||||
if !equalInt64s(cntA.CpuIds, cntB.CpuIds) {
|
||||
return false
|
||||
}
|
||||
|
||||
if !equalContainerDevices(cntA.Devices, cntB.Devices) {
|
||||
return false
|
||||
}
|
||||
|
||||
if !euqalDynamicResources(cntA.DynamicResources, cntB.DynamicResources) {
|
||||
return false
|
||||
}
|
||||
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
@ -25,7 +25,7 @@ import (
|
||||
"k8s.io/kubelet/pkg/apis/podresources/v1alpha1"
|
||||
)
|
||||
|
||||
// podResourcesServerV1alpha1 implements PodResourcesListerServer
|
||||
// v1alpha1PodResourcesServer implements PodResourcesListerServer
|
||||
type v1alpha1PodResourcesServer struct {
|
||||
podsProvider PodsProvider
|
||||
devicesProvider DevicesProvider
|
||||
@ -33,10 +33,10 @@ type v1alpha1PodResourcesServer struct {
|
||||
|
||||
// NewV1alpha1PodResourcesServer returns a PodResourcesListerServer which lists pods provided by the PodsProvider
|
||||
// with device information provided by the DevicesProvider
|
||||
func NewV1alpha1PodResourcesServer(podsProvider PodsProvider, devicesProvider DevicesProvider) v1alpha1.PodResourcesListerServer {
|
||||
func NewV1alpha1PodResourcesServer(providers PodResourcesProviders) v1alpha1.PodResourcesListerServer {
|
||||
return &v1alpha1PodResourcesServer{
|
||||
podsProvider: podsProvider,
|
||||
devicesProvider: devicesProvider,
|
||||
podsProvider: providers.Pods,
|
||||
devicesProvider: providers.Devices,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -134,7 +134,11 @@ func TestListPodResourcesV1alpha1(t *testing.T) {
|
||||
mockDevicesProvider.EXPECT().GetDevices(string(podUID), containerName).Return(tc.devices).AnyTimes()
|
||||
mockDevicesProvider.EXPECT().UpdateAllocatedDevices().Return().AnyTimes()
|
||||
|
||||
server := NewV1alpha1PodResourcesServer(mockPodsProvider, mockDevicesProvider)
|
||||
providers := PodResourcesProviders{
|
||||
Pods: mockPodsProvider,
|
||||
Devices: mockDevicesProvider,
|
||||
}
|
||||
server := NewV1alpha1PodResourcesServer(providers)
|
||||
resp, err := server.List(context.TODO(), &v1alpha1.ListPodResourcesRequest{})
|
||||
if err != nil {
|
||||
t.Errorf("want err = %v, got %q", nil, err)
|
||||
|
@ -114,6 +114,21 @@ func (m *MockPodsProvider) EXPECT() *MockPodsProviderMockRecorder {
|
||||
return m.recorder
|
||||
}
|
||||
|
||||
// GetPodByName mocks base method.
|
||||
func (m *MockPodsProvider) GetPodByName(namespace, name string) (*v1.Pod, bool) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetPodByName", namespace, name)
|
||||
ret0, _ := ret[0].(*v1.Pod)
|
||||
ret1, _ := ret[1].(bool)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetPodByName indicates an expected call of GetPodByName.
|
||||
func (mr *MockPodsProviderMockRecorder) GetPodByName(namespace, name interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPodByName", reflect.TypeOf((*MockPodsProvider)(nil).GetPodByName), namespace, name)
|
||||
}
|
||||
|
||||
// GetPods mocks base method.
|
||||
func (m *MockPodsProvider) GetPods() []*v1.Pod {
|
||||
m.ctrl.T.Helper()
|
||||
@ -229,3 +244,40 @@ func (mr *MockMemoryProviderMockRecorder) GetMemory(podUID, containerName interf
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMemory", reflect.TypeOf((*MockMemoryProvider)(nil).GetMemory), podUID, containerName)
|
||||
}
|
||||
|
||||
// MockDynamicResourcesProvider is a mock of DynamicResourcesProvider interface.
|
||||
type MockDynamicResourcesProvider struct {
|
||||
ctrl *gomock.Controller
|
||||
recorder *MockDynamicResourcesProviderMockRecorder
|
||||
}
|
||||
|
||||
// MockDynamicResourcesProviderMockRecorder is the mock recorder for MockDynamicResourcesProvider.
|
||||
type MockDynamicResourcesProviderMockRecorder struct {
|
||||
mock *MockDynamicResourcesProvider
|
||||
}
|
||||
|
||||
// NewMockDynamicResourcesProvider creates a new mock instance.
|
||||
func NewMockDynamicResourcesProvider(ctrl *gomock.Controller) *MockDynamicResourcesProvider {
|
||||
mock := &MockDynamicResourcesProvider{ctrl: ctrl}
|
||||
mock.recorder = &MockDynamicResourcesProviderMockRecorder{mock}
|
||||
return mock
|
||||
}
|
||||
|
||||
// EXPECT returns an object that allows the caller to indicate expected use.
|
||||
func (m *MockDynamicResourcesProvider) EXPECT() *MockDynamicResourcesProviderMockRecorder {
|
||||
return m.recorder
|
||||
}
|
||||
|
||||
// GetDynamicResources mocks base method.
|
||||
func (m *MockDynamicResourcesProvider) GetDynamicResources(pod *v1.Pod, container *v1.Container) []*v10.DynamicResource {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetDynamicResources", pod, container)
|
||||
ret0, _ := ret[0].([]*v10.DynamicResource)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// GetDynamicResources indicates an expected call of GetDynamicResources.
|
||||
func (mr *MockDynamicResourcesProviderMockRecorder) GetDynamicResources(pod, container interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDynamicResources", reflect.TypeOf((*MockDynamicResourcesProvider)(nil).GetDynamicResources), pod, container)
|
||||
}
|
||||
|
@ -35,6 +35,7 @@ type DevicesProvider interface {
|
||||
// PodsProvider knows how to provide the pods admitted by the node
|
||||
type PodsProvider interface {
|
||||
GetPods() []*v1.Pod
|
||||
GetPodByName(namespace, name string) (*v1.Pod, bool)
|
||||
}
|
||||
|
||||
// CPUsProvider knows how to provide the cpus used by the given container
|
||||
@ -51,3 +52,16 @@ type MemoryProvider interface {
|
||||
// GetAllocatableMemory returns the allocatable memory from the node
|
||||
GetAllocatableMemory() []*podresourcesapi.ContainerMemory
|
||||
}
|
||||
|
||||
type DynamicResourcesProvider interface {
|
||||
// GetDynamicResources returns information about dynamic resources assigned to pods and containers
|
||||
GetDynamicResources(pod *v1.Pod, container *v1.Container) []*podresourcesapi.DynamicResource
|
||||
}
|
||||
|
||||
type PodResourcesProviders struct {
|
||||
Pods PodsProvider
|
||||
Devices DevicesProvider
|
||||
Cpus CPUsProvider
|
||||
Memory MemoryProvider
|
||||
DynamicResources DynamicResourcesProvider
|
||||
}
|
||||
|
@ -127,10 +127,11 @@ type ContainerManager interface {
|
||||
// might need to unprepare resources.
|
||||
PodMightNeedToUnprepareResources(UID types.UID) bool
|
||||
|
||||
// Implements the podresources Provider API for CPUs, Memory and Devices
|
||||
// Implements the PodResources Provider API
|
||||
podresources.CPUsProvider
|
||||
podresources.DevicesProvider
|
||||
podresources.MemoryProvider
|
||||
podresources.DynamicResourcesProvider
|
||||
}
|
||||
|
||||
type NodeConfig struct {
|
||||
|
@ -965,6 +965,41 @@ func (cm *containerManagerImpl) GetAllocatableMemory() []*podresourcesapi.Contai
|
||||
return containerMemoryFromBlock(cm.memoryManager.GetAllocatableMemory())
|
||||
}
|
||||
|
||||
func (cm *containerManagerImpl) GetDynamicResources(pod *v1.Pod, container *v1.Container) []*podresourcesapi.DynamicResource {
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DynamicResourceAllocation) {
|
||||
return []*podresourcesapi.DynamicResource{}
|
||||
}
|
||||
|
||||
var containerDynamicResources []*podresourcesapi.DynamicResource
|
||||
containerClaimInfos, err := cm.draManager.GetContainerClaimInfos(pod, container)
|
||||
if err != nil {
|
||||
klog.ErrorS(err, "Unable to get container claim info state")
|
||||
return []*podresourcesapi.DynamicResource{}
|
||||
}
|
||||
for _, containerClaimInfo := range containerClaimInfos {
|
||||
var claimResources []*podresourcesapi.ClaimResource
|
||||
// TODO: Currently we maintain a list of ClaimResources, each of which contains
|
||||
// a set of CDIDevices from a different kubelet plugin. In the future we may want to
|
||||
// include the name of the kubelet plugin and/or other types of resources that are
|
||||
// not CDIDevices (assuming the DRAmanager supports this).
|
||||
for _, klPluginCdiDevices := range containerClaimInfo.CDIDevices {
|
||||
var cdiDevices []*podresourcesapi.CDIDevice
|
||||
for _, cdiDevice := range klPluginCdiDevices {
|
||||
cdiDevices = append(cdiDevices, &podresourcesapi.CDIDevice{Name: cdiDevice})
|
||||
}
|
||||
claimResources = append(claimResources, &podresourcesapi.ClaimResource{CDIDevices: cdiDevices})
|
||||
}
|
||||
containerDynamicResource := podresourcesapi.DynamicResource{
|
||||
ClassName: containerClaimInfo.ClassName,
|
||||
ClaimName: containerClaimInfo.ClaimName,
|
||||
ClaimNamespace: containerClaimInfo.Namespace,
|
||||
ClaimResources: claimResources,
|
||||
}
|
||||
containerDynamicResources = append(containerDynamicResources, &containerDynamicResource)
|
||||
}
|
||||
return containerDynamicResources
|
||||
}
|
||||
|
||||
func (cm *containerManagerImpl) ShouldResetExtendedResourceCapacity() bool {
|
||||
return cm.deviceManager.ShouldResetExtendedResourceCapacity()
|
||||
}
|
||||
|
@ -159,6 +159,10 @@ func (cm *containerManagerStub) GetAllocatableMemory() []*podresourcesapi.Contai
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cm *containerManagerStub) GetDynamicResources(pod *v1.Pod, container *v1.Container) []*podresourcesapi.DynamicResource {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cm *containerManagerStub) GetNodeAllocatableAbsolute() v1.ResourceList {
|
||||
return nil
|
||||
}
|
||||
|
@ -253,6 +253,10 @@ func (cm *containerManagerImpl) GetNodeAllocatableAbsolute() v1.ResourceList {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cm *containerManagerImpl) GetDynamicResources(pod *v1.Pod, container *v1.Container) []*podresourcesapi.DynamicResource {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cm *containerManagerImpl) PrepareDynamicResources(pod *v1.Pod) error {
|
||||
return nil
|
||||
}
|
||||
|
@ -26,9 +26,9 @@ import (
|
||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||
)
|
||||
|
||||
// claimInfo holds information required
|
||||
// ClaimInfo holds information required
|
||||
// to prepare and unprepare a resource claim.
|
||||
type claimInfo struct {
|
||||
type ClaimInfo struct {
|
||||
sync.RWMutex
|
||||
state.ClaimInfoState
|
||||
// annotations is a list of container annotations associated with
|
||||
@ -36,14 +36,14 @@ type claimInfo struct {
|
||||
annotations []kubecontainer.Annotation
|
||||
}
|
||||
|
||||
func (res *claimInfo) addPodReference(podUID types.UID) {
|
||||
func (res *ClaimInfo) addPodReference(podUID types.UID) {
|
||||
res.Lock()
|
||||
defer res.Unlock()
|
||||
|
||||
res.PodUIDs.Insert(string(podUID))
|
||||
}
|
||||
|
||||
func (res *claimInfo) deletePodReference(podUID types.UID) {
|
||||
func (res *ClaimInfo) deletePodReference(podUID types.UID) {
|
||||
res.Lock()
|
||||
defer res.Unlock()
|
||||
|
||||
@ -54,24 +54,25 @@ func (res *claimInfo) deletePodReference(podUID types.UID) {
|
||||
type claimInfoCache struct {
|
||||
sync.RWMutex
|
||||
state state.CheckpointState
|
||||
claimInfo map[string]*claimInfo
|
||||
claimInfo map[string]*ClaimInfo
|
||||
}
|
||||
|
||||
func newClaimInfo(driverName string, claimUID types.UID, claimName, namespace string, podUIDs sets.Set[string]) *claimInfo {
|
||||
func newClaimInfo(driverName, className string, claimUID types.UID, claimName, namespace string, podUIDs sets.Set[string]) *ClaimInfo {
|
||||
claimInfoState := state.ClaimInfoState{
|
||||
DriverName: driverName,
|
||||
ClassName: className,
|
||||
ClaimUID: claimUID,
|
||||
ClaimName: claimName,
|
||||
Namespace: namespace,
|
||||
PodUIDs: podUIDs,
|
||||
}
|
||||
claimInfo := claimInfo{
|
||||
claimInfo := ClaimInfo{
|
||||
ClaimInfoState: claimInfoState,
|
||||
}
|
||||
return &claimInfo
|
||||
}
|
||||
|
||||
func (info *claimInfo) addCDIDevices(pluginName string, cdiDevices []string) error {
|
||||
func (info *ClaimInfo) addCDIDevices(pluginName string, cdiDevices []string) error {
|
||||
// NOTE: Passing CDI device names as annotations is a temporary solution
|
||||
// It will be removed after all runtimes are updated
|
||||
// to get CDI device names from the ContainerConfig.CDIDevices field
|
||||
@ -104,12 +105,13 @@ func newClaimInfoCache(stateDir, checkpointName string) (*claimInfoCache, error)
|
||||
|
||||
cache := &claimInfoCache{
|
||||
state: stateImpl,
|
||||
claimInfo: make(map[string]*claimInfo),
|
||||
claimInfo: make(map[string]*ClaimInfo),
|
||||
}
|
||||
|
||||
for _, entry := range curState {
|
||||
info := newClaimInfo(
|
||||
entry.DriverName,
|
||||
entry.ClassName,
|
||||
entry.ClaimUID,
|
||||
entry.ClaimName,
|
||||
entry.Namespace,
|
||||
@ -127,14 +129,14 @@ func newClaimInfoCache(stateDir, checkpointName string) (*claimInfoCache, error)
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
func (cache *claimInfoCache) add(res *claimInfo) {
|
||||
func (cache *claimInfoCache) add(res *ClaimInfo) {
|
||||
cache.Lock()
|
||||
defer cache.Unlock()
|
||||
|
||||
cache.claimInfo[res.ClaimName+res.Namespace] = res
|
||||
}
|
||||
|
||||
func (cache *claimInfoCache) get(claimName, namespace string) *claimInfo {
|
||||
func (cache *claimInfoCache) get(claimName, namespace string) *ClaimInfo {
|
||||
cache.RLock()
|
||||
defer cache.RUnlock()
|
||||
|
||||
|
@ -122,6 +122,7 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error {
|
||||
// Create a claimInfo object to store the relevant claim info.
|
||||
claimInfo := newClaimInfo(
|
||||
resourceClaim.Status.DriverName,
|
||||
resourceClaim.Spec.ResourceClassName,
|
||||
resourceClaim.UID,
|
||||
resourceClaim.Name,
|
||||
resourceClaim.Namespace,
|
||||
@ -288,3 +289,24 @@ func (m *ManagerImpl) UnprepareResources(pod *v1.Pod) error {
|
||||
func (m *ManagerImpl) PodMightNeedToUnprepareResources(UID types.UID) bool {
|
||||
return m.cache.hasPodReference(UID)
|
||||
}
|
||||
|
||||
// GetCongtainerClaimInfos gets Container's ClaimInfo
|
||||
func (m *ManagerImpl) GetContainerClaimInfos(pod *v1.Pod, container *v1.Container) ([]*ClaimInfo, error) {
|
||||
claimInfos := make([]*ClaimInfo, 0, len(pod.Spec.ResourceClaims))
|
||||
|
||||
for i, podResourceClaim := range pod.Spec.ResourceClaims {
|
||||
claimName := resourceclaim.Name(pod, &pod.Spec.ResourceClaims[i])
|
||||
|
||||
for _, claim := range container.Resources.Claims {
|
||||
if podResourceClaim.Name != claim.Name {
|
||||
continue
|
||||
}
|
||||
claimInfo := m.cache.get(claimName, pod.Namespace)
|
||||
if claimInfo == nil {
|
||||
return nil, fmt.Errorf("unable to get resource for namespace: %s, claim: %s", pod.Namespace, claimName)
|
||||
}
|
||||
claimInfos = append(claimInfos, claimInfo)
|
||||
}
|
||||
}
|
||||
return claimInfos, nil
|
||||
}
|
||||
|
@ -39,6 +39,9 @@ type ClaimInfoState struct {
|
||||
// Name of the DRA driver
|
||||
DriverName string
|
||||
|
||||
// ClassName is a resource class of the claim
|
||||
ClassName string
|
||||
|
||||
// ClaimUID is an UID of the resource claim
|
||||
ClaimUID types.UID
|
||||
|
||||
|
@ -50,11 +50,12 @@ func TestCheckpointGetOrCreate(t *testing.T) {
|
||||
},
|
||||
{
|
||||
"Restore checkpoint - single claim",
|
||||
`{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":1988120167}`,
|
||||
`{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":153446146}`,
|
||||
"",
|
||||
[]ClaimInfoState{
|
||||
{
|
||||
DriverName: "test-driver.cdi.k8s.io",
|
||||
ClassName: "class-name",
|
||||
ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7",
|
||||
ClaimName: "example",
|
||||
Namespace: "default",
|
||||
@ -67,11 +68,12 @@ func TestCheckpointGetOrCreate(t *testing.T) {
|
||||
},
|
||||
{
|
||||
"Restore checkpoint - single claim - multiple devices",
|
||||
`{"version":"v1","entries":[{"DriverName":"meta-test-driver.cdi.k8s.io","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver-1.cdi.k8s.io":["example-1.com/example-1=cdi-example-1"],"test-driver-2.cdi.k8s.io":["example-2.com/example-2=cdi-example-2"]}}],"checksum":2113538068}`,
|
||||
`{"version":"v1","entries":[{"DriverName":"meta-test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver-1.cdi.k8s.io":["example-1.com/example-1=cdi-example-1"],"test-driver-2.cdi.k8s.io":["example-2.com/example-2=cdi-example-2"]}}],"checksum":1363630443}`,
|
||||
"",
|
||||
[]ClaimInfoState{
|
||||
{
|
||||
DriverName: "meta-test-driver.cdi.k8s.io",
|
||||
ClassName: "class-name",
|
||||
ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7",
|
||||
ClaimName: "example",
|
||||
Namespace: "default",
|
||||
@ -85,11 +87,12 @@ func TestCheckpointGetOrCreate(t *testing.T) {
|
||||
},
|
||||
{
|
||||
"Restore checkpoint - multiple claims",
|
||||
`{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example-1","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example-1"]}},{"DriverName":"test-driver.cdi.k8s.io","ClaimUID":"4cf8db2d-06c0-7d70-1a51-e59b25b2c16c","ClaimName":"example-2","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example-2"]}}],"checksum":666680545}`,
|
||||
`{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name-1","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example-1","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example-1"]}},{"DriverName":"test-driver.cdi.k8s.io","ClaimUID":"4cf8db2d-06c0-7d70-1a51-e59b25b2c16c","ClassName":"class-name-2","ClaimName":"example-2","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example-2"]}}],"checksum":1978566460}`,
|
||||
"",
|
||||
[]ClaimInfoState{
|
||||
{
|
||||
DriverName: "test-driver.cdi.k8s.io",
|
||||
ClassName: "class-name-1",
|
||||
ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7",
|
||||
ClaimName: "example-1",
|
||||
Namespace: "default",
|
||||
@ -100,6 +103,7 @@ func TestCheckpointGetOrCreate(t *testing.T) {
|
||||
},
|
||||
{
|
||||
DriverName: "test-driver.cdi.k8s.io",
|
||||
ClassName: "class-name-2",
|
||||
ClaimUID: "4cf8db2d-06c0-7d70-1a51-e59b25b2c16c",
|
||||
ClaimName: "example-2",
|
||||
Namespace: "default",
|
||||
@ -112,7 +116,7 @@ func TestCheckpointGetOrCreate(t *testing.T) {
|
||||
},
|
||||
{
|
||||
"Restore checkpoint - invalid checksum",
|
||||
`{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":1988120168}`,
|
||||
`{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":1988120168}`,
|
||||
"checkpoint is corrupted",
|
||||
[]ClaimInfoState{},
|
||||
},
|
||||
@ -167,6 +171,7 @@ func TestCheckpointGetOrCreate(t *testing.T) {
|
||||
func TestCheckpointStateStore(t *testing.T) {
|
||||
claimInfoState := ClaimInfoState{
|
||||
DriverName: "test-driver.cdi.k8s.io",
|
||||
ClassName: "class-name",
|
||||
ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7",
|
||||
ClaimName: "example",
|
||||
Namespace: "default",
|
||||
@ -176,7 +181,7 @@ func TestCheckpointStateStore(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
expectedCheckpoint := `{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":1988120167}`
|
||||
expectedCheckpoint := `{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":153446146}`
|
||||
|
||||
// create temp dir
|
||||
testingDir, err := os.MkdirTemp("", "dramanager_state_test")
|
||||
|
@ -38,6 +38,9 @@ type Manager interface {
|
||||
// PodMightNeedToUnprepareResources returns true if the pod with the given UID
|
||||
// might need to unprepare resources.
|
||||
PodMightNeedToUnprepareResources(UID types.UID) bool
|
||||
|
||||
// GetContainerClaimInfos gets Container ClaimInfo objects
|
||||
GetContainerClaimInfos(pod *v1.Pod, container *v1.Container) ([]*ClaimInfo, error)
|
||||
}
|
||||
|
||||
// ContainerInfo contains information required by the runtime to consume prepared resources.
|
||||
|
@ -232,6 +232,10 @@ func (cm *FakeContainerManager) GetAllocatableMemory() []*podresourcesapi.Contai
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cm *FakeContainerManager) GetDynamicResources(pod *v1.Pod, container *v1.Container) []*podresourcesapi.DynamicResource {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cm *FakeContainerManager) GetNodeAllocatableAbsolute() v1.ResourceList {
|
||||
cm.Lock()
|
||||
defer cm.Unlock()
|
||||
|
@ -2783,11 +2783,12 @@ func (kl *Kubelet) ListenAndServePodResources() {
|
||||
return
|
||||
}
|
||||
|
||||
providers := server.PodResourcesProviders{
|
||||
Pods: kl.podManager,
|
||||
Devices: kl.containerManager,
|
||||
Cpus: kl.containerManager,
|
||||
Memory: kl.containerManager,
|
||||
providers := podresources.PodResourcesProviders{
|
||||
Pods: kl.podManager,
|
||||
Devices: kl.containerManager,
|
||||
Cpus: kl.containerManager,
|
||||
Memory: kl.containerManager,
|
||||
DynamicResources: kl.containerManager,
|
||||
}
|
||||
|
||||
server.ListenAndServePodResources(socket, providers)
|
||||
|
@ -78,6 +78,8 @@ const (
|
||||
PodResourcesEndpointRequestsGetAllocatableKey = "pod_resources_endpoint_requests_get_allocatable"
|
||||
PodResourcesEndpointErrorsListKey = "pod_resources_endpoint_errors_list"
|
||||
PodResourcesEndpointErrorsGetAllocatableKey = "pod_resources_endpoint_errors_get_allocatable"
|
||||
PodResourcesEndpointRequestsGetKey = "pod_resources_endpoint_requests_get"
|
||||
PodResourcesEndpointErrorsGetKey = "pod_resources_endpoint_errors_get"
|
||||
|
||||
// Metrics keys for RuntimeClass
|
||||
RunPodSandboxDurationKey = "run_podsandbox_duration_seconds"
|
||||
@ -441,6 +443,30 @@ var (
|
||||
[]string{"server_api_version"},
|
||||
)
|
||||
|
||||
// PodResourcesEndpointRequestsGetCount is a Counter that tracks the number of requests to the PodResource Get() endpoint.
|
||||
// Broken down by server API version.
|
||||
PodResourcesEndpointRequestsGetCount = metrics.NewCounterVec(
|
||||
&metrics.CounterOpts{
|
||||
Subsystem: KubeletSubsystem,
|
||||
Name: PodResourcesEndpointRequestsGetKey,
|
||||
Help: "Number of requests to the PodResource Get endpoint. Broken down by server api version.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"server_api_version"},
|
||||
)
|
||||
|
||||
// PodResourcesEndpointErrorsGetCount is a Counter that tracks the number of errors returned by he PodResource List() endpoint.
|
||||
// Broken down by server API version.
|
||||
PodResourcesEndpointErrorsGetCount = metrics.NewCounterVec(
|
||||
&metrics.CounterOpts{
|
||||
Subsystem: KubeletSubsystem,
|
||||
Name: PodResourcesEndpointErrorsGetKey,
|
||||
Help: "Number of requests to the PodResource Get endpoint which returned error. Broken down by server api version.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"server_api_version"},
|
||||
)
|
||||
|
||||
// RunPodSandboxDuration is a Histogram that tracks the duration (in seconds) it takes to run Pod Sandbox operations.
|
||||
// Broken down by RuntimeClass.Handler.
|
||||
RunPodSandboxDuration = metrics.NewHistogramVec(
|
||||
@ -759,6 +785,10 @@ func Register(collectors ...metrics.StableCollector) {
|
||||
legacyregistry.MustRegister(PodResourcesEndpointErrorsListCount)
|
||||
legacyregistry.MustRegister(PodResourcesEndpointErrorsGetAllocatableCount)
|
||||
}
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPodResourcesGet) {
|
||||
legacyregistry.MustRegister(PodResourcesEndpointRequestsGetCount)
|
||||
legacyregistry.MustRegister(PodResourcesEndpointErrorsGetCount)
|
||||
}
|
||||
}
|
||||
legacyregistry.MustRegister(StartedPodsTotal)
|
||||
legacyregistry.MustRegister(StartedPodsErrorsTotal)
|
||||
|
@ -218,11 +218,11 @@ type PodResourcesProviders struct {
|
||||
}
|
||||
|
||||
// ListenAndServePodResources initializes a gRPC server to serve the PodResources service
|
||||
func ListenAndServePodResources(socket string, providers PodResourcesProviders) {
|
||||
func ListenAndServePodResources(socket string, providers podresources.PodResourcesProviders) {
|
||||
server := grpc.NewServer(podresourcesgrpc.WithRateLimiter(podresourcesgrpc.DefaultQPS, podresourcesgrpc.DefaultBurstTokens))
|
||||
|
||||
podresourcesapiv1alpha1.RegisterPodResourcesListerServer(server, podresources.NewV1alpha1PodResourcesServer(providers.Pods, providers.Devices))
|
||||
podresourcesapi.RegisterPodResourcesListerServer(server, podresources.NewV1PodResourcesServer(providers.Pods, providers.Devices, providers.Cpus, providers.Memory))
|
||||
podresourcesapiv1alpha1.RegisterPodResourcesListerServer(server, podresources.NewV1alpha1PodResourcesServer(providers))
|
||||
podresourcesapi.RegisterPodResourcesListerServer(server, podresources.NewV1PodResourcesServer(providers))
|
||||
|
||||
l, err := util.CreateListener(socket)
|
||||
if err != nil {
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -20,6 +20,7 @@ option (gogoproto.goproto_unrecognized_all) = false;
|
||||
service PodResourcesLister {
|
||||
rpc List(ListPodResourcesRequest) returns (ListPodResourcesResponse) {}
|
||||
rpc GetAllocatableResources(AllocatableResourcesRequest) returns (AllocatableResourcesResponse) {}
|
||||
rpc Get(GetPodResourcesRequest) returns (GetPodResourcesResponse) {}
|
||||
}
|
||||
|
||||
message AllocatableResourcesRequest {}
|
||||
@ -52,6 +53,7 @@ message ContainerResources {
|
||||
repeated ContainerDevices devices = 2;
|
||||
repeated int64 cpu_ids = 3;
|
||||
repeated ContainerMemory memory = 4;
|
||||
repeated DynamicResource dynamic_resources = 5;
|
||||
}
|
||||
|
||||
// ContainerMemory contains information about memory and hugepages assigned to a container
|
||||
@ -77,3 +79,36 @@ message TopologyInfo {
|
||||
message NUMANode {
|
||||
int64 ID = 1;
|
||||
}
|
||||
|
||||
// DynamicResource contains information about the devices assigned to a container by DRA
|
||||
message DynamicResource {
|
||||
string class_name = 1;
|
||||
string claim_name = 2;
|
||||
string claim_namespace = 3;
|
||||
repeated ClaimResource claim_resources = 4;
|
||||
}
|
||||
|
||||
// ClaimResource contains per plugin resource information
|
||||
message ClaimResource {
|
||||
repeated CDIDevice cdi_devices = 1 [(gogoproto.customname) = "CDIDevices"];
|
||||
}
|
||||
|
||||
// CDIDevice specifies a CDI device information
|
||||
message CDIDevice {
|
||||
// Fully qualified CDI device name
|
||||
// for example: vendor.com/gpu=gpudevice1
|
||||
// see more details in the CDI specification:
|
||||
// https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md
|
||||
string name = 1;
|
||||
}
|
||||
|
||||
// GetPodResourcesRequest contains information about the pod
|
||||
message GetPodResourcesRequest {
|
||||
string pod_name = 1;
|
||||
string pod_namespace = 2;
|
||||
}
|
||||
|
||||
// GetPodResourcesResponse contains information about the pod the devices
|
||||
message GetPodResourcesResponse {
|
||||
PodResources pod_resources = 1;
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user