github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-08-08 23:37:11 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #132028 from ffromani/podresources-list-active-pods

Browse Source 
podresources: list: use active pods
This commit is contained in:
Kubernetes Prow Robot
2025-07-14 12:06:24 -07:00
committed by GitHub
parent 1127e02789 8f92a81787
commit bf0be9fb56
9 changed files with 955 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
pkg/features/kube_features.go
View File

@@ -443,6 +443,27 @@ const (
// Enable POD resources API with Get method
KubeletPodResourcesGet featuregate.Feature = "KubeletPodResourcesGet"
// owner: @ffromani
// Deprecated: v1.34
//
// issue: https://github.com/kubernetes/kubernetes/issues/119423
// Disables restricted output for the podresources API list endpoint.
// "Restricted" output only includes the pods which are actually running and thus they
// hold resources. Turns out this was originally the intended behavior, see:
// https://github.com/kubernetes/kubernetes/pull/79409#issuecomment-505975671
// This behavior was lost over time and interaction with memory manager creates
// an unfixable bug because the endpoint returns spurious stale information the clients
// cannot filter out, because the API doesn't provide enough context. See:
// https://github.com/kubernetes/kubernetes/issues/132020
// The endpoint has returning extra information for long time, but that information
// is also useless for the purpose of this API. Nevertheless, we are changing a long-established
// albeit buggy behavior, so users observing any regressions can use the
// KubeletPodResourcesListUseActivePods/ feature gate (default on) to restore the old behavior.
// Please file issues if you hit issues and have to use this Feature Gate.
// The Feature Gate will be locked to true in +4 releases (1.38) and then removed (1.39)
// if there are no bug reported.
KubeletPodResourcesListUseActivePods featuregate.Feature = "KubeletPodResourcesListUseActivePods"
// owner: @hoskeri
//
// Restores previous behavior where Kubelet fails self registration if node create returns 403 Forbidden.
@@ -1281,6 +1302,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
KubeletPodResourcesListUseActivePods: {
{Version: version.MustParse("1.0"), Default: false, PreRelease: featuregate.GA},
{Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.Deprecated}, // lock to default in 1.38, remove in 1.39
},
KubeletRegistrationGetOnExistsOnly: {
{Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.GA},
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Deprecated},

13
pkg/kubelet/apis/podresources/server_v1.go
View File

@@ -22,6 +22,7 @@ import (
v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
kubefeatures "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/metrics"
@@ -36,17 +37,21 @@ type v1PodResourcesServer struct {
cpusProvider CPUsProvider
memoryProvider MemoryProvider
dynamicResourcesProvider DynamicResourcesProvider
useActivePods bool
}
// NewV1PodResourcesServer returns a PodResourcesListerServer which lists pods provided by the PodsProvider
// with device information provided by the DevicesProvider
func NewV1PodResourcesServer(providers PodResourcesProviders) podresourcesv1.PodResourcesListerServer {
useActivePods := utilfeature.DefaultFeatureGate.Enabled(kubefeatures.KubeletPodResourcesListUseActivePods)
klog.InfoS("podresources", "method", "list", "useActivePods", useActivePods)
return &v1PodResourcesServer{
podsProvider: providers.Pods,
devicesProvider: providers.Devices,
cpusProvider: providers.Cpus,
memoryProvider: providers.Memory,
dynamicResourcesProvider: providers.DynamicResources,
useActivePods: useActivePods,
}
}
@@ -55,7 +60,13 @@ func (p *v1PodResourcesServer) List(ctx context.Context, req *podresourcesv1.Lis
metrics.PodResourcesEndpointRequestsTotalCount.WithLabelValues("v1").Inc()
metrics.PodResourcesEndpointRequestsListCount.WithLabelValues("v1").Inc()
pods := p.podsProvider.GetPods()
var pods []*v1.Pod
if p.useActivePods {
pods = p.podsProvider.GetActivePods()
} else {
pods = p.podsProvider.GetPods()
}
podResources := make([]*podresourcesv1.PodResources, len(pods))
p.devicesProvider.UpdateAllocatedDevices()

157
pkg/kubelet/apis/podresources/server_v1_test.go
View File

@@ -19,10 +19,12 @@ package podresources
import (
"context"
"fmt"
"sort"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/mock"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -221,6 +223,7 @@ func TestListPodResourcesV1(t *testing.T) {
mockDynamicResourcesProvider := podresourcetest.NewMockDynamicResourcesProvider(t)
mockPodsProvider.EXPECT().GetPods().Return(tc.pods).Maybe()
mockPodsProvider.EXPECT().GetActivePods().Return(tc.pods).Maybe()
mockDevicesProvider.EXPECT().GetDevices(string(podUID), containerName).Return(tc.devices).Maybe()
mockCPUsProvider.EXPECT().GetCPUs(string(podUID), containerName).Return(tc.cpus).Maybe()
mockMemoryProvider.EXPECT().GetMemory(string(podUID), containerName).Return(tc.memory).Maybe()
@@ -249,6 +252,159 @@ func TestListPodResourcesV1(t *testing.T) {
}
}
func makePod(idx int) *v1.Pod {
podNamespace := "pod-namespace"
podName := fmt.Sprintf("pod-name-%d", idx)
podUID := types.UID(fmt.Sprintf("pod-uid-%d", idx))
containerName := fmt.Sprintf("container-name-%d", idx)
containers := []v1.Container{
{
Name: containerName,
},
}
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: podNamespace,
UID: podUID,
},
Spec: v1.PodSpec{
Containers: containers,
},
}
}
func collectNamespacedNamesFromPods(pods []*v1.Pod) []string {
ret := make([]string, 0, len(pods))
for _, pod := range pods {
ret = append(ret, pod.Namespace+"/"+pod.Name)
}
sort.Strings(ret)
return ret
}
func collectNamespacedNamesFromPodResources(prs []*podresourcesapi.PodResources) []string {
ret := make([]string, 0, len(prs))
for _, pr := range prs {
ret = append(ret, pr.Namespace+"/"+pr.Name)
}
sort.Strings(ret)
return ret
}
func TestListPodResourcesUsesOnlyActivePodsV1(t *testing.T) {
numaID := int64(1)
// we abuse the fact that we don't care about the assignments,
// so we reuse the same for all pods which is actually wrong.
devs := []*podresourcesapi.ContainerDevices{
{
ResourceName: "resource",
DeviceIds: []string{"dev0"},
Topology: &podresourcesapi.TopologyInfo{Nodes: []*podresourcesapi.NUMANode{{ID: numaID}}},
},
}
cpus := []int64{1, 9}
mems := []*podresourcesapi.ContainerMemory{
{
MemoryType: "memory",
Size_: 1073741824,
Topology: &podresourcesapi.TopologyInfo{Nodes: []*podresourcesapi.NUMANode{{ID: numaID}}},
},
{
MemoryType: "hugepages-1Gi",
Size_: 1073741824,
Topology: &podresourcesapi.TopologyInfo{Nodes: []*podresourcesapi.NUMANode{{ID: numaID}}},
},
}
for _, tc := range []struct {
desc string
pods []*v1.Pod
activePods []*v1.Pod
}{
{
desc: "no pods",
pods: []*v1.Pod{},
activePods: []*v1.Pod{},
},
{
desc: "no differences",
pods: []*v1.Pod{
makePod(1),
makePod(2),
makePod(3),
makePod(4),
makePod(5),
},
activePods: []*v1.Pod{
makePod(1),
makePod(2),
makePod(3),
makePod(4),
makePod(5),
},
},
{
desc: "some terminated pods",
pods: []*v1.Pod{
makePod(1),
makePod(2),
makePod(3),
makePod(4),
makePod(5),
makePod(6),
makePod(7),
},
activePods: []*v1.Pod{
makePod(1),
makePod(3),
makePod(4),
makePod(5),
makePod(6),
},
},
} {
t.Run(tc.desc, func(t *testing.T) {
mockDevicesProvider := podresourcetest.NewMockDevicesProvider(t)
mockPodsProvider := podresourcetest.NewMockPodsProvider(t)
mockCPUsProvider := podresourcetest.NewMockCPUsProvider(t)
mockMemoryProvider := podresourcetest.NewMockMemoryProvider(t)
mockDynamicResourcesProvider := podresourcetest.NewMockDynamicResourcesProvider(t)
mockPodsProvider.EXPECT().GetPods().Return(tc.pods).Maybe()
mockPodsProvider.EXPECT().GetActivePods().Return(tc.activePods).Maybe()
mockDevicesProvider.EXPECT().GetDevices(mock.Anything, mock.Anything).Return(devs).Maybe()
mockCPUsProvider.EXPECT().GetCPUs(mock.Anything, mock.Anything).Return(cpus).Maybe()
mockMemoryProvider.EXPECT().GetMemory(mock.Anything, mock.Anything).Return(mems).Maybe()
mockDevicesProvider.EXPECT().UpdateAllocatedDevices().Return().Maybe()
mockCPUsProvider.EXPECT().GetAllocatableCPUs().Return([]int64{}).Maybe()
mockDevicesProvider.EXPECT().GetAllocatableDevices().Return([]*podresourcesapi.ContainerDevices{}).Maybe()
mockMemoryProvider.EXPECT().GetAllocatableMemory().Return([]*podresourcesapi.ContainerMemory{}).Maybe()
providers := PodResourcesProviders{
Pods: mockPodsProvider,
Devices: mockDevicesProvider,
Cpus: mockCPUsProvider,
Memory: mockMemoryProvider,
DynamicResources: mockDynamicResourcesProvider,
}
server := NewV1PodResourcesServer(providers)
resp, err := server.List(context.TODO(), &podresourcesapi.ListPodResourcesRequest{})
if err != nil {
t.Errorf("want err = %v, got %q", nil, err)
}
expectedNames := collectNamespacedNamesFromPods(tc.activePods)
gotNames := collectNamespacedNamesFromPodResources(resp.GetPodResources())
if diff := cmp.Diff(expectedNames, gotNames, cmpopts.EquateEmpty()); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestListPodResourcesWithInitContainersV1(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.KubeletPodResourcesDynamicResources, true)
@@ -423,6 +579,7 @@ func TestListPodResourcesWithInitContainersV1(t *testing.T) {
mockDynamicResourcesProvider := podresourcetest.NewMockDynamicResourcesProvider(t)
mockPodsProvider.EXPECT().GetPods().Return(tc.pods).Maybe()
mockPodsProvider.EXPECT().GetActivePods().Return(tc.pods).Maybe()
tc.mockFunc(tc.pods, mockDevicesProvider, mockCPUsProvider, mockMemoryProvider, mockDynamicResourcesProvider)
providers := PodResourcesProviders{

47
pkg/kubelet/apis/podresources/testing/pods_provider.go
View File

@@ -37,6 +37,53 @@ func (_m *MockPodsProvider) EXPECT() *MockPodsProvider_Expecter {
return &MockPodsProvider_Expecter{mock: &_m.Mock}
}
// GetActivePods provides a mock function with no fields
func (_m *MockPodsProvider) GetActivePods() []*v1.Pod {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for GetActivePods")
}
var r0 []*v1.Pod
if rf, ok := ret.Get(0).(func() []*v1.Pod); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*v1.Pod)
}
}
return r0
}
// MockPodsProvider_GetActivePods_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetActivePods'
type MockPodsProvider_GetActivePods_Call struct {
*mock.Call
}
// GetActivePods is a helper method to define mock.On call
func (_e *MockPodsProvider_Expecter) GetActivePods() *MockPodsProvider_GetActivePods_Call {
return &MockPodsProvider_GetActivePods_Call{Call: _e.mock.On("GetActivePods")}
}
func (_c *MockPodsProvider_GetActivePods_Call) Run(run func()) *MockPodsProvider_GetActivePods_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockPodsProvider_GetActivePods_Call) Return(_a0 []*v1.Pod) *MockPodsProvider_GetActivePods_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockPodsProvider_GetActivePods_Call) RunAndReturn(run func() []*v1.Pod) *MockPodsProvider_GetActivePods_Call {
_c.Call.Return(run)
return _c
}
// GetPodByName provides a mock function with given fields: namespace, name
func (_m *MockPodsProvider) GetPodByName(namespace string, name string) (*v1.Pod, bool) {
ret := _m.Called(namespace, name)

1
pkg/kubelet/apis/podresources/types.go
View File

@@ -34,6 +34,7 @@ type DevicesProvider interface {
// PodsProvider knows how to provide the pods admitted by the node
type PodsProvider interface {
GetActivePods() []*v1.Pod
GetPods() []*v1.Pod
GetPodByName(namespace, name string) (*v1.Pod, bool)
}

18
pkg/kubelet/kubelet.go
View File

@@ -2892,6 +2892,22 @@ func (kl *Kubelet) ListenAndServeReadOnly(address net.IP, port uint, tp trace.Tr
server.ListenAndServeKubeletReadOnlyServer(kl, kl.resourceAnalyzer, kl.containerManager.GetHealthCheckers(), kl.flagz, address, port, tp)
}
type kubeletPodsProvider struct {
kl *Kubelet
}
func (pp *kubeletPodsProvider) GetActivePods() []*v1.Pod {
return pp.kl.GetActivePods()
}
func (pp *kubeletPodsProvider) GetPods() []*v1.Pod {
return pp.kl.podManager.GetPods()
}
func (pp *kubeletPodsProvider) GetPodByName(namespace, name string) (*v1.Pod, bool) {
return pp.kl.podManager.GetPodByName(namespace, name)
}
// ListenAndServePodResources runs the kubelet podresources grpc service
func (kl *Kubelet) ListenAndServePodResources() {
endpoint, err := util.LocalEndpoint(kl.getPodResourcesDir(), podresources.Socket)
@@ -2901,7 +2917,7 @@ func (kl *Kubelet) ListenAndServePodResources() {
}
providers := podresources.PodResourcesProviders{
Pods: kl.podManager,
Pods: &kubeletPodsProvider{kl: kl},
Devices: kl.containerManager,
Cpus: kl.containerManager,
Memory: kl.containerManager,

10
test/compatibility_lifecycle/reference/versioned_feature_list.yaml
View File

@@ -737,6 +737,16 @@
lockToDefault: false
preRelease: Alpha
version: "1.27"
- name: KubeletPodResourcesListUseActivePods
versionedSpecs:
- default: false
lockToDefault: false
preRelease: GA
version: "1.0"
- default: true
lockToDefault: false
preRelease: Deprecated
version: "1.34"
- name: KubeletPSI
versionedSpecs:
- default: false

5
test/e2e_node/memory_manager_test.go
View File

@@ -1,6 +1,3 @@
//go:build linux
// +build linux
/*
Copyright 2017 The Kubernetes Authors.
@@ -242,7 +239,7 @@ func getAllNUMANodes() []int {
}
// Serial because the test updates kubelet configuration.
var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.WithSerial(), feature.MemoryManager, func() {
var _ = SIGDescribe("Memory Manager", "[LinuxOnly]", framework.WithDisruptive(), framework.WithSerial(), feature.MemoryManager, func() {
// TODO: add more complex tests that will include interaction between CPUManager, MemoryManager and TopologyManager
var (
allNUMANodes []int

715
test/e2e_node/podresources_test.go
View File

@@ -24,6 +24,13 @@ import (
"strings"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
"github.com/onsi/gomega/gstruct"
"github.com/onsi/gomega/types"
"google.golang.org/grpc"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -39,10 +46,6 @@ import (
admissionapi "k8s.io/pod-security-admission/api"
"k8s.io/utils/cpuset"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
"github.com/onsi/gomega/gstruct"
"github.com/onsi/gomega/types"
"k8s.io/kubernetes/test/e2e/feature"
"k8s.io/kubernetes/test/e2e/framework"
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
@@ -61,14 +64,34 @@ type podDesc struct {
resourceName string
resourceAmount int
cpuRequest int // cpuRequest is in millicores
memRequest int // memRequest is in megabytes
cpuLimit int // cpuLimit is in millicores
memLimit int // memLimit is in megabytes
initContainers []initContainerDesc
mainCntCommand []string
restartPolicy *v1.RestartPolicy
}
func (desc podDesc) CpuRequestQty() resource.Quantity {
func (desc podDesc) CPURequestQty() resource.Quantity {
qty := resource.NewMilliQuantity(int64(desc.cpuRequest), resource.DecimalSI)
return *qty
}
func (desc podDesc) MemRequestQty() resource.Quantity {
qty := resource.NewQuantity(int64(desc.memRequest)*1024*1024, resource.DecimalSI)
return *qty
}
func (desc podDesc) CPULimitQty() resource.Quantity {
qty := resource.NewMilliQuantity(int64(desc.cpuLimit), resource.DecimalSI)
return *qty
}
func (desc podDesc) MemLimitQty() resource.Quantity {
qty := resource.NewQuantity(int64(desc.memLimit)*1024*1024, resource.DecimalSI)
return *qty
}
func (desc podDesc) CpuRequestExclusive() int {
if (desc.cpuRequest % 1000) != 0 {
// exclusive cpus are request only if the quantity is integral;
@@ -116,6 +139,56 @@ func (desc initContainerDesc) RequiresDevices() bool {
return desc.resourceName != "" && desc.resourceAmount > 0
}
// TODO: replace makePodResourcesTestPod reimplementing in terms of this function
func makeFullPodResourcesTestPod(desc podDesc) *v1.Pod {
cnt := v1.Container{
Name: desc.cntName,
Image: busyboxImage,
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{},
Limits: v1.ResourceList{},
},
Command: []string{"sh", "-c", "sleep 1d"},
}
if desc.cpuLimit > 0 {
cnt.Resources.Limits[v1.ResourceCPU] = desc.CPULimitQty()
}
if desc.cpuRequest > 0 {
cnt.Resources.Requests[v1.ResourceCPU] = desc.CPURequestQty()
}
if desc.memLimit > 0 {
cnt.Resources.Limits[v1.ResourceMemory] = desc.MemLimitQty()
}
if desc.memRequest > 0 {
cnt.Resources.Requests[v1.ResourceMemory] = desc.MemRequestQty()
}
if desc.RequiresDevices() {
devQty := resource.NewQuantity(int64(desc.resourceAmount), resource.DecimalSI)
cnt.Resources.Requests[v1.ResourceName(desc.resourceName)] = *devQty
cnt.Resources.Limits[v1.ResourceName(desc.resourceName)] = *devQty
}
if len(desc.mainCntCommand) > 0 {
cnt.Command = desc.mainCntCommand
}
restartPolicy := v1.RestartPolicyNever
if desc.restartPolicy != nil {
restartPolicy = *desc.restartPolicy
}
// TODO: support for init containers - currently unneeded
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: desc.podName,
},
Spec: v1.PodSpec{
RestartPolicy: restartPolicy,
Containers: []v1.Container{
cnt,
},
},
}
}
func makePodResourcesTestPod(desc podDesc) *v1.Pod {
cnt := v1.Container{
Name: desc.cntName,
@@ -127,7 +200,7 @@ func makePodResourcesTestPod(desc podDesc) *v1.Pod {
Command: []string{"sh", "-c", "sleep 1d"},
}
if desc.RequiresCPU() {
cpuRequestQty := desc.CpuRequestQty()
cpuRequestQty := desc.CPURequestQty()
cnt.Resources.Requests[v1.ResourceCPU] = cpuRequestQty
cnt.Resources.Limits[v1.ResourceCPU] = cpuRequestQty
// we don't really care, we only need to be in guaranteed QoS
@@ -138,6 +211,13 @@ func makePodResourcesTestPod(desc podDesc) *v1.Pod {
cnt.Resources.Requests[v1.ResourceName(desc.resourceName)] = resource.MustParse(fmt.Sprintf("%d", desc.resourceAmount))
cnt.Resources.Limits[v1.ResourceName(desc.resourceName)] = resource.MustParse(fmt.Sprintf("%d", desc.resourceAmount))
}
if len(desc.mainCntCommand) > 0 {
cnt.Command = desc.mainCntCommand
}
restartPolicy := v1.RestartPolicyNever
if desc.restartPolicy != nil {
restartPolicy = *desc.restartPolicy
}
var initCnts []v1.Container
for _, cntDesc := range desc.initContainers {
@@ -174,7 +254,7 @@ func makePodResourcesTestPod(desc podDesc) *v1.Pod {
Name: desc.podName,
},
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
RestartPolicy: restartPolicy,
InitContainers: initCnts,
Containers: []v1.Container{
cnt,
@@ -272,6 +352,30 @@ func findContainerDeviceByName(devs []*kubeletpodresourcesv1.ContainerDevices, r
return nil
}
func matchPodDescWithResourcesNamesOnly(expected []podDesc, found podResMap) error {
framework.Logf("got %d pods expected %d", len(found), len(expected))
if len(found) != len(expected) {
return fmt.Errorf("found %d items expected %d", len(found), len(expected))
}
for _, podReq := range expected {
framework.Logf("matching: %#v", podReq)
if _, ok := found[podReq.podName]; !ok {
return fmt.Errorf("no pod resources for pod %q", podReq.podName)
}
}
return nil
}
func matchPodDescAndCountWithResources(expected []podDesc, found podResMap) error {
framework.Logf("got %d pods expected %d", len(found), len(expected))
if len(found) != len(expected) {
return fmt.Errorf("found %d items expected %d", len(found), len(expected))
}
return matchPodDescWithResources(expected, found)
}
func matchPodDescWithResources(expected []podDesc, found podResMap) error {
for _, podReq := range expected {
framework.Logf("matching: %#v", podReq)
@@ -847,7 +951,27 @@ var _ = SIGDescribe("POD Resources", framework.WithSerial(), feature.PodResource
f := framework.NewDefaultFramework("podresources-test")
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
reservedSystemCPUs := cpuset.New(1)
var reservedSystemCPUs cpuset.CPUSet
var memoryQuantity resource.Quantity
var defaultKubeParams *memoryManagerKubeletParams
ginkgo.BeforeEach(func() {
reservedSystemCPUs = cpuset.New(1)
memoryQuantity = resource.MustParse("1100Mi")
defaultKubeParams = &memoryManagerKubeletParams{
systemReservedMemory: []kubeletconfig.MemoryReservation{
{
NumaNode: 0,
Limits: v1.ResourceList{
resourceMemory: memoryQuantity,
},
},
},
systemReserved: map[string]string{resourceMemory: "500Mi"},
kubeReserved: map[string]string{resourceMemory: "500Mi"},
evictionHard: map[string]string{evictionHardMemory: "100Mi"},
}
})
ginkgo.Context("with SRIOV devices in the system", func() {
ginkgo.BeforeEach(func() {
@@ -1024,27 +1148,6 @@ var _ = SIGDescribe("POD Resources", framework.WithSerial(), feature.PodResource
podresourcesGetAllocatableResourcesTests(ctx, cli, nil, onlineCPUs, reservedSystemCPUs)
podresourcesGetTests(ctx, f, cli, true)
})
ginkgo.It("should account for resources of pods in terminal phase", func(ctx context.Context) {
pd := podDesc{
cntName: "e2e-test-cnt",
podName: "e2e-test-pod",
cpuRequest: 1000,
}
pod := makePodResourcesTestPod(pd)
pod.Spec.Containers[0].Command = []string{"sh", "-c", "/bin/true"}
pod = e2epod.NewPodClient(f).Create(ctx, pod)
defer e2epod.NewPodClient(f).DeleteSync(ctx, pod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete)
err := e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
framework.ExpectNoError(err)
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)
cli, conn, err := podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
framework.ExpectNoError(err)
defer conn.Close()
// although the pod moved into terminal state, PodResourcesAPI still list its cpus
expectPodResources(ctx, 1, cli, []podDesc{pd})
})
})
})
@@ -1081,6 +1184,556 @@ var _ = SIGDescribe("POD Resources", framework.WithSerial(), feature.PodResource
})
})
ginkgo.When("checking core resource managers assignments", func() {
var podresConn *grpc.ClientConn
var podMap map[string]*v1.Pod
var cpuAlloc int64
ginkgo.BeforeEach(func(ctx context.Context) {
podMap = make(map[string]*v1.Pod)
})
ginkgo.AfterEach(func(ctx context.Context) {
if podresConn != nil {
framework.ExpectNoError(podresConn.Close())
}
deletePodsAsync(ctx, f, podMap)
})
ginkgo.JustBeforeEach(func(ctx context.Context) {
// this is a very rough check. We just want to rule out system that does NOT have enough resources
_, cpuAlloc, _ = getLocalNodeCPUDetails(ctx, f)
if cpuAlloc < minCoreCount {
e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
}
})
ginkgo.When("listing with restricted list output enabled", func() {
tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {
kubeParams := *defaultKubeParams
kubeParams.policy = staticPolicy
updateKubeletConfigWithMemoryManagerParams(initialConfig, &kubeParams)
initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 10 * time.Minute} // set it long enough it is practically disabled
cpus := reservedSystemCPUs.String()
framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
initialConfig.ReservedSystemCPUs = cpus
})
ginkgo.It("should report only the running pods regardless of the QoS", func(ctx context.Context) {
// TODO: compute automatically
if cpuAlloc < 8 {
e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, 8)
}
restartNever := v1.RestartPolicyNever
restartAlways := v1.RestartPolicyAlways
descs := []podDesc{
{
cntName: "e2e-test-cnt-gu-term-1",
podName: "e2e-test-pod-gu-term-1",
cpuLimit: 1000,
memLimit: 256,
restartPolicy: &restartNever,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
},
{
cntName: "e2e-test-cnt-gu-term-2",
podName: "e2e-test-pod-gu-term-2",
cpuLimit: 200,
memLimit: 256,
restartPolicy: &restartNever,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
},
{
cntName: "e2e-test-cnt-gu-run-1",
podName: "e2e-test-pod-gu-run-1",
cpuLimit: 200,
memLimit: 256,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-gu-run-2",
podName: "e2e-test-pod-gu-run-2",
cpuLimit: 1000,
memLimit: 256,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-bu-term-1",
podName: "e2e-test-pod-bu-term-1",
cpuRequest: 400,
memRequest: 256,
restartPolicy: &restartNever,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
},
{
cntName: "e2e-test-cnt-bu-run-1",
podName: "e2e-test-pod-bu-run-1",
cpuRequest: 400,
memRequest: 256,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-be-term-1",
podName: "e2e-test-pod-be-term-1",
restartPolicy: &restartNever,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
},
{
cntName: "e2e-test-cnt-be-run-1",
podName: "e2e-test-pod-be-run-1",
restartPolicy: &restartAlways,
},
}
expected := []podDesc{
{
cntName: "e2e-test-cnt-gu-run-1",
podName: "e2e-test-pod-gu-run-1",
cpuLimit: 200,
memLimit: 256,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-gu-run-2",
podName: "e2e-test-pod-gu-run-2",
cpuLimit: 1000,
memLimit: 256,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-bu-run-1",
podName: "e2e-test-pod-bu-run-1",
cpuRequest: 400,
memRequest: 256,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-be-run-1",
podName: "e2e-test-pod-be-run-1",
restartPolicy: &restartAlways,
},
}
for _, desc := range descs {
pod := makeFullPodResourcesTestPod(desc)
if desc.restartPolicy == &restartAlways {
pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
podMap[string(pod.UID)] = pod
} else {
pod = e2epod.NewPodClient(f).Create(ctx, pod)
pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
podMap[string(pod.UID)] = pod
err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
framework.ExpectNoError(err)
}
}
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)
var cli kubeletpodresourcesv1.PodResourcesListerClient
cli, podresConn, err = podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
framework.ExpectNoError(err)
gomega.Consistently(func(ctx context.Context) error {
found, err := getPodResourcesValues(ctx, cli)
if err != nil {
return err
}
return matchPodDescAndCountWithResources(expected, found)
}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
})
ginkgo.It("should report only the running pods among many", func(ctx context.Context) {
// TODO: compute automatically
if cpuAlloc < 8 {
e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, 8)
}
restartNever := v1.RestartPolicyNever
restartAlways := v1.RestartPolicyAlways
descs := []podDesc{
{
cntName: "e2e-test-cnt-payload-gu-1",
podName: "e2e-test-pod-payload-gu-1",
cpuRequest: 1000,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-mixed-cpu-mem",
podName: "e2e-test-pod-mixed-cpu-mem",
cpuRequest: 1000,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
restartPolicy: &restartNever,
},
{
cntName: "e2e-test-cnt-payload-gu-2",
podName: "e2e-test-pod-payload-gu-2",
cpuRequest: 1000,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-payload-gu-3",
podName: "e2e-test-pod-payload-gu-3",
cpuRequest: 1000,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-mixed-mem",
podName: "e2e-test-pod-mixed-mem",
cpuRequest: 1000,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
restartPolicy: &restartNever,
},
}
expected := []podDesc{
{
cntName: "e2e-test-cnt-payload-gu-1",
podName: "e2e-test-pod-payload-gu-1",
cpuRequest: 1000,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-payload-gu-2",
podName: "e2e-test-pod-payload-gu-2",
cpuRequest: 1000,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-payload-gu-3",
podName: "e2e-test-pod-payload-gu-3",
cpuRequest: 1000,
restartPolicy: &restartAlways,
},
}
for _, desc := range descs {
pod := makePodResourcesTestPod(desc)
if desc.restartPolicy == &restartAlways {
pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
podMap[string(pod.UID)] = pod
} else {
pod = e2epod.NewPodClient(f).Create(ctx, pod)
pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
podMap[string(pod.UID)] = pod
err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
framework.ExpectNoError(err)
}
}
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)
var cli kubeletpodresourcesv1.PodResourcesListerClient
cli, podresConn, err = podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
framework.ExpectNoError(err)
gomega.Consistently(func(ctx context.Context) error {
found, err := getPodResourcesValues(ctx, cli)
if err != nil {
return err
}
return matchPodDescAndCountWithResources(expected, found)
}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
})
ginkgo.DescribeTable("all pods terminated and requiring", func(ctx context.Context, cpuReqMilli, podCount int) {
cpuReqTot := int64((cpuReqMilli * podCount) / 1000)
ginkgo.By(fmt.Sprintf("needed: %d cores", cpuReqTot))
if cpuAlloc < cpuReqTot {
e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, cpuReqTot)
}
restartNever := v1.RestartPolicyNever
for idx := range podCount {
pd := podDesc{
cntName: fmt.Sprintf("e2e-test-cnt-%d", idx),
podName: fmt.Sprintf("e2e-test-pod-%d", idx),
cpuRequest: cpuReqMilli,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
restartPolicy: &restartNever,
}
pod := makePodResourcesTestPod(pd)
pod = e2epod.NewPodClient(f).Create(ctx, pod)
pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
podMap[string(pod.UID)] = pod
err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
framework.ExpectNoError(err)
}
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)
var cli kubeletpodresourcesv1.PodResourcesListerClient
cli, podresConn, err = podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
framework.ExpectNoError(err)
gomega.Consistently(func(ctx context.Context) error {
found, err := getPodResourcesValues(ctx, cli)
if err != nil {
return err
}
if len(found) > 0 {
return fmt.Errorf("returned unexpected pods: %v", found)
}
return nil
}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
},
ginkgo.Entry("cpu and mem single", context.TODO(), 1000, 1),
ginkgo.Entry("cpu and mem multi", context.TODO(), 1000, 3),
ginkgo.Entry("mem only single", context.TODO(), 200, 1),
ginkgo.Entry("mem only multi", context.TODO(), 200, 3),
)
})
ginkgo.When("listing with restricted list output disabled for backward compatible defaults", func() {
tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {
kubeParams := *defaultKubeParams
kubeParams.policy = staticPolicy
updateKubeletConfigWithMemoryManagerParams(initialConfig, &kubeParams)
initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 10 * time.Minute} // set it long enough it is practically disabled
cpus := reservedSystemCPUs.String()
framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
initialConfig.ReservedSystemCPUs = cpus
initialConfig.FeatureGates["KubeletPodResourcesListUseActivePods"] = false
})
ginkgo.It("should report all the known pods regardless of the QoS", func(ctx context.Context) {
// TODO: compute automatically
if cpuAlloc < 8 {
e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, 8)
}
restartNever := v1.RestartPolicyNever
restartAlways := v1.RestartPolicyAlways
descs := []podDesc{
{
cntName: "e2e-test-cnt-gu-term-1",
podName: "e2e-test-pod-gu-term-1",
cpuLimit: 1000,
memLimit: 256,
restartPolicy: &restartNever,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
},
{
cntName: "e2e-test-cnt-gu-term-2",
podName: "e2e-test-pod-gu-term-2",
cpuLimit: 200,
memLimit: 256,
restartPolicy: &restartNever,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
},
{
cntName: "e2e-test-cnt-gu-run-1",
podName: "e2e-test-pod-gu-run-1",
cpuLimit: 200,
memLimit: 256,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-gu-run-2",
podName: "e2e-test-pod-gu-run-2",
cpuLimit: 1000,
memLimit: 256,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-bu-term-1",
podName: "e2e-test-pod-bu-term-1",
cpuRequest: 400,
memRequest: 256,
restartPolicy: &restartNever,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
},
{
cntName: "e2e-test-cnt-bu-run-1",
podName: "e2e-test-pod-bu-run-1",
cpuRequest: 400,
memRequest: 256,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-be-term-1",
podName: "e2e-test-pod-be-term-1",
restartPolicy: &restartNever,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
},
{
cntName: "e2e-test-cnt-be-run-1",
podName: "e2e-test-pod-be-run-1",
restartPolicy: &restartAlways,
},
}
expected := descs
for _, desc := range descs {
pod := makeFullPodResourcesTestPod(desc)
if desc.restartPolicy == &restartAlways {
pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
podMap[string(pod.UID)] = pod
} else {
pod = e2epod.NewPodClient(f).Create(ctx, pod)
pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
podMap[string(pod.UID)] = pod
err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
framework.ExpectNoError(err)
}
}
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)
var cli kubeletpodresourcesv1.PodResourcesListerClient
cli, podresConn, err = podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
framework.ExpectNoError(err)
gomega.Consistently(func(ctx context.Context) error {
found, err := getPodResourcesValues(ctx, cli)
if err != nil {
return err
}
return matchPodDescAndCountWithResources(expected, found)
}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
})
ginkgo.It("should report all the known pods", func(ctx context.Context) {
// TODO: compute automatically
if cpuAlloc < 8 {
e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, 8)
}
restartNever := v1.RestartPolicyNever
restartAlways := v1.RestartPolicyAlways
descs := []podDesc{
{
cntName: "e2e-test-cnt-payload-gu-1",
podName: "e2e-test-pod-payload-gu-1",
cpuRequest: 1000,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-mixed-cpu-mem",
podName: "e2e-test-pod-mixed-cpu-mem",
cpuRequest: 1000,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
restartPolicy: &restartNever,
},
{
cntName: "e2e-test-cnt-payload-gu-2",
podName: "e2e-test-pod-payload-gu-2",
cpuRequest: 1000,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-payload-gu-3",
podName: "e2e-test-pod-payload-gu-3",
cpuRequest: 1000,
restartPolicy: &restartAlways,
},
{
cntName: "e2e-test-cnt-mixed-mem",
podName: "e2e-test-pod-mixed-mem",
cpuRequest: 1000,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
restartPolicy: &restartNever,
},
}
expected := descs
for _, desc := range descs {
pod := makePodResourcesTestPod(desc)
if desc.restartPolicy == &restartAlways {
pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
podMap[string(pod.UID)] = pod
} else {
pod = e2epod.NewPodClient(f).Create(ctx, pod)
pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
podMap[string(pod.UID)] = pod
err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
framework.ExpectNoError(err)
}
}
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)
var cli kubeletpodresourcesv1.PodResourcesListerClient
cli, podresConn, err = podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
framework.ExpectNoError(err)
gomega.Consistently(func(ctx context.Context) error {
found, err := getPodResourcesValues(ctx, cli)
if err != nil {
return err
}
// all but the last pod will have inconsistent resource reporting.
// See: https://github.com/kubernetes/kubernetes/issues/132020#issuecomment-2921478368
// this is why we introduced the FeatureGate flag in the first place
return matchPodDescWithResourcesNamesOnly(expected, found)
}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
})
ginkgo.DescribeTable("all pods terminated and requiring", func(ctx context.Context, cpuReqMilli, podCount int) {
cpuReqTot := int64((cpuReqMilli * podCount) / 1000)
ginkgo.By(fmt.Sprintf("needed: %d cores", cpuReqTot))
if cpuAlloc < cpuReqTot {
e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, cpuReqTot)
}
restartNever := v1.RestartPolicyNever
var expected []podDesc
for idx := range podCount {
pd := podDesc{
cntName: fmt.Sprintf("e2e-test-cnt-%d", idx),
podName: fmt.Sprintf("e2e-test-pod-%d", idx),
cpuRequest: cpuReqMilli,
mainCntCommand: []string{"sh", "-c", "/bin/true"},
restartPolicy: &restartNever,
}
expected = append(expected, pd)
pod := makePodResourcesTestPod(pd)
pod = e2epod.NewPodClient(f).Create(ctx, pod)
pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
podMap[string(pod.UID)] = pod
err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
framework.ExpectNoError(err)
}
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)
var cli kubeletpodresourcesv1.PodResourcesListerClient
cli, podresConn, err = podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
framework.ExpectNoError(err)
gomega.Consistently(func(ctx context.Context) error {
found, err := getPodResourcesValues(ctx, cli)
if err != nil {
return err
}
// all but the last pod will have inconsistent resource reporting.
// See: https://github.com/kubernetes/kubernetes/issues/132020#issuecomment-2921478368
// this is why we introduced the FeatureGate in the first place
return matchPodDescWithResourcesNamesOnly(expected, found)
}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
},
ginkgo.Entry("cpu and mem single", context.TODO(), 1000, 1),
ginkgo.Entry("cpu and mem multi", context.TODO(), 1000, 3),
ginkgo.Entry("mem only single", context.TODO(), 200, 1),
ginkgo.Entry("mem only multi", context.TODO(), 200, 3),
)
})
})
ginkgo.Context("with a topology-unaware device plugin, which reports resources w/o hardware topology", func() {
ginkgo.Context("with CPU manager Static policy", func() {
ginkgo.BeforeEach(func(ctx context.Context) {
@@ -1142,7 +1795,7 @@ var _ = SIGDescribe("POD Resources", framework.WithSerial(), feature.PodResource
desc,
})
expectPodResources(ctx, 1, cli, []podDesc{desc})
expectPodResources(ctx, 0, cli, []podDesc{desc})
ginkgo.By("Restarting Kubelet")
restartKubelet(ctx, true)
@@ -1152,7 +1805,7 @@ var _ = SIGDescribe("POD Resources", framework.WithSerial(), feature.PodResource
ginkgo.By("Wait for node to be ready")
waitForTopologyUnawareResources(ctx, f)
expectPodResources(ctx, 1, cli, []podDesc{desc})
expectPodResources(ctx, 0, cli, []podDesc{desc})
tpd.deletePodsForTest(ctx, f)
})
})