From 86bc1bf0a452c303a25341958f2615d33abd06ad Mon Sep 17 00:00:00 2001 From: Natasha Sarkar Date: Fri, 11 Jul 2025 22:53:32 +0000 Subject: [PATCH 1/3] more complex e2e test for deferred resizes --- test/e2e/node/pod_resize.go | 217 ++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/test/e2e/node/pod_resize.go b/test/e2e/node/pod_resize.go index 99375702f96..e99a4ddf341 100644 --- a/test/e2e/node/pod_resize.go +++ b/test/e2e/node/pod_resize.go @@ -24,6 +24,7 @@ import ( v1 "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -768,6 +769,201 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { delErr5 := e2epod.DeletePodWithWait(ctx, f.ClientSet, testPod5) framework.ExpectNoError(delErr5, "failed to delete pod %s", testPod5.Name) }) + + ginkgo.It("pod-resize-retry-deferred-test-3", func(ctx context.Context) { + // Deferred resize E2E test case #3: + // 1. Create pod1, pod2, pod3, and pod4 on node, each starting with 1/4 of the node's allocatable CPU and and 1/5 the memory. + // 2. Resize pod2 to request more CPU than available (1/3), along with a decrease in memory (1/24), verify the resize is deferred. + // 3. Resize pod3 to request more memory than available (2/3), along with a decrease in CPU (1/24), verify the resize is deferred. + // 4. Resize pod4 to request more CPU than available (5/8), verify the resize is deferred. + // 5. The deferred resizes above are chosen carefully such that: + // - deletion of pod1 should mxwke room for pod2's resize (but not pod3 or pod4). + // - pod2's resize should make room for pod3's resize (but not pod4). + // - pod3's resize should make room for pod4's resize. + // 6. Delete pod1, verify the chain of deferred resizes is actuated. + + podClient := e2epod.NewPodClient(f) + nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) + framework.ExpectNoError(err, "failed to get running nodes") + gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty()) + framework.Logf("Found %d schedulable nodes", len(nodes.Items)) + + ginkgo.By("Find node CPU and memory resources available for allocation!") + node := nodes.Items[0] + + nodeAllocatableMilliCPU, initNodeAvailableMilliCPU := getNodeAllocatableAndAvailableMilliCPUValues(ctx, f, &node) + framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", + node.Name, nodeAllocatableMilliCPU, initNodeAvailableMilliCPU) + + nodeAllocatableMem, initNodeAvailableMem := getNodeAllocatableAndAvailableMemoryValues(ctx, f, &node) + framework.Logf("Node '%s': NodeAllocatable Memory = %d. Memory currently available to allocate = %d.", + node.Name, nodeAllocatableMem, initNodeAvailableMem) + + initialCPUQuantity := resource.NewMilliQuantity(initNodeAvailableMilliCPU/4, resource.DecimalSI) + initialMemoryQuantity := resource.NewQuantity(initNodeAvailableMem/5, resource.DecimalSI) + framework.Logf("initial CPU request is '%dm'", initialCPUQuantity.MilliValue()) + framework.Logf("initial Memory request is '%d'", initialMemoryQuantity.Value()) + + c := []podresize.ResizableContainerInfo{ + { + Name: "c", + Resources: &cgroups.ContainerResources{ + CPUReq: initialCPUQuantity.String(), + MemReq: initialMemoryQuantity.String(), + }, + }, + } + + tStamp := strconv.Itoa(time.Now().Nanosecond()) + testPod1 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod1", tStamp, c) + testPod1 = e2epod.MustMixinRestrictedPodSecurity(testPod1) + e2epod.SetNodeAffinity(&testPod1.Spec, node.Name) + + testPod2 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod2", tStamp, c) + testPod2 = e2epod.MustMixinRestrictedPodSecurity(testPod2) + e2epod.SetNodeAffinity(&testPod2.Spec, node.Name) + + testPod3 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod3", tStamp, c) + testPod3 = e2epod.MustMixinRestrictedPodSecurity(testPod3) + e2epod.SetNodeAffinity(&testPod3.Spec, node.Name) + + testPod4 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod4", tStamp, c) + testPod4 = e2epod.MustMixinRestrictedPodSecurity(testPod4) + e2epod.SetNodeAffinity(&testPod4.Spec, node.Name) + + testPods := []*v1.Pod{testPod1, testPod2, testPod3, testPod4} + + for i, pod := range testPods { + ginkgo.By(fmt.Sprintf("Create pod '%s' that fits the node '%s'", pod.Name, node.Name)) + testPods[i] = podClient.CreateSync(ctx, pod) + gomega.Expect(testPods[i].Status.Phase).To(gomega.Equal(v1.PodRunning)) + gomega.Expect(testPods[i].Generation).To(gomega.BeEquivalentTo(1)) + } + + testPod2CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableMilliCPU/3, resource.DecimalSI) + testPod2MemoryQuantityResized := resource.NewQuantity(initNodeAvailableMem/24, resource.DecimalSI) + + testPod3CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableMilliCPU/24, resource.DecimalSI) + testPod3MemoryQuantityResized := resource.NewQuantity(2*initNodeAvailableMem/3, resource.DecimalSI) + + testPod4CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableMilliCPU/2+initNodeAvailableMilliCPU/8, resource.DecimalSI) + testPod4MemoryQuantityResized := initialMemoryQuantity + + patchTestpod2ToDeferred := fmt.Sprintf(`{ + "spec": { + "containers": [ + { + "name": "c", + "resources": { + "requests": { + "cpu": "%dm", + "memory": "%d" + } + } + } + ] + } + }`, testPod2CPUQuantityResized.MilliValue(), testPod2MemoryQuantityResized.Value()) + + patchTestpod3ToDeferred := fmt.Sprintf(`{ + "spec": { + "containers": [ + { + "name": "c", + "resources": { + "requests": { + "cpu": "%dm", + "memory": "%d" + } + } + } + ] + } + }`, testPod3CPUQuantityResized.MilliValue(), testPod3MemoryQuantityResized.Value()) + + patchTestpod4ToDeferred := fmt.Sprintf(`{ + "spec": { + "containers": [ + { + "name": "c", + "resources": { + "requests": { + "cpu": "%dm", + "memory": "%d" + } + } + } + ] + } + }`, testPod4CPUQuantityResized.MilliValue(), testPod4MemoryQuantityResized.Value()) + + patches := []string{patchTestpod2ToDeferred, patchTestpod3ToDeferred, patchTestpod4ToDeferred} + + for i := range patches { + testPod := testPods[i+1] + patch := patches[i] + ginkgo.By(fmt.Sprintf("Resize pod '%s' that cannot fit node due to insufficient CPU or memory", testPod.Name)) + testPod, err = f.ClientSet.CoreV1().Pods(testPod.Namespace).Patch(ctx, + testPod.Name, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "resize") + framework.ExpectNoError(err, "failed to patch pod for resize") + waitForPodDeferred(ctx, f, testPod) + gomega.Expect(testPod.Generation).To(gomega.BeEquivalentTo(2)) + } + + ginkgo.By("deleting pod 1") + delErr1 := e2epod.DeletePodWithWait(ctx, f.ClientSet, testPod1) + framework.ExpectNoError(delErr1, "failed to delete pod %s", testPod1.Name) + + ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod deletion '%s'", testPod2.Name, testPod1.Name)) + expected := []podresize.ResizableContainerInfo{ + { + Name: "c", + Resources: &cgroups.ContainerResources{ + CPUReq: testPod2CPUQuantityResized.String(), + MemReq: testPod2MemoryQuantityResized.String(), + }, + }, + } + resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod2, expected) + podresize.ExpectPodResized(ctx, f, resizedPod, expected) + + ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod resize '%s'", testPod3.Name, testPod2.Name)) + expected = []podresize.ResizableContainerInfo{ + { + Name: "c", + Resources: &cgroups.ContainerResources{ + CPUReq: testPod3CPUQuantityResized.String(), + MemReq: testPod3MemoryQuantityResized.String(), + }, + }, + } + resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod3, expected) + podresize.ExpectPodResized(ctx, f, resizedPod, expected) + + ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod resize '%s'", testPod4.Name, testPod3.Name)) + expected = []podresize.ResizableContainerInfo{ + { + Name: "c", + Resources: &cgroups.ContainerResources{ + CPUReq: testPod4CPUQuantityResized.String(), + MemReq: testPod4MemoryQuantityResized.String(), + }, + }, + } + resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod4, expected) + podresize.ExpectPodResized(ctx, f, resizedPod, expected) + + ginkgo.By("deleting pods") + for _, testPod := range testPods { + delErr := f.ClientSet.CoreV1().Pods(testPod.Namespace).Delete(ctx, testPod.Name, metav1.DeleteOptions{}) + if delErr != nil && !apierrors.IsNotFound(delErr) { + framework.ExpectNoError(delErr, "failed to delete pod %s", testPod.Name) + } + } + for _, testPod := range testPods { + framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, testPod.Name, testPod.Namespace, e2epod.PodDeleteTimeout)) + } + }) } var _ = SIGDescribe(framework.WithSerial(), "Pod InPlace Resize Container (scheduler-focused)", framework.WithFeatureGate(features.InPlacePodVerticalScaling), func() { @@ -838,6 +1034,27 @@ func getNodeAllocatableAndAvailableMilliCPUValues(ctx context.Context, f *framew return nodeAllocatableMilliCPU, nodeAvailableMilliCPU } +// Calculate available memory. nodeAvailableMemory = nodeAllocatableMemory - sum(podAllocatedMemory) +func getNodeAllocatableAndAvailableMemoryValues(ctx context.Context, f *framework.Framework, n *v1.Node) (int64, int64) { + nodeAllocatableMem := n.Status.Allocatable.Memory().Value() + gomega.Expect(n.Status.Allocatable).ShouldNot(gomega.BeEmpty(), "allocatable") + podAllocatedMem := int64(0) + + // Exclude pods that are in the Succeeded or Failed states + selector := fmt.Sprintf("spec.nodeName=%s,status.phase!=%v,status.phase!=%v", n.Name, v1.PodSucceeded, v1.PodFailed) + listOptions := metav1.ListOptions{FieldSelector: selector} + podList, err := f.ClientSet.CoreV1().Pods(metav1.NamespaceAll).List(ctx, listOptions) + + framework.ExpectNoError(err, "failed to get running pods") + framework.Logf("Found %d pods on node '%s'", len(podList.Items), n.Name) + for _, pod := range podList.Items { + podRequestMem := resourceapi.GetResourceRequest(&pod, v1.ResourceMemory) + podAllocatedMem += podRequestMem + } + nodeAvailableMem := nodeAllocatableMem - podAllocatedMem + return nodeAllocatableMem, nodeAvailableMem +} + func waitForPodDeferred(ctx context.Context, f *framework.Framework, testPod *v1.Pod) { framework.ExpectNoError(e2epod.WaitForPodCondition(ctx, f.ClientSet, testPod.Namespace, testPod.Name, "display pod resize status as deferred", f.Timeouts.PodStart, func(pod *v1.Pod) (bool, error) { return helpers.IsPodResizeDeferred(pod), nil From f456a70bdeacab0f63744e170970dab578e21f94 Mon Sep 17 00:00:00 2001 From: Natasha Sarkar Date: Thu, 17 Jul 2025 19:19:05 +0000 Subject: [PATCH 2/3] use CreateBatch and MakeResizePatch --- test/e2e/node/pod_resize.go | 129 +++++++++++------------------------- 1 file changed, 40 insertions(+), 89 deletions(-) diff --git a/test/e2e/node/pod_resize.go b/test/e2e/node/pod_resize.go index e99a4ddf341..1a4b706f19c 100644 --- a/test/e2e/node/pod_resize.go +++ b/test/e2e/node/pod_resize.go @@ -777,7 +777,7 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { // 3. Resize pod3 to request more memory than available (2/3), along with a decrease in CPU (1/24), verify the resize is deferred. // 4. Resize pod4 to request more CPU than available (5/8), verify the resize is deferred. // 5. The deferred resizes above are chosen carefully such that: - // - deletion of pod1 should mxwke room for pod2's resize (but not pod3 or pod4). + // - deletion of pod1 should make room for pod2's resize (but not pod3 or pod4). // - pod2's resize should make room for pod3's resize (but not pod4). // - pod3's resize should make room for pod4's resize. // 6. Delete pod1, verify the chain of deferred resizes is actuated. @@ -832,13 +832,8 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { e2epod.SetNodeAffinity(&testPod4.Spec, node.Name) testPods := []*v1.Pod{testPod1, testPod2, testPod3, testPod4} - - for i, pod := range testPods { - ginkgo.By(fmt.Sprintf("Create pod '%s' that fits the node '%s'", pod.Name, node.Name)) - testPods[i] = podClient.CreateSync(ctx, pod) - gomega.Expect(testPods[i].Status.Phase).To(gomega.Equal(v1.PodRunning)) - gomega.Expect(testPods[i].Generation).To(gomega.BeEquivalentTo(1)) - } + ginkgo.By(fmt.Sprintf("Create pods that fits the node '%s'", node.Name)) + podClient.CreateBatch(ctx, testPods) testPod2CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableMilliCPU/3, resource.DecimalSI) testPod2MemoryQuantityResized := resource.NewQuantity(initNodeAvailableMem/24, resource.DecimalSI) @@ -849,55 +844,39 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { testPod4CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableMilliCPU/2+initNodeAvailableMilliCPU/8, resource.DecimalSI) testPod4MemoryQuantityResized := initialMemoryQuantity - patchTestpod2ToDeferred := fmt.Sprintf(`{ - "spec": { - "containers": [ - { - "name": "c", - "resources": { - "requests": { - "cpu": "%dm", - "memory": "%d" - } - } - } - ] - } - }`, testPod2CPUQuantityResized.MilliValue(), testPod2MemoryQuantityResized.Value()) + expectedTestPod2Resized := []podresize.ResizableContainerInfo{ + { + Name: "c", + Resources: &cgroups.ContainerResources{ + CPUReq: testPod2CPUQuantityResized.String(), + MemReq: testPod2MemoryQuantityResized.String(), + }, + }, + } + expectedTestPod3Resized := []podresize.ResizableContainerInfo{ + { + Name: "c", + Resources: &cgroups.ContainerResources{ + CPUReq: testPod3CPUQuantityResized.String(), + MemReq: testPod3MemoryQuantityResized.String(), + }, + }, + } + expectedTestPod4Resized := []podresize.ResizableContainerInfo{ + { + Name: "c", + Resources: &cgroups.ContainerResources{ + CPUReq: testPod4CPUQuantityResized.String(), + MemReq: testPod4MemoryQuantityResized.String(), + }, + }, + } - patchTestpod3ToDeferred := fmt.Sprintf(`{ - "spec": { - "containers": [ - { - "name": "c", - "resources": { - "requests": { - "cpu": "%dm", - "memory": "%d" - } - } - } - ] - } - }`, testPod3CPUQuantityResized.MilliValue(), testPod3MemoryQuantityResized.Value()) + patchTestPod2ToDeferred := podresize.MakeResizePatch(c, expectedTestPod2Resized) + patchTestPod3ToDeferred := podresize.MakeResizePatch(c, expectedTestPod3Resized) + patchTestPod4ToDeferred := podresize.MakeResizePatch(c, expectedTestPod4Resized) - patchTestpod4ToDeferred := fmt.Sprintf(`{ - "spec": { - "containers": [ - { - "name": "c", - "resources": { - "requests": { - "cpu": "%dm", - "memory": "%d" - } - } - } - ] - } - }`, testPod4CPUQuantityResized.MilliValue(), testPod4MemoryQuantityResized.Value()) - - patches := []string{patchTestpod2ToDeferred, patchTestpod3ToDeferred, patchTestpod4ToDeferred} + patches := []string{string(patchTestPod2ToDeferred), string(patchTestPod3ToDeferred), string(patchTestPod4ToDeferred)} for i := range patches { testPod := testPods[i+1] @@ -907,7 +886,6 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { testPod.Name, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "resize") framework.ExpectNoError(err, "failed to patch pod for resize") waitForPodDeferred(ctx, f, testPod) - gomega.Expect(testPod.Generation).To(gomega.BeEquivalentTo(2)) } ginkgo.By("deleting pod 1") @@ -915,43 +893,16 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { framework.ExpectNoError(delErr1, "failed to delete pod %s", testPod1.Name) ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod deletion '%s'", testPod2.Name, testPod1.Name)) - expected := []podresize.ResizableContainerInfo{ - { - Name: "c", - Resources: &cgroups.ContainerResources{ - CPUReq: testPod2CPUQuantityResized.String(), - MemReq: testPod2MemoryQuantityResized.String(), - }, - }, - } - resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod2, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) + resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod2, expectedTestPod2Resized) + podresize.ExpectPodResized(ctx, f, resizedPod, expectedTestPod2Resized) ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod resize '%s'", testPod3.Name, testPod2.Name)) - expected = []podresize.ResizableContainerInfo{ - { - Name: "c", - Resources: &cgroups.ContainerResources{ - CPUReq: testPod3CPUQuantityResized.String(), - MemReq: testPod3MemoryQuantityResized.String(), - }, - }, - } - resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod3, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) + resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod3, expectedTestPod3Resized) + podresize.ExpectPodResized(ctx, f, resizedPod, expectedTestPod3Resized) ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod resize '%s'", testPod4.Name, testPod3.Name)) - expected = []podresize.ResizableContainerInfo{ - { - Name: "c", - Resources: &cgroups.ContainerResources{ - CPUReq: testPod4CPUQuantityResized.String(), - MemReq: testPod4MemoryQuantityResized.String(), - }, - }, - } - resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod4, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) + resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod4, expectedTestPod4Resized) + podresize.ExpectPodResized(ctx, f, resizedPod, expectedTestPod4Resized) ginkgo.By("deleting pods") for _, testPod := range testPods { From 465f7281419184e4c19e3e198c881b6e8fc8938e Mon Sep 17 00:00:00 2001 From: Natasha Sarkar Date: Thu, 17 Jul 2025 19:19:42 +0000 Subject: [PATCH 3/3] use gomega.BeComparableTo instead of Equal for resource quantity check in resize test --- test/e2e/common/node/framework/podresize/resize.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/common/node/framework/podresize/resize.go b/test/e2e/common/node/framework/podresize/resize.go index 88c645fc615..56c4a0e061f 100644 --- a/test/e2e/common/node/framework/podresize/resize.go +++ b/test/e2e/common/node/framework/podresize/resize.go @@ -168,7 +168,7 @@ func VerifyPodResources(gotPod *v1.Pod, wantInfo []ResizableContainerInfo) { if wantCtr.Name != gotCtr.Name { continue } - gomega.Expect(v1.Container{Name: gotCtr.Name, Resources: gotCtr.Resources}).To(gomega.Equal(v1.Container{Name: wantCtr.Name, Resources: wantCtr.Resources})) + gomega.Expect(gotCtr.Resources).To(gomega.BeComparableTo(wantCtr.Resources)) } } } @@ -202,7 +202,8 @@ func verifyPodContainersStatusResources(gotCtrStatuses []v1.ContainerStatus, wan errs = append(errs, fmt.Errorf("container status %d name %q != expected name %q", i, gotCtrStatus.Name, wantCtr.Name)) continue } - if err := framework.Gomega().Expect(*gotCtrStatus.Resources).To(gomega.Equal(wantCtr.Resources)); err != nil { + + if err := framework.Gomega().Expect(*gotCtrStatus.Resources).To(gomega.BeComparableTo(wantCtr.Resources)); err != nil { errs = append(errs, fmt.Errorf("container[%s] status resources mismatch: %w", wantCtr.Name, err)) } }