From efddaf6561f2bbcc85e832f97848e75d36d57642 Mon Sep 17 00:00:00 2001 From: KhushAhuja Date: Sun, 8 Mar 2026 17:36:18 +0530 Subject: [PATCH] test/e2e: deflake pod resize cgroup value verification Replace the manual 3-retry loop (with no delay) in VerifyCgroupValue with framework.Gomega().Eventually() + HandleRetry, matching the pattern used for oom_score_adj deflake in #137329. This gives proper polling with backoff when exec fails during container restarts. --- .../common/node/framework/cgroups/cgroups.go | 70 ++++++++----------- .../common/node/framework/podresize/resize.go | 2 +- test/e2e/common/node/pod_level_resources.go | 8 +-- 3 files changed, 35 insertions(+), 45 deletions(-) diff --git a/test/e2e/common/node/framework/cgroups/cgroups.go b/test/e2e/common/node/framework/cgroups/cgroups.go index f8a96d673d2..2e9ae7bc6f7 100644 --- a/test/e2e/common/node/framework/cgroups/cgroups.go +++ b/test/e2e/common/node/framework/cgroups/cgroups.go @@ -216,7 +216,7 @@ func getExpectedMemLimitString(memLimit *resource.Quantity, podOnCgroupv2 bool) return expectedMemLimitString } -func verifyContainerCPUWeight(f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { +func verifyContainerCPUWeight(ctx context.Context, f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { cpuWeightCgPath := getCgroupCPURequestPath(cgroupFsPath, podOnCgroupv2) cpuLim := expectedResources.Limits.Cpu() if cpuLim.IsZero() && pod.Spec.Resources != nil { @@ -232,26 +232,26 @@ func verifyContainerCPUWeight(f *framework.Framework, pod *v1.Pod, containerName } expectedCPUShares := getExpectedCPUShares(cpuReq, cpuLim, podOnCgroupv2) - if err := VerifyCgroupValue(f, pod, containerName, cpuWeightCgPath, expectedCPUShares...); err != nil { + if err := VerifyCgroupValue(ctx, f, pod, containerName, cpuWeightCgPath, expectedCPUShares...); err != nil { return fmt.Errorf("failed to verify cpu request cgroup value: %w", err) } return nil } -func VerifyContainerCPULimit(f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { +func VerifyContainerCPULimit(ctx context.Context, f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { cpuLimCgPath := getCgroupCPULimitPath(cgroupFsPath, podOnCgroupv2) cpuLim := expectedResources.Limits.Cpu() if cpuLim.IsZero() && pod.Spec.Resources != nil { cpuLim = pod.Spec.Resources.Limits.Cpu() } expectedCPULimits := getCPULimitCgroupExpectations(cpuLim, podOnCgroupv2) - if err := VerifyCgroupValue(f, pod, containerName, cpuLimCgPath, expectedCPULimits...); err != nil { + if err := VerifyCgroupValue(ctx, f, pod, containerName, cpuLimCgPath, expectedCPULimits...); err != nil { return fmt.Errorf("failed to verify cpu limit cgroup value: %w", err) } return nil } -func VerifyContainerMemoryLimit(f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { +func VerifyContainerMemoryLimit(ctx context.Context, f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { memLimCgPath := getCgroupMemLimitPath(cgroupFsPath, podOnCgroupv2) memLim := expectedResources.Limits.Memory() if memLim.IsZero() && pod.Spec.Resources != nil { @@ -261,21 +261,21 @@ func VerifyContainerMemoryLimit(f *framework.Framework, pod *v1.Pod, containerNa if expectedMemLim == "0" { return nil } - if err := VerifyCgroupValue(f, pod, containerName, memLimCgPath, expectedMemLim); err != nil { + if err := VerifyCgroupValue(ctx, f, pod, containerName, memLimCgPath, expectedMemLim); err != nil { return fmt.Errorf("failed to verify memory limit cgroup value: %w", err) } return nil } -func VerifyContainerCgroupValues(f *framework.Framework, pod *v1.Pod, tc *v1.Container, podOnCgroupv2 bool) error { +func VerifyContainerCgroupValues(ctx context.Context, f *framework.Framework, pod *v1.Pod, tc *v1.Container, podOnCgroupv2 bool) error { var errs []error - errs = append(errs, VerifyContainerMemoryLimit(f, pod, tc.Name, &tc.Resources, podOnCgroupv2)) - errs = append(errs, VerifyContainerCPULimit(f, pod, tc.Name, &tc.Resources, podOnCgroupv2)) - errs = append(errs, verifyContainerCPUWeight(f, pod, tc.Name, &tc.Resources, podOnCgroupv2)) + errs = append(errs, VerifyContainerMemoryLimit(ctx, f, pod, tc.Name, &tc.Resources, podOnCgroupv2)) + errs = append(errs, VerifyContainerCPULimit(ctx, f, pod, tc.Name, &tc.Resources, podOnCgroupv2)) + errs = append(errs, verifyContainerCPUWeight(ctx, f, pod, tc.Name, &tc.Resources, podOnCgroupv2)) return utilerrors.NewAggregate(errs) } -func verifyPodCPUWeight(f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { +func verifyPodCPUWeight(ctx context.Context, f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { podCgPath, err := getPodCgroupPath(f, pod, podOnCgroupv2, "cpu") if err != nil { if podCgPath, err = getPodCgroupPath(f, pod, podOnCgroupv2, "cpu,cpuacct"); err != nil { @@ -290,13 +290,13 @@ func verifyPodCPUWeight(f *framework.Framework, pod *v1.Pod, expectedResources * cpuWeightCgPath = fmt.Sprintf("%s/%s", podCgPath, cgroupCPUSharesFile) } expectedCPUShares := getExpectedCPUShares(expectedResources.Requests.Cpu(), expectedResources.Limits.Cpu(), podOnCgroupv2) - if err := VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, cpuWeightCgPath, expectedCPUShares...); err != nil { + if err := VerifyCgroupValue(ctx, f, pod, pod.Spec.Containers[0].Name, cpuWeightCgPath, expectedCPUShares...); err != nil { return fmt.Errorf("pod cgroup cpu weight verification failed: %w", err) } return nil } -func verifyPodCPULimit(f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { +func verifyPodCPULimit(ctx context.Context, f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { podCgPath, err := getPodCgroupPath(f, pod, podOnCgroupv2, "cpu") if err != nil { if podCgPath, err = getPodCgroupPath(f, pod, podOnCgroupv2, "cpu,cpuacct"); err != nil { @@ -311,13 +311,13 @@ func verifyPodCPULimit(f *framework.Framework, pod *v1.Pod, expectedResources *v cpuLimCgPath = fmt.Sprintf("%s/%s", podCgPath, cgroupCPUQuotaFile) } expectedCPULimits := getCPULimitCgroupExpectations(expectedResources.Limits.Cpu(), podOnCgroupv2) - if err := VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, cpuLimCgPath, expectedCPULimits...); err != nil { + if err := VerifyCgroupValue(ctx, f, pod, pod.Spec.Containers[0].Name, cpuLimCgPath, expectedCPULimits...); err != nil { return fmt.Errorf("pod cgroup cpu limit verification failed: %w", err) } return nil } -func verifyPodMemoryLimit(f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { +func verifyPodMemoryLimit(ctx context.Context, f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error { podCgPath, err := getPodCgroupPath(f, pod, podOnCgroupv2, "memory") if err != nil { return err @@ -334,7 +334,7 @@ func verifyPodMemoryLimit(f *framework.Framework, pod *v1.Pod, expectedResources return nil } - if err := VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, memLimCgPath, expectedMemLim); err != nil { + if err := VerifyCgroupValue(ctx, f, pod, pod.Spec.Containers[0].Name, memLimCgPath, expectedMemLim); err != nil { return fmt.Errorf("pod cgroup memory limit verification failed: %w", err) } return nil @@ -349,9 +349,9 @@ func VerifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod, // Verify cgroup values expectedResources := info.ResourceRequirements() var errs []error - errs = append(errs, verifyPodCPUWeight(f, pod, expectedResources, onCgroupV2)) - errs = append(errs, verifyPodCPULimit(f, pod, expectedResources, onCgroupV2)) - errs = append(errs, verifyPodMemoryLimit(f, pod, expectedResources, onCgroupV2)) + errs = append(errs, verifyPodCPUWeight(ctx, f, pod, expectedResources, onCgroupV2)) + errs = append(errs, verifyPodCPULimit(ctx, f, pod, expectedResources, onCgroupV2)) + errs = append(errs, verifyPodMemoryLimit(ctx, f, pod, expectedResources, onCgroupV2)) return utilerrors.NewAggregate(errs) } @@ -374,32 +374,22 @@ func BuildPodResourceInfo(podCPURequestMilliValue, podCPULimitMilliValue, podMem // the specified container of the pod. It execs into the container to retrieve the // cgroup value, and ensures that the retrieved cgroup value is equivalent to at // least one of the values in expectedCgValues. -func VerifyCgroupValue(f *framework.Framework, pod *v1.Pod, cName, cgPath string, expectedCgValues ...string) error { +func VerifyCgroupValue(ctx context.Context, f *framework.Framework, pod *v1.Pod, cName, cgPath string, expectedCgValues ...string) error { cmd := fmt.Sprintf("head -n 1 %s", cgPath) framework.Logf("Namespace %s Pod %s Container %s - looking for one of the expected cgroup values %s in path %s", pod.Namespace, pod.Name, cName, expectedCgValues, cgPath) - const maxRetries = 3 - var cgValue string - var err error - for i := range maxRetries { - cgValue, _, err = e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd) - if err == nil { - cgValue = strings.Trim(cgValue, "\n") - break - } else { - framework.Logf("[Attempt %d of %d] Failed to read cgroup value %q for container %q: %v", i+1, maxRetries, cgPath, cName, err) + return framework.Gomega().Eventually(ctx, framework.HandleRetry(func(ctx context.Context) (error, error) { + cgValue, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd) + if err != nil { + return fmt.Errorf("failed to read cgroup value %q for container %q: %w", cgPath, cName, err), nil } - } - if err != nil { - return fmt.Errorf("failed to read cgroup value %q for container %q after %d attempts: %w", cgPath, cName, maxRetries, err) - } - - if err := framework.Gomega().Expect(cgValue).To(gomega.BeElementOf(expectedCgValues)); err != nil { - return fmt.Errorf("value of cgroup %q for container %q was %q; expected one of %q", cgPath, cName, cgValue, expectedCgValues) - } - - return nil + cgValue = strings.Trim(cgValue, "\n") + if err := framework.Gomega().Expect(cgValue).To(gomega.BeElementOf(expectedCgValues)); err != nil { + return nil, fmt.Errorf("value of cgroup %q for container %q was %q; expected one of %q: %w", cgPath, cName, cgValue, expectedCgValues, err) + } + return nil, nil + })).WithTimeout(framework.PollShortTimeout).Should(gomega.Succeed()) } // VerifyOomScoreAdjValue verifies that oom_score_adj for pid 1 (pidof init/systemd -> app) diff --git a/test/e2e/common/node/framework/podresize/resize.go b/test/e2e/common/node/framework/podresize/resize.go index 0dc9ce2572e..0ccee72ea16 100644 --- a/test/e2e/common/node/framework/podresize/resize.go +++ b/test/e2e/common/node/framework/podresize/resize.go @@ -300,7 +300,7 @@ func VerifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework var errs []error for _, ci := range tcInfo { tc := makeResizableContainer(ci) - errs = append(errs, cgroups.VerifyContainerCgroupValues(f, pod, &tc, onCgroupv2)) + errs = append(errs, cgroups.VerifyContainerCgroupValues(ctx, f, pod, &tc, onCgroupv2)) } return utilerrors.NewAggregate(errs) } diff --git a/test/e2e/common/node/pod_level_resources.go b/test/e2e/common/node/pod_level_resources.go index 0cd60e8047b..743fa7a1c7f 100644 --- a/test/e2e/common/node/pod_level_resources.go +++ b/test/e2e/common/node/pod_level_resources.go @@ -440,7 +440,7 @@ func podLevelResourcesTests(f *framework.Framework) { framework.ExpectNoError(err, "failed to verify pod's cgroup values: %v", err) ginkgo.By("verifying containers cgroup limits are same as pod container's cgroup limits") - err = verifyContainersCgroupLimits(f, pod) + err = verifyContainersCgroupLimits(ctx, f, pod) framework.ExpectNoError(err, "failed to verify containers cgroup values: %v", err) ginkgo.By("deleting pods") @@ -450,12 +450,12 @@ func podLevelResourcesTests(f *framework.Framework) { } } -func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error { +func verifyContainersCgroupLimits(ctx context.Context, f *framework.Framework, pod *v1.Pod) error { var errs []error for _, container := range pod.Spec.Containers { if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Memory() != nil && container.Resources.Limits.Memory() == nil { - err := cgroups.VerifyContainerMemoryLimit(f, pod, container.Name, &container.Resources, true) + err := cgroups.VerifyContainerMemoryLimit(ctx, f, pod, container.Name, &container.Resources, true) if err != nil { errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err)) } @@ -463,7 +463,7 @@ func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error { if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Cpu() != nil && container.Resources.Limits.Cpu() == nil { - err := cgroups.VerifyContainerCPULimit(f, pod, container.Name, &container.Resources, true) + err := cgroups.VerifyContainerCPULimit(ctx, f, pod, container.Name, &container.Resources, true) if err != nil { errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err)) }