github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-07-17 11:28:58 +00:00
Code Issues Projects Releases Wiki Activity

test/e2e: deflake pod resize cgroup value verification

Browse Source 
Replace the manual 3-retry loop (with no delay) in VerifyCgroupValue
with framework.Gomega().Eventually() + HandleRetry, matching the
pattern used for oom_score_adj deflake in #137329. This gives proper
polling with backoff when exec fails during container restarts.
This commit is contained in:
KhushAhuja
2026-03-08 17:36:18 +05:30
committed by khush ahuja
parent dfbe79674a
commit efddaf6561
3 changed files with 35 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
test/e2e/common/node/framework/cgroups/cgroups.go
View File

@@ -216,7 +216,7 @@ func getExpectedMemLimitString(memLimit *resource.Quantity, podOnCgroupv2 bool)
return expectedMemLimitString
}
func verifyContainerCPUWeight(f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
func verifyContainerCPUWeight(ctx context.Context, f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
cpuWeightCgPath := getCgroupCPURequestPath(cgroupFsPath, podOnCgroupv2)
cpuLim := expectedResources.Limits.Cpu()
if cpuLim.IsZero() && pod.Spec.Resources != nil {
@@ -232,26 +232,26 @@ func verifyContainerCPUWeight(f *framework.Framework, pod *v1.Pod, containerName
}
expectedCPUShares := getExpectedCPUShares(cpuReq, cpuLim, podOnCgroupv2)
if err := VerifyCgroupValue(f, pod, containerName, cpuWeightCgPath, expectedCPUShares...); err != nil {
if err := VerifyCgroupValue(ctx, f, pod, containerName, cpuWeightCgPath, expectedCPUShares...); err != nil {
return fmt.Errorf("failed to verify cpu request cgroup value: %w", err)
}
return nil
}
func VerifyContainerCPULimit(f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
func VerifyContainerCPULimit(ctx context.Context, f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
cpuLimCgPath := getCgroupCPULimitPath(cgroupFsPath, podOnCgroupv2)
cpuLim := expectedResources.Limits.Cpu()
if cpuLim.IsZero() && pod.Spec.Resources != nil {
cpuLim = pod.Spec.Resources.Limits.Cpu()
}
expectedCPULimits := getCPULimitCgroupExpectations(cpuLim, podOnCgroupv2)
if err := VerifyCgroupValue(f, pod, containerName, cpuLimCgPath, expectedCPULimits...); err != nil {
if err := VerifyCgroupValue(ctx, f, pod, containerName, cpuLimCgPath, expectedCPULimits...); err != nil {
return fmt.Errorf("failed to verify cpu limit cgroup value: %w", err)
}
return nil
}
func VerifyContainerMemoryLimit(f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
func VerifyContainerMemoryLimit(ctx context.Context, f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
memLimCgPath := getCgroupMemLimitPath(cgroupFsPath, podOnCgroupv2)
memLim := expectedResources.Limits.Memory()
if memLim.IsZero() && pod.Spec.Resources != nil {
@@ -261,21 +261,21 @@ func VerifyContainerMemoryLimit(f *framework.Framework, pod *v1.Pod, containerNa
if expectedMemLim == "0" {
return nil
}
if err := VerifyCgroupValue(f, pod, containerName, memLimCgPath, expectedMemLim); err != nil {
if err := VerifyCgroupValue(ctx, f, pod, containerName, memLimCgPath, expectedMemLim); err != nil {
return fmt.Errorf("failed to verify memory limit cgroup value: %w", err)
}
return nil
}
func VerifyContainerCgroupValues(f *framework.Framework, pod *v1.Pod, tc *v1.Container, podOnCgroupv2 bool) error {
func VerifyContainerCgroupValues(ctx context.Context, f *framework.Framework, pod *v1.Pod, tc *v1.Container, podOnCgroupv2 bool) error {
var errs []error
errs = append(errs, VerifyContainerMemoryLimit(f, pod, tc.Name, &tc.Resources, podOnCgroupv2))
errs = append(errs, VerifyContainerCPULimit(f, pod, tc.Name, &tc.Resources, podOnCgroupv2))
errs = append(errs, verifyContainerCPUWeight(f, pod, tc.Name, &tc.Resources, podOnCgroupv2))
errs = append(errs, VerifyContainerMemoryLimit(ctx, f, pod, tc.Name, &tc.Resources, podOnCgroupv2))
errs = append(errs, VerifyContainerCPULimit(ctx, f, pod, tc.Name, &tc.Resources, podOnCgroupv2))
errs = append(errs, verifyContainerCPUWeight(ctx, f, pod, tc.Name, &tc.Resources, podOnCgroupv2))
return utilerrors.NewAggregate(errs)
}
func verifyPodCPUWeight(f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
func verifyPodCPUWeight(ctx context.Context, f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
podCgPath, err := getPodCgroupPath(f, pod, podOnCgroupv2, "cpu")
if err != nil {
if podCgPath, err = getPodCgroupPath(f, pod, podOnCgroupv2, "cpu,cpuacct"); err != nil {
@@ -290,13 +290,13 @@ func verifyPodCPUWeight(f *framework.Framework, pod *v1.Pod, expectedResources *
cpuWeightCgPath = fmt.Sprintf("%s/%s", podCgPath, cgroupCPUSharesFile)
}
expectedCPUShares := getExpectedCPUShares(expectedResources.Requests.Cpu(), expectedResources.Limits.Cpu(), podOnCgroupv2)
if err := VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, cpuWeightCgPath, expectedCPUShares...); err != nil {
if err := VerifyCgroupValue(ctx, f, pod, pod.Spec.Containers[0].Name, cpuWeightCgPath, expectedCPUShares...); err != nil {
return fmt.Errorf("pod cgroup cpu weight verification failed: %w", err)
}
return nil
}
func verifyPodCPULimit(f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
func verifyPodCPULimit(ctx context.Context, f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
podCgPath, err := getPodCgroupPath(f, pod, podOnCgroupv2, "cpu")
if err != nil {
if podCgPath, err = getPodCgroupPath(f, pod, podOnCgroupv2, "cpu,cpuacct"); err != nil {
@@ -311,13 +311,13 @@ func verifyPodCPULimit(f *framework.Framework, pod *v1.Pod, expectedResources *v
cpuLimCgPath = fmt.Sprintf("%s/%s", podCgPath, cgroupCPUQuotaFile)
}
expectedCPULimits := getCPULimitCgroupExpectations(expectedResources.Limits.Cpu(), podOnCgroupv2)
if err := VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, cpuLimCgPath, expectedCPULimits...); err != nil {
if err := VerifyCgroupValue(ctx, f, pod, pod.Spec.Containers[0].Name, cpuLimCgPath, expectedCPULimits...); err != nil {
return fmt.Errorf("pod cgroup cpu limit verification failed: %w", err)
}
return nil
}
func verifyPodMemoryLimit(f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
func verifyPodMemoryLimit(ctx context.Context, f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
podCgPath, err := getPodCgroupPath(f, pod, podOnCgroupv2, "memory")
if err != nil {
return err
@@ -334,7 +334,7 @@ func verifyPodMemoryLimit(f *framework.Framework, pod *v1.Pod, expectedResources
return nil
}
if err := VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, memLimCgPath, expectedMemLim); err != nil {
if err := VerifyCgroupValue(ctx, f, pod, pod.Spec.Containers[0].Name, memLimCgPath, expectedMemLim); err != nil {
return fmt.Errorf("pod cgroup memory limit verification failed: %w", err)
}
return nil
@@ -349,9 +349,9 @@ func VerifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod,
// Verify cgroup values
expectedResources := info.ResourceRequirements()
var errs []error
errs = append(errs, verifyPodCPUWeight(f, pod, expectedResources, onCgroupV2))
errs = append(errs, verifyPodCPULimit(f, pod, expectedResources, onCgroupV2))
errs = append(errs, verifyPodMemoryLimit(f, pod, expectedResources, onCgroupV2))
errs = append(errs, verifyPodCPUWeight(ctx, f, pod, expectedResources, onCgroupV2))
errs = append(errs, verifyPodCPULimit(ctx, f, pod, expectedResources, onCgroupV2))
errs = append(errs, verifyPodMemoryLimit(ctx, f, pod, expectedResources, onCgroupV2))
return utilerrors.NewAggregate(errs)
}
@@ -374,32 +374,22 @@ func BuildPodResourceInfo(podCPURequestMilliValue, podCPULimitMilliValue, podMem
// the specified container of the pod. It execs into the container to retrieve the
// cgroup value, and ensures that the retrieved cgroup value is equivalent to at
// least one of the values in expectedCgValues.
func VerifyCgroupValue(f *framework.Framework, pod *v1.Pod, cName, cgPath string, expectedCgValues ...string) error {
func VerifyCgroupValue(ctx context.Context, f *framework.Framework, pod *v1.Pod, cName, cgPath string, expectedCgValues ...string) error {
cmd := fmt.Sprintf("head -n 1 %s", cgPath)
framework.Logf("Namespace %s Pod %s Container %s - looking for one of the expected cgroup values %s in path %s",
pod.Namespace, pod.Name, cName, expectedCgValues, cgPath)
const maxRetries = 3
var cgValue string
var err error
for i := range maxRetries {
cgValue, _, err = e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd)
if err == nil {
cgValue = strings.Trim(cgValue, "\n")
break
} else {
framework.Logf("[Attempt %d of %d] Failed to read cgroup value %q for container %q: %v", i+1, maxRetries, cgPath, cName, err)
return framework.Gomega().Eventually(ctx, framework.HandleRetry(func(ctx context.Context) (error, error) {
cgValue, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd)
if err != nil {
return fmt.Errorf("failed to read cgroup value %q for container %q: %w", cgPath, cName, err), nil
}
}
if err != nil {
return fmt.Errorf("failed to read cgroup value %q for container %q after %d attempts: %w", cgPath, cName, maxRetries, err)
}
if err := framework.Gomega().Expect(cgValue).To(gomega.BeElementOf(expectedCgValues)); err != nil {
return fmt.Errorf("value of cgroup %q for container %q was %q; expected one of %q", cgPath, cName, cgValue, expectedCgValues)
}
return nil
cgValue = strings.Trim(cgValue, "\n")
if err := framework.Gomega().Expect(cgValue).To(gomega.BeElementOf(expectedCgValues)); err != nil {
return nil, fmt.Errorf("value of cgroup %q for container %q was %q; expected one of %q: %w", cgPath, cName, cgValue, expectedCgValues, err)
}
return nil, nil
})).WithTimeout(framework.PollShortTimeout).Should(gomega.Succeed())
}
// VerifyOomScoreAdjValue verifies that oom_score_adj for pid 1 (pidof init/systemd -> app)

2
test/e2e/common/node/framework/podresize/resize.go
View File

@@ -300,7 +300,7 @@ func VerifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework
var errs []error
for _, ci := range tcInfo {
tc := makeResizableContainer(ci)
errs = append(errs, cgroups.VerifyContainerCgroupValues(f, pod, &tc, onCgroupv2))
errs = append(errs, cgroups.VerifyContainerCgroupValues(ctx, f, pod, &tc, onCgroupv2))
}
return utilerrors.NewAggregate(errs)
}

8
test/e2e/common/node/pod_level_resources.go
View File

@@ -440,7 +440,7 @@ func podLevelResourcesTests(f *framework.Framework) {
framework.ExpectNoError(err, "failed to verify pod's cgroup values: %v", err)
ginkgo.By("verifying containers cgroup limits are same as pod container's cgroup limits")
err = verifyContainersCgroupLimits(f, pod)
err = verifyContainersCgroupLimits(ctx, f, pod)
framework.ExpectNoError(err, "failed to verify containers cgroup values: %v", err)
ginkgo.By("deleting pods")
@@ -450,12 +450,12 @@ func podLevelResourcesTests(f *framework.Framework) {
}
}
func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error {
func verifyContainersCgroupLimits(ctx context.Context, f *framework.Framework, pod *v1.Pod) error {
var errs []error
for _, container := range pod.Spec.Containers {
if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Memory() != nil &&
container.Resources.Limits.Memory() == nil {
err := cgroups.VerifyContainerMemoryLimit(f, pod, container.Name, &container.Resources, true)
err := cgroups.VerifyContainerMemoryLimit(ctx, f, pod, container.Name, &container.Resources, true)
if err != nil {
errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err))
}
@@ -463,7 +463,7 @@ func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error {
if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Cpu() != nil &&
container.Resources.Limits.Cpu() == nil {
err := cgroups.VerifyContainerCPULimit(f, pod, container.Name, &container.Resources, true)
err := cgroups.VerifyContainerCPULimit(ctx, f, pod, container.Name, &container.Resources, true)
if err != nil {
errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err))
}