From 8504c0d4e3b0b5d2e623ac9d218be66cc1f57ee2 Mon Sep 17 00:00:00 2001 From: Natasha Sarkar Date: Sun, 1 Mar 2026 16:46:26 +0000 Subject: [PATCH] deflake pod resize oom_score_adj check --- .../common/node/framework/cgroups/cgroups.go | 4 ++-- .../common/node/framework/podresize/resize.go | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/test/e2e/common/node/framework/cgroups/cgroups.go b/test/e2e/common/node/framework/cgroups/cgroups.go index 001127afe34..f8a96d673d2 100644 --- a/test/e2e/common/node/framework/cgroups/cgroups.go +++ b/test/e2e/common/node/framework/cgroups/cgroups.go @@ -411,11 +411,11 @@ func VerifyOomScoreAdjValue(f *framework.Framework, pod *v1.Pod, cName, expected pod.Namespace, pod.Name, cName, expectedOomScoreAdj) oomScoreAdj, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd) if err != nil { - return fmt.Errorf("failed to find expected value %s for container app process", expectedOomScoreAdj) + return fmt.Errorf("failed to find expected oom_score_adj value %s for container %s in pod %s", expectedOomScoreAdj, cName, pod.Name) } oomScoreAdj = strings.Trim(oomScoreAdj, "\n") if oomScoreAdj != expectedOomScoreAdj { - return fmt.Errorf("oom_score_adj value %s not equal to expected %s", oomScoreAdj, expectedOomScoreAdj) + return fmt.Errorf("oom_score_adj value %s not equal to expected %s for container %s in pod %s", oomScoreAdj, expectedOomScoreAdj, cName, pod.Name) } return nil } diff --git a/test/e2e/common/node/framework/podresize/resize.go b/test/e2e/common/node/framework/podresize/resize.go index fc6c92452ba..0dc9ce2572e 100644 --- a/test/e2e/common/node/framework/podresize/resize.go +++ b/test/e2e/common/node/framework/podresize/resize.go @@ -305,22 +305,22 @@ func VerifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework return utilerrors.NewAggregate(errs) } -func verifyPodRestarts(f *framework.Framework, pod *v1.Pod, wantInfo []ResizableContainerInfo) error { +func verifyPodRestarts(ctx context.Context, f *framework.Framework, pod *v1.Pod, wantInfo []ResizableContainerInfo) error { ginkgo.GinkgoHelper() initCtrStatuses, ctrStatuses := separateContainerStatuses(wantInfo) errs := []error{} - if err := verifyContainerRestarts(f, pod, pod.Status.InitContainerStatuses, initCtrStatuses); err != nil { + if err := verifyContainerRestarts(ctx, f, pod, pod.Status.InitContainerStatuses, initCtrStatuses); err != nil { errs = append(errs, err) } - if err := verifyContainerRestarts(f, pod, pod.Status.ContainerStatuses, ctrStatuses); err != nil { + if err := verifyContainerRestarts(ctx, f, pod, pod.Status.ContainerStatuses, ctrStatuses); err != nil { errs = append(errs, err) } return utilerrors.NewAggregate(errs) } -func verifyContainerRestarts(f *framework.Framework, pod *v1.Pod, gotStatuses []v1.ContainerStatus, wantStatuses []v1.ContainerStatus) error { +func verifyContainerRestarts(ctx context.Context, f *framework.Framework, pod *v1.Pod, gotStatuses []v1.ContainerStatus, wantStatuses []v1.ContainerStatus) error { ginkgo.GinkgoHelper() if len(gotStatuses) != len(wantStatuses) { @@ -333,7 +333,7 @@ func verifyContainerRestarts(f *framework.Framework, pod *v1.Pod, gotStatuses [] if gotStatus.RestartCount != wantStatuses[i].RestartCount { errs = append(errs, fmt.Errorf("unexpected number of restarts for container %s: got %d, want %d", gotStatus.Name, gotStatus.RestartCount, wantStatuses[i].RestartCount)) } else if gotStatus.RestartCount > 0 { - if err := verifyOomScoreAdj(f, pod, gotStatus.Name); err != nil { + if err := verifyOomScoreAdj(ctx, f, pod, gotStatus.Name); err != nil { errs = append(errs, err) } } @@ -341,7 +341,7 @@ func verifyContainerRestarts(f *framework.Framework, pod *v1.Pod, gotStatuses [] return utilerrors.NewAggregate(errs) } -func verifyOomScoreAdj(f *framework.Framework, pod *v1.Pod, containerName string) error { +func verifyOomScoreAdj(ctx context.Context, f *framework.Framework, pod *v1.Pod, containerName string) error { container := e2epod.FindContainerInPod(pod, containerName) if container == nil { return fmt.Errorf("failed to find container %s in pod %s", containerName, pod.Name) @@ -356,7 +356,9 @@ func verifyOomScoreAdj(f *framework.Framework, pod *v1.Pod, containerName string oomScoreAdj := kubeqos.GetContainerOOMScoreAdjust(pod, container, int64(nodeMemoryCapacity.Value())) expectedOomScoreAdj := strconv.FormatInt(int64(oomScoreAdj), 10) - return cgroups.VerifyOomScoreAdjValue(f, pod, container.Name, expectedOomScoreAdj) + return framework.Gomega().Eventually(ctx, framework.HandleRetry(func(ctx context.Context) (error, error) { + return cgroups.VerifyOomScoreAdjValue(f, pod, container.Name, expectedOomScoreAdj), nil + })).WithTimeout(framework.PollShortTimeout).Should(gomega.Succeed()) } func WaitForPodResizeActuation(ctx context.Context, f *framework.Framework, podClient *e2epod.PodClient, pod *v1.Pod, expectedContainers []ResizableContainerInfo) *v1.Pod { @@ -438,7 +440,7 @@ func ExpectPodResized(ctx context.Context, f *framework.Framework, resizedPod *v errs = append(errs, fmt.Errorf("container status resources don't match expected: %w", formatErrors(resourceErrs))) } - if restartErrs := verifyPodRestarts(f, resizedPod, expectedContainers); restartErrs != nil { + if restartErrs := verifyPodRestarts(ctx, f, resizedPod, expectedContainers); restartErrs != nil { errs = append(errs, fmt.Errorf("container restart counts don't match expected: %w", formatErrors(restartErrs))) }