From 10efa46fbb79864d72a0f6defa983c21b5064f80 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Sat, 28 Mar 2026 15:27:25 -0400 Subject: [PATCH] e2e_node: wait for pod drain before asserting zero pods in Memory Manager Metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Memory Manager Metrics BeforeEach asserts that zero pods are running on the node after a kubelet config update. This hard assertion flakes when a preceding serial test's namespace deletion hasn't completed yet — framework namespace cleanup is async and the kubelet restart in updateKubeletConfig can delay in-flight pod termination. CI logs show leftover pods from MemoryQoS tests (memqos-burstable, memqos-no-limit, etc.), Probe Stress tests (50-container pods), and Summary API PSI tests (memory-pressure-pod), all still Running when the assertion fires 4-7ms after the previous test finishes. Replace the immediate Expect(count).To(BeZero()) with an Eventually poll (2 minute timeout, 5 second interval) that gives pods time to drain after the kubelet restart. The existing printAllPodsOnNode diagnostic output is preserved inside the poll for debugging. Signed-off-by: Davanum Srinivas --- test/e2e_node/memory_manager_metrics_test.go | 26 +++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/test/e2e_node/memory_manager_metrics_test.go b/test/e2e_node/memory_manager_metrics_test.go index f085ac4f04c..e7461670fc8 100644 --- a/test/e2e_node/memory_manager_metrics_test.go +++ b/test/e2e_node/memory_manager_metrics_test.go @@ -79,8 +79,21 @@ var _ = SIGDescribe("Memory Manager Metrics", framework.WithSerial(), feature.Me updateKubeletConfig(ctx, f, oldCfg, true) }) - count := printAllPodsOnNode(ctx, f.ClientSet, framework.TestContext.NodeName) - gomega.Expect(count).To(gomega.BeZero(), "unexpected pods on %q, please check output above", framework.TestContext.NodeName) + // Wait for pods from previous serial tests to finish terminating. + // Framework namespace deletion is async, and the kubelet restart + // above can delay in-flight pod cleanup. Poll instead of asserting + // immediately so we don't flake on test ordering. + gomega.Eventually(ctx, func(ctx context.Context) error { + count, err := printAllPodsOnNode(ctx, f.ClientSet, framework.TestContext.NodeName) + if err != nil { + return err + } + if count != 0 { + return fmt.Errorf("unexpected pods on %q: %d still present", framework.TestContext.NodeName, count) + } + return nil + }).WithTimeout(2*time.Minute).WithPolling(5*time.Second).Should( + gomega.Succeed(), "unexpected pods on %q, please check output above", framework.TestContext.NodeName) }) ginkgo.It("should report zero pinning counters after a fresh restart", func(ctx context.Context) { @@ -160,7 +173,8 @@ var _ = SIGDescribe("Memory Manager Metrics", framework.WithSerial(), feature.Me }), ) - printAllPodsOnNode(ctx, f.ClientSet, framework.TestContext.NodeName) + _, err := printAllPodsOnNode(ctx, f.ClientSet, framework.TestContext.NodeName) + framework.ExpectNoError(err) // we updated the kubelet config in BeforeEach, so we can assume we start fresh. // being [Serial], we can also assume noone else but us is running pods. @@ -213,7 +227,7 @@ func validateMetrics(values e2emetrics.KubeletMetrics, totalKey, errorKey string // printAllPodsOnNode outputs status of all kubelet pods into log. // Note considering the e2e_node environment we will always have exactly 1 node, but still. -func printAllPodsOnNode(ctx context.Context, c clientset.Interface, nodeName string) int { +func printAllPodsOnNode(ctx context.Context, c clientset.Interface, nodeName string) (int, error) { nodeSelector := fields.Set{ "spec.nodeName": nodeName, }.AsSelector().String() @@ -223,7 +237,7 @@ func printAllPodsOnNode(ctx context.Context, c clientset.Interface, nodeName str }) if err != nil { framework.Logf("Unable to retrieve pods for node %v: %v", nodeName, err) - return 0 + return 0, err } count := 0 framework.Logf("begin listing pods: %d found", len(podList.Items)) @@ -237,5 +251,5 @@ func printAllPodsOnNode(ctx context.Context, c clientset.Interface, nodeName str count++ } framework.Logf("end listing pods: %d found", len(podList.Items)) - return count + return count, nil }