From d6cd51e5c042c92f4a51d63ffbdd3bda669e1e50 Mon Sep 17 00:00:00 2001 From: David Porter Date: Tue, 8 Mar 2022 19:54:25 -0800 Subject: [PATCH] test: Verify that nodes do not transition to Failed while ready Signed-off-by: David Porter --- test/e2e_node/node_shutdown_linux_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/e2e_node/node_shutdown_linux_test.go b/test/e2e_node/node_shutdown_linux_test.go index 4d3b93b98fe..18793c3a932 100644 --- a/test/e2e_node/node_shutdown_linux_test.go +++ b/test/e2e_node/node_shutdown_linux_test.go @@ -30,6 +30,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/fields" + "k8s.io/kubectl/pkg/util/podutils" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -128,6 +129,11 @@ var _ = SIGDescribe("GracefulNodeShutdown [Serial] [NodeFeature:GracefulNodeShut framework.ExpectEqual(len(list.Items), len(pods), "the number of pods is not as expected") for _, pod := range list.Items { + if isPodStatusAffectedByIssue108594(&pod) { + framework.Logf("Detected invalid pod state for pod %q: pod status: %+v", pod.Name, pod.Status) + framework.Failf("failing test due to detecting invalid pod status") + } + if kubelettypes.IsCriticalPod(&pod) { if isPodShutdown(&pod) { framework.Logf("Expecting critical pod to be running, but it's not currently. Pod: %q, Pod Status %+v", pod.Name, pod.Status) @@ -155,6 +161,10 @@ var _ = SIGDescribe("GracefulNodeShutdown [Serial] [NodeFeature:GracefulNodeShut framework.ExpectEqual(len(list.Items), len(pods), "the number of pods is not as expected") for _, pod := range list.Items { + if isPodStatusAffectedByIssue108594(&pod) { + framework.Logf("Detected invalid pod state for pod %q: pod status: %+v", pod.Name, pod.Status) + framework.Failf("failing test due to detecting invalid pod status") + } if !isPodShutdown(&pod) { framework.Logf("Expecting pod to be shutdown, but it's not currently: Pod: %q, Pod Status %+v", pod.Name, pod.Status) return fmt.Errorf("pod should be shutdown, phase: %s", pod.Status.Phase) @@ -541,3 +551,8 @@ func isPodShutdown(pod *v1.Pod) bool { return pod.Status.Message == podShutdownMessage && pod.Status.Reason == podShutdownReason && hasContainersNotReadyCondition && pod.Status.Phase == v1.PodFailed } + +// Pods should never report failed phase and have ready condition = true (https://github.com/kubernetes/kubernetes/issues/108594) +func isPodStatusAffectedByIssue108594(pod *v1.Pod) bool { + return pod.Status.Phase == v1.PodFailed && podutils.IsPodReady(pod) +}