From 29fd58ad0eae18c51c3f5efe58cbb44de0ffd2b7 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Wed, 23 Nov 2016 05:39:25 +0300 Subject: [PATCH] Fix running e2e with 'Completed' kube-system pods --- test/e2e/e2e.go | 2 +- test/e2e/framework/util.go | 25 +++++++++++++++++-------- test/e2e/mesos.go | 2 +- test/e2e/resize_nodes.go | 2 +- test/e2e/scheduler_predicates.go | 2 +- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 22449966cb3..d77d7b46a2d 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -123,7 +123,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { // test pods from running, and tests that ensure all pods are running and // ready will fail). podStartupTimeout := framework.TestContext.SystemPodsStartupTimeout - if err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, int32(framework.TestContext.MinStartupPods), podStartupTimeout, framework.ImagePullerLabels); err != nil { + if err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, int32(framework.TestContext.MinStartupPods), podStartupTimeout, framework.ImagePullerLabels, true); err != nil { framework.DumpAllNamespaceInfo(c, api.NamespaceSystem) framework.LogFailedContainers(c, api.NamespaceSystem, framework.Logf) framework.RunKubernetesServiceTestContainer(c, v1.NamespaceDefault) diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index a48b9cf7bcd..b4f31c26808 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -496,8 +496,8 @@ var ReadyReplicaVersion = version.MustParse("v1.4.0") // even if there are minPods pods, some of which are in Running/Ready // and some in Success. This is to allow the client to decide if "Success" // means "Ready" or not. -func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods int32, timeout time.Duration, ignoreLabels map[string]string) error { - +// If skipSucceeded is true, any pods that are Succeeded are not counted. +func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods int32, timeout time.Duration, ignoreLabels map[string]string, skipSucceeded bool) error { // This can be removed when we no longer have 1.3 servers running with upgrade tests. hasReadyReplicas, err := ServerVersionGTE(ReadyReplicaVersion, c.Discovery()) if err != nil { @@ -561,13 +561,22 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods int32, ti Logf("%v in state %v, ignoring", pod.Name, pod.Status.Phase) continue } - if res, err := testutils.PodRunningReady(&pod); res && err == nil { + res, err := testutils.PodRunningReady(&pod) + switch { + case res && err == nil: nOk++ - } else { - if pod.Status.Phase != v1.PodFailed { - Logf("The status of Pod %s is %s (Ready = false), waiting for it to be either Running (with Ready = true) or Failed", pod.ObjectMeta.Name, pod.Status.Phase) - badPods = append(badPods, pod) - } else if _, ok := pod.Annotations[v1.CreatedByAnnotation]; !ok { + case pod.Status.Phase == v1.PodSucceeded && skipSucceeded: + continue + case pod.Status.Phase == v1.PodSucceeded: + Logf("The status of Pod %s is Succeeded which is unexpected", pod.ObjectMeta.Name) + badPods = append(badPods, pod) + // it doesn't make sense to wait for this pod + return false, errors.New("unexpected Succeeded pod state") + case pod.Status.Phase != v1.PodFailed: + Logf("The status of Pod %s is %s (Ready = false), waiting for it to be either Running (with Ready = true) or Failed", pod.ObjectMeta.Name, pod.Status.Phase) + badPods = append(badPods, pod) + default: + if _, ok := pod.Annotations[v1.CreatedByAnnotation]; !ok { Logf("Pod %s is Failed, but it's not controlled by a controller", pod.ObjectMeta.Name) badPods = append(badPods, pod) } diff --git a/test/e2e/mesos.go b/test/e2e/mesos.go index bb65d1597d3..9d4662f4123 100644 --- a/test/e2e/mesos.go +++ b/test/e2e/mesos.go @@ -68,7 +68,7 @@ var _ = framework.KubeDescribe("Mesos", func() { nodelist := framework.GetReadySchedulableNodesOrDie(client) const ns = "static-pods" numpods := int32(len(nodelist.Items)) - framework.ExpectNoError(framework.WaitForPodsRunningReady(client, ns, numpods, wait.ForeverTestTimeout, map[string]string{}), + framework.ExpectNoError(framework.WaitForPodsRunningReady(client, ns, numpods, wait.ForeverTestTimeout, map[string]string{}, false), fmt.Sprintf("number of static pods in namespace %s is %d", ns, numpods)) }) diff --git a/test/e2e/resize_nodes.go b/test/e2e/resize_nodes.go index 65e54c9d9ef..789862cb6a1 100644 --- a/test/e2e/resize_nodes.go +++ b/test/e2e/resize_nodes.go @@ -285,7 +285,7 @@ var _ = framework.KubeDescribe("Nodes [Disruptive]", func() { // Many e2e tests assume that the cluster is fully healthy before they start. Wait until // the cluster is restored to health. By("waiting for system pods to successfully restart") - err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, systemPodsNo, framework.PodReadyBeforeTimeout, ignoreLabels) + err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, systemPodsNo, framework.PodReadyBeforeTimeout, ignoreLabels, true) Expect(err).NotTo(HaveOccurred()) By("waiting for image prepulling pods to complete") framework.WaitForPodsSuccess(c, api.NamespaceSystem, framework.ImagePullerLabels, imagePrePullingTimeout) diff --git a/test/e2e/scheduler_predicates.go b/test/e2e/scheduler_predicates.go index ac507c5a805..46ed85e0b38 100644 --- a/test/e2e/scheduler_predicates.go +++ b/test/e2e/scheduler_predicates.go @@ -90,7 +90,7 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() { } } - err = framework.WaitForPodsRunningReady(cs, api.NamespaceSystem, int32(systemPodsNo), framework.PodReadyBeforeTimeout, ignoreLabels) + err = framework.WaitForPodsRunningReady(cs, api.NamespaceSystem, int32(systemPodsNo), framework.PodReadyBeforeTimeout, ignoreLabels, true) Expect(err).NotTo(HaveOccurred()) for _, node := range nodeList.Items {