diff --git a/test/e2e/framework/statefulset_utils.go b/test/e2e/framework/statefulset_utils.go index 91d7c79b690..4ab4ad4d29a 100644 --- a/test/e2e/framework/statefulset_utils.go +++ b/test/e2e/framework/statefulset_utils.go @@ -265,14 +265,21 @@ func (s *StatefulSetTester) GetPodList(ss *apps.StatefulSet) *v1.PodList { // ConfirmStatefulPodCount asserts that the current number of Pods in ss is count waiting up to timeout for ss to // to scale to count. -func (s *StatefulSetTester) ConfirmStatefulPodCount(count int, ss *apps.StatefulSet, timeout time.Duration) { +func (s *StatefulSetTester) ConfirmStatefulPodCount(count int, ss *apps.StatefulSet, timeout time.Duration, hard bool) { start := time.Now() deadline := start.Add(timeout) for t := time.Now(); t.Before(deadline); t = time.Now() { podList := s.GetPodList(ss) statefulPodCount := len(podList.Items) if statefulPodCount != count { - Failf("StatefulSet %v scaled unexpectedly scaled to %d -> %d replicas: %+v", ss.Name, count, len(podList.Items), podList) + logPodStates(podList.Items) + if hard { + Failf("StatefulSet %v scaled unexpectedly scaled to %d -> %d replicas", ss.Name, count, len(podList.Items)) + } else { + Logf("StatefulSet %v has not reached scale %d, at %d", ss.Name, count, statefulPodCount) + } + time.Sleep(1 * time.Second) + continue } Logf("Verifying statefulset %v doesn't scale past %d for another %+v", ss.Name, count, deadline.Sub(t)) time.Sleep(1 * time.Second) diff --git a/test/e2e/statefulset.go b/test/e2e/statefulset.go index 432f2e4bee2..866d960b5a1 100644 --- a/test/e2e/statefulset.go +++ b/test/e2e/statefulset.go @@ -311,7 +311,7 @@ var _ = framework.KubeDescribe("StatefulSet", func() { sst.UpdateReplicas(ss, 1) By("Verifying that the 2nd pod wont be removed if it is not running and ready") - sst.ConfirmStatefulPodCount(2, ss, 10*time.Second) + sst.ConfirmStatefulPodCount(2, ss, 10*time.Second, true) expectedPodName := ss.Name + "-1" expectedPod, err := f.ClientSet.Core().Pods(ns).Get(expectedPodName, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -369,7 +369,7 @@ var _ = framework.KubeDescribe("StatefulSet", func() { sst.WaitForRunningAndNotReady(*ss.Spec.Replicas, ss) sst.WaitForStatus(ss, 0) sst.UpdateReplicas(ss, 3) - sst.ConfirmStatefulPodCount(1, ss, 10*time.Second) + sst.ConfirmStatefulPodCount(1, ss, 10*time.Second, true) By("Scaling up stateful set " + ssName + " to 3 replicas and waiting until all of them will be running in namespace " + ns) sst.RestoreProbe(ss, testProbe) @@ -400,7 +400,7 @@ var _ = framework.KubeDescribe("StatefulSet", func() { sst.WaitForStatus(ss, 0) sst.WaitForRunningAndNotReady(3, ss) sst.UpdateReplicas(ss, 0) - sst.ConfirmStatefulPodCount(3, ss, 10*time.Second) + sst.ConfirmStatefulPodCount(3, ss, 10*time.Second, true) By("Scaling down stateful set " + ssName + " to 0 replicas and waiting until none of pods will run in namespace" + ns) sst.RestoreProbe(ss, testProbe) @@ -422,6 +422,47 @@ var _ = framework.KubeDescribe("StatefulSet", func() { Expect(err).NotTo(HaveOccurred()) }) + It("Burst scaling should run to completion even with unhealthy pods", func() { + psLabels := klabels.Set(labels) + + By("Creating stateful set " + ssName + " in namespace " + ns) + testProbe := &v1.Probe{Handler: v1.Handler{HTTPGet: &v1.HTTPGetAction{ + Path: "/index.html", + Port: intstr.IntOrString{IntVal: 80}}}} + ss := framework.NewStatefulSet(ssName, ns, headlessSvcName, 1, nil, nil, psLabels) + ss.Spec.PodManagementPolicy = apps.ParallelPodManagement + ss.Spec.Template.Spec.Containers[0].ReadinessProbe = testProbe + ss, err := c.Apps().StatefulSets(ns).Create(ss) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting until all stateful set " + ssName + " replicas will be running in namespace " + ns) + sst := framework.NewStatefulSetTester(c) + sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss) + + By("Confirming that stateful set scale up will not halt with unhealthy stateful pod") + sst.BreakProbe(ss, testProbe) + sst.WaitForRunningAndNotReady(*ss.Spec.Replicas, ss) + sst.WaitForStatus(ss, 0) + sst.UpdateReplicas(ss, 3) + sst.ConfirmStatefulPodCount(3, ss, 10*time.Second, false) + + By("Scaling up stateful set " + ssName + " to 3 replicas and waiting until all of them will be running in namespace " + ns) + sst.RestoreProbe(ss, testProbe) + sst.WaitForRunningAndReady(3, ss) + + By("Scale down will not halt with unhealthy stateful pod") + sst.BreakProbe(ss, testProbe) + sst.WaitForStatus(ss, 0) + sst.WaitForRunningAndNotReady(3, ss) + sst.UpdateReplicas(ss, 0) + sst.ConfirmStatefulPodCount(0, ss, 10*time.Second, false) + + By("Scaling down stateful set " + ssName + " to 0 replicas and waiting until none of pods will run in namespace" + ns) + sst.RestoreProbe(ss, testProbe) + sst.Scale(ss, 0) + sst.WaitForStatus(ss, 0) + }) + It("Should recreate evicted statefulset", func() { podName := "test-pod" statefulPodName := ssName + "-0"