From 3bc767602439e5e89a11004907cb55135ac70f1f Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Mon, 21 Aug 2017 11:29:25 -0700 Subject: [PATCH] StatefulSet: Deflake e2e "Saturate" phase. The "Saturate" phase of StatefulSet e2e tests verifies orderly startup by controlling when each Pod is allowed to report Ready. If a Pod unexepectedly goes down during the test, the replacement Pod created by the controller will forget if it was already allowed to report Ready. After this change, the signal that allows each Pod to report Ready is persisted in the Pod's PVC. Thus, the replacement Pod will remember that it was already told to proceed to a Ready state. --- test/e2e/apps/statefulset.go | 10 +++++----- test/e2e/framework/statefulset_utils.go | 4 ++-- test/e2e/upgrades/apps/statefulset.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/e2e/apps/statefulset.go b/test/e2e/apps/statefulset.go index b0c8cbdffc3..6a0ed6e1f22 100644 --- a/test/e2e/apps/statefulset.go +++ b/test/e2e/apps/statefulset.go @@ -115,7 +115,7 @@ var _ = SIGDescribe("StatefulSet", func() { By("Restarting statefulset " + ss.Name) sst.Restart(ss) - sst.Saturate(ss) + sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss) By("Verifying statefulset mounted data directory is usable") framework.ExpectNoError(sst.CheckMount(ss, "/data")) @@ -233,13 +233,13 @@ var _ = SIGDescribe("StatefulSet", func() { sst.DeleteStatefulPodAtIndex(0, ss) By("Confirming stateful pod at index 0 is recreated.") - sst.WaitForRunning(2, 0, ss) + sst.WaitForRunning(2, 1, ss) - By("Deleting unhealthy stateful pod at index 1.") - sst.DeleteStatefulPodAtIndex(1, ss) + By("Resuming stateful pod at index 1.") + sst.ResumeNextPod(ss) By("Confirming all stateful pods in statefulset are created.") - sst.Saturate(ss) + sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss) }) It("should perform rolling updates and roll backs of template modifications", func() { diff --git a/test/e2e/framework/statefulset_utils.go b/test/e2e/framework/statefulset_utils.go index b33cc332c54..1ba66024307 100644 --- a/test/e2e/framework/statefulset_utils.go +++ b/test/e2e/framework/statefulset_utils.go @@ -548,7 +548,7 @@ func (s *StatefulSetTester) RestorePodHttpProbe(ss *apps.StatefulSet, pod *v1.Po var pauseProbe = &v1.Probe{ Handler: v1.Handler{ - Exec: &v1.ExecAction{Command: []string{"test", "-f", "/tmp/statefulset-continue"}}, + Exec: &v1.ExecAction{Command: []string{"test", "-f", "/data/statefulset-continue"}}, }, PeriodSeconds: 1, SuccessThreshold: 1, @@ -586,7 +586,7 @@ func (s *StatefulSetTester) ResumeNextPod(ss *apps.StatefulSet) { if resumedPod != "" { Failf("Found multiple paused stateful pods: %v and %v", pod.Name, resumedPod) } - _, err := RunHostCmd(pod.Namespace, pod.Name, "touch /tmp/statefulset-continue") + _, err := RunHostCmd(pod.Namespace, pod.Name, "touch /data/statefulset-continue") ExpectNoError(err) Logf("Resumed pod %v", pod.Name) resumedPod = pod.Name diff --git a/test/e2e/upgrades/apps/statefulset.go b/test/e2e/upgrades/apps/statefulset.go index 89116f4f908..6b3224290ea 100644 --- a/test/e2e/upgrades/apps/statefulset.go +++ b/test/e2e/upgrades/apps/statefulset.go @@ -110,5 +110,5 @@ func (t *StatefulSetUpgradeTest) verify() { func (t *StatefulSetUpgradeTest) restart() { By("Restarting statefulset " + t.set.Name) t.tester.Restart(t.set) - t.tester.Saturate(t.set) + t.tester.WaitForRunningAndReady(*t.set.Spec.Replicas, t.set) }