From db69638911894117b74875d4558092c7274ac263 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Fri, 25 May 2018 18:16:44 -0500 Subject: [PATCH] Add StatefulSet rollback and rolling update test with PVCs Signed-off-by: Jose A. Rivera --- test/e2e/apps/statefulset.go | 235 +++++++++++++++++++---------------- 1 file changed, 125 insertions(+), 110 deletions(-) diff --git a/test/e2e/apps/statefulset.go b/test/e2e/apps/statefulset.go index 09b14721168..5f9c21a1bfa 100644 --- a/test/e2e/apps/statefulset.go +++ b/test/e2e/apps/statefulset.go @@ -248,6 +248,14 @@ var _ = SIGDescribe("StatefulSet", func() { sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss) }) + // This can't be Conformance yet because it depends on a default + // StorageClass and a dynamic provisioner. + It("should perform rolling updates and roll backs of template modifications with PVCs", func() { + By("Creating a new StatefulSet with PVCs") + *(ss.Spec.Replicas) = 3 + rollbackTest(c, ns, ss) + }) + /* Release : v1.9 Testname: StatefulSet, Rolling Update @@ -256,116 +264,7 @@ var _ = SIGDescribe("StatefulSet", func() { framework.ConformanceIt("should perform rolling updates and roll backs of template modifications", func() { By("Creating a new StatefulSet") ss := framework.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels) - sst := framework.NewStatefulSetTester(c) - sst.SetHttpProbe(ss) - ss, err := c.AppsV1().StatefulSets(ns).Create(ss) - Expect(err).NotTo(HaveOccurred()) - sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss) - ss = sst.WaitForStatus(ss) - currentRevision, updateRevision := ss.Status.CurrentRevision, ss.Status.UpdateRevision - Expect(currentRevision).To(Equal(updateRevision), - fmt.Sprintf("StatefulSet %s/%s created with update revision %s not equal to current revision %s", - ss.Namespace, ss.Name, updateRevision, currentRevision)) - pods := sst.GetPodList(ss) - for i := range pods.Items { - Expect(pods.Items[i].Labels[apps.StatefulSetRevisionLabel]).To(Equal(currentRevision), - fmt.Sprintf("Pod %s/%s revision %s is not equal to current revision %s", - pods.Items[i].Namespace, - pods.Items[i].Name, - pods.Items[i].Labels[apps.StatefulSetRevisionLabel], - currentRevision)) - } - sst.SortStatefulPods(pods) - sst.BreakPodHttpProbe(ss, &pods.Items[1]) - Expect(err).NotTo(HaveOccurred()) - ss, pods = sst.WaitForPodNotReady(ss, pods.Items[1].Name) - newImage := NewNginxImage - oldImage := ss.Spec.Template.Spec.Containers[0].Image - - By(fmt.Sprintf("Updating StatefulSet template: update image from %s to %s", oldImage, newImage)) - Expect(oldImage).NotTo(Equal(newImage), "Incorrect test setup: should update to a different image") - ss, err = framework.UpdateStatefulSetWithRetries(c, ns, ss.Name, func(update *apps.StatefulSet) { - update.Spec.Template.Spec.Containers[0].Image = newImage - }) - Expect(err).NotTo(HaveOccurred()) - - By("Creating a new revision") - ss = sst.WaitForStatus(ss) - currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision - Expect(currentRevision).NotTo(Equal(updateRevision), - "Current revision should not equal update revision during rolling update") - - By("Updating Pods in reverse ordinal order") - pods = sst.GetPodList(ss) - sst.SortStatefulPods(pods) - sst.RestorePodHttpProbe(ss, &pods.Items[1]) - ss, pods = sst.WaitForPodReady(ss, pods.Items[1].Name) - ss, pods = sst.WaitForRollingUpdate(ss) - Expect(ss.Status.CurrentRevision).To(Equal(updateRevision), - fmt.Sprintf("StatefulSet %s/%s current revision %s does not equal update revision %s on update completion", - ss.Namespace, - ss.Name, - ss.Status.CurrentRevision, - updateRevision)) - for i := range pods.Items { - Expect(pods.Items[i].Spec.Containers[0].Image).To(Equal(newImage), - fmt.Sprintf(" Pod %s/%s has image %s not have new image %s", - pods.Items[i].Namespace, - pods.Items[i].Name, - pods.Items[i].Spec.Containers[0].Image, - newImage)) - Expect(pods.Items[i].Labels[apps.StatefulSetRevisionLabel]).To(Equal(updateRevision), - fmt.Sprintf("Pod %s/%s revision %s is not equal to update revision %s", - pods.Items[i].Namespace, - pods.Items[i].Name, - pods.Items[i].Labels[apps.StatefulSetRevisionLabel], - updateRevision)) - } - - By("Rolling back to a previous revision") - sst.BreakPodHttpProbe(ss, &pods.Items[1]) - Expect(err).NotTo(HaveOccurred()) - ss, pods = sst.WaitForPodNotReady(ss, pods.Items[1].Name) - priorRevision := currentRevision - currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision - ss, err = framework.UpdateStatefulSetWithRetries(c, ns, ss.Name, func(update *apps.StatefulSet) { - update.Spec.Template.Spec.Containers[0].Image = oldImage - }) - Expect(err).NotTo(HaveOccurred()) - ss = sst.WaitForStatus(ss) - currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision - Expect(currentRevision).NotTo(Equal(updateRevision), - "Current revision should not equal update revision during roll back") - Expect(priorRevision).To(Equal(updateRevision), - "Prior revision should equal update revision during roll back") - - By("Rolling back update in reverse ordinal order") - pods = sst.GetPodList(ss) - sst.SortStatefulPods(pods) - sst.RestorePodHttpProbe(ss, &pods.Items[1]) - ss, pods = sst.WaitForPodReady(ss, pods.Items[1].Name) - ss, pods = sst.WaitForRollingUpdate(ss) - Expect(ss.Status.CurrentRevision).To(Equal(priorRevision), - fmt.Sprintf("StatefulSet %s/%s current revision %s does not equal prior revision %s on rollback completion", - ss.Namespace, - ss.Name, - ss.Status.CurrentRevision, - updateRevision)) - - for i := range pods.Items { - Expect(pods.Items[i].Spec.Containers[0].Image).To(Equal(oldImage), - fmt.Sprintf("Pod %s/%s has image %s not equal to previous image %s", - pods.Items[i].Namespace, - pods.Items[i].Name, - pods.Items[i].Spec.Containers[0].Image, - oldImage)) - Expect(pods.Items[i].Labels[apps.StatefulSetRevisionLabel]).To(Equal(priorRevision), - fmt.Sprintf("Pod %s/%s revision %s is not equal to prior revision %s", - pods.Items[i].Namespace, - pods.Items[i].Name, - pods.Items[i].Labels[apps.StatefulSetRevisionLabel], - priorRevision)) - } + rollbackTest(c, ns, ss) }) /* @@ -1176,3 +1075,119 @@ func pollReadWithTimeout(statefulPod statefulPodTester, statefulPodNumber int, k } return err } + +// This function is used by two tests to test StatefulSet rollbacks: one using +// PVCs and one using no storage. +func rollbackTest(c clientset.Interface, ns string, ss *apps.StatefulSet) { + sst := framework.NewStatefulSetTester(c) + sst.SetHttpProbe(ss) + ss, err := c.AppsV1().StatefulSets(ns).Create(ss) + Expect(err).NotTo(HaveOccurred()) + sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss) + ss = sst.WaitForStatus(ss) + currentRevision, updateRevision := ss.Status.CurrentRevision, ss.Status.UpdateRevision + Expect(currentRevision).To(Equal(updateRevision), + fmt.Sprintf("StatefulSet %s/%s created with update revision %s not equal to current revision %s", + ss.Namespace, ss.Name, updateRevision, currentRevision)) + pods := sst.GetPodList(ss) + for i := range pods.Items { + Expect(pods.Items[i].Labels[apps.StatefulSetRevisionLabel]).To(Equal(currentRevision), + fmt.Sprintf("Pod %s/%s revision %s is not equal to current revision %s", + pods.Items[i].Namespace, + pods.Items[i].Name, + pods.Items[i].Labels[apps.StatefulSetRevisionLabel], + currentRevision)) + } + sst.SortStatefulPods(pods) + err = sst.BreakPodHttpProbe(ss, &pods.Items[1]) + Expect(err).NotTo(HaveOccurred()) + ss, pods = sst.WaitForPodNotReady(ss, pods.Items[1].Name) + newImage := NewNginxImage + oldImage := ss.Spec.Template.Spec.Containers[0].Image + + By(fmt.Sprintf("Updating StatefulSet template: update image from %s to %s", oldImage, newImage)) + Expect(oldImage).NotTo(Equal(newImage), "Incorrect test setup: should update to a different image") + ss, err = framework.UpdateStatefulSetWithRetries(c, ns, ss.Name, func(update *apps.StatefulSet) { + update.Spec.Template.Spec.Containers[0].Image = newImage + }) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a new revision") + ss = sst.WaitForStatus(ss) + currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision + Expect(currentRevision).NotTo(Equal(updateRevision), + "Current revision should not equal update revision during rolling update") + + By("Updating Pods in reverse ordinal order") + pods = sst.GetPodList(ss) + sst.SortStatefulPods(pods) + err = sst.RestorePodHttpProbe(ss, &pods.Items[1]) + Expect(err).NotTo(HaveOccurred()) + ss, pods = sst.WaitForPodReady(ss, pods.Items[1].Name) + ss, pods = sst.WaitForRollingUpdate(ss) + Expect(ss.Status.CurrentRevision).To(Equal(updateRevision), + fmt.Sprintf("StatefulSet %s/%s current revision %s does not equal update revision %s on update completion", + ss.Namespace, + ss.Name, + ss.Status.CurrentRevision, + updateRevision)) + for i := range pods.Items { + Expect(pods.Items[i].Spec.Containers[0].Image).To(Equal(newImage), + fmt.Sprintf(" Pod %s/%s has image %s not have new image %s", + pods.Items[i].Namespace, + pods.Items[i].Name, + pods.Items[i].Spec.Containers[0].Image, + newImage)) + Expect(pods.Items[i].Labels[apps.StatefulSetRevisionLabel]).To(Equal(updateRevision), + fmt.Sprintf("Pod %s/%s revision %s is not equal to update revision %s", + pods.Items[i].Namespace, + pods.Items[i].Name, + pods.Items[i].Labels[apps.StatefulSetRevisionLabel], + updateRevision)) + } + + By("Rolling back to a previous revision") + err = sst.BreakPodHttpProbe(ss, &pods.Items[1]) + Expect(err).NotTo(HaveOccurred()) + ss, pods = sst.WaitForPodNotReady(ss, pods.Items[1].Name) + priorRevision := currentRevision + currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision + ss, err = framework.UpdateStatefulSetWithRetries(c, ns, ss.Name, func(update *apps.StatefulSet) { + update.Spec.Template.Spec.Containers[0].Image = oldImage + }) + Expect(err).NotTo(HaveOccurred()) + ss = sst.WaitForStatus(ss) + currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision + Expect(currentRevision).NotTo(Equal(updateRevision), + "Current revision should not equal update revision during roll back") + Expect(priorRevision).To(Equal(updateRevision), + "Prior revision should equal update revision during roll back") + + By("Rolling back update in reverse ordinal order") + pods = sst.GetPodList(ss) + sst.SortStatefulPods(pods) + sst.RestorePodHttpProbe(ss, &pods.Items[1]) + ss, pods = sst.WaitForPodReady(ss, pods.Items[1].Name) + ss, pods = sst.WaitForRollingUpdate(ss) + Expect(ss.Status.CurrentRevision).To(Equal(priorRevision), + fmt.Sprintf("StatefulSet %s/%s current revision %s does not equal prior revision %s on rollback completion", + ss.Namespace, + ss.Name, + ss.Status.CurrentRevision, + updateRevision)) + + for i := range pods.Items { + Expect(pods.Items[i].Spec.Containers[0].Image).To(Equal(oldImage), + fmt.Sprintf("Pod %s/%s has image %s not equal to previous image %s", + pods.Items[i].Namespace, + pods.Items[i].Name, + pods.Items[i].Spec.Containers[0].Image, + oldImage)) + Expect(pods.Items[i].Labels[apps.StatefulSetRevisionLabel]).To(Equal(priorRevision), + fmt.Sprintf("Pod %s/%s revision %s is not equal to prior revision %s", + pods.Items[i].Namespace, + pods.Items[i].Name, + pods.Items[i].Labels[apps.StatefulSetRevisionLabel], + priorRevision)) + } +}