From 45de9fbe34b18cceb2bc929d96b6b4cae25c647a Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Tue, 15 Nov 2016 14:42:36 -0800 Subject: [PATCH 1/2] Add e2e test for statefulset updates --- .../e2e-image-puller.manifest | 2 +- test/e2e/framework/util.go | 25 ++++++ test/e2e/kubectl.go | 1 + test/e2e/petset.go | 80 +++++++++++++++---- 4 files changed, 92 insertions(+), 16 deletions(-) diff --git a/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest b/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest index 277035bdee8..26777f29171 100644 --- a/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest +++ b/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest @@ -27,7 +27,7 @@ spec: command: - /bin/sh - -c - - "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.7 gcr.io/google_containers/mounttest-user:0.3 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.0 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.6 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;" + - "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.7 gcr.io/google_containers/mounttest-user:0.3 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.0 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.6 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;" securityContext: privileged: true volumeMounts: diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index b1e0363e703..7be188de72a 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -48,6 +48,7 @@ import ( "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apimachinery/registered" + "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/extensions" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5" @@ -3360,6 +3361,30 @@ func UpdateReplicationControllerWithRetries(c clientset.Interface, namespace, na return rc, pollErr } +type updateStatefulSetFunc func(*apps.StatefulSet) + +func UpdateStatefulSetWithRetries(c clientset.Interface, namespace, name string, applyUpdate updateStatefulSetFunc) (statefulSet *apps.StatefulSet, err error) { + statefulSets := c.Apps().StatefulSets(namespace) + var updateErr error + pollErr := wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) { + if statefulSet, err = statefulSets.Get(name); err != nil { + return false, err + } + // Apply the update, then attempt to push it to the apiserver. + applyUpdate(statefulSet) + if statefulSet, err = statefulSets.Update(statefulSet); err == nil { + Logf("Updating stateful set %s", name) + return true, nil + } + updateErr = err + return false, nil + }) + if pollErr == wait.ErrWaitTimeout { + pollErr = fmt.Errorf("couldn't apply the provided updated to stateful set %q: %v", name, updateErr) + } + return statefulSet, pollErr +} + // NodeAddresses returns the first address of the given type of each node. func NodeAddresses(nodelist *api.NodeList, addrType api.NodeAddressType) []string { hosts := []string{} diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index b4d275032b1..17e6d7516b8 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -87,6 +87,7 @@ const ( runJobTimeout = 5 * time.Minute busyboxImage = "gcr.io/google_containers/busybox:1.24" nginxImage = "gcr.io/google_containers/nginx-slim:0.7" + newNginxImage = "gcr.io/google_containers/nginx-slim:0.8" kubeCtlManifestPath = "test/e2e/testing-manifests/kubectl" redisControllerFilename = "redis-master-controller.json" redisServiceFilename = "redis-master-service.json" diff --git a/test/e2e/petset.go b/test/e2e/petset.go index 757d24eee6a..26ef0e2a476 100644 --- a/test/e2e/petset.go +++ b/test/e2e/petset.go @@ -82,9 +82,15 @@ var _ = framework.KubeDescribe("StatefulSet [Slow] [Feature:PetSet]", func() { "baz": "blah", } headlessSvcName := "test" + var petMounts, podMounts []api.VolumeMount + var ps *apps.StatefulSet BeforeEach(func() { - By("creating service " + headlessSvcName + " in namespace " + ns) + petMounts = []api.VolumeMount{{Name: "datadir", MountPath: "/data/"}} + podMounts = []api.VolumeMount{{Name: "home", MountPath: "/home"}} + ps = newStatefulSet(psName, ns, headlessSvcName, 2, petMounts, podMounts, labels) + + By("Creating service " + headlessSvcName + " in namespace " + ns) headlessService := createServiceSpec(headlessSvcName, "", true, labels) _, err := c.Core().Services(ns).Create(headlessService) Expect(err).NotTo(HaveOccurred()) @@ -99,10 +105,8 @@ var _ = framework.KubeDescribe("StatefulSet [Slow] [Feature:PetSet]", func() { }) It("should provide basic identity [Feature:StatefulSet]", func() { - By("creating statefulset " + psName + " in namespace " + ns) - petMounts := []api.VolumeMount{{Name: "datadir", MountPath: "/data/"}} - podMounts := []api.VolumeMount{{Name: "home", MountPath: "/home"}} - ps := newStatefulSet(psName, ns, headlessSvcName, 3, petMounts, podMounts, labels) + By("Creating statefulset " + psName + " in namespace " + ns) + ps.Spec.Replicas = 3 setInitializedAnnotation(ps, "false") _, err := c.Apps().StatefulSets(ns).Create(ps) @@ -136,12 +140,10 @@ var _ = framework.KubeDescribe("StatefulSet [Slow] [Feature:PetSet]", func() { }) It("should handle healthy pet restarts during scale [Feature:PetSet]", func() { - By("creating statefulset " + psName + " in namespace " + ns) - - petMounts := []api.VolumeMount{{Name: "datadir", MountPath: "/data/"}} - podMounts := []api.VolumeMount{{Name: "home", MountPath: "/home"}} - ps := newStatefulSet(psName, ns, headlessSvcName, 2, petMounts, podMounts, labels) + By("Creating statefulset " + psName + " in namespace " + ns) + ps.Spec.Replicas = 2 setInitializedAnnotation(ps, "false") + _, err := c.Apps().StatefulSets(ns).Create(ps) Expect(err).NotTo(HaveOccurred()) @@ -171,6 +173,41 @@ var _ = framework.KubeDescribe("StatefulSet [Slow] [Feature:PetSet]", func() { By("Confirming all pets in statefulset are created.") pst.saturate(ps) }) + + It("should allow template updates", func() { + By("Creating stateful set " + psName + " in namespace " + ns) + ps.Spec.Replicas = 2 + + ps, err := c.Apps().StatefulSets(ns).Create(ps) + Expect(err).NotTo(HaveOccurred()) + + pst := statefulSetTester{c: c} + + pst.waitForRunning(ps.Spec.Replicas, ps) + + newImage := newNginxImage + oldImage := ps.Spec.Template.Spec.Containers[0].Image + By(fmt.Sprintf("Updating stateful set template: update image from %s to %s", oldImage, newImage)) + Expect(oldImage).NotTo(Equal(newImage), "Incorrect test setup: should update to a different image") + _, err = framework.UpdateStatefulSetWithRetries(c, ns, ps.Name, func(update *apps.StatefulSet) { + update.Spec.Template.Spec.Containers[0].Image = newImage + }) + Expect(err).NotTo(HaveOccurred()) + + updateIndex := 0 + By(fmt.Sprintf("Deleting stateful pod at index %d", updateIndex)) + pst.deletePetAtIndex(updateIndex, ps) + + By("Waiting for all stateful pods to be running again") + pst.waitForRunning(ps.Spec.Replicas, ps) + + By(fmt.Sprintf("Verifying stateful pod at index %d is updated", updateIndex)) + verify := func(pod *api.Pod) { + podImage := pod.Spec.Containers[0].Image + Expect(podImage).To(Equal(newImage), fmt.Sprintf("Expected stateful pod image %s updated to %s", podImage, newImage)) + } + pst.verifyPodAtIndex(updateIndex, ps, verify) + }) }) framework.KubeDescribe("Deploy clustered applications [Slow] [Feature:PetSet]", func() { @@ -593,15 +630,28 @@ func (p *statefulSetTester) saturate(ps *apps.StatefulSet) { } func (p *statefulSetTester) deletePetAtIndex(index int, ps *apps.StatefulSet) { - // TODO: we won't use "-index" as the name strategy forever, - // pull the name out from an identity mapper. - name := fmt.Sprintf("%v-%v", ps.Name, index) + name := getPodNameAtIndex(index, ps) noGrace := int64(0) if err := p.c.Core().Pods(ps.Namespace).Delete(name, &api.DeleteOptions{GracePeriodSeconds: &noGrace}); err != nil { - framework.Failf("Failed to delete pet %v for StatefulSet %v: %v", name, ps.Name, ps.Namespace, err) + framework.Failf("Failed to delete pet %v for StatefulSet %v/%v: %v", name, ps.Namespace, ps.Name, err) } } +type verifyPodFunc func(*api.Pod) + +func (p *statefulSetTester) verifyPodAtIndex(index int, ps *apps.StatefulSet, verify verifyPodFunc) { + name := getPodNameAtIndex(index, ps) + pod, err := p.c.Core().Pods(ps.Namespace).Get(name) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to get stateful pod %s for StatefulSet %s/%s", name, ps.Namespace, ps.Name)) + verify(pod) +} + +func getPodNameAtIndex(index int, ps *apps.StatefulSet) string { + // TODO: we won't use "-index" as the name strategy forever, + // pull the name out from an identity mapper. + return fmt.Sprintf("%v-%v", ps.Name, index) +} + func (p *statefulSetTester) scale(ps *apps.StatefulSet, count int32) error { name := ps.Name ns := ps.Namespace @@ -913,7 +963,7 @@ func newStatefulSet(name, ns, governingSvcName string, replicas int32, petMounts Containers: []api.Container{ { Name: "nginx", - Image: "gcr.io/google_containers/nginx-slim:0.7", + Image: nginxImage, VolumeMounts: mounts, }, }, From b47508700cad0ee4aa600dfa9e827d4c6a855ee7 Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Tue, 15 Nov 2016 16:14:02 -0800 Subject: [PATCH 2/2] (Auto-gen) Update bazel --- test/e2e/framework/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 1052de6bb66..d3204b1896c 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -38,6 +38,7 @@ go_library( "//pkg/api/v1:go_default_library", "//pkg/api/validation:go_default_library", "//pkg/apimachinery/registered:go_default_library", + "//pkg/apis/apps:go_default_library", "//pkg/apis/extensions:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library",