From c0981963d9433e75b37803c1be1b74bccdd7900d Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 2 Sep 2016 18:23:51 +0300 Subject: [PATCH] Verify evicted pods managed by petset controller will be recreated Spawn pet set with single replica and simple pod. They will have conflicting hostPort definitions, and spawned on the same node. As the result pet set pod, it will be created after simple pod, will be in Failed state. Pet set controller will try to re-create it. After verifying that pet set pod failed and was recreated atleast once, we will remove pod with conflicting hostPort and wait until pet set pod will be in running state. Change-Id: I5903f5881f8606c696bd390df58b06ece33be88a --- pkg/controller/petset/pet.go | 8 +-- test/e2e/petset.go | 121 ++++++++++++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 7 deletions(-) diff --git a/pkg/controller/petset/pet.go b/pkg/controller/petset/pet.go index 0ca7e69325b..0127d1a5802 100644 --- a/pkg/controller/petset/pet.go +++ b/pkg/controller/petset/pet.go @@ -96,8 +96,8 @@ func (p *petSyncer) Sync(pet *pcb) error { if err := p.SyncPVCs(pet); err != nil { return err } - // if pet was evicted - we need to remove old one because of consistent naming - if exists && isEvicted(realPet.pod) { + // if pet failed - we need to remove old one because of consistent naming + if exists && realPet.pod.Status.Phase == api.PodFailed { glog.V(4).Infof("Delete evicted pod %v", realPet.pod.Name) if err := p.petClient.Delete(realPet); err != nil { return err @@ -317,7 +317,3 @@ func (d *defaultPetHealthChecker) isHealthy(pod *api.Pod) bool { func (d *defaultPetHealthChecker) isDying(pod *api.Pod) bool { return pod != nil && pod.DeletionTimestamp != nil } - -func isEvicted(pod *api.Pod) bool { - return pod != nil && pod.Status.Phase == api.PodFailed && pod.Status.Reason == "Evicted" -} diff --git a/test/e2e/petset.go b/test/e2e/petset.go index 87732359f80..17cfaa46fb1 100644 --- a/test/e2e/petset.go +++ b/test/e2e/petset.go @@ -37,16 +37,20 @@ import ( "k8s.io/kubernetes/pkg/controller/petset" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" utilyaml "k8s.io/kubernetes/pkg/util/yaml" + "k8s.io/kubernetes/pkg/watch" "k8s.io/kubernetes/test/e2e/framework" ) const ( petsetPoll = 10 * time.Second // Some pets install base packages via wget - petsetTimeout = 10 * time.Minute + petsetTimeout = 10 * time.Minute + // Timeout for pet pods to change state + petPodTimeout = 5 * time.Minute zookeeperManifestPath = "test/e2e/testing-manifests/petset/zookeeper" mysqlGaleraManifestPath = "test/e2e/testing-manifests/petset/mysql-galera" redisManifestPath = "test/e2e/testing-manifests/petset/redis" @@ -243,6 +247,121 @@ var _ = framework.KubeDescribe("PetSet [Slow] [Feature:PetSet]", func() { }) }) +var _ = framework.KubeDescribe("Pet set recreate [Slow] [Feature:PetSet]", func() { + f := framework.NewDefaultFramework("pet-set-recreate") + var c *client.Client + var ns string + + labels := map[string]string{ + "foo": "bar", + "baz": "blah", + } + headlessSvcName := "test" + podName := "test-pod" + petSetName := "web" + petPodName := "web-0" + + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce", "vagrant") + By("creating service " + headlessSvcName + " in namespace " + f.Namespace.Name) + headlessService := createServiceSpec(headlessSvcName, "", true, labels) + _, err := f.Client.Services(f.Namespace.Name).Create(headlessService) + framework.ExpectNoError(err) + c = f.Client + ns = f.Namespace.Name + }) + + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + dumpDebugInfo(c, ns) + } + By("Deleting all petset in ns " + ns) + deleteAllPetSets(c, ns) + }) + + It("should recreate evicted petset", func() { + By("looking for a node to schedule pet set and pod") + nodes := framework.GetReadySchedulableNodesOrDie(f.Client) + node := nodes.Items[0] + + By("creating pod with conflicting port in namespace " + f.Namespace.Name) + conflictingPort := api.ContainerPort{HostPort: 21017, ContainerPort: 21017, Name: "conflict"} + pod := &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: podName, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "nginx", + Image: "gcr.io/google_containers/nginx-slim:0.7", + Ports: []api.ContainerPort{conflictingPort}, + }, + }, + NodeName: node.Name, + }, + } + pod, err := f.Client.Pods(f.Namespace.Name).Create(pod) + framework.ExpectNoError(err) + + By("creating petset with conflicting port in namespace " + f.Namespace.Name) + ps := newPetSet(petSetName, f.Namespace.Name, headlessSvcName, 1, nil, nil, labels) + petContainer := &ps.Spec.Template.Spec.Containers[0] + petContainer.Ports = append(petContainer.Ports, conflictingPort) + ps.Spec.Template.Spec.NodeName = node.Name + _, err = f.Client.Apps().PetSets(f.Namespace.Name).Create(ps) + framework.ExpectNoError(err) + + By("waiting until pod " + podName + " will start running in namespace " + f.Namespace.Name) + if err := f.WaitForPodRunning(podName); err != nil { + framework.Failf("Pod %v did not start running: %v", podName, err) + } + + var initialPetPodUID types.UID + By("waiting until pet pod " + petPodName + " will be recreated and deleted at least once in namespace " + f.Namespace.Name) + w, err := f.Client.Pods(f.Namespace.Name).Watch(api.SingleObject(api.ObjectMeta{Name: petPodName})) + framework.ExpectNoError(err) + // we need to get UID from pod in any state and wait until pet set controller will remove pod atleast once + _, err = watch.Until(petPodTimeout, w, func(event watch.Event) (bool, error) { + pod := event.Object.(*api.Pod) + switch event.Type { + case watch.Deleted: + framework.Logf("Observed delete event for pet pod %v in namespace %v", pod.Name, pod.Namespace) + if initialPetPodUID == "" { + return false, nil + } + return true, nil + } + framework.Logf("Observed pet pod in namespace: %v, name: %v, uid: %v, status phase: %v. Waiting for petset controller to delete.", + pod.Namespace, pod.Name, pod.UID, pod.Status.Phase) + initialPetPodUID = pod.UID + return false, nil + }) + if err != nil { + framework.Failf("Pod %v expected to be re-created atleast once", petPodName) + } + + By("removing pod with conflicting port in namespace " + f.Namespace.Name) + err = f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0)) + framework.ExpectNoError(err) + + By("waiting when pet pod " + petPodName + " will be recreated in namespace " + f.Namespace.Name + " and will be in running state") + // we may catch delete event, thats why we are waiting for running phase like this, and not with watch.Until + Eventually(func() error { + petPod, err := f.Client.Pods(f.Namespace.Name).Get(petPodName) + if err != nil { + return err + } + if petPod.Status.Phase != api.PodRunning { + return fmt.Errorf("Pod %v is not in running phase: %v", petPod.Name, petPod.Status.Phase) + } else if petPod.UID == initialPetPodUID { + return fmt.Errorf("Pod %v wasn't recreated: %v == %v", petPod.Name, petPod.UID, initialPetPodUID) + } + return nil + }, petPodTimeout, 2*time.Second).Should(BeNil()) + }) +}) + func dumpDebugInfo(c *client.Client, ns string) { pl, _ := c.Pods(ns).List(api.ListOptions{LabelSelector: labels.Everything()}) for _, p := range pl.Items {