Merge pull request #31777 from dshulyak/evict_pet

Automatic merge from submit-queue

Delete evicted pet

If pet was evicted by kubelet - it will stuck in this state forever.
By analogy to regular pod we need to re-create pet so that it will
be re-scheduled to another node, so in order to re-create pet
and preserve consitent naming we will delete it in petset controller
and create after that.

fixes: https://github.com/kubernetes/kubernetes/issues/31098

pkg/controller/petset/pet.go

@@ -105,7 +105,13 @@ func (p *petSyncer) Sync(pet *pcb) error {
 	if err := p.SyncPVCs(pet); err != nil {
 		return err
 	}
-	if exists {
+	// if pet failed - we need to remove old one because of consistent naming
+	if exists && realPet.pod.Status.Phase == api.PodFailed {
+		glog.V(4).Infof("Delete evicted pod %v", realPet.pod.Name)
+		if err := p.petClient.Delete(realPet); err != nil {
+			return err
+		}
+	} else if exists {
 		if !p.isHealthy(realPet.pod) {
 			glog.Infof("PetSet %v waiting on unhealthy pet %v", pet.parent.Name, realPet.pod.Name)
 		}

test/e2e/petset.go

@@ -37,9 +37,11 @@ import (
 	"k8s.io/kubernetes/pkg/controller/petset"
 	"k8s.io/kubernetes/pkg/labels"
 	"k8s.io/kubernetes/pkg/runtime"
+	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/util/wait"
 	utilyaml "k8s.io/kubernetes/pkg/util/yaml"
+	"k8s.io/kubernetes/pkg/watch"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -47,6 +49,8 @@ const (
 	petsetPoll = 10 * time.Second
 	// Some pets install base packages via wget
 	petsetTimeout = 10 * time.Minute
+	// Timeout for pet pods to change state
+	petPodTimeout           = 5 * time.Minute
 	zookeeperManifestPath   = "test/e2e/testing-manifests/petset/zookeeper"
 	mysqlGaleraManifestPath = "test/e2e/testing-manifests/petset/mysql-galera"
 	redisManifestPath       = "test/e2e/testing-manifests/petset/redis"
@@ -245,6 +249,121 @@ var _ = framework.KubeDescribe("PetSet [Slow] [Feature:PetSet]", func() {
 	})
 })
 
+var _ = framework.KubeDescribe("Pet set recreate [Slow] [Feature:PetSet]", func() {
+	f := framework.NewDefaultFramework("pet-set-recreate")
+	var c *client.Client
+	var ns string
+
+	labels := map[string]string{
+		"foo": "bar",
+		"baz": "blah",
+	}
+	headlessSvcName := "test"
+	podName := "test-pod"
+	petSetName := "web"
+	petPodName := "web-0"
+
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce", "vagrant")
+		By("creating service " + headlessSvcName + " in namespace " + f.Namespace.Name)
+		headlessService := createServiceSpec(headlessSvcName, "", true, labels)
+		_, err := f.Client.Services(f.Namespace.Name).Create(headlessService)
+		framework.ExpectNoError(err)
+		c = f.Client
+		ns = f.Namespace.Name
+	})
+
+	AfterEach(func() {
+		if CurrentGinkgoTestDescription().Failed {
+			dumpDebugInfo(c, ns)
+		}
+		By("Deleting all petset in ns " + ns)
+		deleteAllPetSets(c, ns)
+	})
+
+	It("should recreate evicted petset", func() {
+		By("looking for a node to schedule pet set and pod")
+		nodes := framework.GetReadySchedulableNodesOrDie(f.Client)
+		node := nodes.Items[0]
+
+		By("creating pod with conflicting port in namespace " + f.Namespace.Name)
+		conflictingPort := api.ContainerPort{HostPort: 21017, ContainerPort: 21017, Name: "conflict"}
+		pod := &api.Pod{
+			ObjectMeta: api.ObjectMeta{
+				Name: podName,
+			},
+			Spec: api.PodSpec{
+				Containers: []api.Container{
+					{
+						Name:  "nginx",
+						Image: "gcr.io/google_containers/nginx-slim:0.7",
+						Ports: []api.ContainerPort{conflictingPort},
+					},
+				},
+				NodeName: node.Name,
+			},
+		}
+		pod, err := f.Client.Pods(f.Namespace.Name).Create(pod)
+		framework.ExpectNoError(err)
+
+		By("creating petset with conflicting port in namespace " + f.Namespace.Name)
+		ps := newPetSet(petSetName, f.Namespace.Name, headlessSvcName, 1, nil, nil, labels)
+		petContainer := &ps.Spec.Template.Spec.Containers[0]
+		petContainer.Ports = append(petContainer.Ports, conflictingPort)
+		ps.Spec.Template.Spec.NodeName = node.Name
+		_, err = f.Client.Apps().PetSets(f.Namespace.Name).Create(ps)
+		framework.ExpectNoError(err)
+
+		By("waiting until pod " + podName + " will start running in namespace " + f.Namespace.Name)
+		if err := f.WaitForPodRunning(podName); err != nil {
+			framework.Failf("Pod %v did not start running: %v", podName, err)
+		}
+
+		var initialPetPodUID types.UID
+		By("waiting until pet pod " + petPodName + " will be recreated and deleted at least once in namespace " + f.Namespace.Name)
+		w, err := f.Client.Pods(f.Namespace.Name).Watch(api.SingleObject(api.ObjectMeta{Name: petPodName}))
+		framework.ExpectNoError(err)
+		// we need to get UID from pod in any state and wait until pet set controller will remove pod atleast once
+		_, err = watch.Until(petPodTimeout, w, func(event watch.Event) (bool, error) {
+			pod := event.Object.(*api.Pod)
+			switch event.Type {
+			case watch.Deleted:
+				framework.Logf("Observed delete event for pet pod %v in namespace %v", pod.Name, pod.Namespace)
+				if initialPetPodUID == "" {
+					return false, nil
+				}
+				return true, nil
+			}
+			framework.Logf("Observed pet pod in namespace: %v, name: %v, uid: %v, status phase: %v. Waiting for petset controller to delete.",
+				pod.Namespace, pod.Name, pod.UID, pod.Status.Phase)
+			initialPetPodUID = pod.UID
+			return false, nil
+		})
+		if err != nil {
+			framework.Failf("Pod %v expected to be re-created atleast once", petPodName)
+		}
+
+		By("removing pod with conflicting port in namespace " + f.Namespace.Name)
+		err = f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0))
+		framework.ExpectNoError(err)
+
+		By("waiting when pet pod " + petPodName + " will be recreated in namespace " + f.Namespace.Name + " and will be in running state")
+		// we may catch delete event, thats why we are waiting for running phase like this, and not with watch.Until
+		Eventually(func() error {
+			petPod, err := f.Client.Pods(f.Namespace.Name).Get(petPodName)
+			if err != nil {
+				return err
+			}
+			if petPod.Status.Phase != api.PodRunning {
+				return fmt.Errorf("Pod %v is not in running phase: %v", petPod.Name, petPod.Status.Phase)
+			} else if petPod.UID == initialPetPodUID {
+				return fmt.Errorf("Pod %v wasn't recreated: %v == %v", petPod.Name, petPod.UID, initialPetPodUID)
+			}
+			return nil
+		}, petPodTimeout, 2*time.Second).Should(BeNil())
+	})
+})
+
 func dumpDebugInfo(c *client.Client, ns string) {
 	pl, _ := c.Pods(ns).List(api.ListOptions{LabelSelector: labels.Everything()})
 	for _, p := range pl.Items {