Merge pull request #31777 from dshulyak/evict_pet

Automatic merge from submit-queue

Delete evicted pet

If pet was evicted by kubelet - it will stuck in this state forever.
By analogy to regular pod we need to re-create pet so that it will
be re-scheduled to another node, so in order to re-create pet
and preserve consitent naming we will delete it in petset controller
and create after that.

fixes: https://github.com/kubernetes/kubernetes/issues/31098
This commit is contained in:
Kubernetes Submit Queue 2016-10-04 01:32:02 -07:00 committed by GitHub
commit f79a53a734
2 changed files with 127 additions and 2 deletions

View File

@ -105,7 +105,13 @@ func (p *petSyncer) Sync(pet *pcb) error {
if err := p.SyncPVCs(pet); err != nil {
return err
}
if exists {
// if pet failed - we need to remove old one because of consistent naming
if exists && realPet.pod.Status.Phase == api.PodFailed {
glog.V(4).Infof("Delete evicted pod %v", realPet.pod.Name)
if err := p.petClient.Delete(realPet); err != nil {
return err
}
} else if exists {
if !p.isHealthy(realPet.pod) {
glog.Infof("PetSet %v waiting on unhealthy pet %v", pet.parent.Name, realPet.pod.Name)
}

View File

@ -37,16 +37,20 @@ import (
"k8s.io/kubernetes/pkg/controller/petset"
"k8s.io/kubernetes/pkg/labels"
"k8s.io/kubernetes/pkg/runtime"
"k8s.io/kubernetes/pkg/types"
"k8s.io/kubernetes/pkg/util/sets"
"k8s.io/kubernetes/pkg/util/wait"
utilyaml "k8s.io/kubernetes/pkg/util/yaml"
"k8s.io/kubernetes/pkg/watch"
"k8s.io/kubernetes/test/e2e/framework"
)
const (
petsetPoll = 10 * time.Second
// Some pets install base packages via wget
petsetTimeout = 10 * time.Minute
petsetTimeout = 10 * time.Minute
// Timeout for pet pods to change state
petPodTimeout = 5 * time.Minute
zookeeperManifestPath = "test/e2e/testing-manifests/petset/zookeeper"
mysqlGaleraManifestPath = "test/e2e/testing-manifests/petset/mysql-galera"
redisManifestPath = "test/e2e/testing-manifests/petset/redis"
@ -245,6 +249,121 @@ var _ = framework.KubeDescribe("PetSet [Slow] [Feature:PetSet]", func() {
})
})
var _ = framework.KubeDescribe("Pet set recreate [Slow] [Feature:PetSet]", func() {
f := framework.NewDefaultFramework("pet-set-recreate")
var c *client.Client
var ns string
labels := map[string]string{
"foo": "bar",
"baz": "blah",
}
headlessSvcName := "test"
podName := "test-pod"
petSetName := "web"
petPodName := "web-0"
BeforeEach(func() {
framework.SkipUnlessProviderIs("gce", "vagrant")
By("creating service " + headlessSvcName + " in namespace " + f.Namespace.Name)
headlessService := createServiceSpec(headlessSvcName, "", true, labels)
_, err := f.Client.Services(f.Namespace.Name).Create(headlessService)
framework.ExpectNoError(err)
c = f.Client
ns = f.Namespace.Name
})
AfterEach(func() {
if CurrentGinkgoTestDescription().Failed {
dumpDebugInfo(c, ns)
}
By("Deleting all petset in ns " + ns)
deleteAllPetSets(c, ns)
})
It("should recreate evicted petset", func() {
By("looking for a node to schedule pet set and pod")
nodes := framework.GetReadySchedulableNodesOrDie(f.Client)
node := nodes.Items[0]
By("creating pod with conflicting port in namespace " + f.Namespace.Name)
conflictingPort := api.ContainerPort{HostPort: 21017, ContainerPort: 21017, Name: "conflict"}
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: podName,
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "nginx",
Image: "gcr.io/google_containers/nginx-slim:0.7",
Ports: []api.ContainerPort{conflictingPort},
},
},
NodeName: node.Name,
},
}
pod, err := f.Client.Pods(f.Namespace.Name).Create(pod)
framework.ExpectNoError(err)
By("creating petset with conflicting port in namespace " + f.Namespace.Name)
ps := newPetSet(petSetName, f.Namespace.Name, headlessSvcName, 1, nil, nil, labels)
petContainer := &ps.Spec.Template.Spec.Containers[0]
petContainer.Ports = append(petContainer.Ports, conflictingPort)
ps.Spec.Template.Spec.NodeName = node.Name
_, err = f.Client.Apps().PetSets(f.Namespace.Name).Create(ps)
framework.ExpectNoError(err)
By("waiting until pod " + podName + " will start running in namespace " + f.Namespace.Name)
if err := f.WaitForPodRunning(podName); err != nil {
framework.Failf("Pod %v did not start running: %v", podName, err)
}
var initialPetPodUID types.UID
By("waiting until pet pod " + petPodName + " will be recreated and deleted at least once in namespace " + f.Namespace.Name)
w, err := f.Client.Pods(f.Namespace.Name).Watch(api.SingleObject(api.ObjectMeta{Name: petPodName}))
framework.ExpectNoError(err)
// we need to get UID from pod in any state and wait until pet set controller will remove pod atleast once
_, err = watch.Until(petPodTimeout, w, func(event watch.Event) (bool, error) {
pod := event.Object.(*api.Pod)
switch event.Type {
case watch.Deleted:
framework.Logf("Observed delete event for pet pod %v in namespace %v", pod.Name, pod.Namespace)
if initialPetPodUID == "" {
return false, nil
}
return true, nil
}
framework.Logf("Observed pet pod in namespace: %v, name: %v, uid: %v, status phase: %v. Waiting for petset controller to delete.",
pod.Namespace, pod.Name, pod.UID, pod.Status.Phase)
initialPetPodUID = pod.UID
return false, nil
})
if err != nil {
framework.Failf("Pod %v expected to be re-created atleast once", petPodName)
}
By("removing pod with conflicting port in namespace " + f.Namespace.Name)
err = f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0))
framework.ExpectNoError(err)
By("waiting when pet pod " + petPodName + " will be recreated in namespace " + f.Namespace.Name + " and will be in running state")
// we may catch delete event, thats why we are waiting for running phase like this, and not with watch.Until
Eventually(func() error {
petPod, err := f.Client.Pods(f.Namespace.Name).Get(petPodName)
if err != nil {
return err
}
if petPod.Status.Phase != api.PodRunning {
return fmt.Errorf("Pod %v is not in running phase: %v", petPod.Name, petPod.Status.Phase)
} else if petPod.UID == initialPetPodUID {
return fmt.Errorf("Pod %v wasn't recreated: %v == %v", petPod.Name, petPod.UID, initialPetPodUID)
}
return nil
}, petPodTimeout, 2*time.Second).Should(BeNil())
})
})
func dumpDebugInfo(c *client.Client, ns string) {
pl, _ := c.Pods(ns).List(api.ListOptions{LabelSelector: labels.Everything()})
for _, p := range pl.Items {