Merge pull request #33389 from Random-Liu/lifecycle-hook

Automatic merge from submit-queue

CRI: Fix lifecycle hook and add container lifecycle node e2e test

This PR:
1) Adds pod spec missing handling in kuberuntime. (1st commit)
2) Adds container lifecycle hook node e2e test. (2nd commit)

@yujuhong @feiskyer
This commit is contained in:
Kubernetes Submit Queue 2016-09-26 10:48:35 -07:00 committed by GitHub
commit 19a2a10354
6 changed files with 271 additions and 24 deletions

View File

@ -104,7 +104,7 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb
if handlerErr != nil {
err := fmt.Errorf("PostStart handler: %v", handlerErr)
m.generateContainerEvent(kubeContainerID, api.EventTypeWarning, events.FailedPostStartHook, msg)
m.killContainer(pod, kubeContainerID, container, "FailedPostStartHook", nil)
m.killContainer(pod, kubeContainerID, container.Name, "FailedPostStartHook", nil)
return "PostStart Hook Failed", err
}
}
@ -429,24 +429,83 @@ func (m *kubeGenericRuntimeManager) executePreStopHook(pod *api.Pod, containerID
return int64(unversioned.Now().Sub(start.Time).Seconds())
}
// restoreSpecsFromContainerLabels restores all information needed for killing a container. In some
// case we may not have pod and container spec when killing a container, e.g. pod is deleted during
// kubelet restart.
// To solve this problem, we've already written necessary information into container labels. Here we
// just need to retrieve them from container labels and restore the specs.
// TODO(random-liu): Add a node e2e test to test this behaviour.
// TODO(random-liu): Change the lifecycle handler to just accept information needed, so that we can
// just pass the needed function not create the fake object.
func (m *kubeGenericRuntimeManager) restoreSpecsFromContainerLabels(containerID kubecontainer.ContainerID) (*api.Pod, *api.Container, error) {
var pod *api.Pod
var container *api.Container
s, err := m.runtimeService.ContainerStatus(containerID.ID)
if err != nil {
return nil, nil, err
}
l := getContainerInfoFromLabels(s.Labels)
a := getContainerInfoFromAnnotations(s.Annotations)
// Notice that the followings are not full spec. The container killing code should not use
// un-restored fields.
pod = &api.Pod{
ObjectMeta: api.ObjectMeta{
UID: l.PodUID,
Name: l.PodName,
Namespace: l.PodNamespace,
DeletionGracePeriodSeconds: a.PodDeletionGracePeriod,
},
Spec: api.PodSpec{
TerminationGracePeriodSeconds: a.PodTerminationGracePeriod,
},
}
container = &api.Container{
Name: l.ContainerName,
Ports: a.ContainerPorts,
TerminationMessagePath: a.TerminationMessagePath,
}
if a.PreStopHandler != nil {
container.Lifecycle = &api.Lifecycle{
PreStop: a.PreStopHandler,
}
}
return pod, container, nil
}
// killContainer kills a container through the following steps:
// * Run the pre-stop lifecycle hooks (if applicable).
// * Stop the container.
func (m *kubeGenericRuntimeManager) killContainer(pod *api.Pod, containerID kubecontainer.ContainerID, containerSpec *api.Container, reason string, gracePeriodOverride *int64) error {
gracePeriod := int64(minimumGracePeriodInSeconds)
func (m *kubeGenericRuntimeManager) killContainer(pod *api.Pod, containerID kubecontainer.ContainerID, containerName string, reason string, gracePeriodOverride *int64) error {
var containerSpec *api.Container
if pod != nil {
switch {
case pod.DeletionGracePeriodSeconds != nil:
gracePeriod = *pod.DeletionGracePeriodSeconds
case pod.Spec.TerminationGracePeriodSeconds != nil:
gracePeriod = *pod.Spec.TerminationGracePeriodSeconds
for i, c := range pod.Spec.Containers {
if containerName == c.Name {
containerSpec = &pod.Spec.Containers[i]
break
}
}
} else {
// Restore necessary information if one of the specs is nil.
restoredPod, restoredContainer, err := m.restoreSpecsFromContainerLabels(containerID)
if err != nil {
return err
}
pod, containerSpec = restoredPod, restoredContainer
}
// From this point , pod and container must be non-nil.
gracePeriod := int64(minimumGracePeriodInSeconds)
switch {
case pod.DeletionGracePeriodSeconds != nil:
gracePeriod = *pod.DeletionGracePeriodSeconds
case pod.Spec.TerminationGracePeriodSeconds != nil:
gracePeriod = *pod.Spec.TerminationGracePeriodSeconds
}
glog.V(2).Infof("Killing container %q with %d second grace period", containerID.String(), gracePeriod)
// Run the pre-stop lifecycle hooks if applicable.
if pod != nil && containerSpec != nil && containerSpec.Lifecycle != nil && containerSpec.Lifecycle.PreStop != nil {
if containerSpec.Lifecycle != nil && containerSpec.Lifecycle.PreStop != nil {
gracePeriod = gracePeriod - m.executePreStopHook(pod, containerID, containerSpec, gracePeriod)
}
// always give containers a minimal shutdown window to avoid unnecessary SIGKILLs
@ -485,19 +544,8 @@ func (m *kubeGenericRuntimeManager) killContainersWithSyncResult(pod *api.Pod, r
go func(container *kubecontainer.Container) {
defer utilruntime.HandleCrash()
defer wg.Done()
var containerSpec *api.Container
if pod != nil {
for i, c := range pod.Spec.Containers {
if container.Name == c.Name {
containerSpec = &pod.Spec.Containers[i]
break
}
}
}
killContainerResult := kubecontainer.NewSyncResult(kubecontainer.KillContainer, container.Name)
if err := m.killContainer(pod, container.ID, containerSpec, "Need to kill Pod", gracePeriodOverride); err != nil {
if err := m.killContainer(pod, container.ID, container.Name, "Need to kill Pod", gracePeriodOverride); err != nil {
killContainerResult.Fail(kubecontainer.ErrKillContainer, err.Error())
}
containerResults <- killContainerResult

View File

@ -499,7 +499,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *api.Pod, _ api.PodStatus, podSt
glog.V(3).Infof("Killing unwanted container %q(id=%q) for pod %q", containerInfo.name, containerID, format.Pod(pod))
killContainerResult := kubecontainer.NewSyncResult(kubecontainer.KillContainer, containerInfo.name)
result.AddSyncResult(killContainerResult)
if err := m.killContainer(pod, containerID, containerInfo.container, containerInfo.message, nil); err != nil {
if err := m.killContainer(pod, containerID, containerInfo.name, containerInfo.message, nil); err != nil {
killContainerResult.Fail(kubecontainer.ErrKillContainer, err.Error())
glog.Errorf("killContainer %q(id=%q) for pod %q failed: %v", containerInfo.name, containerID, format.Pod(pod), err)
return

View File

@ -63,14 +63,14 @@ func (hr *HandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod
output := ioutils.WriteCloserWrapper(&buffer)
err := hr.commandRunner.ExecInContainer(containerID, handler.Exec.Command, nil, output, output, false, nil)
if err != nil {
msg := fmt.Sprintf("Exec lifecycle hook (%v) for Container %q in Pod %q failed - %q", handler.Exec.Command, container.Name, format.Pod(pod), buffer.String())
msg := fmt.Sprintf("Exec lifecycle hook (%v) for Container %q in Pod %q failed - error: %v, message: %q", handler.Exec.Command, container.Name, format.Pod(pod), err, buffer.String())
glog.V(1).Infof(msg)
}
return msg, err
case handler.HTTPGet != nil:
msg, err := hr.runHTTPHandler(pod, container, handler)
if err != nil {
msg := fmt.Sprintf("Http lifecycle hook (%s) for Container %q in Pod %q failed - %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), msg)
msg := fmt.Sprintf("Http lifecycle hook (%s) for Container %q in Pod %q failed - error: %v, message: %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), err, msg)
glog.V(1).Infof(msg)
}
return msg, err

View File

@ -149,3 +149,19 @@ func (c *PodClient) mungeSpec(pod *api.Pod) {
}
// TODO(random-liu): Move pod wait function into this file
// WaitForSuccess waits for pod to success.
func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
f := c.f
Expect(waitForPodCondition(f.Client, f.Namespace.Name, name, "success or failure", timeout,
func(pod *api.Pod) (bool, error) {
switch pod.Status.Phase {
case api.PodFailed:
return true, fmt.Errorf("pod %q failed with reason: %q, message: %q", name, pod.Status.Reason, pod.Status.Message)
case api.PodSucceeded:
return true, nil
default:
return false, nil
}
},
)).To(Succeed(), "wait for pod %q to success", name)
}

View File

@ -1455,6 +1455,7 @@ func waitForPodTerminatedInNamespace(c *client.Client, podName, reason, namespac
func waitForPodSuccessInNamespaceTimeout(c *client.Client, podName string, contName string, namespace string, timeout time.Duration) error {
return waitForPodCondition(c, namespace, podName, "success or failure", timeout, func(pod *api.Pod) (bool, error) {
// Cannot use pod.Status.Phase == api.PodSucceeded/api.PodFailed due to #2632
// TODO: This was not true from long time ago. We can use api.PodSucceeded now.
ci, ok := api.GetContainerStatus(pod.Status.ContainerStatuses, contName)
if !ok {
Logf("No Status.Info for container '%s' in pod '%s' yet", contName, podName)

View File

@ -0,0 +1,182 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
"fmt"
"time"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/util/intstr"
"k8s.io/kubernetes/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
)
var _ = framework.KubeDescribe("Container Lifecycle Hook", func() {
f := framework.NewDefaultFramework("container-lifecycle-hook")
var podClient *framework.PodClient
var file string
const podWaitTimeout = 2 * time.Minute
testPodWithHook := func(podWithHook *api.Pod) {
podCheckHook := getLifecycleHookTestPod("pod-check-hook",
// Wait until the file is created.
[]string{"sh", "-c", fmt.Sprintf("while [ ! -e %s ]; do sleep 1; done", file)},
)
By("create the pod with lifecycle hook")
podClient.CreateSync(podWithHook)
if podWithHook.Spec.Containers[0].Lifecycle.PostStart != nil {
By("create the hook check pod")
podClient.Create(podCheckHook)
By("wait for the hook check pod to success")
podClient.WaitForSuccess(podCheckHook.Name, podWaitTimeout)
}
By("delete the pod with lifecycle hook")
podClient.DeleteSync(podWithHook.Name, api.NewDeleteOptions(15), podWaitTimeout)
if podWithHook.Spec.Containers[0].Lifecycle.PreStop != nil {
By("create the hook check pod")
podClient.Create(podCheckHook)
By("wait for the prestop check pod to success")
podClient.WaitForSuccess(podCheckHook.Name, podWaitTimeout)
}
}
Context("when create a pod with lifecycle hook", func() {
BeforeEach(func() {
podClient = f.PodClient()
file = "/tmp/test-" + string(uuid.NewUUID())
})
AfterEach(func() {
By("cleanup the temporary file created in the test.")
cleanupPod := getLifecycleHookTestPod("pod-clean-up", []string{"rm", file})
podClient.Create(cleanupPod)
podClient.WaitForSuccess(cleanupPod.Name, podWaitTimeout)
})
Context("when it is exec hook", func() {
It("should execute poststart exec hook properly", func() {
podWithHook := getLifecycleHookTestPod("pod-with-poststart-exec-hook",
// Block forever
[]string{"tail", "-f", "/dev/null"},
)
podWithHook.Spec.Containers[0].Lifecycle = &api.Lifecycle{
PostStart: &api.Handler{
Exec: &api.ExecAction{Command: []string{"touch", file}},
},
}
testPodWithHook(podWithHook)
})
It("should execute prestop exec hook properly", func() {
podWithHook := getLifecycleHookTestPod("pod-with-prestop-exec-hook",
// Block forever
[]string{"tail", "-f", "/dev/null"},
)
podWithHook.Spec.Containers[0].Lifecycle = &api.Lifecycle{
PreStop: &api.Handler{
Exec: &api.ExecAction{Command: []string{"touch", file}},
},
}
testPodWithHook(podWithHook)
})
})
Context("when it is http hook", func() {
var targetIP string
BeforeEach(func() {
By("cleanup the container to handle the HTTPGet hook request.")
podHandleHookRequest := getLifecycleHookTestPod("pod-handle-http-request",
[]string{"sh", "-c",
// Create test file when receive request on 1234.
fmt.Sprintf("echo -e \"HTTP/1.1 200 OK\n\" | nc -l -p 1234; touch %s", file),
},
)
podHandleHookRequest.Spec.Containers[0].Ports = []api.ContainerPort{
{
ContainerPort: 1234,
Protocol: api.ProtocolTCP,
},
}
podHandleHookRequest = podClient.CreateSync(podHandleHookRequest)
targetIP = podHandleHookRequest.Status.PodIP
})
It("should execute poststart http hook properly", func() {
podWithHook := getLifecycleHookTestPod("pod-with-poststart-http-hook",
// Block forever
[]string{"tail", "-f", "/dev/null"},
)
podWithHook.Spec.Containers[0].Lifecycle = &api.Lifecycle{
PostStart: &api.Handler{
HTTPGet: &api.HTTPGetAction{
Host: targetIP,
Port: intstr.FromInt(1234),
},
},
}
testPodWithHook(podWithHook)
})
It("should execute prestop http hook properly", func() {
podWithHook := getLifecycleHookTestPod("pod-with-prestop-http-hook",
// Block forever
[]string{"tail", "-f", "/dev/null"},
)
podWithHook.Spec.Containers[0].Lifecycle = &api.Lifecycle{
PreStop: &api.Handler{
HTTPGet: &api.HTTPGetAction{
Host: targetIP,
Port: intstr.FromInt(1234),
},
},
}
testPodWithHook(podWithHook)
})
})
})
})
func getLifecycleHookTestPod(name string, cmd []string) *api.Pod {
return &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: name,
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: name,
Image: "gcr.io/google_containers/busybox:1.24",
VolumeMounts: []api.VolumeMount{
{
Name: "tmpfs",
MountPath: "/tmp",
},
},
Command: cmd,
},
},
RestartPolicy: api.RestartPolicyNever,
Volumes: []api.Volume{
{
Name: "tmpfs",
VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/tmp"}},
},
},
},
}
}