diff --git a/pkg/kubelet/container/event.go b/pkg/kubelet/container/event.go index 6694239e9a9..949dfcccc0e 100644 --- a/pkg/kubelet/container/event.go +++ b/pkg/kubelet/container/event.go @@ -62,4 +62,8 @@ const ( // Config event reason list FailedValidation = "FailedValidation" + + // Lifecycle hooks + FailedPostStartHook = "FailedPostStartHook" + FailedPreStopHook = "FailedPreStopHook" ) diff --git a/pkg/kubelet/container/helpers.go b/pkg/kubelet/container/helpers.go index 581713b19a1..ed0311df2a8 100644 --- a/pkg/kubelet/container/helpers.go +++ b/pkg/kubelet/container/helpers.go @@ -34,7 +34,7 @@ import ( // HandlerRunner runs a lifecycle handler for a container. type HandlerRunner interface { - Run(containerID ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) error + Run(containerID ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) (string, error) } // RuntimeHelper wraps kubelet to make container runtime diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index fea46a5a52b..0a6c9a87b5e 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -1315,8 +1315,9 @@ func (dm *DockerManager) killContainer(containerID kubecontainer.ContainerID, co go func() { defer close(done) defer utilruntime.HandleCrash() - if err := dm.runner.Run(containerID, pod, container, container.Lifecycle.PreStop); err != nil { + if msg, err := dm.runner.Run(containerID, pod, container, container.Lifecycle.PreStop); err != nil { glog.Errorf("preStop hook for container %q failed: %v", name, err) + dm.generateFailedContainerEvent(containerID, pod.Name, kubecontainer.FailedPreStopHook, msg) } }() select { @@ -1362,6 +1363,15 @@ func (dm *DockerManager) killContainer(containerID kubecontainer.ContainerID, co return err } +func (dm *DockerManager) generateFailedContainerEvent(containerID kubecontainer.ContainerID, podName, reason, message string) { + ref, ok := dm.containerRefManager.GetRef(containerID) + if !ok { + glog.Warningf("No ref for pod '%q'", podName) + return + } + dm.recorder.Event(ref, api.EventTypeWarning, reason, message) +} + var errNoPodOnContainer = fmt.Errorf("no pod information labels on Docker container") // containerAndPodFromLabels tries to load the appropriate container info off of a Docker container's labels @@ -1477,9 +1487,10 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe } if container.Lifecycle != nil && container.Lifecycle.PostStart != nil { - handlerErr := dm.runner.Run(id, pod, container, container.Lifecycle.PostStart) + msg, handlerErr := dm.runner.Run(id, pod, container, container.Lifecycle.PostStart) if handlerErr != nil { err := fmt.Errorf("PostStart handler: %v", handlerErr) + dm.generateFailedContainerEvent(id, pod.Name, kubecontainer.FailedPostStartHook, msg) dm.KillContainerInPod(id, container, pod, err.Error(), nil) return kubecontainer.ContainerID{}, err } diff --git a/pkg/kubelet/lifecycle/fake_handler_runner.go b/pkg/kubelet/lifecycle/fake_handler_runner.go index 90304632e65..501fb79caa2 100644 --- a/pkg/kubelet/lifecycle/fake_handler_runner.go +++ b/pkg/kubelet/lifecycle/fake_handler_runner.go @@ -35,12 +35,12 @@ func NewFakeHandlerRunner() *FakeHandlerRunner { return &FakeHandlerRunner{HandlerRuns: []string{}} } -func (hr *FakeHandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) error { +func (hr *FakeHandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) (string, error) { hr.Lock() defer hr.Unlock() if hr.Err != nil { - return hr.Err + return "", hr.Err } switch { @@ -51,9 +51,9 @@ func (hr *FakeHandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api case handler.TCPSocket != nil: hr.HandlerRuns = append(hr.HandlerRuns, fmt.Sprintf("tcp-socket on pod: %v, container: %v: %v", format.Pod(pod), container.Name, containerID.String())) default: - return fmt.Errorf("Invalid handler: %v", handler) + return "", fmt.Errorf("Invalid handler: %v", handler) } - return nil + return "", nil } func (hr *FakeHandlerRunner) Reset() { diff --git a/pkg/kubelet/lifecycle/handlers.go b/pkg/kubelet/lifecycle/handlers.go index be93b6972c3..011a264ad00 100644 --- a/pkg/kubelet/lifecycle/handlers.go +++ b/pkg/kubelet/lifecycle/handlers.go @@ -52,26 +52,32 @@ func NewHandlerRunner(httpGetter kubetypes.HttpGetter, commandRunner kubecontain } } -func (hr *HandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) error { +func (hr *HandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) (string, error) { switch { case handler.Exec != nil: - var buffer bytes.Buffer + var ( + buffer bytes.Buffer + msg string + ) output := ioutils.WriteCloserWrapper(&buffer) err := hr.commandRunner.ExecInContainer(containerID, handler.Exec.Command, nil, output, output, false) if err != nil { - glog.V(1).Infof("Exec lifecycle hook (%v) for Container %q in Pod %q failed - %q", handler.Exec.Command, container.Name, format.Pod(pod), buffer.String()) + msg := fmt.Sprintf("Exec lifecycle hook (%v) for Container %q in Pod %q failed - %q", handler.Exec.Command, container.Name, format.Pod(pod), buffer.String()) + glog.V(1).Infof(msg) } - return err + return msg, err case handler.HTTPGet != nil: msg, err := hr.runHTTPHandler(pod, container, handler) if err != nil { - glog.V(1).Infof("Http lifecycle hook (%s) for Container %q in Pod %q failed - %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), msg) + msg := fmt.Sprintf("Http lifecycle hook (%s) for Container %q in Pod %q failed - %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), msg) + glog.V(1).Infof(msg) } - return err + return msg, err default: err := fmt.Errorf("Invalid handler: %v", handler) - glog.Errorf("Cannot run handler: %v", err) - return err + msg := fmt.Sprintf("Cannot run handler: %v", err) + glog.Errorf(msg) + return msg, err } } diff --git a/pkg/kubelet/lifecycle/handlers_test.go b/pkg/kubelet/lifecycle/handlers_test.go index 1c9e9272029..3fc2389ee8d 100644 --- a/pkg/kubelet/lifecycle/handlers_test.go +++ b/pkg/kubelet/lifecycle/handlers_test.go @@ -19,8 +19,10 @@ package lifecycle import ( "fmt" "io" + "io/ioutil" "net/http" "reflect" + "strings" "testing" "k8s.io/kubernetes/pkg/api" @@ -110,7 +112,7 @@ func TestRunHandlerExec(t *testing.T) { pod.ObjectMeta.Name = "podFoo" pod.ObjectMeta.Namespace = "nsFoo" pod.Spec.Containers = []api.Container{container} - err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart) + _, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -121,13 +123,14 @@ func TestRunHandlerExec(t *testing.T) { } type fakeHTTP struct { - url string - err error + url string + err error + resp *http.Response } func (f *fakeHTTP) Get(url string) (*http.Response, error) { f.url = url - return nil, f.err + return f.resp, f.err } func TestRunHandlerHttp(t *testing.T) { @@ -153,7 +156,7 @@ func TestRunHandlerHttp(t *testing.T) { pod.ObjectMeta.Name = "podFoo" pod.ObjectMeta.Namespace = "nsFoo" pod.Spec.Containers = []api.Container{container} - err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart) + _, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart) if err != nil { t.Errorf("unexpected error: %v", err) @@ -180,7 +183,7 @@ func TestRunHandlerNil(t *testing.T) { pod.ObjectMeta.Name = podName pod.ObjectMeta.Namespace = podNamespace pod.Spec.Containers = []api.Container{container} - err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart) + _, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart) if err == nil { t.Errorf("expect error, but got nil") } @@ -188,14 +191,13 @@ func TestRunHandlerNil(t *testing.T) { func TestRunHandlerHttpFailure(t *testing.T) { expectedErr := fmt.Errorf("fake http error") - fakeHttp := fakeHTTP{err: expectedErr} - handlerRunner := &HandlerRunner{ - httpGetter: &fakeHttp, - commandRunner: &fakeContainerCommandRunner{}, - containerManager: nil, + expectedResp := http.Response{ + Body: ioutil.NopCloser(strings.NewReader(expectedErr.Error())), } + fakeHttp := fakeHTTP{err: expectedErr, resp: &expectedResp} + handlerRunner := NewHandlerRunner(&fakeHttp, &fakeContainerCommandRunner{}, nil) containerName := "containerFoo" - + containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"} container := api.Container{ Name: containerName, Lifecycle: &api.Lifecycle{ @@ -212,12 +214,12 @@ func TestRunHandlerHttpFailure(t *testing.T) { pod.ObjectMeta.Name = "podFoo" pod.ObjectMeta.Namespace = "nsFoo" pod.Spec.Containers = []api.Container{container} - msg, err := handlerRunner.runHTTPHandler(&pod, &container, container.Lifecycle.PostStart) + msg, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart) if err == nil { t.Errorf("expected error: %v", expectedErr) } - if msg != "" { - t.Errorf("expected empty error message") + if msg != expectedErr.Error() { + t.Errorf("unexpected error message: %q; expected %q", msg, expectedErr) } if fakeHttp.url != "http://foo:8080/bar" { t.Errorf("unexpected url: %s", fakeHttp.url) diff --git a/pkg/kubelet/rkt/rkt.go b/pkg/kubelet/rkt/rkt.go index 9e310ae5c78..878898dfd44 100644 --- a/pkg/kubelet/rkt/rkt.go +++ b/pkg/kubelet/rkt/rkt.go @@ -1184,7 +1184,16 @@ func (r *Runtime) RunPod(pod *api.Pod, pullSecrets []api.Secret) error { func (r *Runtime) runPreStopHook(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container) error { glog.V(4).Infof("rkt: Running pre-stop hook for container %q of pod %q", container.Name, format.Pod(pod)) - return r.runner.Run(containerID, pod, container, container.Lifecycle.PreStop) + msg, err := r.runner.Run(containerID, pod, container, container.Lifecycle.PreStop) + if err != nil { + ref, ok := r.containerRefManager.GetRef(containerID) + if !ok { + glog.Warningf("No ref for container %q", containerID) + } else { + r.recorder.Eventf(ref, api.EventTypeWarning, kubecontainer.FailedPreStopHook, msg) + } + } + return err } func (r *Runtime) runPostStartHook(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container) error { @@ -1215,7 +1224,16 @@ func (r *Runtime) runPostStartHook(containerID kubecontainer.ContainerID, pod *a return fmt.Errorf("rkt: Pod %q doesn't become running in %v: %v", format.Pod(pod), timeout, err) } - return r.runner.Run(containerID, pod, container, container.Lifecycle.PostStart) + msg, err := r.runner.Run(containerID, pod, container, container.Lifecycle.PostStart) + if err != nil { + ref, ok := r.containerRefManager.GetRef(containerID) + if !ok { + glog.Warningf("No ref for container %q", containerID) + } else { + r.recorder.Eventf(ref, api.EventTypeWarning, kubecontainer.FailedPostStartHook, msg) + } + } + return err } type lifecycleHookType string