diff --git a/pkg/api/conversion.go b/pkg/api/conversion.go index a43bd8f0554..ebdce3d5180 100644 --- a/pkg/api/conversion.go +++ b/pkg/api/conversion.go @@ -34,6 +34,14 @@ func init() { out.Spec.RestartPolicy = in.RestartPolicy out.Name = in.ID out.UID = in.UUID + // TODO(dchen1107): Move this conversion to pkg/api/v1beta[123]/conversion.go + // along with fixing #1502 + for i := range out.Spec.Containers { + ctr := &out.Spec.Containers[i] + if len(ctr.TerminationMessagePath) == 0 { + ctr.TerminationMessagePath = TerminationMessagePathDefault + } + } return nil }, func(in *BoundPod, out *ContainerManifest, s conversion.Scope) error { @@ -43,6 +51,12 @@ func init() { out.Version = "v1beta2" out.ID = in.Name out.UUID = in.UID + for i := range out.Containers { + ctr := &out.Containers[i] + if len(ctr.TerminationMessagePath) == 0 { + ctr.TerminationMessagePath = TerminationMessagePathDefault + } + } return nil }, func(in *ContainerManifestList, out *BoundPods, s conversion.Scope) error { diff --git a/pkg/api/types.go b/pkg/api/types.go index cd98cd93227..10ab6221d96 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -121,6 +121,8 @@ const ( NamespaceDefault string = "default" // NamespaceAll is the default argument to specify on a context when you want to list or filter resources across all namespaces NamespaceAll string = "" + // TerminationMessagePathDefault means the default path to capture the application termination message running in a container + TerminationMessagePathDefault string = "/dev/termination-log" ) // Volume represents a named volume in a pod that may be accessed by any containers in the pod. @@ -301,6 +303,8 @@ type Container struct { VolumeMounts []VolumeMount `json:"volumeMounts,omitempty" yaml:"volumeMounts,omitempty"` LivenessProbe *LivenessProbe `json:"livenessProbe,omitempty" yaml:"livenessProbe,omitempty"` Lifecycle *Lifecycle `json:"lifecycle,omitempty" yaml:"lifecycle,omitempty"` + // Optional: Defaults to /dev/termination-log + TerminationMessagePath string `json:"terminationMessagePath,omitempty" yaml:"terminationMessagePath,omitempty"` // Optional: Default to false. Privileged bool `json:"privileged,omitempty" yaml:"privileged,omitempty"` // Optional: Policy for pulling images for this container @@ -364,6 +368,7 @@ type ContainerStateTerminated struct { ExitCode int `json:"exitCode" yaml:"exitCode"` Signal int `json:"signal,omitempty" yaml:"signal,omitempty"` Reason string `json:"reason,omitempty" yaml:"reason,omitempty"` + Message string `json:"message,omitempty" yaml:"message,omitempty"` StartedAt time.Time `json:"startedAt,omitempty" yaml:"startedAt,omitempty"` FinishedAt time.Time `json:"finishedAt,omitempty" yaml:"finishedAt,omitempty"` } diff --git a/pkg/api/v1beta1/types.go b/pkg/api/v1beta1/types.go index ad6d9ea36f6..2ae145f5b8f 100644 --- a/pkg/api/v1beta1/types.go +++ b/pkg/api/v1beta1/types.go @@ -256,6 +256,8 @@ type Container struct { VolumeMounts []VolumeMount `yaml:"volumeMounts,omitempty" json:"volumeMounts,omitempty"` LivenessProbe *LivenessProbe `yaml:"livenessProbe,omitempty" json:"livenessProbe,omitempty"` Lifecycle *Lifecycle `yaml:"lifecycle,omitempty" json:"lifecycle,omitempty"` + // Optional: Defaults to /dev/termination-log + TerminationMessagePath string `yaml:"terminationMessagePath,omitempty" json:"terminationMessagePath,omitempty"` // Optional: Default to false. Privileged bool `json:"privileged,omitempty" yaml:"privileged,omitempty"` // Optional: Policy for pulling images for this container @@ -330,6 +332,7 @@ type ContainerStateTerminated struct { ExitCode int `json:"exitCode" yaml:"exitCode"` Signal int `json:"signal,omitempty" yaml:"signal,omitempty"` Reason string `json:"reason,omitempty" yaml:"reason,omitempty"` + Message string `json:"message,omitempty" yaml:"message,omitempty"` StartedAt time.Time `json:"startedAt,omitempty" yaml:"startedAt,omitempty"` FinishedAt time.Time `json:"finishedAt,omitempty" yaml:"finishedAt,omitempty"` } diff --git a/pkg/api/v1beta2/types.go b/pkg/api/v1beta2/types.go index 6cdd91592dc..f58ae83b46b 100644 --- a/pkg/api/v1beta2/types.go +++ b/pkg/api/v1beta2/types.go @@ -222,6 +222,8 @@ type Container struct { VolumeMounts []VolumeMount `yaml:"volumeMounts,omitempty" json:"volumeMounts,omitempty"` LivenessProbe *LivenessProbe `yaml:"livenessProbe,omitempty" json:"livenessProbe,omitempty"` Lifecycle *Lifecycle `yaml:"lifecycle,omitempty" json:"lifecycle,omitempty"` + // Optional: Defaults to /dev/termination-log + TerminationMessagePath string `yaml:"terminationMessagePath,omitempty" json:"terminationMessagePath,omitempty"` // Optional: Default to false. Privileged bool `json:"privileged,omitempty" yaml:"privileged,omitempty"` // Optional: Policy for pulling images for this container @@ -295,6 +297,7 @@ type ContainerStateTerminated struct { ExitCode int `json:"exitCode" yaml:"exitCode"` Signal int `json:"signal,omitempty" yaml:"signal,omitempty"` Reason string `json:"reason,omitempty" yaml:"reason,omitempty"` + Message string `json:"message,omitempty" yaml:"message,omitempty"` StartedAt time.Time `json:"startedAt,omitempty" yaml:"startedAt,omitempty"` FinishedAt time.Time `json:"finishedAt,omitempty" yaml:"finishedAt,omitempty"` } diff --git a/pkg/api/v1beta3/types.go b/pkg/api/v1beta3/types.go index 7be1b377dc1..eedd95a1fb2 100644 --- a/pkg/api/v1beta3/types.go +++ b/pkg/api/v1beta3/types.go @@ -334,6 +334,8 @@ type Container struct { VolumeMounts []VolumeMount `json:"volumeMounts,omitempty" yaml:"volumeMounts,omitempty"` LivenessProbe *LivenessProbe `json:"livenessProbe,omitempty" yaml:"livenessProbe,omitempty"` Lifecycle *Lifecycle `json:"lifecycle,omitempty" yaml:"lifecycle,omitempty"` + // Optional: Defaults to /dev/termination-log + TerminationMessagePath string `json:"terminationMessagePath,omitempty" yaml:"terminationMessagePath,omitempty"` // Optional: Default to false. Privileged bool `json:"privileged,omitempty" yaml:"privileged,omitempty"` // Optional: Policy for pulling images for this container @@ -395,6 +397,7 @@ type ContainerStateTerminated struct { ExitCode int `json:"exitCode" yaml:"exitCode"` Signal int `json:"signal,omitempty" yaml:"signal,omitempty"` Reason string `json:"reason,omitempty" yaml:"reason,omitempty"` + Message string `json:"message,omitempty" yaml:"message,omitempty"` StartedAt time.Time `json:"startedAt,omitempty" yaml:"startedAt,omitempty"` FinishedAt time.Time `json:"finishedAt,omitempty" yaml:"finishedAt,omitempty"` } diff --git a/pkg/kubelet/config/file_test.go b/pkg/kubelet/config/file_test.go index f34073a69e2..bf6d6584955 100644 --- a/pkg/kubelet/config/file_test.go +++ b/pkg/kubelet/config/file_test.go @@ -40,6 +40,7 @@ func ExampleManifestAndPod(id string) (api.ContainerManifest, api.BoundPod) { { Name: "c" + id, Image: "foo", + TerminationMessagePath: "/somepath", }, }, Volumes: []api.Volume{ @@ -62,6 +63,7 @@ func ExampleManifestAndPod(id string) (api.ContainerManifest, api.BoundPod) { { Name: "c" + id, Image: "foo", + TerminationMessagePath: "/somepath", }, }, Volumes: []api.Volume{ @@ -124,7 +126,7 @@ func TestReadFromFile(t *testing.T) { Namespace: "default", }, Spec: api.PodSpec{ - Containers: []api.Container{{Image: "test/image"}}, + Containers: []api.Container{{Image: "test/image", TerminationMessagePath: "/dev/termination-log"}}, }, }) if !reflect.DeepEqual(expected, update) { diff --git a/pkg/kubelet/config/http_test.go b/pkg/kubelet/config/http_test.go index 7071f53dcc3..525d2020af9 100644 --- a/pkg/kubelet/config/http_test.go +++ b/pkg/kubelet/config/http_test.go @@ -146,14 +146,24 @@ func TestExtractFromHTTP(t *testing.T) { Name: "1", Namespace: "default", }, - Spec: api.PodSpec{Containers: []api.Container{{Name: "1", Image: "foo"}}}, + Spec: api.PodSpec{ + Containers: []api.Container{{ + Name: "1", + Image: "foo", + TerminationMessagePath: "/dev/termination-log"}}, + }, }, api.BoundPod{ ObjectMeta: api.ObjectMeta{ Name: "bar", Namespace: "default", }, - Spec: api.PodSpec{Containers: []api.Container{{Name: "1", Image: "foo"}}}, + Spec: api.PodSpec{ + Containers: []api.Container{{ + Name: "1", + Image: "foo", + TerminationMessagePath: "/dev/termination-log"}}, + }, }), }, { diff --git a/pkg/kubelet/dockertools/docker.go b/pkg/kubelet/dockertools/docker.go index c7bcce4c6ec..d6d2896032f 100644 --- a/pkg/kubelet/dockertools/docker.go +++ b/pkg/kubelet/dockertools/docker.go @@ -23,6 +23,7 @@ import ( "fmt" "hash/adler32" "io" + "io/ioutil" "math/rand" "os" "os/exec" @@ -364,8 +365,9 @@ var ( ErrContainerCannotRun = errors.New("Container cannot run") ) -func inspectContainer(client DockerInterface, dockerID, containerName string) (*api.ContainerStatus, error) { +func inspectContainer(client DockerInterface, dockerID, containerName, tPath string) (*api.ContainerStatus, error) { inspectResult, err := client.InspectContainer(dockerID) + if err != nil { return nil, err } @@ -396,6 +398,17 @@ func inspectContainer(client DockerInterface, dockerID, containerName string) (* StartedAt: inspectResult.State.StartedAt, FinishedAt: inspectResult.State.FinishedAt, } + if tPath != "" { + path, found := inspectResult.Volumes[tPath] + if found { + data, err := ioutil.ReadFile(path) + if err != nil { + glog.Errorf("Error on reading termination-log %s(%v)", path, err) + } else { + containerStatus.State.Termination.Message = string(data) + } + } + } waiting = false } @@ -414,6 +427,11 @@ func inspectContainer(client DockerInterface, dockerID, containerName string) (* // GetDockerPodInfo returns docker info for all containers in the pod/manifest. func GetDockerPodInfo(client DockerInterface, manifest api.PodSpec, podFullName, uuid string) (api.PodInfo, error) { info := api.PodInfo{} + expectedContainers := make(map[string]api.Container) + for _, container := range manifest.Containers { + expectedContainers[container.Name] = container + } + expectedContainers["net"] = api.Container{} containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) if err != nil { @@ -428,6 +446,14 @@ func GetDockerPodInfo(client DockerInterface, manifest api.PodSpec, podFullName, if uuid != "" && dockerUUID != uuid { continue } + c, found := expectedContainers[dockerContainerName] + terminationMessagePath := "" + if !found { + // TODO(dchen1107): should figure out why not continue here + // continue + } else { + terminationMessagePath = c.TerminationMessagePath + } // We assume docker return us a list of containers in time order if containerStatus, found := info[dockerContainerName]; found { containerStatus.RestartCount += 1 @@ -435,7 +461,7 @@ func GetDockerPodInfo(client DockerInterface, manifest api.PodSpec, podFullName, continue } - containerStatus, err := inspectContainer(client, value.ID, dockerContainerName) + containerStatus, err := inspectContainer(client, value.ID, dockerContainerName, terminationMessagePath) if err != nil { return nil, err } diff --git a/pkg/kubelet/dockertools/fake_docker_client.go b/pkg/kubelet/dockertools/fake_docker_client.go index bb27573100b..22fb88fa1d3 100644 --- a/pkg/kubelet/dockertools/fake_docker_client.go +++ b/pkg/kubelet/dockertools/fake_docker_client.go @@ -109,6 +109,11 @@ func (f *FakeDockerClient) StartContainer(id string, hostConfig *docker.HostConf f.Lock() defer f.Unlock() f.called = append(f.called, "start") + f.Container = &docker.Container{ + ID: id, + Config: &docker.Config{Image: "testimage"}, + HostConfig: hostConfig, + } return f.Err } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index f89f4190b1b..c7f24915f92 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "net/http" + "os" "path" "sort" "strconv" @@ -295,7 +296,6 @@ func makeBinds(pod *api.BoundPod, container *api.Container, podVolumes volumeMap } return binds } - func makePortsAndBindings(container *api.Container) (map[docker.Port]struct{}, map[docker.Port][]docker.PortBinding) { exposedPorts := map[docker.Port]struct{}{} portBindings := map[docker.Port][]docker.PortBinding{} @@ -463,6 +463,21 @@ func (kl *Kubelet) runContainer(pod *api.BoundPod, container *api.Container, pod if err != nil { return "", err } + if len(container.TerminationMessagePath) != 0 { + p := path.Join(kl.rootDirectory, pod.Name, container.Name) + if err := os.MkdirAll(p, 0750); err != nil { + glog.Errorf("Error on creating %s(%v)", p, err) + } else { + containerLogPath := path.Join(p, dockerContainer.ID) + fs, err := os.Create(containerLogPath) + if err != nil { + glog.Errorf("Error on creating termination-log file: %s(%v)", containerLogPath, err) + } + defer fs.Close() + b := fmt.Sprintf("%s:%s", containerLogPath, container.TerminationMessagePath) + binds = append(binds, b) + } + } privileged := false if capabilities.Get().AllowPrivileged { privileged = container.Privileged diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 827a25a0833..6436f4ea3b0 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -186,6 +186,44 @@ func TestSyncPodsDoesNothing(t *testing.T) { verifyCalls(t, fakeDocker, []string{"list", "list", "inspect_container", "inspect_container"}) } +func TestSyncPodsWithTerminationLog(t *testing.T) { + kubelet, _, fakeDocker := newTestKubelet(t) + container := api.Container{ + Name: "bar", + TerminationMessagePath: "/dev/somepath", + } + fakeDocker.ContainerList = []docker.APIContainers{} + err := kubelet.SyncPods([]api.BoundPod{ + { + ObjectMeta: api.ObjectMeta{ + Name: "foo", + Namespace: "new", + Annotations: map[string]string{ConfigSourceAnnotationKey: "test"}, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + container, + }, + }, + }, + }) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + kubelet.drainWorkers() + verifyCalls(t, fakeDocker, []string{ + "list", "create", "start", "list", "inspect_container", "list", "create", "start"}) + + fakeDocker.Lock() + parts := strings.Split(fakeDocker.Container.HostConfig.Binds[0], ":") + if fakeDocker.Container.HostConfig == nil || + !matchString(t, "/tmp/kubelet/foo/bar/k8s_bar\\.[a-f0-9]", parts[0]) || + parts[1] != "/dev/somepath" { + t.Errorf("Unexpected containers created %v", fakeDocker.Container) + } + fakeDocker.Unlock() +} + // drainWorkers waits until all workers are done. Should only used for testing. func (kl *Kubelet) drainWorkers() { for {