Merge pull request #2225 from dchen1107/deathrattle

Capture application termination messages/output

pkg/api/conversion.go

@@ -34,6 +34,14 @@ func init() {
 			out.Spec.RestartPolicy = in.RestartPolicy
 			out.Name = in.ID
 			out.UID = in.UUID
+			// TODO(dchen1107): Move this conversion to pkg/api/v1beta[123]/conversion.go
+			// along with fixing #1502
+			for i := range out.Spec.Containers {
+				ctr := &out.Spec.Containers[i]
+				if len(ctr.TerminationMessagePath) == 0 {
+					ctr.TerminationMessagePath = TerminationMessagePathDefault
+				}
+			}
 			return nil
 		},
 		func(in *BoundPod, out *ContainerManifest, s conversion.Scope) error {
@@ -43,6 +51,12 @@ func init() {
 			out.Version = "v1beta2"
 			out.ID = in.Name
 			out.UUID = in.UID
+			for i := range out.Containers {
+				ctr := &out.Containers[i]
+				if len(ctr.TerminationMessagePath) == 0 {
+					ctr.TerminationMessagePath = TerminationMessagePathDefault
+				}
+			}
 			return nil
 		},
 		func(in *ContainerManifestList, out *BoundPods, s conversion.Scope) error {

pkg/api/types.go

@@ -121,6 +121,8 @@ const (
 	NamespaceDefault string = "default"
 	// NamespaceAll is the default argument to specify on a context when you want to list or filter resources across all namespaces
 	NamespaceAll string = ""
+	// TerminationMessagePathDefault means the default path to capture the application termination message running in a container
+	TerminationMessagePathDefault string = "/dev/termination-log"
 )
 
 // Volume represents a named volume in a pod that may be accessed by any containers in the pod.
@@ -301,6 +303,8 @@ type Container struct {
 	VolumeMounts  []VolumeMount  `json:"volumeMounts,omitempty" yaml:"volumeMounts,omitempty"`
 	LivenessProbe *LivenessProbe `json:"livenessProbe,omitempty" yaml:"livenessProbe,omitempty"`
 	Lifecycle     *Lifecycle     `json:"lifecycle,omitempty" yaml:"lifecycle,omitempty"`
+	// Optional: Defaults to /dev/termination-log
+	TerminationMessagePath string `json:"terminationMessagePath,omitempty" yaml:"terminationMessagePath,omitempty"`
 	// Optional: Default to false.
 	Privileged bool `json:"privileged,omitempty" yaml:"privileged,omitempty"`
 	// Optional: Policy for pulling images for this container
@@ -364,6 +368,7 @@ type ContainerStateTerminated struct {
 	ExitCode   int       `json:"exitCode" yaml:"exitCode"`
 	Signal     int       `json:"signal,omitempty" yaml:"signal,omitempty"`
 	Reason     string    `json:"reason,omitempty" yaml:"reason,omitempty"`
+	Message    string    `json:"message,omitempty" yaml:"message,omitempty"`
 	StartedAt  time.Time `json:"startedAt,omitempty" yaml:"startedAt,omitempty"`
 	FinishedAt time.Time `json:"finishedAt,omitempty" yaml:"finishedAt,omitempty"`
 }

pkg/api/v1beta1/types.go

@@ -256,6 +256,8 @@ type Container struct {
 	VolumeMounts  []VolumeMount  `yaml:"volumeMounts,omitempty" json:"volumeMounts,omitempty"`
 	LivenessProbe *LivenessProbe `yaml:"livenessProbe,omitempty" json:"livenessProbe,omitempty"`
 	Lifecycle     *Lifecycle     `yaml:"lifecycle,omitempty" json:"lifecycle,omitempty"`
+	// Optional: Defaults to /dev/termination-log
+	TerminationMessagePath string `yaml:"terminationMessagePath,omitempty" json:"terminationMessagePath,omitempty"`
 	// Optional: Default to false.
 	Privileged bool `json:"privileged,omitempty" yaml:"privileged,omitempty"`
 	// Optional: Policy for pulling images for this container
@@ -330,6 +332,7 @@ type ContainerStateTerminated struct {
 	ExitCode   int       `json:"exitCode" yaml:"exitCode"`
 	Signal     int       `json:"signal,omitempty" yaml:"signal,omitempty"`
 	Reason     string    `json:"reason,omitempty" yaml:"reason,omitempty"`
+	Message    string    `json:"message,omitempty" yaml:"message,omitempty"`
 	StartedAt  time.Time `json:"startedAt,omitempty" yaml:"startedAt,omitempty"`
 	FinishedAt time.Time `json:"finishedAt,omitempty" yaml:"finishedAt,omitempty"`
 }

pkg/api/v1beta2/types.go

@@ -222,6 +222,8 @@ type Container struct {
 	VolumeMounts  []VolumeMount  `yaml:"volumeMounts,omitempty" json:"volumeMounts,omitempty"`
 	LivenessProbe *LivenessProbe `yaml:"livenessProbe,omitempty" json:"livenessProbe,omitempty"`
 	Lifecycle     *Lifecycle     `yaml:"lifecycle,omitempty" json:"lifecycle,omitempty"`
+	// Optional: Defaults to /dev/termination-log
+	TerminationMessagePath string `yaml:"terminationMessagePath,omitempty" json:"terminationMessagePath,omitempty"`
 	// Optional: Default to false.
 	Privileged bool `json:"privileged,omitempty" yaml:"privileged,omitempty"`
 	// Optional: Policy for pulling images for this container
@@ -295,6 +297,7 @@ type ContainerStateTerminated struct {
 	ExitCode   int       `json:"exitCode" yaml:"exitCode"`
 	Signal     int       `json:"signal,omitempty" yaml:"signal,omitempty"`
 	Reason     string    `json:"reason,omitempty" yaml:"reason,omitempty"`
+	Message    string    `json:"message,omitempty" yaml:"message,omitempty"`
 	StartedAt  time.Time `json:"startedAt,omitempty" yaml:"startedAt,omitempty"`
 	FinishedAt time.Time `json:"finishedAt,omitempty" yaml:"finishedAt,omitempty"`
 }

pkg/api/v1beta3/types.go

@@ -334,6 +334,8 @@ type Container struct {
 	VolumeMounts  []VolumeMount  `json:"volumeMounts,omitempty" yaml:"volumeMounts,omitempty"`
 	LivenessProbe *LivenessProbe `json:"livenessProbe,omitempty" yaml:"livenessProbe,omitempty"`
 	Lifecycle     *Lifecycle     `json:"lifecycle,omitempty" yaml:"lifecycle,omitempty"`
+	// Optional: Defaults to /dev/termination-log
+	TerminationMessagePath string `json:"terminationMessagePath,omitempty" yaml:"terminationMessagePath,omitempty"`
 	// Optional: Default to false.
 	Privileged bool `json:"privileged,omitempty" yaml:"privileged,omitempty"`
 	// Optional: Policy for pulling images for this container
@@ -395,6 +397,7 @@ type ContainerStateTerminated struct {
 	ExitCode   int       `json:"exitCode" yaml:"exitCode"`
 	Signal     int       `json:"signal,omitempty" yaml:"signal,omitempty"`
 	Reason     string    `json:"reason,omitempty" yaml:"reason,omitempty"`
+	Message    string    `json:"message,omitempty" yaml:"message,omitempty"`
 	StartedAt  time.Time `json:"startedAt,omitempty" yaml:"startedAt,omitempty"`
 	FinishedAt time.Time `json:"finishedAt,omitempty" yaml:"finishedAt,omitempty"`
 }

pkg/kubelet/config/file_test.go

@@ -40,6 +40,7 @@ func ExampleManifestAndPod(id string) (api.ContainerManifest, api.BoundPod) {
 			{
 				Name:  "c" + id,
 				Image: "foo",
+				TerminationMessagePath: "/somepath",
 			},
 		},
 		Volumes: []api.Volume{
@@ -62,6 +63,7 @@ func ExampleManifestAndPod(id string) (api.ContainerManifest, api.BoundPod) {
 				{
 					Name:  "c" + id,
 					Image: "foo",
+					TerminationMessagePath: "/somepath",
 				},
 			},
 			Volumes: []api.Volume{
@@ -124,7 +126,7 @@ func TestReadFromFile(t *testing.T) {
 				Namespace: "default",
 			},
 			Spec: api.PodSpec{
-				Containers: []api.Container{{Image: "test/image"}},
+				Containers: []api.Container{{Image: "test/image", TerminationMessagePath: "/dev/termination-log"}},
 			},
 		})
 		if !reflect.DeepEqual(expected, update) {

pkg/kubelet/config/http_test.go

@@ -146,14 +146,24 @@ func TestExtractFromHTTP(t *testing.T) {
 						Name:      "1",
 						Namespace: "default",
 					},
-					Spec: api.PodSpec{Containers: []api.Container{{Name: "1", Image: "foo"}}},
+					Spec: api.PodSpec{
+						Containers: []api.Container{{
+							Name:  "1",
+							Image: "foo",
+							TerminationMessagePath: "/dev/termination-log"}},
+					},
 				},
 				api.BoundPod{
 					ObjectMeta: api.ObjectMeta{
 						Name:      "bar",
 						Namespace: "default",
 					},
-					Spec: api.PodSpec{Containers: []api.Container{{Name: "1", Image: "foo"}}},
+					Spec: api.PodSpec{
+						Containers: []api.Container{{
+							Name:  "1",
+							Image: "foo",
+							TerminationMessagePath: "/dev/termination-log"}},
+					},
 				}),
 		},
 		{

pkg/kubelet/dockertools/docker.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"hash/adler32"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"os"
 	"os/exec"
@@ -364,8 +365,9 @@ var (
 	ErrContainerCannotRun = errors.New("Container cannot run")
 )
 
-func inspectContainer(client DockerInterface, dockerID, containerName string) (*api.ContainerStatus, error) {
+func inspectContainer(client DockerInterface, dockerID, containerName, tPath string) (*api.ContainerStatus, error) {
 	inspectResult, err := client.InspectContainer(dockerID)
+
 	if err != nil {
 		return nil, err
 	}
@@ -396,6 +398,17 @@ func inspectContainer(client DockerInterface, dockerID, containerName string) (*
 			StartedAt:  inspectResult.State.StartedAt,
 			FinishedAt: inspectResult.State.FinishedAt,
 		}
+		if tPath != "" {
+			path, found := inspectResult.Volumes[tPath]
+			if found {
+				data, err := ioutil.ReadFile(path)
+				if err != nil {
+					glog.Errorf("Error on reading termination-log %s(%v)", path, err)
+				} else {
+					containerStatus.State.Termination.Message = string(data)
+				}
+			}
+		}
 		waiting = false
 	}
 
@@ -414,6 +427,11 @@ func inspectContainer(client DockerInterface, dockerID, containerName string) (*
 // GetDockerPodInfo returns docker info for all containers in the pod/manifest.
 func GetDockerPodInfo(client DockerInterface, manifest api.PodSpec, podFullName, uuid string) (api.PodInfo, error) {
 	info := api.PodInfo{}
+	expectedContainers := make(map[string]api.Container)
+	for _, container := range manifest.Containers {
+		expectedContainers[container.Name] = container
+	}
+	expectedContainers["net"] = api.Container{}
 
 	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
 	if err != nil {
@@ -428,6 +446,14 @@ func GetDockerPodInfo(client DockerInterface, manifest api.PodSpec, podFullName,
 		if uuid != "" && dockerUUID != uuid {
 			continue
 		}
+		c, found := expectedContainers[dockerContainerName]
+		terminationMessagePath := ""
+		if !found {
+			// TODO(dchen1107): should figure out why not continue here
+			// continue
+		} else {
+			terminationMessagePath = c.TerminationMessagePath
+		}
 		// We assume docker return us a list of containers in time order
 		if containerStatus, found := info[dockerContainerName]; found {
 			containerStatus.RestartCount += 1
@@ -435,7 +461,7 @@ func GetDockerPodInfo(client DockerInterface, manifest api.PodSpec, podFullName,
 			continue
 		}
 
-		containerStatus, err := inspectContainer(client, value.ID, dockerContainerName)
+		containerStatus, err := inspectContainer(client, value.ID, dockerContainerName, terminationMessagePath)
 		if err != nil {
 			return nil, err
 		}

pkg/kubelet/dockertools/fake_docker_client.go

@@ -109,6 +109,11 @@ func (f *FakeDockerClient) StartContainer(id string, hostConfig *docker.HostConf
 	f.Lock()
 	defer f.Unlock()
 	f.called = append(f.called, "start")
+	f.Container = &docker.Container{
+		ID:         id,
+		Config:     &docker.Config{Image: "testimage"},
+		HostConfig: hostConfig,
+	}
 	return f.Err
 }
 

pkg/kubelet/kubelet.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"path"
 	"sort"
 	"strconv"
@@ -295,7 +296,6 @@ func makeBinds(pod *api.BoundPod, container *api.Container, podVolumes volumeMap
 	}
 	return binds
 }
-
 func makePortsAndBindings(container *api.Container) (map[docker.Port]struct{}, map[docker.Port][]docker.PortBinding) {
 	exposedPorts := map[docker.Port]struct{}{}
 	portBindings := map[docker.Port][]docker.PortBinding{}
@@ -463,6 +463,21 @@ func (kl *Kubelet) runContainer(pod *api.BoundPod, container *api.Container, pod
 	if err != nil {
 		return "", err
 	}
+	if len(container.TerminationMessagePath) != 0 {
+		p := path.Join(kl.rootDirectory, pod.Name, container.Name)
+		if err := os.MkdirAll(p, 0750); err != nil {
+			glog.Errorf("Error on creating %s(%v)", p, err)
+		} else {
+			containerLogPath := path.Join(p, dockerContainer.ID)
+			fs, err := os.Create(containerLogPath)
+			if err != nil {
+				glog.Errorf("Error on creating termination-log file: %s(%v)", containerLogPath, err)
+			}
+			defer fs.Close()
+			b := fmt.Sprintf("%s:%s", containerLogPath, container.TerminationMessagePath)
+			binds = append(binds, b)
+		}
+	}
 	privileged := false
 	if capabilities.Get().AllowPrivileged {
 		privileged = container.Privileged

pkg/kubelet/kubelet_test.go

@@ -186,6 +186,44 @@ func TestSyncPodsDoesNothing(t *testing.T) {
 	verifyCalls(t, fakeDocker, []string{"list", "list", "inspect_container", "inspect_container"})
 }
 
+func TestSyncPodsWithTerminationLog(t *testing.T) {
+	kubelet, _, fakeDocker := newTestKubelet(t)
+	container := api.Container{
+		Name: "bar",
+		TerminationMessagePath: "/dev/somepath",
+	}
+	fakeDocker.ContainerList = []docker.APIContainers{}
+	err := kubelet.SyncPods([]api.BoundPod{
+		{
+			ObjectMeta: api.ObjectMeta{
+				Name:        "foo",
+				Namespace:   "new",
+				Annotations: map[string]string{ConfigSourceAnnotationKey: "test"},
+			},
+			Spec: api.PodSpec{
+				Containers: []api.Container{
+					container,
+				},
+			},
+		},
+	})
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	kubelet.drainWorkers()
+	verifyCalls(t, fakeDocker, []string{
+		"list", "create", "start", "list", "inspect_container", "list", "create", "start"})
+
+	fakeDocker.Lock()
+	parts := strings.Split(fakeDocker.Container.HostConfig.Binds[0], ":")
+	if fakeDocker.Container.HostConfig == nil ||
+		!matchString(t, "/tmp/kubelet/foo/bar/k8s_bar\\.[a-f0-9]", parts[0]) ||
+		parts[1] != "/dev/somepath" {
+		t.Errorf("Unexpected containers created %v", fakeDocker.Container)
+	}
+	fakeDocker.Unlock()
+}
+
 // drainWorkers waits until all workers are done.  Should only used for testing.
 func (kl *Kubelet) drainWorkers() {
 	for {