From f6f9372d1a02dd86c494ea5048b01804883dd09d Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Mon, 15 Jun 2015 14:20:17 -0700 Subject: [PATCH 1/2] Apply oom_score_adj (0) to PID of user containers by default. --- pkg/kubelet/dockertools/manager.go | 43 +++++++++++++++++++----------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index dee4da35e77..341aaeb4904 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -48,9 +48,11 @@ import ( ) const ( - // The oom_score_adj of the POD infrastructure container. The default is 0, so - // any value below that makes it *less* likely to get OOM killed. - podOomScoreAdj = -100 + // The oom_score_adj of the POD infrastructure container. The default is 0 for + // any other docker containers, so any value below that makes it *less* likely + // to get OOM killed. + podOomScoreAdj = -100 + userContainerOomScoreAdj = 0 maxReasonCacheEntries = 200 @@ -1181,6 +1183,28 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe if err = dm.os.Symlink(containerLogFile, symlinkFile); err != nil { glog.Errorf("Failed to create symbolic link to the log file of pod %q container %q: %v", podFullName, container.Name, err) } + + // Set OOM score of POD container to lower than those of the other containers + // which have OOM score 0 by default in the pod. This ensures that it is + // killed only as a last resort. + containerInfo, err := dm.client.InspectContainer(string(id)) + if err != nil { + return "", err + } + + // Ensure the PID actually exists, else we'll move ourselves. + if containerInfo.State.Pid == 0 { + return "", fmt.Errorf("failed to get init PID for Docker container %q", string(id)) + } + if container.Name == PodInfraContainerName { + util.ApplyOomScoreAdj(containerInfo.State.Pid, podOomScoreAdj) + } else { + // Children processes of docker daemon will inheritant the OOM score from docker + // daemon process. We explicitly apply OOM score 0 by default to the user + // containers to avoid daemons or POD containers are killed by oom killer. + util.ApplyOomScoreAdj(containerInfo.State.Pid, userContainerOomScoreAdj) + } + return kubeletTypes.DockerID(id), err } @@ -1235,19 +1259,6 @@ func (dm *DockerManager) createPodInfraContainer(pod *api.Pod) (kubeletTypes.Doc return "", err } - // Set OOM score of POD container to lower than those of the other - // containers in the pod. This ensures that it is killed only as a last - // resort. - containerInfo, err := dm.client.InspectContainer(string(id)) - if err != nil { - return "", err - } - - // Ensure the PID actually exists, else we'll move ourselves. - if containerInfo.State.Pid == 0 { - return "", fmt.Errorf("failed to get init PID for Docker pod infra container %q", string(id)) - } - util.ApplyOomScoreAdj(containerInfo.State.Pid, podOomScoreAdj) return id, nil } From 14f78b74ccf9f87ab2b958bc3108d63b7b29b21f Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Mon, 15 Jun 2015 14:38:45 -0700 Subject: [PATCH 2/2] Fix the unittests cause by applying oom_score_adj (0) to the user containers. --- pkg/kubelet/dockertools/manager_test.go | 18 +++++++++--------- pkg/kubelet/kubelet_test.go | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/kubelet/dockertools/manager_test.go b/pkg/kubelet/dockertools/manager_test.go index c959fdb5706..30db5b4aca8 100644 --- a/pkg/kubelet/dockertools/manager_test.go +++ b/pkg/kubelet/dockertools/manager_test.go @@ -898,7 +898,7 @@ func TestSyncPodCreateNetAndContainer(t *testing.T) { // Create pod infra container. "create", "start", "inspect_container", // Create container. - "create", "start", + "create", "start", "inspect_container", }) fakeDocker.Lock() @@ -947,7 +947,7 @@ func TestSyncPodCreatesNetAndContainerPullsImage(t *testing.T) { // Create pod infra container. "create", "start", "inspect_container", // Create container. - "create", "start", + "create", "start", "inspect_container", }) fakeDocker.Lock() @@ -999,7 +999,7 @@ func TestSyncPodWithPodInfraCreatesContainer(t *testing.T) { // Inspect pod infra container (but does not create)" "inspect_container", // Create container. - "create", "start", + "create", "start", "inspect_container", }) fakeDocker.Lock() @@ -1040,7 +1040,7 @@ func TestSyncPodDeletesWithNoPodInfraContainer(t *testing.T) { // Create pod infra container. "create", "start", "inspect_container", // Create container. - "create", "start", + "create", "start", "inspect_container", }) // A map iteration is used to delete containers, so must not depend on @@ -1165,7 +1165,7 @@ func TestSyncPodBadHash(t *testing.T) { // Check the pod infra container. "inspect_container", // Kill and restart the bad hash container. - "inspect_container", "stop", "create", "start", + "inspect_container", "stop", "create", "start", "inspect_container", }) if err := fakeDocker.AssertStopped([]string{"1234"}); err != nil { @@ -1225,7 +1225,7 @@ func TestSyncPodsUnhealthy(t *testing.T) { // Kill the unhealthy container. "inspect_container", "stop", // Restart the unhealthy container. - "create", "start", + "create", "start", "inspect_container", }) if err := fakeDocker.AssertStopped([]string{"1234"}); err != nil { @@ -1410,7 +1410,7 @@ func TestSyncPodWithRestartPolicy(t *testing.T) { // Check the pod infra container. "inspect_container", // Restart both containers. - "create", "start", "create", "start", + "create", "start", "inspect_container", "create", "start", "inspect_container", }, []string{"succeeded", "failed"}, []string{}, @@ -1421,7 +1421,7 @@ func TestSyncPodWithRestartPolicy(t *testing.T) { // Check the pod infra container. "inspect_container", // Restart the failed container. - "create", "start", + "create", "start", "inspect_container", }, []string{"failed"}, []string{}, @@ -1834,7 +1834,7 @@ func TestSyncPodWithPodInfraCreatesContainerCallsHandler(t *testing.T) { // Check the pod infra container. "inspect_container", // Create container. - "create", "start", + "create", "start", "inspect_container", }) fakeDocker.Lock() diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 5be39fd1793..5a9a90470e0 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -484,7 +484,7 @@ func TestSyncPodsWithTerminationLog(t *testing.T) { // Create pod infra container. "create", "start", "inspect_container", // Create container. - "create", "start", + "create", "start", "inspect_container", // Get pod status. "list", "inspect_container", "inspect_container", // Get pods for deleting orphaned volumes.