diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index 9818814e8ec..2fb6bf67f29 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -1357,6 +1357,38 @@ func containerAndPodFromLabels(inspect *docker.Container) (pod *api.Pod, contain return } +func (dm *DockerManager) applyOOMScoreAdj(container *api.Container, containerInfo *docker.Container) error { + cgroupName, err := dm.procFs.GetFullContainerName(containerInfo.State.Pid) + if err != nil { + if err == os.ErrNotExist { + // Container exited. We cannot do anything about it. Ignore this error. + glog.V(2).Infof("Failed to apply OOM score adj on container %q with ID %q. Init process does not exist.", containerInfo.Name, containerInfo.ID) + return nil + } + return err + } + // Set OOM score of the container based on the priority of the container. + // Processes in lower-priority pods should be killed first if the system runs out of memory. + // The main pod infrastructure container is considered high priority, since if it is killed the + // whole pod will die. + // TODO: Cache this value. + var oomScoreAdj int + if containerInfo.Name == PodInfraContainerName { + oomScoreAdj = qos.PodInfraOOMAdj + } else { + oomScoreAdj = qos.GetContainerOOMScoreAdjust(container, int64(dm.machineInfo.MemoryCapacity)) + } + if err = dm.oomAdjuster.ApplyOOMScoreAdjContainer(cgroupName, oomScoreAdj, 5); err != nil { + if err == os.ErrNotExist { + // Container exited. We cannot do anything about it. Ignore this error. + glog.V(2).Infof("Failed to apply OOM score adj on container %q with ID %q. Init process does not exist.", containerInfo.Name, containerInfo.ID) + return nil + } + return err + } + return nil +} + // Run a single container from a pod. Returns the docker container ID // If do not need to pass labels, just pass nil. func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Container, netMode, ipcMode, pidMode string, restartCount int) (kubecontainer.ContainerID, error) { @@ -1418,24 +1450,9 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe return kubecontainer.ContainerID{}, fmt.Errorf("can't get init PID for container %q", id) } - // Set OOM score of the container based on the priority of the container. - // Processes in lower-priority pods should be killed first if the system runs out of memory. - // The main pod infrastructure container is considered high priority, since if it is killed the - // whole pod will die. - var oomScoreAdj int - if container.Name == PodInfraContainerName { - oomScoreAdj = qos.PodInfraOOMAdj - } else { - oomScoreAdj = qos.GetContainerOOMScoreAdjust(container, int64(dm.machineInfo.MemoryCapacity)) + if err := dm.applyOOMScoreAdj(container, containerInfo); err != nil { + return kubecontainer.ContainerID{}, fmt.Errorf("failed to apply oom-score-adj to container %q- %v", err, containerInfo.Name) } - cgroupName, err := dm.procFs.GetFullContainerName(containerInfo.State.Pid) - if err != nil { - return kubecontainer.ContainerID{}, fmt.Errorf("GetFullContainerName: %v", err) - } - if err = dm.oomAdjuster.ApplyOOMScoreAdjContainer(cgroupName, oomScoreAdj, 5); err != nil { - return kubecontainer.ContainerID{}, fmt.Errorf("ApplyOOMScoreAdjContainer: %v", err) - } - // The addNDotsOption call appends the ndots option to the resolv.conf file generated by docker. // This resolv.conf file is shared by all containers of the same pod, and needs to be modified only once per pod. // we modify it when the pause container is created since it is the first container created in the pod since it holds diff --git a/pkg/util/oom/oom_linux.go b/pkg/util/oom/oom_linux.go index 9a89dbde261..5503bad8c2c 100644 --- a/pkg/util/oom/oom_linux.go +++ b/pkg/util/oom/oom_linux.go @@ -21,6 +21,7 @@ package oom import ( "fmt" "io/ioutil" + "os" "path" "strconv" @@ -48,7 +49,18 @@ func getPids(cgroupName string) ([]int, error) { return fsManager.GetPids() } +func syscallNotExists(err error) bool { + if err == nil { + return false + } + if e, ok := err.(*os.SyscallError); ok && os.IsNotExist(e) { + return true + } + return false +} + // Writes 'value' to /proc//oom_score_adj. PID = 0 means self +// Returns os.ErrNotExist if the `pid` does not exist. func applyOOMScoreAdj(pid int, oomScoreAdj int) error { if pid < 0 { return fmt.Errorf("invalid PID %d specified for oom_score_adj", pid) @@ -61,20 +73,18 @@ func applyOOMScoreAdj(pid int, oomScoreAdj int) error { pidStr = strconv.Itoa(pid) } - oomScoreAdjPath := path.Join("/proc", pidStr, "oom_score_adj") maxTries := 2 + oomScoreAdjPath := path.Join("/proc", pidStr, "oom_score_adj") + value := strconv.Itoa(oomScoreAdj) var err error for i := 0; i < maxTries; i++ { - _, readErr := ioutil.ReadFile(oomScoreAdjPath) - if readErr != nil { - err = fmt.Errorf("failed to read oom_score_adj: %v", readErr) - } else if writeErr := ioutil.WriteFile(oomScoreAdjPath, []byte(strconv.Itoa(oomScoreAdj)), 0700); writeErr != nil { - err = fmt.Errorf("failed to set oom_score_adj to %d: %v", oomScoreAdj, writeErr) - } else { - return nil + if err = ioutil.WriteFile(oomScoreAdjPath, []byte(value), 0700); err != nil { + if syscallNotExists(err) { + return os.ErrNotExist + } + err = fmt.Errorf("failed to apply oom-score-adj to pid %d (%v)", err) } } - return err } @@ -86,6 +96,10 @@ func (oomAdjuster *OOMAdjuster) applyOOMScoreAdjContainer(cgroupName string, oom continueAdjusting := false pidList, err := oomAdjuster.pidLister(cgroupName) if err != nil { + if syscallNotExists(err) { + // Nothing to do since the container doesn't exist anymore. + return os.ErrNotExist + } continueAdjusting = true glog.Errorf("Error getting process list for cgroup %s: %+v", cgroupName, err) } else if len(pidList) == 0 { @@ -97,6 +111,7 @@ func (oomAdjuster *OOMAdjuster) applyOOMScoreAdjContainer(cgroupName string, oom if err = oomAdjuster.ApplyOOMScoreAdj(pid, oomScoreAdj); err == nil { adjustedProcessSet[pid] = true } + // Processes can come and go while we try to apply oom score adjust value. So ignore errors here. } } } diff --git a/pkg/util/procfs/procfs.go b/pkg/util/procfs/procfs.go index c0a45725fbf..cc432255fb2 100644 --- a/pkg/util/procfs/procfs.go +++ b/pkg/util/procfs/procfs.go @@ -19,6 +19,7 @@ package procfs import ( "fmt" "io/ioutil" + "os" "path" "strconv" "strings" @@ -48,6 +49,9 @@ func (pfs *ProcFS) GetFullContainerName(pid int) (string, error) { filePath := path.Join("/proc", strconv.Itoa(pid), "cgroup") content, err := ioutil.ReadFile(filePath) if err != nil { + if e, ok := err.(*os.SyscallError); ok && os.IsNotExist(e) { + return "", os.ErrNotExist + } return "", err } return containerNameFromProcCgroup(string(content))