From 73b2d831254756a3595625432a6c23398e180472 Mon Sep 17 00:00:00 2001 From: mattjmcnaughton Date: Mon, 20 Jan 2020 10:22:31 -0500 Subject: [PATCH] Refactor docker specific oom const out of qos pkg Previously, `pkg/kubelet/qos` contained two different docker-specific OOM constants. One set the oom adj for sandbox docker containers and the other set the oom adj for the docker daemon. Move both to be closer to their actual usages in `dockershim`. This change addresses a TODO and leads us towards the overall goal of making `pkg/kubelet` runtime agnostic except for `dockershim`. --- pkg/kubelet/dockershim/BUILD | 1 - pkg/kubelet/dockershim/cm/BUILD | 2 -- pkg/kubelet/dockershim/cm/container_manager_linux.go | 7 ++++--- pkg/kubelet/dockershim/docker_sandbox.go | 11 ++++++++--- pkg/kubelet/qos/policy.go | 7 ------- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/pkg/kubelet/dockershim/BUILD b/pkg/kubelet/dockershim/BUILD index 751d42eeead..b86a408530c 100644 --- a/pkg/kubelet/dockershim/BUILD +++ b/pkg/kubelet/dockershim/BUILD @@ -51,7 +51,6 @@ go_library( "//pkg/kubelet/dockershim/network/kubenet:go_default_library", "//pkg/kubelet/kuberuntime:go_default_library", "//pkg/kubelet/leaky:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//pkg/kubelet/server/streaming:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/cache:go_default_library", diff --git a/pkg/kubelet/dockershim/cm/BUILD b/pkg/kubelet/dockershim/cm/BUILD index bdf49dcae40..fa9b5981131 100644 --- a/pkg/kubelet/dockershim/cm/BUILD +++ b/pkg/kubelet/dockershim/cm/BUILD @@ -18,7 +18,6 @@ go_library( "@io_bazel_rules_go//go/platform:android": [ "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/dockershim/libdocker:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", @@ -40,7 +39,6 @@ go_library( "@io_bazel_rules_go//go/platform:linux": [ "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/dockershim/libdocker:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", diff --git a/pkg/kubelet/dockershim/cm/container_manager_linux.go b/pkg/kubelet/dockershim/cm/container_manager_linux.go index 37ab7dcb69d..da38ec0d1b1 100644 --- a/pkg/kubelet/dockershim/cm/container_manager_linux.go +++ b/pkg/kubelet/dockershim/cm/container_manager_linux.go @@ -31,7 +31,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" kubecm "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) @@ -45,8 +44,10 @@ const ( // The minimum memory limit allocated to docker container: 150Mi minDockerMemoryLimit = 150 * 1024 * 1024 - // The Docker OOM score adjustment. - dockerOOMScoreAdj = qos.DockerOOMScoreAdj + // The OOM score adjustment for the docker process (i.e. the docker + // daemon). Essentially, makes docker very unlikely to experience an oom + // kill. + dockerOOMScoreAdj = -999 ) var ( diff --git a/pkg/kubelet/dockershim/docker_sandbox.go b/pkg/kubelet/dockershim/docker_sandbox.go index 1824cdbadbf..ac22c4d7add 100644 --- a/pkg/kubelet/dockershim/docker_sandbox.go +++ b/pkg/kubelet/dockershim/docker_sandbox.go @@ -34,7 +34,6 @@ import ( "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" - "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -44,6 +43,13 @@ const ( // Various default sandbox resources requests/limits. defaultSandboxCPUshares int64 = 2 + // defaultSandboxOOMAdj is the oom score adjustment for the docker + // sandbox container. Using this OOM adj makes it very unlikely, but not + // impossible, that the defaultSandox will experience an oom kill. -998 + // is chosen to signify sandbox should be OOM killed before other more + // vital processes like the docker daemon, the kubelet, etc... + defaultSandboxOOMAdj int = -998 + // Name of the underlying container runtime runtimeName = "docker" ) @@ -643,8 +649,7 @@ func (ds *dockerService) makeSandboxDockerConfig(c *runtimeapi.PodSandboxConfig, createConfig.Config.ExposedPorts = exposedPorts hc.PortBindings = portBindings - // TODO: Get rid of the dependency on kubelet internal package. - hc.OomScoreAdj = qos.PodInfraOOMAdj + hc.OomScoreAdj = defaultSandboxOOMAdj // Apply resource options. if err := ds.applySandboxResources(hc, c.GetLinux()); err != nil { diff --git a/pkg/kubelet/qos/policy.go b/pkg/kubelet/qos/policy.go index b1e8ed998bc..f005fc59d50 100644 --- a/pkg/kubelet/qos/policy.go +++ b/pkg/kubelet/qos/policy.go @@ -23,15 +23,8 @@ import ( ) const ( - // PodInfraOOMAdj is very docker specific. For arbitrary runtime, it may not make - // sense to set sandbox level oom score, e.g. a sandbox could only be a namespace - // without a process. - // TODO: Handle infra container oom score adj in a runtime agnostic way. - PodInfraOOMAdj int = -998 // KubeletOOMScoreAdj is the OOM score adjustment for Kubelet KubeletOOMScoreAdj int = -999 - // DockerOOMScoreAdj is the OOM score adjustment for Docker - DockerOOMScoreAdj int = -999 // KubeProxyOOMScoreAdj is the OOM score adjustment for kube-proxy KubeProxyOOMScoreAdj int = -999 guaranteedOOMScoreAdj int = -998