Merge pull request #41349 from derekwaynecarr/enable-pod-cgroups

Automatic merge from submit-queue (batch tested with PRs 41349, 41532, 41256, 41587, 41657)

Enable pod level cgroups by default

**What this PR does / why we need it**:
It enables pod level cgroups by default.

**Special notes for your reviewer**:
This is intended to be enabled by default on 2/14/2017 per the plan outlined here:
https://github.com/kubernetes/community/pull/314

**Release note**:
```release-note
Each pod has its own associated cgroup by default.
```

cmd/kubelet/app/server.go

@@ -452,6 +452,10 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
 		if s.SystemCgroups != "" && s.CgroupRoot == "" {
 			return fmt.Errorf("invalid configuration: system container was specified and cgroup root was not specified")
 		}
+		if s.CgroupsPerQOS && s.CgroupRoot == "" {
+			glog.Infof("--cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /")
+			s.CgroupRoot = "/"
+		}
 		kubeDeps.ContainerManager, err = cm.NewContainerManager(
 			kubeDeps.Mounter,
 			kubeDeps.CAdvisorInterface,

hack/local-up-cluster.sh

@@ -36,10 +36,7 @@ NET_PLUGIN_DIR=${NET_PLUGIN_DIR:-""}
 SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/24}
 FIRST_SERVICE_CLUSTER_IP=${FIRST_SERVICE_CLUSTER_IP:-10.0.0.1}
 # if enabled, must set CGROUP_ROOT
-CGROUPS_PER_QOS=${CGROUPS_PER_QOS:-false}
-# this is not defaulted to preserve backward compatibility.
-# if EXPERIMENTAL_CGROUPS_PER_QOS is enabled, recommend setting to /
-CGROUP_ROOT=${CGROUP_ROOT:-""}
+CGROUPS_PER_QOS=${CGROUPS_PER_QOS:-true}
 # name of the cgroup driver, i.e. cgroupfs or systemd
 CGROUP_DRIVER=${CGROUP_DRIVER:-""}
 # owner of client certs, default to current user if not specified
@@ -594,7 +591,6 @@ function start_kubelet {
         --enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}" \
         --cgroups-per-qos=${CGROUPS_PER_QOS} \
         --cgroup-driver=${CGROUP_DRIVER} \
-        --cgroup-root=${CGROUP_ROOT} \
         --keep-terminated-pod-volumes=true \
         --eviction-hard=${EVICTION_HARD} \
         --eviction-soft=${EVICTION_SOFT} \

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -204,9 +204,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 	if obj.CertDirectory == "" {
 		obj.CertDirectory = "/var/run/kubernetes"
 	}
-	if obj.CgroupsPerQOS == nil {
-		obj.CgroupsPerQOS = boolVar(false)
-	}
 	if obj.ContainerRuntime == "" {
 		obj.ContainerRuntime = "docker"
 	}
@@ -395,22 +392,12 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 		obj.IPTablesDropBit = &temp
 	}
 	if obj.CgroupsPerQOS == nil {
-		temp := false
+		temp := true
 		obj.CgroupsPerQOS = &temp
 	}
 	if obj.CgroupDriver == "" {
 		obj.CgroupDriver = "cgroupfs"
 	}
-	// NOTE: this is for backwards compatibility with earlier releases where cgroup-root was optional.
-	// if cgroups per qos is not enabled, and cgroup-root is not specified, we need to default to the
-	// container runtime default and not default to the root cgroup.
-	if obj.CgroupsPerQOS != nil {
-		if *obj.CgroupsPerQOS {
-			if obj.CgroupRoot == "" {
-				obj.CgroupRoot = "/"
-			}
-		}
-	}
 	if obj.EnableCRI == nil {
 		obj.EnableCRI = boolVar(true)
 	}

pkg/kubelet/cm/cgroup_manager_linux.go

@@ -49,25 +49,37 @@ func ConvertCgroupNameToSystemd(cgroupName CgroupName, outputToCgroupFs bool) st
 	name := string(cgroupName)
 	result := ""
 	if name != "" && name != "/" {
-		// systemd treats - as a step in the hierarchy, we convert all - to _
-		name = strings.Replace(name, "-", "_", -1)
 		parts := strings.Split(name, "/")
+		results := []string{}
 		for _, part := range parts {
-			// ignore leading stuff for now
+			// ignore leading stuff
 			if part == "" {
 				continue
 			}
-			if len(result) > 0 {
-				result = result + "-"
+			// detect if we are given a systemd style name.
+			// if so, we do not want to do double encoding.
+			if strings.HasSuffix(part, ".slice") {
+				part = strings.TrimSuffix(part, ".slice")
+				separatorIndex := strings.LastIndex(part, "-")
+				if separatorIndex >= 0 && separatorIndex < len(part) {
+					part = part[separatorIndex+1:]
+				}
+			} else {
+				// systemd treats - as a step in the hierarchy, we convert all - to _
+				part = strings.Replace(part, "-", "_", -1)
 			}
-			result = result + part
+			results = append(results, part)
 		}
+		// each part is appended with systemd style -
+		result = strings.Join(results, "-")
 	} else {
 		// root converts to -
 		result = "-"
 	}
 	// always have a .slice suffix
-	result = result + ".slice"
+	if !strings.HasSuffix(result, ".slice") {
+		result = result + ".slice"
+	}
 
 	// if the caller desired the result in cgroupfs format...
 	if outputToCgroupFs {

pkg/kubelet/cm/cgroup_manager_linux_test.go

@@ -29,6 +29,26 @@ func TestLibcontainerAdapterAdaptToSystemd(t *testing.T) {
 			input:    "/",
 			expected: "-.slice",
 		},
+		{
+			input:    "/system.slice",
+			expected: "system.slice",
+		},
+		{
+			input:    "/system.slice/Burstable",
+			expected: "system-Burstable.slice",
+		},
+		{
+			input:    "/Burstable.slice/Burstable-pod_123.slice",
+			expected: "Burstable-pod_123.slice",
+		},
+		{
+			input:    "/test.slice/test-a.slice/test-a-b.slice",
+			expected: "test-a-b.slice",
+		},
+		{
+			input:    "/test.slice/test-a.slice/test-a-b.slice/Burstable",
+			expected: "test-a-b-Burstable.slice",
+		},
 		{
 			input:    "/Burstable",
 			expected: "Burstable.slice",

pkg/kubelet/cm/container_manager_linux.go

@@ -220,6 +220,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
 		if !cgroupManager.Exists(CgroupName(nodeConfig.CgroupRoot)) {
 			return nil, fmt.Errorf("invalid configuration: cgroup-root doesn't exist: %v", err)
 		}
+		glog.Infof("container manager verified cgroup-root exists: %v", nodeConfig.CgroupRoot)
 	}
 	return &containerManagerImpl{
 		cadvisorInterface: cadvisorInterface,

pkg/kubelet/cm/pod_container_manager_linux.go

@@ -194,6 +194,9 @@ func (m *podContainerManagerImpl) GetAllPodsFromCgroups() (map[types.UID]CgroupN
 			qc := path.Join(val, qcConversion)
 			dirInfo, err := ioutil.ReadDir(qc)
 			if err != nil {
+				if os.IsNotExist(err) {
+					continue
+				}
 				return nil, fmt.Errorf("failed to read the cgroup directory %v : %v", qc, err)
 			}
 			for i := range dirInfo {

test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties

@@ -5,5 +5,5 @@ GCE_PROJECT=k8s-jkns-ci-node-e2e
 CLEANUP=true
 GINKGO_FLAGS='--skip="\[Flaky\]"'
 TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'
-KUBELET_ARGS='--cgroups-per-qos=false --cgroup-root=/'
+KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/'
 PARALLELISM=1

test/e2e_node/jenkins/conformance/jenkins-conformance.properties

@@ -3,4 +3,4 @@ GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml
 GCE_ZONE=us-central1-f
 GCE_PROJECT=k8s-jkns-ci-node-e2e
 CLEANUP=true
-KUBELET_ARGS='--cgroups-per-qos=false --cgroup-root=/'
+KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/'