Merge pull request #102218 from kolyshkin/cgroup-cleanups

pkg/kubelet/cm: cgroup-related cleanups

pkg/kubelet/cm/cgroup_manager_linux.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cm
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -29,6 +30,7 @@ import (
 	libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
 	cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
 	cgroupfs2 "github.com/opencontainers/runc/libcontainer/cgroups/fs2"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
 	cgroupsystemd "github.com/opencontainers/runc/libcontainer/cgroups/systemd"
 	libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs"
 	libcontainerdevices "github.com/opencontainers/runc/libcontainer/devices"
@@ -330,27 +332,6 @@ func (m *cgroupManagerImpl) Destroy(cgroupConfig *CgroupConfig) error {
 	return nil
 }
 
-type subsystem interface {
-	// Name returns the name of the subsystem.
-	Name() string
-	// Set the cgroup represented by cgroup.
-	Set(path string, cgroup *libcontainerconfigs.Resources) error
-	// GetStats returns the statistics associated with the cgroup
-	GetStats(path string, stats *libcontainercgroups.Stats) error
-}
-
-// getSupportedSubsystems returns a map of subsystem and if it must be mounted for the kubelet to function.
-func getSupportedSubsystems() map[subsystem]bool {
-	supportedSubsystems := map[subsystem]bool{
-		&cgroupfs.MemoryGroup{}: true,
-		&cgroupfs.CpuGroup{}:    true,
-		&cgroupfs.PidsGroup{}:   true,
-	}
-	// not all hosts support hugetlb cgroup, and in the absent of hugetlb, we will fail silently by reporting no capacity.
-	supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false
-	return supportedSubsystems
-}
-
 // getCpuWeight converts from the range [2, 262144] to [1, 10000]
 func getCpuWeight(cpuShares *uint64) uint64 {
 	if cpuShares == nil {
@@ -393,51 +374,6 @@ func getSupportedUnifiedControllers() sets.String {
 	return supportedControllers.Intersection(availableRootControllers)
 }
 
-// propagateControllers on an unified hierarchy enables all the supported controllers for the specified cgroup
-func propagateControllers(path string) error {
-	if err := os.MkdirAll(filepath.Join(cmutil.CgroupRoot, path), 0755); err != nil {
-		return fmt.Errorf("failed to create cgroup %q : %v", path, err)
-	}
-
-	// Retrieve all the supported controllers from the cgroup root
-	controllersFileContent, err := ioutil.ReadFile(filepath.Join(cmutil.CgroupRoot, "cgroup.controllers"))
-	if err != nil {
-		return fmt.Errorf("failed to read controllers from %q : %v", cmutil.CgroupRoot, err)
-	}
-
-	supportedControllers := getSupportedUnifiedControllers()
-
-	// The retrieved content looks like: "cpuset cpu io memory hugetlb pids".  Prepend each of the controllers
-	// with '+', so we have something like "+cpuset +cpu +io +memory +hugetlb +pids"
-	controllers := ""
-	for _, controller := range strings.Fields(string(controllersFileContent)) {
-		// ignore controllers we don't care about
-		if !supportedControllers.Has(controller) {
-			continue
-		}
-
-		sep := " +"
-		if controllers == "" {
-			sep = "+"
-		}
-		controllers = controllers + sep + controller
-	}
-
-	current := cmutil.CgroupRoot
-
-	// Write the controllers list to each "cgroup.subtree_control" file until it reaches the parent cgroup.
-	// For the /foo/bar/baz cgroup, controllers must be enabled sequentially in the files:
-	// - /sys/fs/cgroup/foo/cgroup.subtree_control
-	// - /sys/fs/cgroup/foo/bar/cgroup.subtree_control
-	for _, p := range strings.Split(filepath.Dir(path), "/") {
-		current = filepath.Join(current, p)
-		if err := ioutil.WriteFile(filepath.Join(current, "cgroup.subtree_control"), []byte(controllers), 0755); err != nil {
-			return fmt.Errorf("failed to enable controllers on %q: %v", cmutil.CgroupRoot, err)
-		}
-	}
-	return nil
-}
-
 func (m *cgroupManagerImpl) toResources(resourceConfig *ResourceConfig) *libcontainerconfigs.Resources {
 	resources := &libcontainerconfigs.Resources{
 		Devices: []*libcontainerdevices.Rule{
@@ -535,10 +471,6 @@ func (m *cgroupManagerImpl) Update(cgroupConfig *CgroupConfig) error {
 	}
 
 	if unified {
-		if err := propagateControllers(libcontainerCgroupConfig.Path); err != nil {
-			return err
-		}
-
 		supportedControllers := getSupportedUnifiedControllers()
 		if !supportedControllers.Has("hugetlb") {
 			resources.HugetlbLimit = nil
@@ -669,53 +601,21 @@ func (m *cgroupManagerImpl) ReduceCPULimits(cgroupName CgroupName) error {
 	return m.Update(containerConfig)
 }
 
-func getStatsSupportedSubsystems(cgroupPaths map[string]string) (*libcontainercgroups.Stats, error) {
+// MemoryUsage returns the current memory usage of the specified cgroup,
-	stats := libcontainercgroups.NewStats()
+// as read from cgroupfs.
-	for sys, required := range getSupportedSubsystems() {
+func (m *cgroupManagerImpl) MemoryUsage(name CgroupName) (int64, error) {
-		if _, ok := cgroupPaths[sys.Name()]; !ok {
+	var path, file string
-			if required {
-				return nil, fmt.Errorf("failed to find subsystem mount for required subsystem: %v", sys.Name())
-			}
-			// the cgroup is not mounted, but its not required so continue...
-			klog.V(6).InfoS("Unable to find subsystem mount for optional subsystem", "subsystemName", sys.Name())
-			continue
-		}
-		if err := sys.GetStats(cgroupPaths[sys.Name()], stats); err != nil {
-			return nil, fmt.Errorf("failed to get stats for supported subsystems : %v", err)
-		}
-	}
-	return stats, nil
-}
-
-func toResourceStats(stats *libcontainercgroups.Stats) *ResourceStats {
-	return &ResourceStats{
-		MemoryStats: &MemoryStats{
-			Usage: int64(stats.MemoryStats.Usage.Usage),
-		},
-	}
-}
-
-// Get sets the ResourceParameters of the specified cgroup as read from the cgroup fs
-func (m *cgroupManagerImpl) GetResourceStats(name CgroupName) (*ResourceStats, error) {
-	var err error
-	var stats *libcontainercgroups.Stats
 	if libcontainercgroups.IsCgroup2UnifiedMode() {
-		cgroupPath := m.buildCgroupUnifiedPath(name)
+		path = m.buildCgroupUnifiedPath(name)
-		manager, err := cgroupfs2.NewManager(nil, cgroupPath, false)
+		file = "memory.current"
-		if err != nil {
-			return nil, fmt.Errorf("failed to create cgroup v2 manager: %v", err)
-		}
-
-		stats, err = manager.GetStats()
-		if err != nil {
-			return nil, fmt.Errorf("failed to get stats for cgroup %v: %v", name, err)
-		}
 	} else {
-		cgroupPaths := m.buildCgroupPaths(name)
+		mp, ok := m.subsystems.MountPoints["memory"]
-		stats, err = getStatsSupportedSubsystems(cgroupPaths)
+		if !ok { // should not happen
-		if err != nil {
+			return -1, errors.New("no cgroup v1 mountpoint for memory controller found")
-			return nil, fmt.Errorf("failed to get stats supported cgroup subsystems for cgroup %v: %v", name, err)
 		}
+		path = mp + "/" + m.Name(name)
+		file = "memory.usage_in_bytes"
 	}
-	return toResourceStats(stats), nil
+	val, err := fscommon.GetCgroupParamUint(path, file)
+	return int64(val), err
 }

pkg/kubelet/cm/cgroup_manager_unsupported.go

@@ -18,10 +18,12 @@ limitations under the License.
 
 package cm
 
-import "fmt"
+import "errors"
 
 type unsupportedCgroupManager struct{}
 
+var errNotSupported = errors.New("Cgroup Manager is not supported in this build")
+
 // Make sure that unsupportedCgroupManager implements the CgroupManager interface
 var _ CgroupManager = &unsupportedCgroupManager{}
 
@@ -51,11 +53,11 @@ func (m *unsupportedCgroupManager) Update(_ *CgroupConfig) error {
 }
 
 func (m *unsupportedCgroupManager) Create(_ *CgroupConfig) error {
-	return fmt.Errorf("Cgroup Manager is not supported in this build")
+	return errNotSupported
 }
 
-func (m *unsupportedCgroupManager) GetResourceStats(name CgroupName) (*ResourceStats, error) {
+func (m *unsupportedCgroupManager) MemoryUsage(_ CgroupName) (int64, error) {
-	return nil, fmt.Errorf("Cgroup Manager is not supported in this build")
+	return -1, errNotSupported
 }
 
 func (m *unsupportedCgroupManager) Pids(_ CgroupName) []int {

pkg/kubelet/cm/qos_container_manager_linux.go

@@ -28,7 +28,7 @@ import (
 
 	units "github.com/docker/go-units"
 	cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api/v1/resource"
 	v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
@@ -247,12 +247,11 @@ func (m *qosContainerManagerImpl) retrySetMemoryReserve(configs map[v1.PodQOSCla
 	// Attempt to set the limit near the current usage to put pressure
 	// on the cgroup and prevent further growth.
 	for qos, config := range configs {
-		stats, err := m.cgroupManager.GetResourceStats(config.Name)
+		usage, err := m.cgroupManager.MemoryUsage(config.Name)
 		if err != nil {
 			klog.V(2).InfoS("Failed to get resource stats", "err", err)
 			return
 		}
-		usage := stats.MemoryStats.Usage
 
 		// Because there is no good way to determine of the original Update()
 		// on the memory resource was successful, we determine failure of the

pkg/kubelet/cm/types.go

@@ -17,7 +17,7 @@ limitations under the License.
 package cm
 
 import (
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 )
 
@@ -53,18 +53,6 @@ type CgroupConfig struct {
 	ResourceParameters *ResourceConfig
 }
 
-// MemoryStats holds the on-demand statistics from the memory cgroup
-type MemoryStats struct {
-	// Memory usage (in bytes).
-	Usage int64
-}
-
-// ResourceStats holds on-demand statistics from various cgroup subsystems
-type ResourceStats struct {
-	// Memory statistics.
-	MemoryStats *MemoryStats
-}
-
 // CgroupManager allows for cgroup management.
 // Supports Cgroup Creation ,Deletion and Updates.
 type CgroupManager interface {
@@ -90,8 +78,8 @@ type CgroupManager interface {
 	Pids(name CgroupName) []int
 	// ReduceCPULimits reduces the CPU CFS values to the minimum amount of shares.
 	ReduceCPULimits(cgroupName CgroupName) error
-	// GetResourceStats returns statistics of the specified cgroup as read from the cgroup fs.
+	// MemoryUsage returns current memory usage of the specified cgroup, as read from the cgroupfs.
-	GetResourceStats(name CgroupName) (*ResourceStats, error)
+	MemoryUsage(name CgroupName) (int64, error)
 }
 
 // QOSContainersInfo stores the names of containers per qos