diff --git a/pkg/kubelet/cm/cgroup_manager_linux.go b/pkg/kubelet/cm/cgroup_manager_linux.go index bd7415a945b..7d1687948ef 100644 --- a/pkg/kubelet/cm/cgroup_manager_linux.go +++ b/pkg/kubelet/cm/cgroup_manager_linux.go @@ -17,6 +17,7 @@ limitations under the License. package cm import ( + "errors" "fmt" "io/ioutil" "os" @@ -29,6 +30,7 @@ import ( libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups" cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs" cgroupfs2 "github.com/opencontainers/runc/libcontainer/cgroups/fs2" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" cgroupsystemd "github.com/opencontainers/runc/libcontainer/cgroups/systemd" libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs" libcontainerdevices "github.com/opencontainers/runc/libcontainer/devices" @@ -330,27 +332,6 @@ func (m *cgroupManagerImpl) Destroy(cgroupConfig *CgroupConfig) error { return nil } -type subsystem interface { - // Name returns the name of the subsystem. - Name() string - // Set the cgroup represented by cgroup. - Set(path string, cgroup *libcontainerconfigs.Resources) error - // GetStats returns the statistics associated with the cgroup - GetStats(path string, stats *libcontainercgroups.Stats) error -} - -// getSupportedSubsystems returns a map of subsystem and if it must be mounted for the kubelet to function. -func getSupportedSubsystems() map[subsystem]bool { - supportedSubsystems := map[subsystem]bool{ - &cgroupfs.MemoryGroup{}: true, - &cgroupfs.CpuGroup{}: true, - &cgroupfs.PidsGroup{}: true, - } - // not all hosts support hugetlb cgroup, and in the absent of hugetlb, we will fail silently by reporting no capacity. - supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false - return supportedSubsystems -} - // getCpuWeight converts from the range [2, 262144] to [1, 10000] func getCpuWeight(cpuShares *uint64) uint64 { if cpuShares == nil { @@ -393,51 +374,6 @@ func getSupportedUnifiedControllers() sets.String { return supportedControllers.Intersection(availableRootControllers) } -// propagateControllers on an unified hierarchy enables all the supported controllers for the specified cgroup -func propagateControllers(path string) error { - if err := os.MkdirAll(filepath.Join(cmutil.CgroupRoot, path), 0755); err != nil { - return fmt.Errorf("failed to create cgroup %q : %v", path, err) - } - - // Retrieve all the supported controllers from the cgroup root - controllersFileContent, err := ioutil.ReadFile(filepath.Join(cmutil.CgroupRoot, "cgroup.controllers")) - if err != nil { - return fmt.Errorf("failed to read controllers from %q : %v", cmutil.CgroupRoot, err) - } - - supportedControllers := getSupportedUnifiedControllers() - - // The retrieved content looks like: "cpuset cpu io memory hugetlb pids". Prepend each of the controllers - // with '+', so we have something like "+cpuset +cpu +io +memory +hugetlb +pids" - controllers := "" - for _, controller := range strings.Fields(string(controllersFileContent)) { - // ignore controllers we don't care about - if !supportedControllers.Has(controller) { - continue - } - - sep := " +" - if controllers == "" { - sep = "+" - } - controllers = controllers + sep + controller - } - - current := cmutil.CgroupRoot - - // Write the controllers list to each "cgroup.subtree_control" file until it reaches the parent cgroup. - // For the /foo/bar/baz cgroup, controllers must be enabled sequentially in the files: - // - /sys/fs/cgroup/foo/cgroup.subtree_control - // - /sys/fs/cgroup/foo/bar/cgroup.subtree_control - for _, p := range strings.Split(filepath.Dir(path), "/") { - current = filepath.Join(current, p) - if err := ioutil.WriteFile(filepath.Join(current, "cgroup.subtree_control"), []byte(controllers), 0755); err != nil { - return fmt.Errorf("failed to enable controllers on %q: %v", cmutil.CgroupRoot, err) - } - } - return nil -} - func (m *cgroupManagerImpl) toResources(resourceConfig *ResourceConfig) *libcontainerconfigs.Resources { resources := &libcontainerconfigs.Resources{ Devices: []*libcontainerdevices.Rule{ @@ -535,10 +471,6 @@ func (m *cgroupManagerImpl) Update(cgroupConfig *CgroupConfig) error { } if unified { - if err := propagateControllers(libcontainerCgroupConfig.Path); err != nil { - return err - } - supportedControllers := getSupportedUnifiedControllers() if !supportedControllers.Has("hugetlb") { resources.HugetlbLimit = nil @@ -669,53 +601,21 @@ func (m *cgroupManagerImpl) ReduceCPULimits(cgroupName CgroupName) error { return m.Update(containerConfig) } -func getStatsSupportedSubsystems(cgroupPaths map[string]string) (*libcontainercgroups.Stats, error) { - stats := libcontainercgroups.NewStats() - for sys, required := range getSupportedSubsystems() { - if _, ok := cgroupPaths[sys.Name()]; !ok { - if required { - return nil, fmt.Errorf("failed to find subsystem mount for required subsystem: %v", sys.Name()) - } - // the cgroup is not mounted, but its not required so continue... - klog.V(6).InfoS("Unable to find subsystem mount for optional subsystem", "subsystemName", sys.Name()) - continue - } - if err := sys.GetStats(cgroupPaths[sys.Name()], stats); err != nil { - return nil, fmt.Errorf("failed to get stats for supported subsystems : %v", err) - } - } - return stats, nil -} - -func toResourceStats(stats *libcontainercgroups.Stats) *ResourceStats { - return &ResourceStats{ - MemoryStats: &MemoryStats{ - Usage: int64(stats.MemoryStats.Usage.Usage), - }, - } -} - -// Get sets the ResourceParameters of the specified cgroup as read from the cgroup fs -func (m *cgroupManagerImpl) GetResourceStats(name CgroupName) (*ResourceStats, error) { - var err error - var stats *libcontainercgroups.Stats +// MemoryUsage returns the current memory usage of the specified cgroup, +// as read from cgroupfs. +func (m *cgroupManagerImpl) MemoryUsage(name CgroupName) (int64, error) { + var path, file string if libcontainercgroups.IsCgroup2UnifiedMode() { - cgroupPath := m.buildCgroupUnifiedPath(name) - manager, err := cgroupfs2.NewManager(nil, cgroupPath, false) - if err != nil { - return nil, fmt.Errorf("failed to create cgroup v2 manager: %v", err) - } - - stats, err = manager.GetStats() - if err != nil { - return nil, fmt.Errorf("failed to get stats for cgroup %v: %v", name, err) - } + path = m.buildCgroupUnifiedPath(name) + file = "memory.current" } else { - cgroupPaths := m.buildCgroupPaths(name) - stats, err = getStatsSupportedSubsystems(cgroupPaths) - if err != nil { - return nil, fmt.Errorf("failed to get stats supported cgroup subsystems for cgroup %v: %v", name, err) + mp, ok := m.subsystems.MountPoints["memory"] + if !ok { // should not happen + return -1, errors.New("no cgroup v1 mountpoint for memory controller found") } + path = mp + "/" + m.Name(name) + file = "memory.usage_in_bytes" } - return toResourceStats(stats), nil + val, err := fscommon.GetCgroupParamUint(path, file) + return int64(val), err } diff --git a/pkg/kubelet/cm/cgroup_manager_unsupported.go b/pkg/kubelet/cm/cgroup_manager_unsupported.go index e8aaf2d2ba4..fb7116a4d6d 100644 --- a/pkg/kubelet/cm/cgroup_manager_unsupported.go +++ b/pkg/kubelet/cm/cgroup_manager_unsupported.go @@ -18,10 +18,12 @@ limitations under the License. package cm -import "fmt" +import "errors" type unsupportedCgroupManager struct{} +var errNotSupported = errors.New("Cgroup Manager is not supported in this build") + // Make sure that unsupportedCgroupManager implements the CgroupManager interface var _ CgroupManager = &unsupportedCgroupManager{} @@ -51,11 +53,11 @@ func (m *unsupportedCgroupManager) Update(_ *CgroupConfig) error { } func (m *unsupportedCgroupManager) Create(_ *CgroupConfig) error { - return fmt.Errorf("Cgroup Manager is not supported in this build") + return errNotSupported } -func (m *unsupportedCgroupManager) GetResourceStats(name CgroupName) (*ResourceStats, error) { - return nil, fmt.Errorf("Cgroup Manager is not supported in this build") +func (m *unsupportedCgroupManager) MemoryUsage(_ CgroupName) (int64, error) { + return -1, errNotSupported } func (m *unsupportedCgroupManager) Pids(_ CgroupName) []int { diff --git a/pkg/kubelet/cm/qos_container_manager_linux.go b/pkg/kubelet/cm/qos_container_manager_linux.go index 209b70b717d..0e3ba2c071e 100644 --- a/pkg/kubelet/cm/qos_container_manager_linux.go +++ b/pkg/kubelet/cm/qos_container_manager_linux.go @@ -28,7 +28,7 @@ import ( units "github.com/docker/go-units" cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/v1/resource" v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" @@ -247,12 +247,11 @@ func (m *qosContainerManagerImpl) retrySetMemoryReserve(configs map[v1.PodQOSCla // Attempt to set the limit near the current usage to put pressure // on the cgroup and prevent further growth. for qos, config := range configs { - stats, err := m.cgroupManager.GetResourceStats(config.Name) + usage, err := m.cgroupManager.MemoryUsage(config.Name) if err != nil { klog.V(2).InfoS("Failed to get resource stats", "err", err) return } - usage := stats.MemoryStats.Usage // Because there is no good way to determine of the original Update() // on the memory resource was successful, we determine failure of the diff --git a/pkg/kubelet/cm/types.go b/pkg/kubelet/cm/types.go index e6011797443..b28e2e93fcd 100644 --- a/pkg/kubelet/cm/types.go +++ b/pkg/kubelet/cm/types.go @@ -17,7 +17,7 @@ limitations under the License. package cm import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" ) @@ -53,18 +53,6 @@ type CgroupConfig struct { ResourceParameters *ResourceConfig } -// MemoryStats holds the on-demand statistics from the memory cgroup -type MemoryStats struct { - // Memory usage (in bytes). - Usage int64 -} - -// ResourceStats holds on-demand statistics from various cgroup subsystems -type ResourceStats struct { - // Memory statistics. - MemoryStats *MemoryStats -} - // CgroupManager allows for cgroup management. // Supports Cgroup Creation ,Deletion and Updates. type CgroupManager interface { @@ -90,8 +78,8 @@ type CgroupManager interface { Pids(name CgroupName) []int // ReduceCPULimits reduces the CPU CFS values to the minimum amount of shares. ReduceCPULimits(cgroupName CgroupName) error - // GetResourceStats returns statistics of the specified cgroup as read from the cgroup fs. - GetResourceStats(name CgroupName) (*ResourceStats, error) + // MemoryUsage returns current memory usage of the specified cgroup, as read from the cgroupfs. + MemoryUsage(name CgroupName) (int64, error) } // QOSContainersInfo stores the names of containers per qos