github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-14 13:45:06 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #124216 from iholder101/in-pod-vertical-scaling/update-cgroup-systemd-instead-of-cgroupfs

Browse Source 
[FG:InPlacePodVerticalScaling] Handle systemd cgroup driver by using libcontainer for updating pod cgroup values
This commit is contained in:
Kubernetes Prow Robot
2024-11-04 20:17:29 +00:00
committed by GitHub
parent 05bc270870 f21473b924
commit a8e1f41131
9 changed files with 27 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
pkg/kubelet/cm/cgroup_manager_linux.go
View File

@@ -21,14 +21,13 @@ import (
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
"github.com/opencontainers/runc/libcontainer/cgroups/manager"
libcontainercgroupmanager "github.com/opencontainers/runc/libcontainer/cgroups/manager"
cgroupsystemd "github.com/opencontainers/runc/libcontainer/cgroups/systemd"
libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs"
"k8s.io/klog/v2"
@@ -242,7 +241,7 @@ func (m *cgroupCommon) Destroy(cgroupConfig *CgroupConfig) error {
}()
libcontainerCgroupConfig := m.libctCgroupConfig(cgroupConfig, false)
manager, err := manager.New(libcontainerCgroupConfig)
manager, err := libcontainercgroupmanager.New(libcontainerCgroupConfig)
if err != nil {
return err
}
@@ -255,6 +254,15 @@ func (m *cgroupCommon) Destroy(cgroupConfig *CgroupConfig) error {
return nil
}
func (m *cgroupCommon) SetCgroupConfig(name CgroupName, resourceConfig *ResourceConfig) error {
containerConfig := &CgroupConfig{
Name: name,
ResourceParameters: resourceConfig,
}
return m.Update(containerConfig)
}
// getCPUWeight converts from the range [2, 262144] to [1, 10000]
func getCPUWeight(cpuShares *uint64) uint64 {
if cpuShares == nil {
@@ -362,7 +370,7 @@ func (m *cgroupCommon) Update(cgroupConfig *CgroupConfig) error {
}()
libcontainerCgroupConfig := m.libctCgroupConfig(cgroupConfig, true)
manager, err := manager.New(libcontainerCgroupConfig)
manager, err := libcontainercgroupmanager.New(libcontainerCgroupConfig)
if err != nil {
return fmt.Errorf("failed to create cgroup manager: %v", err)
}
@@ -377,7 +385,7 @@ func (m *cgroupCommon) Create(cgroupConfig *CgroupConfig) error {
}()
libcontainerCgroupConfig := m.libctCgroupConfig(cgroupConfig, true)
manager, err := manager.New(libcontainerCgroupConfig)
manager, err := libcontainercgroupmanager.New(libcontainerCgroupConfig)
if err != nil {
return err
}
@@ -475,12 +483,3 @@ func readCgroupMemoryConfig(cgroupPath string, memLimitFile string) (*ResourceCo
return &ResourceConfig{Memory: &mLim}, nil
}
func writeCgroupMemoryLimit(memoryLimitFileLocation string, resourceConfig *ResourceConfig) error {
memLimit := strconv.FormatInt(*resourceConfig.Memory, 10)
if err := os.WriteFile(memoryLimitFileLocation, []byte(memLimit), 0700); err != nil {
return fmt.Errorf("failed to write %v to %v: %w", memLimit, memoryLimitFileLocation, err)
}
//TODO(vinaykul,InPlacePodVerticalScaling): Add memory request support
return nil
}

2
pkg/kubelet/cm/cgroup_manager_unsupported.go
View File

@@ -89,7 +89,7 @@ func (m *unsupportedCgroupManager) GetCgroupConfig(name CgroupName, resource v1.
return nil, errNotSupported
}
func (m *unsupportedCgroupManager) SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error {
func (m *unsupportedCgroupManager) SetCgroupConfig(name CgroupName, resourceConfig *ResourceConfig) error {
return errNotSupported
}

46
pkg/kubelet/cm/cgroup_v1_manager_linux.go
View File

@@ -19,14 +19,11 @@ package cm
import (
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
)
@@ -123,22 +120,6 @@ func (c *cgroupV1impl) GetCgroupConfig(name CgroupName, resource v1.ResourceName
return nil, fmt.Errorf("unsupported resource %v for cgroup %v", resource, name)
}
// Set resource config for the specified resource type on the cgroup
func (c *cgroupV1impl) SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error {
cgroupPaths := c.buildCgroupPaths(name)
cgroupResourcePath, found := cgroupPaths[string(resource)]
if !found {
return fmt.Errorf("failed to build %v cgroup fs path for cgroup %v", resource, name)
}
switch resource {
case v1.ResourceCPU:
return c.setCgroupCPUConfig(cgroupResourcePath, resourceConfig)
case v1.ResourceMemory:
return c.setCgroupMemoryConfig(cgroupResourcePath, resourceConfig)
}
return nil
}
func (c *cgroupV1impl) getCgroupCPUConfig(cgroupPath string) (*ResourceConfig, error) {
cpuQuotaStr, errQ := fscommon.GetCgroupParamString(cgroupPath, "cpu.cfs_quota_us")
if errQ != nil {
@@ -159,33 +140,6 @@ func (c *cgroupV1impl) getCgroupCPUConfig(cgroupPath string) (*ResourceConfig, e
return &ResourceConfig{CPUShares: &cpuShares, CPUQuota: &cpuQuota, CPUPeriod: &cpuPeriod}, nil
}
func (c *cgroupV1impl) setCgroupCPUConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
var cpuQuotaStr, cpuPeriodStr, cpuSharesStr string
if resourceConfig.CPUQuota != nil {
cpuQuotaStr = strconv.FormatInt(*resourceConfig.CPUQuota, 10)
if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.cfs_quota_us"), []byte(cpuQuotaStr), 0700); err != nil {
return fmt.Errorf("failed to write %v to %v: %w", cpuQuotaStr, cgroupPath, err)
}
}
if resourceConfig.CPUPeriod != nil {
cpuPeriodStr = strconv.FormatUint(*resourceConfig.CPUPeriod, 10)
if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.cfs_period_us"), []byte(cpuPeriodStr), 0700); err != nil {
return fmt.Errorf("failed to write %v to %v: %w", cpuPeriodStr, cgroupPath, err)
}
}
if resourceConfig.CPUShares != nil {
cpuSharesStr = strconv.FormatUint(*resourceConfig.CPUShares, 10)
if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.shares"), []byte(cpuSharesStr), 0700); err != nil {
return fmt.Errorf("failed to write %v to %v: %w", cpuSharesStr, cgroupPath, err)
}
}
return nil
}
func (c *cgroupV1impl) setCgroupMemoryConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
return writeCgroupMemoryLimit(filepath.Join(cgroupPath, cgroupv1MemLimitFile), resourceConfig)
}
func (c *cgroupV1impl) getCgroupMemoryConfig(cgroupPath string) (*ResourceConfig, error) {
return readCgroupMemoryConfig(cgroupPath, cgroupv1MemLimitFile)
}

45
pkg/kubelet/cm/cgroup_v2_manager_linux.go
View File

@@ -98,22 +98,6 @@ func (c *cgroupV2impl) GetCgroupConfig(name CgroupName, resource v1.ResourceName
return nil, fmt.Errorf("unsupported resource %v for cgroup %v", resource, name)
}
// Set resource config for the specified resource type on the cgroup
func (c *cgroupV2impl) SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error {
cgroupPaths := c.buildCgroupPaths(name)
cgroupResourcePath, found := cgroupPaths[string(resource)]
if !found {
return fmt.Errorf("failed to build %v cgroup fs path for cgroup %v", resource, name)
}
switch resource {
case v1.ResourceCPU:
return c.setCgroupCPUConfig(cgroupResourcePath, resourceConfig)
case v1.ResourceMemory:
return c.setCgroupMemoryConfig(cgroupResourcePath, resourceConfig)
}
return nil
}
func (c *cgroupV2impl) getCgroupCPUConfig(cgroupPath string) (*ResourceConfig, error) {
var cpuLimitStr, cpuPeriodStr string
cpuLimitAndPeriod, err := fscommon.GetCgroupParamString(cgroupPath, "cpu.max")
@@ -144,35 +128,6 @@ func (c *cgroupV2impl) getCgroupCPUConfig(cgroupPath string) (*ResourceConfig, e
return &ResourceConfig{CPUShares: &cpuShares, CPUQuota: &cpuLimit, CPUPeriod: &cpuPeriod}, nil
}
func (c *cgroupV2impl) setCgroupCPUConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
if resourceConfig.CPUQuota != nil {
if resourceConfig.CPUPeriod == nil {
return fmt.Errorf("CpuPeriod must be specified in order to set CpuLimit")
}
cpuLimitStr := Cgroup2MaxCpuLimit
if *resourceConfig.CPUQuota > -1 {
cpuLimitStr = strconv.FormatInt(*resourceConfig.CPUQuota, 10)
}
cpuPeriodStr := strconv.FormatUint(*resourceConfig.CPUPeriod, 10)
cpuMaxStr := fmt.Sprintf("%s %s", cpuLimitStr, cpuPeriodStr)
if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.max"), []byte(cpuMaxStr), 0700); err != nil {
return fmt.Errorf("failed to write %v to %v: %w", cpuMaxStr, cgroupPath, err)
}
}
if resourceConfig.CPUShares != nil {
cpuWeight := cpuSharesToCPUWeight(*resourceConfig.CPUShares)
cpuWeightStr := strconv.FormatUint(cpuWeight, 10)
if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.weight"), []byte(cpuWeightStr), 0700); err != nil {
return fmt.Errorf("failed to write %v to %v: %w", cpuWeightStr, cgroupPath, err)
}
}
return nil
}
func (c *cgroupV2impl) setCgroupMemoryConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
return writeCgroupMemoryLimit(filepath.Join(cgroupPath, cgroupv2MemLimitFile), resourceConfig)
}
func (c *cgroupV2impl) getCgroupMemoryConfig(cgroupPath string) (*ResourceConfig, error) {
return readCgroupMemoryConfig(cgroupPath, cgroupv2MemLimitFile)
}

2
pkg/kubelet/cm/container_manager_stub.go
View File

@@ -103,7 +103,7 @@ func (m *podContainerManagerStub) GetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceNam
return nil, fmt.Errorf("not implemented")
}
func (m *podContainerManagerStub) SetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceName, _ *ResourceConfig) error {
func (m *podContainerManagerStub) SetPodCgroupConfig(pod *v1.Pod, resourceConfig *ResourceConfig) error {
return fmt.Errorf("not implemented")
}

2
pkg/kubelet/cm/fake_pod_container_manager.go
View File

@@ -119,7 +119,7 @@ func (cm *FakePodContainerManager) GetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceNa
return nil, nil
}
func (cm *FakePodContainerManager) SetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceName, _ *ResourceConfig) error {
func (cm *FakePodContainerManager) SetPodCgroupConfig(pod *v1.Pod, resourceConfig *ResourceConfig) error {
cm.Lock()
defer cm.Unlock()
cm.CalledFunctions = append(cm.CalledFunctions, "SetPodCgroupConfig")

6
pkg/kubelet/cm/pod_container_manager_linux.go
View File

@@ -134,9 +134,9 @@ func (m *podContainerManagerImpl) GetPodCgroupConfig(pod *v1.Pod, resource v1.Re
return m.cgroupManager.GetCgroupConfig(podCgroupName, resource)
}
func (m *podContainerManagerImpl) SetPodCgroupConfig(pod *v1.Pod, resource v1.ResourceName, resourceConfig *ResourceConfig) error {
func (m *podContainerManagerImpl) SetPodCgroupConfig(pod *v1.Pod, resourceConfig *ResourceConfig) error {
podCgroupName, _ := m.GetPodContainerName(pod)
return m.cgroupManager.SetCgroupConfig(podCgroupName, resource, resourceConfig)
return m.cgroupManager.SetCgroupConfig(podCgroupName, resourceConfig)
}
// Kill one process ID
@@ -350,6 +350,6 @@ func (m *podContainerManagerNoop) GetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceNam
return nil, nil
}
func (m *podContainerManagerNoop) SetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceName, _ *ResourceConfig) error {
func (m *podContainerManagerNoop) SetPodCgroupConfig(_ *v1.Pod, _ *ResourceConfig) error {
return nil
}

4
pkg/kubelet/cm/types.go
View File

@@ -90,7 +90,7 @@ type CgroupManager interface {
// Get the resource config values applied to the cgroup for specified resource type
GetCgroupConfig(name CgroupName, resource v1.ResourceName) (*ResourceConfig, error)
// Set resource config for the specified resource type on the cgroup
SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error
SetCgroupConfig(name CgroupName, resourceConfig *ResourceConfig) error
// Version of the cgroup implementation on the host
Version() int
}
@@ -136,5 +136,5 @@ type PodContainerManager interface {
GetPodCgroupConfig(pod *v1.Pod, resource v1.ResourceName) (*ResourceConfig, error)
// Set resource config values for the specified resource type on the pod cgroup
SetPodCgroupConfig(pod *v1.Pod, resource v1.ResourceName, resourceConfig *ResourceConfig) error
SetPodCgroupConfig(pod *v1.Pod, resourceConfig *ResourceConfig) error
}

11
pkg/kubelet/kuberuntime/kuberuntime_manager.go
View File

@@ -663,15 +663,16 @@ func (m *kubeGenericRuntimeManager) doPodResizeAction(pod *v1.Pod, podStatus *ku
var err error
switch rName {
case v1.ResourceCPU:
podCpuResources := &cm.ResourceConfig{CPUPeriod: podResources.CPUPeriod}
podCPUResources := &cm.ResourceConfig{}
if setLimitValue {
podCpuResources.CPUQuota = podResources.CPUQuota
podCPUResources.CPUPeriod = podResources.CPUPeriod
podCPUResources.CPUQuota = podResources.CPUQuota
} else {
podCpuResources.CPUShares = podResources.CPUShares
podCPUResources.CPUShares = podResources.CPUShares
}
err = pcm.SetPodCgroupConfig(pod, rName, podCpuResources)
err = pcm.SetPodCgroupConfig(pod, podCPUResources)
case v1.ResourceMemory:
err = pcm.SetPodCgroupConfig(pod, rName, podResources)
err = pcm.SetPodCgroupConfig(pod, podResources)
}
if err != nil {
klog.ErrorS(err, "Failed to set cgroup config", "resource", rName, "pod", pod.Name)