diff --git a/pkg/kubelet/cm/cgroup_manager_linux.go b/pkg/kubelet/cm/cgroup_manager_linux.go index 9f8fe3f40f1..6bcb1298c15 100644 --- a/pkg/kubelet/cm/cgroup_manager_linux.go +++ b/pkg/kubelet/cm/cgroup_manager_linux.go @@ -21,14 +21,13 @@ import ( "os" "path" "path/filepath" - "strconv" "strings" "sync" "time" libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" - "github.com/opencontainers/runc/libcontainer/cgroups/manager" + libcontainercgroupmanager "github.com/opencontainers/runc/libcontainer/cgroups/manager" cgroupsystemd "github.com/opencontainers/runc/libcontainer/cgroups/systemd" libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs" "k8s.io/klog/v2" @@ -242,7 +241,7 @@ func (m *cgroupCommon) Destroy(cgroupConfig *CgroupConfig) error { }() libcontainerCgroupConfig := m.libctCgroupConfig(cgroupConfig, false) - manager, err := manager.New(libcontainerCgroupConfig) + manager, err := libcontainercgroupmanager.New(libcontainerCgroupConfig) if err != nil { return err } @@ -255,6 +254,15 @@ func (m *cgroupCommon) Destroy(cgroupConfig *CgroupConfig) error { return nil } +func (m *cgroupCommon) SetCgroupConfig(name CgroupName, resourceConfig *ResourceConfig) error { + containerConfig := &CgroupConfig{ + Name: name, + ResourceParameters: resourceConfig, + } + + return m.Update(containerConfig) +} + // getCPUWeight converts from the range [2, 262144] to [1, 10000] func getCPUWeight(cpuShares *uint64) uint64 { if cpuShares == nil { @@ -362,7 +370,7 @@ func (m *cgroupCommon) Update(cgroupConfig *CgroupConfig) error { }() libcontainerCgroupConfig := m.libctCgroupConfig(cgroupConfig, true) - manager, err := manager.New(libcontainerCgroupConfig) + manager, err := libcontainercgroupmanager.New(libcontainerCgroupConfig) if err != nil { return fmt.Errorf("failed to create cgroup manager: %v", err) } @@ -377,7 +385,7 @@ func (m *cgroupCommon) Create(cgroupConfig *CgroupConfig) error { }() libcontainerCgroupConfig := m.libctCgroupConfig(cgroupConfig, true) - manager, err := manager.New(libcontainerCgroupConfig) + manager, err := libcontainercgroupmanager.New(libcontainerCgroupConfig) if err != nil { return err } @@ -475,12 +483,3 @@ func readCgroupMemoryConfig(cgroupPath string, memLimitFile string) (*ResourceCo return &ResourceConfig{Memory: &mLim}, nil } - -func writeCgroupMemoryLimit(memoryLimitFileLocation string, resourceConfig *ResourceConfig) error { - memLimit := strconv.FormatInt(*resourceConfig.Memory, 10) - if err := os.WriteFile(memoryLimitFileLocation, []byte(memLimit), 0700); err != nil { - return fmt.Errorf("failed to write %v to %v: %w", memLimit, memoryLimitFileLocation, err) - } - //TODO(vinaykul,InPlacePodVerticalScaling): Add memory request support - return nil -} diff --git a/pkg/kubelet/cm/cgroup_manager_unsupported.go b/pkg/kubelet/cm/cgroup_manager_unsupported.go index c2c6e52d2ce..a98ea0c9b5f 100644 --- a/pkg/kubelet/cm/cgroup_manager_unsupported.go +++ b/pkg/kubelet/cm/cgroup_manager_unsupported.go @@ -89,7 +89,7 @@ func (m *unsupportedCgroupManager) GetCgroupConfig(name CgroupName, resource v1. return nil, errNotSupported } -func (m *unsupportedCgroupManager) SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error { +func (m *unsupportedCgroupManager) SetCgroupConfig(name CgroupName, resourceConfig *ResourceConfig) error { return errNotSupported } diff --git a/pkg/kubelet/cm/cgroup_v1_manager_linux.go b/pkg/kubelet/cm/cgroup_v1_manager_linux.go index 7325ba797c9..afd02925833 100644 --- a/pkg/kubelet/cm/cgroup_v1_manager_linux.go +++ b/pkg/kubelet/cm/cgroup_v1_manager_linux.go @@ -19,14 +19,11 @@ package cm import ( "errors" "fmt" - "os" - "path/filepath" "strconv" "strings" libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" ) @@ -123,22 +120,6 @@ func (c *cgroupV1impl) GetCgroupConfig(name CgroupName, resource v1.ResourceName return nil, fmt.Errorf("unsupported resource %v for cgroup %v", resource, name) } -// Set resource config for the specified resource type on the cgroup -func (c *cgroupV1impl) SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error { - cgroupPaths := c.buildCgroupPaths(name) - cgroupResourcePath, found := cgroupPaths[string(resource)] - if !found { - return fmt.Errorf("failed to build %v cgroup fs path for cgroup %v", resource, name) - } - switch resource { - case v1.ResourceCPU: - return c.setCgroupCPUConfig(cgroupResourcePath, resourceConfig) - case v1.ResourceMemory: - return c.setCgroupMemoryConfig(cgroupResourcePath, resourceConfig) - } - return nil -} - func (c *cgroupV1impl) getCgroupCPUConfig(cgroupPath string) (*ResourceConfig, error) { cpuQuotaStr, errQ := fscommon.GetCgroupParamString(cgroupPath, "cpu.cfs_quota_us") if errQ != nil { @@ -159,33 +140,6 @@ func (c *cgroupV1impl) getCgroupCPUConfig(cgroupPath string) (*ResourceConfig, e return &ResourceConfig{CPUShares: &cpuShares, CPUQuota: &cpuQuota, CPUPeriod: &cpuPeriod}, nil } -func (c *cgroupV1impl) setCgroupCPUConfig(cgroupPath string, resourceConfig *ResourceConfig) error { - var cpuQuotaStr, cpuPeriodStr, cpuSharesStr string - if resourceConfig.CPUQuota != nil { - cpuQuotaStr = strconv.FormatInt(*resourceConfig.CPUQuota, 10) - if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.cfs_quota_us"), []byte(cpuQuotaStr), 0700); err != nil { - return fmt.Errorf("failed to write %v to %v: %w", cpuQuotaStr, cgroupPath, err) - } - } - if resourceConfig.CPUPeriod != nil { - cpuPeriodStr = strconv.FormatUint(*resourceConfig.CPUPeriod, 10) - if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.cfs_period_us"), []byte(cpuPeriodStr), 0700); err != nil { - return fmt.Errorf("failed to write %v to %v: %w", cpuPeriodStr, cgroupPath, err) - } - } - if resourceConfig.CPUShares != nil { - cpuSharesStr = strconv.FormatUint(*resourceConfig.CPUShares, 10) - if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.shares"), []byte(cpuSharesStr), 0700); err != nil { - return fmt.Errorf("failed to write %v to %v: %w", cpuSharesStr, cgroupPath, err) - } - } - return nil -} - -func (c *cgroupV1impl) setCgroupMemoryConfig(cgroupPath string, resourceConfig *ResourceConfig) error { - return writeCgroupMemoryLimit(filepath.Join(cgroupPath, cgroupv1MemLimitFile), resourceConfig) -} - func (c *cgroupV1impl) getCgroupMemoryConfig(cgroupPath string) (*ResourceConfig, error) { return readCgroupMemoryConfig(cgroupPath, cgroupv1MemLimitFile) } diff --git a/pkg/kubelet/cm/cgroup_v2_manager_linux.go b/pkg/kubelet/cm/cgroup_v2_manager_linux.go index d8cf58d79cf..ad93e971fef 100644 --- a/pkg/kubelet/cm/cgroup_v2_manager_linux.go +++ b/pkg/kubelet/cm/cgroup_v2_manager_linux.go @@ -98,22 +98,6 @@ func (c *cgroupV2impl) GetCgroupConfig(name CgroupName, resource v1.ResourceName return nil, fmt.Errorf("unsupported resource %v for cgroup %v", resource, name) } -// Set resource config for the specified resource type on the cgroup -func (c *cgroupV2impl) SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error { - cgroupPaths := c.buildCgroupPaths(name) - cgroupResourcePath, found := cgroupPaths[string(resource)] - if !found { - return fmt.Errorf("failed to build %v cgroup fs path for cgroup %v", resource, name) - } - switch resource { - case v1.ResourceCPU: - return c.setCgroupCPUConfig(cgroupResourcePath, resourceConfig) - case v1.ResourceMemory: - return c.setCgroupMemoryConfig(cgroupResourcePath, resourceConfig) - } - return nil -} - func (c *cgroupV2impl) getCgroupCPUConfig(cgroupPath string) (*ResourceConfig, error) { var cpuLimitStr, cpuPeriodStr string cpuLimitAndPeriod, err := fscommon.GetCgroupParamString(cgroupPath, "cpu.max") @@ -144,35 +128,6 @@ func (c *cgroupV2impl) getCgroupCPUConfig(cgroupPath string) (*ResourceConfig, e return &ResourceConfig{CPUShares: &cpuShares, CPUQuota: &cpuLimit, CPUPeriod: &cpuPeriod}, nil } -func (c *cgroupV2impl) setCgroupCPUConfig(cgroupPath string, resourceConfig *ResourceConfig) error { - if resourceConfig.CPUQuota != nil { - if resourceConfig.CPUPeriod == nil { - return fmt.Errorf("CpuPeriod must be specified in order to set CpuLimit") - } - cpuLimitStr := Cgroup2MaxCpuLimit - if *resourceConfig.CPUQuota > -1 { - cpuLimitStr = strconv.FormatInt(*resourceConfig.CPUQuota, 10) - } - cpuPeriodStr := strconv.FormatUint(*resourceConfig.CPUPeriod, 10) - cpuMaxStr := fmt.Sprintf("%s %s", cpuLimitStr, cpuPeriodStr) - if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.max"), []byte(cpuMaxStr), 0700); err != nil { - return fmt.Errorf("failed to write %v to %v: %w", cpuMaxStr, cgroupPath, err) - } - } - if resourceConfig.CPUShares != nil { - cpuWeight := cpuSharesToCPUWeight(*resourceConfig.CPUShares) - cpuWeightStr := strconv.FormatUint(cpuWeight, 10) - if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.weight"), []byte(cpuWeightStr), 0700); err != nil { - return fmt.Errorf("failed to write %v to %v: %w", cpuWeightStr, cgroupPath, err) - } - } - return nil -} - -func (c *cgroupV2impl) setCgroupMemoryConfig(cgroupPath string, resourceConfig *ResourceConfig) error { - return writeCgroupMemoryLimit(filepath.Join(cgroupPath, cgroupv2MemLimitFile), resourceConfig) -} - func (c *cgroupV2impl) getCgroupMemoryConfig(cgroupPath string) (*ResourceConfig, error) { return readCgroupMemoryConfig(cgroupPath, cgroupv2MemLimitFile) } diff --git a/pkg/kubelet/cm/container_manager_stub.go b/pkg/kubelet/cm/container_manager_stub.go index 85d51934b4c..e271366edf8 100644 --- a/pkg/kubelet/cm/container_manager_stub.go +++ b/pkg/kubelet/cm/container_manager_stub.go @@ -103,7 +103,7 @@ func (m *podContainerManagerStub) GetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceNam return nil, fmt.Errorf("not implemented") } -func (m *podContainerManagerStub) SetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceName, _ *ResourceConfig) error { +func (m *podContainerManagerStub) SetPodCgroupConfig(pod *v1.Pod, resourceConfig *ResourceConfig) error { return fmt.Errorf("not implemented") } diff --git a/pkg/kubelet/cm/fake_pod_container_manager.go b/pkg/kubelet/cm/fake_pod_container_manager.go index b4cce4418fc..c3ffae15078 100644 --- a/pkg/kubelet/cm/fake_pod_container_manager.go +++ b/pkg/kubelet/cm/fake_pod_container_manager.go @@ -119,7 +119,7 @@ func (cm *FakePodContainerManager) GetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceNa return nil, nil } -func (cm *FakePodContainerManager) SetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceName, _ *ResourceConfig) error { +func (cm *FakePodContainerManager) SetPodCgroupConfig(pod *v1.Pod, resourceConfig *ResourceConfig) error { cm.Lock() defer cm.Unlock() cm.CalledFunctions = append(cm.CalledFunctions, "SetPodCgroupConfig") diff --git a/pkg/kubelet/cm/pod_container_manager_linux.go b/pkg/kubelet/cm/pod_container_manager_linux.go index dae7f8bd3d4..c67f0cd4842 100644 --- a/pkg/kubelet/cm/pod_container_manager_linux.go +++ b/pkg/kubelet/cm/pod_container_manager_linux.go @@ -134,9 +134,9 @@ func (m *podContainerManagerImpl) GetPodCgroupConfig(pod *v1.Pod, resource v1.Re return m.cgroupManager.GetCgroupConfig(podCgroupName, resource) } -func (m *podContainerManagerImpl) SetPodCgroupConfig(pod *v1.Pod, resource v1.ResourceName, resourceConfig *ResourceConfig) error { +func (m *podContainerManagerImpl) SetPodCgroupConfig(pod *v1.Pod, resourceConfig *ResourceConfig) error { podCgroupName, _ := m.GetPodContainerName(pod) - return m.cgroupManager.SetCgroupConfig(podCgroupName, resource, resourceConfig) + return m.cgroupManager.SetCgroupConfig(podCgroupName, resourceConfig) } // Kill one process ID @@ -350,6 +350,6 @@ func (m *podContainerManagerNoop) GetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceNam return nil, nil } -func (m *podContainerManagerNoop) SetPodCgroupConfig(_ *v1.Pod, _ v1.ResourceName, _ *ResourceConfig) error { +func (m *podContainerManagerNoop) SetPodCgroupConfig(_ *v1.Pod, _ *ResourceConfig) error { return nil } diff --git a/pkg/kubelet/cm/types.go b/pkg/kubelet/cm/types.go index 7a7d9268015..e6338d3af81 100644 --- a/pkg/kubelet/cm/types.go +++ b/pkg/kubelet/cm/types.go @@ -90,7 +90,7 @@ type CgroupManager interface { // Get the resource config values applied to the cgroup for specified resource type GetCgroupConfig(name CgroupName, resource v1.ResourceName) (*ResourceConfig, error) // Set resource config for the specified resource type on the cgroup - SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error + SetCgroupConfig(name CgroupName, resourceConfig *ResourceConfig) error // Version of the cgroup implementation on the host Version() int } @@ -136,5 +136,5 @@ type PodContainerManager interface { GetPodCgroupConfig(pod *v1.Pod, resource v1.ResourceName) (*ResourceConfig, error) // Set resource config values for the specified resource type on the pod cgroup - SetPodCgroupConfig(pod *v1.Pod, resource v1.ResourceName, resourceConfig *ResourceConfig) error + SetPodCgroupConfig(pod *v1.Pod, resourceConfig *ResourceConfig) error } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index 0c7a9b24c63..f523b155d2b 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -663,15 +663,16 @@ func (m *kubeGenericRuntimeManager) doPodResizeAction(pod *v1.Pod, podStatus *ku var err error switch rName { case v1.ResourceCPU: - podCpuResources := &cm.ResourceConfig{CPUPeriod: podResources.CPUPeriod} + podCPUResources := &cm.ResourceConfig{} if setLimitValue { - podCpuResources.CPUQuota = podResources.CPUQuota + podCPUResources.CPUPeriod = podResources.CPUPeriod + podCPUResources.CPUQuota = podResources.CPUQuota } else { - podCpuResources.CPUShares = podResources.CPUShares + podCPUResources.CPUShares = podResources.CPUShares } - err = pcm.SetPodCgroupConfig(pod, rName, podCpuResources) + err = pcm.SetPodCgroupConfig(pod, podCPUResources) case v1.ResourceMemory: - err = pcm.SetPodCgroupConfig(pod, rName, podResources) + err = pcm.SetPodCgroupConfig(pod, podResources) } if err != nil { klog.ErrorS(err, "Failed to set cgroup config", "resource", rName, "pod", pod.Name)