mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-01-05 23:47:50 +00:00
This commit contains the following:
1. Scheduler bug-fix + scheduler-focussed E2E tests 2. Add cgroup v2 support for in-place pod resize 3. Enable full E2E pod resize test for containerd>=1.6.9 and EventedPLEG related changes. Co-Authored-By: Vinay Kulkarni <vskibum@gmail.com>
This commit is contained in:
Chen Wang
committed by
vinay kulkarni
vinay kulkarni
parent
f2bd94a0de
commit
7db339dba2
@@ -32,6 +32,7 @@ import (
|
||||
"github.com/opencontainers/runc/libcontainer/cgroups/manager"
|
||||
cgroupsystemd "github.com/opencontainers/runc/libcontainer/cgroups/systemd"
|
||||
libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/klog/v2"
|
||||
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
|
||||
|
||||
@@ -47,7 +48,8 @@ const (
|
||||
// MemoryMin is memory.min for cgroup v2
|
||||
MemoryMin string = "memory.min"
|
||||
// MemoryHigh is memory.high for cgroup v2
|
||||
MemoryHigh string = "memory.high"
|
||||
MemoryHigh string = "memory.high"
|
||||
Cgroup2MaxCpuLimit string = "max"
|
||||
)
|
||||
|
||||
var RootCgroupName = CgroupName([]string{})
|
||||
@@ -559,85 +561,188 @@ func (m *cgroupManagerImpl) MemoryUsage(name CgroupName) (int64, error) {
|
||||
return int64(val), err
|
||||
}
|
||||
|
||||
// Get the memory limit in bytes applied to the cgroup
|
||||
func (m *cgroupManagerImpl) GetCgroupMemoryConfig(name CgroupName) (uint64, error) {
|
||||
cgroupPaths := m.buildCgroupPaths(name)
|
||||
cgroupMemoryPath, found := cgroupPaths["memory"]
|
||||
if !found {
|
||||
return 0, fmt.Errorf("failed to build memory cgroup fs path for cgroup %v", name)
|
||||
}
|
||||
memLimit, err := fscommon.GetCgroupParamUint(cgroupMemoryPath, "memory.limit_in_bytes")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to get memory.limit_in_bytes for cgroup %v: %v", name, err)
|
||||
}
|
||||
return memLimit, nil
|
||||
// Convert cgroup v1 cpu.shares value to cgroup v2 cpu.weight
|
||||
// https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2254-cgroup-v2#phase-1-convert-from-cgroups-v1-settings-to-v2
|
||||
func CpuSharesToCpuWeight(cpuShares uint64) uint64 {
|
||||
return uint64((((cpuShares - 2) * 9999) / 262142) + 1)
|
||||
}
|
||||
|
||||
// Get the cpu quota, cpu period, and cpu shares applied to the cgroup
|
||||
func (m *cgroupManagerImpl) GetCgroupCpuConfig(name CgroupName) (int64, uint64, uint64, error) {
|
||||
cgroupPaths := m.buildCgroupPaths(name)
|
||||
cgroupCpuPath, found := cgroupPaths["cpu"]
|
||||
if !found {
|
||||
return 0, 0, 0, fmt.Errorf("failed to build CPU cgroup fs path for cgroup %v", name)
|
||||
}
|
||||
cpuQuotaStr, errQ := fscommon.GetCgroupParamString(cgroupCpuPath, "cpu.cfs_quota_us")
|
||||
// Convert cgroup v2 cpu.weight value to cgroup v1 cpu.shares
|
||||
// https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2254-cgroup-v2#phase-1-convert-from-cgroups-v1-settings-to-v2
|
||||
func CpuWeightToCpuShares(cpuWeight uint64) uint64 {
|
||||
return uint64((((cpuWeight - 1) * 262142) / 9999) + 2)
|
||||
}
|
||||
|
||||
func getCgroupv1CpuConfig(cgroupPath string) (*ResourceConfig, error) {
|
||||
cpuQuotaStr, errQ := fscommon.GetCgroupParamString(cgroupPath, "cpu.cfs_quota_us")
|
||||
if errQ != nil {
|
||||
return 0, 0, 0, fmt.Errorf("failed to read CPU quota for cgroup %v: %v", name, errQ)
|
||||
return nil, fmt.Errorf("failed to read CPU quota for cgroup %v: %v", cgroupPath, errQ)
|
||||
}
|
||||
cpuQuota, errInt := strconv.ParseInt(cpuQuotaStr, 10, 64)
|
||||
if errInt != nil {
|
||||
return 0, 0, 0, fmt.Errorf("failed to convert CPU quota as integer for cgroup %v: %v", name, errInt)
|
||||
return nil, fmt.Errorf("failed to convert CPU quota as integer for cgroup %v: %v", cgroupPath, errInt)
|
||||
}
|
||||
cpuPeriod, errP := fscommon.GetCgroupParamUint(cgroupCpuPath, "cpu.cfs_period_us")
|
||||
cpuPeriod, errP := fscommon.GetCgroupParamUint(cgroupPath, "cpu.cfs_period_us")
|
||||
if errP != nil {
|
||||
return 0, 0, 0, fmt.Errorf("failed to read CPU period for cgroup %v: %v", name, errP)
|
||||
return nil, fmt.Errorf("failed to read CPU period for cgroup %v: %v", cgroupPath, errP)
|
||||
}
|
||||
cpuShares, errS := fscommon.GetCgroupParamUint(cgroupCpuPath, "cpu.shares")
|
||||
if errP != nil {
|
||||
return 0, 0, 0, fmt.Errorf("failed to read CPU shares for cgroup %v: %v", name, errS)
|
||||
cpuShares, errS := fscommon.GetCgroupParamUint(cgroupPath, "cpu.shares")
|
||||
if errS != nil {
|
||||
return nil, fmt.Errorf("failed to read CPU shares for cgroup %v: %v", cgroupPath, errS)
|
||||
}
|
||||
return cpuQuota, cpuPeriod, cpuShares, nil
|
||||
return &ResourceConfig{CPUShares: &cpuShares, CPUQuota: &cpuQuota, CPUPeriod: &cpuPeriod}, nil
|
||||
}
|
||||
|
||||
// Set the memory limit in bytes applied to the cgroup
|
||||
func (m *cgroupManagerImpl) SetCgroupMemoryConfig(name CgroupName, memoryLimit int64) error {
|
||||
func getCgroupv2CpuConfig(cgroupPath string) (*ResourceConfig, error) {
|
||||
var cpuLimitStr, cpuPeriodStr string
|
||||
cpuLimitAndPeriod, err := fscommon.GetCgroupParamString(cgroupPath, "cpu.max")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read cpu.max file for cgroup %v: %v", cgroupPath, err)
|
||||
}
|
||||
numItems, errScan := fmt.Sscanf(cpuLimitAndPeriod, "%s %s", &cpuLimitStr, &cpuPeriodStr)
|
||||
if errScan != nil || numItems != 2 {
|
||||
return nil, fmt.Errorf("failed to correctly parse content of cpu.max file ('%s') for cgroup %v: %v",
|
||||
cpuLimitAndPeriod, cgroupPath, errScan)
|
||||
}
|
||||
cpuLimit := int64(-1)
|
||||
if cpuLimitStr != Cgroup2MaxCpuLimit {
|
||||
cpuLimit, err = strconv.ParseInt(cpuLimitStr, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert CPU limit as integer for cgroup %v: %v", cgroupPath, err)
|
||||
}
|
||||
}
|
||||
cpuPeriod, errPeriod := strconv.ParseUint(cpuPeriodStr, 10, 64)
|
||||
if errPeriod != nil {
|
||||
return nil, fmt.Errorf("failed to convert CPU period as integer for cgroup %v: %v", cgroupPath, errPeriod)
|
||||
}
|
||||
cpuWeight, errWeight := fscommon.GetCgroupParamUint(cgroupPath, "cpu.weight")
|
||||
if errWeight != nil {
|
||||
return nil, fmt.Errorf("failed to read CPU weight for cgroup %v: %v", cgroupPath, errWeight)
|
||||
}
|
||||
cpuShares := CpuWeightToCpuShares(cpuWeight)
|
||||
return &ResourceConfig{CPUShares: &cpuShares, CPUQuota: &cpuLimit, CPUPeriod: &cpuPeriod}, nil
|
||||
}
|
||||
|
||||
func getCgroupCpuConfig(cgroupPath string) (*ResourceConfig, error) {
|
||||
if libcontainercgroups.IsCgroup2UnifiedMode() {
|
||||
return getCgroupv2CpuConfig(cgroupPath)
|
||||
} else {
|
||||
return getCgroupv1CpuConfig(cgroupPath)
|
||||
}
|
||||
}
|
||||
|
||||
func getCgroupMemoryConfig(cgroupPath string) (*ResourceConfig, error) {
|
||||
memLimitFile := "memory.limit_in_bytes"
|
||||
if libcontainercgroups.IsCgroup2UnifiedMode() {
|
||||
memLimitFile = "memory.max"
|
||||
}
|
||||
memLimit, err := fscommon.GetCgroupParamUint(cgroupPath, memLimitFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read %s for cgroup %v: %v", memLimitFile, cgroupPath, err)
|
||||
}
|
||||
mLim := int64(memLimit)
|
||||
//TODO(vinaykul,InPlacePodVerticalScaling): Add memory request support
|
||||
return &ResourceConfig{Memory: &mLim}, nil
|
||||
|
||||
}
|
||||
|
||||
// Get the resource config values applied to the cgroup for specified resource type
|
||||
func (m *cgroupManagerImpl) GetCgroupConfig(name CgroupName, resource v1.ResourceName) (*ResourceConfig, error) {
|
||||
cgroupPaths := m.buildCgroupPaths(name)
|
||||
cgroupMemoryPath, found := cgroupPaths["memory"]
|
||||
cgroupResourcePath, found := cgroupPaths[string(resource)]
|
||||
if !found {
|
||||
return fmt.Errorf("failed to build memory cgroup fs path for cgroup %v", name)
|
||||
return nil, fmt.Errorf("failed to build %v cgroup fs path for cgroup %v", resource, name)
|
||||
}
|
||||
memLimit := strconv.FormatInt(memoryLimit, 10)
|
||||
if err := os.WriteFile(filepath.Join(cgroupMemoryPath, "memory.limit_in_bytes"), []byte(memLimit), 0700); err != nil {
|
||||
return fmt.Errorf("failed to write %v to %v: %v", memLimit, cgroupMemoryPath, err)
|
||||
switch resource {
|
||||
case v1.ResourceCPU:
|
||||
return getCgroupCpuConfig(cgroupResourcePath)
|
||||
case v1.ResourceMemory:
|
||||
return getCgroupMemoryConfig(cgroupResourcePath)
|
||||
}
|
||||
return nil
|
||||
return nil, fmt.Errorf("unsupported resource %v for cgroup %v", resource, name)
|
||||
}
|
||||
|
||||
// Set the cpu quota, cpu period, and cpu shares applied to the cgroup
|
||||
func (m *cgroupManagerImpl) SetCgroupCpuConfig(name CgroupName, cpuQuota *int64, cpuPeriod, cpuShares *uint64) error {
|
||||
func setCgroupv1CpuConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
|
||||
var cpuQuotaStr, cpuPeriodStr, cpuSharesStr string
|
||||
cgroupPaths := m.buildCgroupPaths(name)
|
||||
cgroupCpuPath, found := cgroupPaths["cpu"]
|
||||
if !found {
|
||||
return fmt.Errorf("failed to build cpu cgroup fs path for cgroup %v", name)
|
||||
}
|
||||
if cpuQuota != nil {
|
||||
cpuQuotaStr = strconv.FormatInt(*cpuQuota, 10)
|
||||
if err := os.WriteFile(filepath.Join(cgroupCpuPath, "cpu.cfs_quota_us"), []byte(cpuQuotaStr), 0700); err != nil {
|
||||
return fmt.Errorf("failed to write %v to %v: %v", cpuQuotaStr, cgroupCpuPath, err)
|
||||
if resourceConfig.CPUQuota != nil {
|
||||
cpuQuotaStr = strconv.FormatInt(*resourceConfig.CPUQuota, 10)
|
||||
if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.cfs_quota_us"), []byte(cpuQuotaStr), 0700); err != nil {
|
||||
return fmt.Errorf("failed to write %v to %v: %v", cpuQuotaStr, cgroupPath, err)
|
||||
}
|
||||
}
|
||||
if cpuPeriod != nil {
|
||||
cpuPeriodStr = strconv.FormatUint(*cpuPeriod, 10)
|
||||
if err := os.WriteFile(filepath.Join(cgroupCpuPath, "cpu.cfs_period_us"), []byte(cpuPeriodStr), 0700); err != nil {
|
||||
return fmt.Errorf("failed to write %v to %v: %v", cpuPeriodStr, cgroupCpuPath, err)
|
||||
if resourceConfig.CPUPeriod != nil {
|
||||
cpuPeriodStr = strconv.FormatUint(*resourceConfig.CPUPeriod, 10)
|
||||
if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.cfs_period_us"), []byte(cpuPeriodStr), 0700); err != nil {
|
||||
return fmt.Errorf("failed to write %v to %v: %v", cpuPeriodStr, cgroupPath, err)
|
||||
}
|
||||
}
|
||||
if cpuShares != nil {
|
||||
cpuSharesStr = strconv.FormatUint(*cpuShares, 10)
|
||||
if err := os.WriteFile(filepath.Join(cgroupCpuPath, "cpu.shares"), []byte(cpuSharesStr), 0700); err != nil {
|
||||
return fmt.Errorf("failed to write %v to %v: %v", cpuSharesStr, cgroupCpuPath, err)
|
||||
if resourceConfig.CPUShares != nil {
|
||||
cpuSharesStr = strconv.FormatUint(*resourceConfig.CPUShares, 10)
|
||||
if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.shares"), []byte(cpuSharesStr), 0700); err != nil {
|
||||
return fmt.Errorf("failed to write %v to %v: %v", cpuSharesStr, cgroupPath, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func setCgroupv2CpuConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
|
||||
if resourceConfig.CPUQuota != nil {
|
||||
if resourceConfig.CPUPeriod == nil {
|
||||
return fmt.Errorf("CpuPeriod must be specified in order to set CpuLimit")
|
||||
}
|
||||
cpuLimitStr := Cgroup2MaxCpuLimit
|
||||
if *resourceConfig.CPUQuota > -1 {
|
||||
cpuLimitStr = strconv.FormatInt(*resourceConfig.CPUQuota, 10)
|
||||
}
|
||||
cpuPeriodStr := strconv.FormatUint(*resourceConfig.CPUPeriod, 10)
|
||||
cpuMaxStr := fmt.Sprintf("%s %s", cpuLimitStr, cpuPeriodStr)
|
||||
if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.max"), []byte(cpuMaxStr), 0700); err != nil {
|
||||
return fmt.Errorf("failed to write %v to %v: %v", cpuMaxStr, cgroupPath, err)
|
||||
}
|
||||
}
|
||||
if resourceConfig.CPUShares != nil {
|
||||
cpuWeight := CpuSharesToCpuWeight(*resourceConfig.CPUShares)
|
||||
cpuWeightStr := strconv.FormatUint(cpuWeight, 10)
|
||||
if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.weight"), []byte(cpuWeightStr), 0700); err != nil {
|
||||
return fmt.Errorf("failed to write %v to %v: %v", cpuWeightStr, cgroupPath, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func setCgroupCpuConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
|
||||
if libcontainercgroups.IsCgroup2UnifiedMode() {
|
||||
return setCgroupv2CpuConfig(cgroupPath, resourceConfig)
|
||||
} else {
|
||||
return setCgroupv1CpuConfig(cgroupPath, resourceConfig)
|
||||
}
|
||||
}
|
||||
|
||||
func setCgroupMemoryConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
|
||||
memLimitFile := "memory.limit_in_bytes"
|
||||
if libcontainercgroups.IsCgroup2UnifiedMode() {
|
||||
memLimitFile = "memory.max"
|
||||
}
|
||||
memLimit := strconv.FormatInt(*resourceConfig.Memory, 10)
|
||||
if err := os.WriteFile(filepath.Join(cgroupPath, memLimitFile), []byte(memLimit), 0700); err != nil {
|
||||
return fmt.Errorf("failed to write %v to %v/%v: %v", memLimit, cgroupPath, memLimitFile, err)
|
||||
}
|
||||
//TODO(vinaykul,InPlacePodVerticalScaling): Add memory request support
|
||||
return nil
|
||||
}
|
||||
|
||||
// Set resource config for the specified resource type on the cgroup
|
||||
func (m *cgroupManagerImpl) SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error {
|
||||
cgroupPaths := m.buildCgroupPaths(name)
|
||||
cgroupResourcePath, found := cgroupPaths[string(resource)]
|
||||
if !found {
|
||||
return fmt.Errorf("failed to build %v cgroup fs path for cgroup %v", resource, name)
|
||||
}
|
||||
switch resource {
|
||||
case v1.ResourceCPU:
|
||||
return setCgroupCpuConfig(cgroupResourcePath, resourceConfig)
|
||||
case v1.ResourceMemory:
|
||||
return setCgroupMemoryConfig(cgroupResourcePath, resourceConfig)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user