github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-08 03:33:56 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #112123 from paskal/paskal/cfs_clarification

Browse Source 
clarify CPUCFSQuotaPeriod values, set the minimum to 1ms
This commit is contained in:
Kubernetes Prow Robot 2022-09-12 07:01:25 -07:00 committed by GitHub
commit 74469ca4c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 38 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/generated/openapi/zz_generated.openapi.go generated
View File

@ -54495,7 +54495,7 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen
}, },
"cpuCFSQuotaPeriod": { "cpuCFSQuotaPeriod": {
SchemaProps: spec.SchemaProps{ SchemaProps: spec.SchemaProps{
Description: "cpuCFSQuotaPeriod is the CPU CFS quota period value, `cpu.cfs_period_us`. The value must be between 1 us and 1 second, inclusive. Requires the CustomCPUCFSQuotaPeriod feature gate to be enabled. Default: \"100ms\"", Description: "cpuCFSQuotaPeriod is the CPU CFS quota period value, `cpu.cfs_period_us`. The value must be between 1 ms and 1 second, inclusive. Requires the CustomCPUCFSQuotaPeriod feature gate to be enabled. Default: \"100ms\"",
Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"),
}, },
}, },

4
pkg/kubelet/apis/config/validation/validation.go
View File

@ -71,8 +71,8 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur
if !localFeatureGate.Enabled(features.CPUCFSQuotaPeriod) && kc.CPUCFSQuotaPeriod != defaultCFSQuota { if !localFeatureGate.Enabled(features.CPUCFSQuotaPeriod) && kc.CPUCFSQuotaPeriod != defaultCFSQuota {
allErrors = append(allErrors, fmt.Errorf("invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) %v requires feature gate CustomCPUCFSQuotaPeriod", kc.CPUCFSQuotaPeriod)) allErrors = append(allErrors, fmt.Errorf("invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) %v requires feature gate CustomCPUCFSQuotaPeriod", kc.CPUCFSQuotaPeriod))
} }
if localFeatureGate.Enabled(features.CPUCFSQuotaPeriod) && utilvalidation.IsInRange(int(kc.CPUCFSQuotaPeriod.Duration), int(1*time.Microsecond), int(time.Second)) != nil { if localFeatureGate.Enabled(features.CPUCFSQuotaPeriod) && utilvalidation.IsInRange(int(kc.CPUCFSQuotaPeriod.Duration), int(1*time.Millisecond), int(time.Second)) != nil {
allErrors = append(allErrors, fmt.Errorf("invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) %v must be between 1usec and 1sec, inclusive", kc.CPUCFSQuotaPeriod)) allErrors = append(allErrors, fmt.Errorf("invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) %v must be between 1ms and 1sec, inclusive", kc.CPUCFSQuotaPeriod))
} }
if utilvalidation.IsInRange(int(kc.ImageGCHighThresholdPercent), 0, 100) != nil { if utilvalidation.IsInRange(int(kc.ImageGCHighThresholdPercent), 0, 100) != nil {
allErrors = append(allErrors, fmt.Errorf("invalid configuration: imageGCHighThresholdPercent (--image-gc-high-threshold) %v must be between 0 and 100, inclusive", kc.ImageGCHighThresholdPercent)) allErrors = append(allErrors, fmt.Errorf("invalid configuration: imageGCHighThresholdPercent (--image-gc-high-threshold) %v must be between 0 and 100, inclusive", kc.ImageGCHighThresholdPercent))

8
pkg/kubelet/apis/config/validation/validation_test.go
View File

@ -59,7 +59,7 @@ var (
RegistryPullQPS: 5, RegistryPullQPS: 5,
HairpinMode: kubeletconfig.PromiscuousBridge, HairpinMode: kubeletconfig.PromiscuousBridge,
NodeLeaseDurationSeconds: 1, NodeLeaseDurationSeconds: 1,
CPUCFSQuotaPeriod: metav1.Duration{Duration: 25 * time.Microsecond}, CPUCFSQuotaPeriod: metav1.Duration{Duration: 25 * time.Millisecond},
TopologyManagerScope: kubeletconfig.PodTopologyManagerScope, TopologyManagerScope: kubeletconfig.PodTopologyManagerScope,
TopologyManagerPolicy: kubeletconfig.SingleNumaNodeTopologyManagerPolicy, TopologyManagerPolicy: kubeletconfig.SingleNumaNodeTopologyManagerPolicy,
ShutdownGracePeriod: metav1.Duration{Duration: 30 * time.Second}, ShutdownGracePeriod: metav1.Duration{Duration: 30 * time.Second},
@ -145,10 +145,10 @@ func TestValidateKubeletConfiguration(t *testing.T) {
name: "specify CPUCFSQuotaPeriod without enabling CPUCFSQuotaPeriod", name: "specify CPUCFSQuotaPeriod without enabling CPUCFSQuotaPeriod",
configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
conf.FeatureGates = map[string]bool{"CustomCPUCFSQuotaPeriod": false} conf.FeatureGates = map[string]bool{"CustomCPUCFSQuotaPeriod": false}
conf.CPUCFSQuotaPeriod = metav1.Duration{Duration: 200 * time.Microsecond} conf.CPUCFSQuotaPeriod = metav1.Duration{Duration: 200 * time.Millisecond}
return conf return conf
}, },
errMsg: "invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) {200µs} requires feature gate CustomCPUCFSQuotaPeriod", errMsg: "invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) {200ms} requires feature gate CustomCPUCFSQuotaPeriod",
}, },
{ {
name: "invalid CPUCFSQuotaPeriod", name: "invalid CPUCFSQuotaPeriod",
@ -157,7 +157,7 @@ func TestValidateKubeletConfiguration(t *testing.T) {
conf.CPUCFSQuotaPeriod = metav1.Duration{Duration: 2 * time.Second} conf.CPUCFSQuotaPeriod = metav1.Duration{Duration: 2 * time.Second}
return conf return conf
}, },
errMsg: "invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) {2s} must be between 1usec and 1sec, inclusive", errMsg: "invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) {2s} must be between 1ms and 1sec, inclusive",
}, },
{ {
name: "invalid ImageGCHighThresholdPercent", name: "invalid ImageGCHighThresholdPercent",

2
pkg/kubelet/cm/container_manager_linux.go
View File

@ -359,6 +359,8 @@ func (cm *containerManagerImpl) NewPodContainerManager() PodContainerManager {
cgroupManager: cm.cgroupManager, cgroupManager: cm.cgroupManager,
podPidsLimit: cm.ExperimentalPodPidsLimit, podPidsLimit: cm.ExperimentalPodPidsLimit,
enforceCPULimits: cm.EnforceCPULimits, enforceCPULimits: cm.EnforceCPULimits,
// cpuCFSQuotaPeriod is in microseconds. NodeConfig.CPUCFSQuotaPeriod is time.Duration (measured in nano seconds).
// Convert (cm.CPUCFSQuotaPeriod) [nanoseconds] / time.Microsecond (1000) to get cpuCFSQuotaPeriod in microseconds.
cpuCFSQuotaPeriod: uint64(cm.CPUCFSQuotaPeriod / time.Microsecond), cpuCFSQuotaPeriod: uint64(cm.CPUCFSQuotaPeriod / time.Microsecond),
} }
} }

12
pkg/kubelet/cm/helpers_linux.go
View File

@ -44,16 +44,20 @@ const (
SharesPerCPU = 1024 SharesPerCPU = 1024
MilliCPUToCPU = 1000 MilliCPUToCPU = 1000
// 100000 is equivalent to 100usec // 100000 microseconds is equivalent to 100ms
QuotaPeriod = 100000 QuotaPeriod = 100000
// 1000 microseconds is equivalent to 1ms
// defined here:
// https://github.com/torvalds/linux/blob/cac03ac368fabff0122853de2422d4e17a32de08/kernel/sched/core.c#L10546
MinQuotaPeriod = 1000 MinQuotaPeriod = 1000
) )
// MilliCPUToQuota converts milliCPU to CFS quota and period values. // MilliCPUToQuota converts milliCPU to CFS quota and period values.
// Input parameters and resulting value is number of microseconds.
func MilliCPUToQuota(milliCPU int64, period int64) (quota int64) { func MilliCPUToQuota(milliCPU int64, period int64) (quota int64) {
// CFS quota is measured in two values: // CFS quota is measured in two values:
// - cfs_period_us=100usec (the amount of time to measure usage across given by period) // - cfs_period_us=100ms (the amount of time to measure usage across given by period)
// - cfs_quota=20usec (the amount of cpu time allowed to be used across a period) // - cfs_quota=20ms (the amount of cpu time allowed to be used across a period)
// so in the above example, you are limited to 20% of a single CPU // so in the above example, you are limited to 20% of a single CPU
// for multi-cpu environments, you just scale equivalent amounts // for multi-cpu environments, you just scale equivalent amounts
// see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details // see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details

18
pkg/kubelet/cm/helpers_linux_test.go
View File

@ -54,8 +54,8 @@ func getResourceRequirements(requests, limits v1.ResourceList) v1.ResourceRequir
} }
func TestResourceConfigForPod(t *testing.T) { func TestResourceConfigForPod(t *testing.T) {
defaultQuotaPeriod := uint64(100 * time.Millisecond / time.Microsecond) defaultQuotaPeriod := uint64(100 * time.Millisecond / time.Microsecond) // in microseconds
tunedQuotaPeriod := uint64(5 * time.Millisecond / time.Microsecond) tunedQuotaPeriod := uint64(5 * time.Millisecond / time.Microsecond) // in microseconds
minShares := uint64(MinShares) minShares := uint64(MinShares)
burstableShares := MilliCPUToShares(100) burstableShares := MilliCPUToShares(100)
@ -73,7 +73,7 @@ func TestResourceConfigForPod(t *testing.T) {
pod *v1.Pod pod *v1.Pod
expected *ResourceConfig expected *ResourceConfig
enforceCPULimits bool enforceCPULimits bool
quotaPeriod uint64 quotaPeriod uint64 // in microseconds
}{ }{
"besteffort": { "besteffort": {
pod: &v1.Pod{ pod: &v1.Pod{
@ -296,8 +296,8 @@ func TestResourceConfigForPod(t *testing.T) {
} }
func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) { func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
defaultQuotaPeriod := uint64(100 * time.Millisecond / time.Microsecond) defaultQuotaPeriod := uint64(100 * time.Millisecond / time.Microsecond) // in microseconds
tunedQuotaPeriod := uint64(5 * time.Millisecond / time.Microsecond) tunedQuotaPeriod := uint64(5 * time.Millisecond / time.Microsecond) // in microseconds
tunedQuota := int64(1 * time.Millisecond / time.Microsecond) tunedQuota := int64(1 * time.Millisecond / time.Microsecond)
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUCFSQuotaPeriod, true)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUCFSQuotaPeriod, true)()
@ -318,7 +318,7 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
pod *v1.Pod pod *v1.Pod
expected *ResourceConfig expected *ResourceConfig
enforceCPULimits bool enforceCPULimits bool
quotaPeriod uint64 quotaPeriod uint64 // in microseconds
}{ }{
"besteffort": { "besteffort": {
pod: &v1.Pod{ pod: &v1.Pod{
@ -650,8 +650,8 @@ func TestHugePageLimits(t *testing.T) {
} }
func TestResourceConfigForPodWithEnforceMemoryQoS(t *testing.T) { func TestResourceConfigForPodWithEnforceMemoryQoS(t *testing.T) {
defaultQuotaPeriod := uint64(100 * time.Millisecond / time.Microsecond) defaultQuotaPeriod := uint64(100 * time.Millisecond / time.Microsecond) // in microseconds
tunedQuotaPeriod := uint64(5 * time.Millisecond / time.Microsecond) tunedQuotaPeriod := uint64(5 * time.Millisecond / time.Microsecond) // in microseconds
minShares := uint64(MinShares) minShares := uint64(MinShares)
burstableShares := MilliCPUToShares(100) burstableShares := MilliCPUToShares(100)
@ -669,7 +669,7 @@ func TestResourceConfigForPodWithEnforceMemoryQoS(t *testing.T) {
pod *v1.Pod pod *v1.Pod
expected *ResourceConfig expected *ResourceConfig
enforceCPULimits bool enforceCPULimits bool
quotaPeriod uint64 quotaPeriod uint64 // in microseconds
}{ }{
"besteffort": { "besteffort": {
pod: &v1.Pod{ pod: &v1.Pod{

2
pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go
View File

@ -91,7 +91,7 @@ func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS
kubeRuntimeManager := &kubeGenericRuntimeManager{ kubeRuntimeManager := &kubeGenericRuntimeManager{
recorder: recorder, recorder: recorder,
cpuCFSQuota: false, cpuCFSQuota: false,
cpuCFSQuotaPeriod: metav1.Duration{Duration: time.Microsecond * 100}, cpuCFSQuotaPeriod: metav1.Duration{Duration: time.Millisecond * 100},
livenessManager: proberesults.NewManager(), livenessManager: proberesults.NewManager(),
startupManager: proberesults.NewManager(), startupManager: proberesults.NewManager(),
machineInfo: machineInfo, machineInfo: machineInfo,

12
pkg/kubelet/kuberuntime/helpers_linux.go
View File

@ -22,16 +22,20 @@ package kuberuntime
const ( const (
milliCPUToCPU = 1000 milliCPUToCPU = 1000
// 100000 is equivalent to 100usec // 100000 microseconds is equivalent to 100ms
quotaPeriod = 100000 quotaPeriod = 100000
// 1000 microseconds is equivalent to 1ms
// defined here:
// https://github.com/torvalds/linux/blob/cac03ac368fabff0122853de2422d4e17a32de08/kernel/sched/core.c#L10546
minQuotaPeriod = 1000 minQuotaPeriod = 1000
) )
// milliCPUToQuota converts milliCPU to CFS quota and period values // milliCPUToQuota converts milliCPU to CFS quota and period values
// Input parameters and resulting value is number of microseconds.
func milliCPUToQuota(milliCPU int64, period int64) (quota int64) { func milliCPUToQuota(milliCPU int64, period int64) (quota int64) {
// CFS quota is measured in two values: // CFS quota is measured in two values:
// - cfs_period_us=100usec (the amount of time to measure usage across given by period) // - cfs_period_us=100ms (the amount of time to measure usage across)
// - cfs_quota=20usec (the amount of cpu time allowed to be used across a period) // - cfs_quota=20ms (the amount of cpu time allowed to be used across a period)
// so in the above example, you are limited to 20% of a single CPU // so in the above example, you are limited to 20% of a single CPU
// for multi-cpu environments, you just scale equivalent amounts // for multi-cpu environments, you just scale equivalent amounts
// see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details // see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details

2
pkg/kubelet/kuberuntime/kuberuntime_container_linux.go
View File

@ -166,6 +166,8 @@ func (m *kubeGenericRuntimeManager) calculateLinuxResources(cpuRequest, cpuLimit
// to allow full usage of cpu resource. // to allow full usage of cpu resource.
cpuPeriod := int64(quotaPeriod) cpuPeriod := int64(quotaPeriod)
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUCFSQuotaPeriod) { if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUCFSQuotaPeriod) {
// kubeGenericRuntimeManager.cpuCFSQuotaPeriod is provided in time.Duration,
// but we need to convert it to number of microseconds which is used by kernel.
cpuPeriod = int64(m.cpuCFSQuotaPeriod.Duration / time.Microsecond) cpuPeriod = int64(m.cpuCFSQuotaPeriod.Duration / time.Microsecond)
} }
cpuQuota := milliCPUToQuota(cpuLimit.MilliValue(), cpuPeriod) cpuQuota := milliCPUToQuota(cpuLimit.MilliValue(), cpuPeriod)

2
staging/src/k8s.io/kubelet/config/v1beta1/types.go
View File

@ -440,7 +440,7 @@ type KubeletConfiguration struct {
// +optional // +optional
CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"`
// cpuCFSQuotaPeriod is the CPU CFS quota period value, `cpu.cfs_period_us`. // cpuCFSQuotaPeriod is the CPU CFS quota period value, `cpu.cfs_period_us`.
// The value must be between 1 us and 1 second, inclusive. // The value must be between 1 ms and 1 second, inclusive.
// Requires the CustomCPUCFSQuotaPeriod feature gate to be enabled. // Requires the CustomCPUCFSQuotaPeriod feature gate to be enabled.
// Default: "100ms" // Default: "100ms"
// +optional // +optional