From 5126192548c9f0b8e86000ed4d98c2a9529cfccd Mon Sep 17 00:00:00 2001 From: Dmitry Verkhoturov Date: Fri, 29 Jul 2022 23:02:35 +0200 Subject: [PATCH 1/2] clarify cpu.cfs_period_us default value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cpu.cfs_period_us is 100μs by default despite having an "ms" unit for some unfortunate reason. Documentation: https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management The desired effect of that change is more clarity on the default value so users would be aware that the 10ms custom value would be not 0.1x of the default, but 100x of it. --- .../scheme/testdata/KubeletConfiguration/after/v1beta1.yaml | 2 +- .../KubeletConfiguration/roundtrip/default/v1beta1.yaml | 2 +- pkg/kubelet/apis/config/validation/validation_test.go | 6 +++--- pkg/kubelet/cm/helpers_linux.go | 6 +++--- pkg/kubelet/kuberuntime/helpers_linux.go | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml index 3cb76b89923..edea8030375 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml @@ -19,7 +19,7 @@ containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true -cpuCFSQuotaPeriod: 100ms +cpuCFSQuotaPeriod: 100us cpuManagerPolicy: none cpuManagerReconcilePeriod: 10s enableControllerAttachDetach: true diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml index 3cb76b89923..edea8030375 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml @@ -19,7 +19,7 @@ containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true -cpuCFSQuotaPeriod: 100ms +cpuCFSQuotaPeriod: 100us cpuManagerPolicy: none cpuManagerReconcilePeriod: 10s enableControllerAttachDetach: true diff --git a/pkg/kubelet/apis/config/validation/validation_test.go b/pkg/kubelet/apis/config/validation/validation_test.go index 26c517d1442..3a3e108c70b 100644 --- a/pkg/kubelet/apis/config/validation/validation_test.go +++ b/pkg/kubelet/apis/config/validation/validation_test.go @@ -58,7 +58,7 @@ var ( RegistryPullQPS: 5, HairpinMode: kubeletconfig.PromiscuousBridge, NodeLeaseDurationSeconds: 1, - CPUCFSQuotaPeriod: metav1.Duration{Duration: 25 * time.Millisecond}, + CPUCFSQuotaPeriod: metav1.Duration{Duration: 25 * time.Microsecond}, TopologyManagerScope: kubeletconfig.PodTopologyManagerScope, TopologyManagerPolicy: kubeletconfig.SingleNumaNodeTopologyManagerPolicy, ShutdownGracePeriod: metav1.Duration{Duration: 30 * time.Second}, @@ -144,10 +144,10 @@ func TestValidateKubeletConfiguration(t *testing.T) { name: "specify CPUCFSQuotaPeriod without enabling CPUCFSQuotaPeriod", configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { conf.FeatureGates = map[string]bool{"CustomCPUCFSQuotaPeriod": false} - conf.CPUCFSQuotaPeriod = metav1.Duration{Duration: 200 * time.Millisecond} + conf.CPUCFSQuotaPeriod = metav1.Duration{Duration: 200 * time.Microsecond} return conf }, - errMsg: "invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) {200ms} requires feature gate CustomCPUCFSQuotaPeriod", + errMsg: "invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) {200us} requires feature gate CustomCPUCFSQuotaPeriod", }, { name: "invalid CPUCFSQuotaPeriod", diff --git a/pkg/kubelet/cm/helpers_linux.go b/pkg/kubelet/cm/helpers_linux.go index 25ff3f13b82..5c6e5938525 100644 --- a/pkg/kubelet/cm/helpers_linux.go +++ b/pkg/kubelet/cm/helpers_linux.go @@ -44,7 +44,7 @@ const ( SharesPerCPU = 1024 MilliCPUToCPU = 1000 - // 100000 is equivalent to 100ms + // 100000 is equivalent to 100usec QuotaPeriod = 100000 MinQuotaPeriod = 1000 ) @@ -52,8 +52,8 @@ const ( // MilliCPUToQuota converts milliCPU to CFS quota and period values. func MilliCPUToQuota(milliCPU int64, period int64) (quota int64) { // CFS quota is measured in two values: - // - cfs_period_us=100ms (the amount of time to measure usage across given by period) - // - cfs_quota=20ms (the amount of cpu time allowed to be used across a period) + // - cfs_period_us=100usec (the amount of time to measure usage across given by period) + // - cfs_quota=20usec (the amount of cpu time allowed to be used across a period) // so in the above example, you are limited to 20% of a single CPU // for multi-cpu environments, you just scale equivalent amounts // see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details diff --git a/pkg/kubelet/kuberuntime/helpers_linux.go b/pkg/kubelet/kuberuntime/helpers_linux.go index 3f83ff86e4a..33616de3850 100644 --- a/pkg/kubelet/kuberuntime/helpers_linux.go +++ b/pkg/kubelet/kuberuntime/helpers_linux.go @@ -22,7 +22,7 @@ package kuberuntime const ( milliCPUToCPU = 1000 - // 100000 is equivalent to 100ms + // 100000 is equivalent to 100usec quotaPeriod = 100000 minQuotaPeriod = 1000 ) @@ -30,8 +30,8 @@ const ( // milliCPUToQuota converts milliCPU to CFS quota and period values func milliCPUToQuota(milliCPU int64, period int64) (quota int64) { // CFS quota is measured in two values: - // - cfs_period_us=100ms (the amount of time to measure usage across) - // - cfs_quota=20ms (the amount of cpu time allowed to be used across a period) + // - cfs_period_us=100usec (the amount of time to measure usage across given by period) + // - cfs_quota=20usec (the amount of cpu time allowed to be used across a period) // so in the above example, you are limited to 20% of a single CPU // for multi-cpu environments, you just scale equivalent amounts // see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details From 32df800ba7a678e00a2c8025194ad95638013680 Mon Sep 17 00:00:00 2001 From: Dmitry Verkhoturov Date: Mon, 1 Aug 2022 21:25:21 +0200 Subject: [PATCH 2/2] change CPUCFSQuotaPeriod default value to 100us to match Linux default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cpu.cfs_period_us is 100μs by default despite having an "ms" unit for some unfortunate reason. Documentation: https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management The desired effect of that change is to match k8s default `CPUCFSQuotaPeriod` value (100ms before that change) with one used in k8s without the `CustomCPUCFSQuotaPeriod` flag enabled and Linux CFS (100us, 1000x smaller than 100ms). --- .../scheme/testdata/KubeletConfiguration/after/v1beta1.yaml | 2 +- .../KubeletConfiguration/roundtrip/default/v1beta1.yaml | 2 +- pkg/kubelet/apis/config/validation/validation_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml index edea8030375..3cb76b89923 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml @@ -19,7 +19,7 @@ containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true -cpuCFSQuotaPeriod: 100us +cpuCFSQuotaPeriod: 100ms cpuManagerPolicy: none cpuManagerReconcilePeriod: 10s enableControllerAttachDetach: true diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml index edea8030375..3cb76b89923 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml @@ -19,7 +19,7 @@ containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true -cpuCFSQuotaPeriod: 100us +cpuCFSQuotaPeriod: 100ms cpuManagerPolicy: none cpuManagerReconcilePeriod: 10s enableControllerAttachDetach: true diff --git a/pkg/kubelet/apis/config/validation/validation_test.go b/pkg/kubelet/apis/config/validation/validation_test.go index 3a3e108c70b..95cbcde5d02 100644 --- a/pkg/kubelet/apis/config/validation/validation_test.go +++ b/pkg/kubelet/apis/config/validation/validation_test.go @@ -147,7 +147,7 @@ func TestValidateKubeletConfiguration(t *testing.T) { conf.CPUCFSQuotaPeriod = metav1.Duration{Duration: 200 * time.Microsecond} return conf }, - errMsg: "invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) {200us} requires feature gate CustomCPUCFSQuotaPeriod", + errMsg: "invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) {200µs} requires feature gate CustomCPUCFSQuotaPeriod", }, { name: "invalid CPUCFSQuotaPeriod",