diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml index c6d01ae38fe..c6f7b27a26e 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml @@ -78,6 +78,7 @@ ComponentConfigs: CacheAuthorizedTTL: 5m0s CacheUnauthorizedTTL: 30s CPUCFSQuota: true + CPUCFSQuotaPeriod: 0s CPUManagerPolicy: none CPUManagerReconcilePeriod: 10s CgroupDriver: cgroupfs diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml index 06fd21a2ce1..01f4137dd71 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml @@ -91,6 +91,7 @@ kubeletConfiguration: containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true + cpuCFSQuotaPeriod: 0s cpuManagerPolicy: none cpuManagerReconcilePeriod: 10s enableControllerAttachDetach: true diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml index 6db892537d2..f7be2f9bd71 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml @@ -107,6 +107,7 @@ containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true +cpuCFSQuotaPeriod: 0s cpuManagerPolicy: none cpuManagerReconcilePeriod: 10s enableControllerAttachDetach: true diff --git a/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml b/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml index a659926db69..ab89f85b5d8 100644 --- a/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml +++ b/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml @@ -102,6 +102,7 @@ containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true +cpuCFSQuotaPeriod: 0s cpuManagerPolicy: none cpuManagerReconcilePeriod: 10s enableControllerAttachDetach: true diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index d06e978d813..39a31f46b9a 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -530,6 +530,7 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig fs.StringVar(&c.ResolverConfig, "resolv-conf", c.ResolverConfig, "Resolver configuration file used as the basis for the container DNS resolution configuration.") fs.BoolVar(&c.CPUCFSQuota, "cpu-cfs-quota", c.CPUCFSQuota, "Enable CPU CFS quota enforcement for containers that specify CPU limits") + fs.DurationVar(&c.CPUCFSQuotaPeriod.Duration, "cpu-cfs-quota-period", c.CPUCFSQuotaPeriod.Duration, "Sets CPU CFS quota period value, cpu.cfs_period_us, defaults to Linux Kernel default") fs.BoolVar(&c.EnableControllerAttachDetach, "enable-controller-attach-detach", c.EnableControllerAttachDetach, "Enables the Attach/Detach controller to manage attachment/detachment of volumes scheduled to this node, and disables kubelet from executing any attach/detach operations") fs.BoolVar(&c.MakeIPTablesUtilChains, "make-iptables-util-chains", c.MakeIPTablesUtilChains, "If true, kubelet will ensure iptables utility rules are present on host.") fs.Int32Var(&c.IPTablesMasqueradeBit, "iptables-masquerade-bit", c.IPTablesMasqueradeBit, "The bit of the fwmark space to mark packets for SNAT. Must be within the range [0, 31]. Please match this parameter with corresponding parameter in kube-proxy.") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index f0cf3ac7cb5..77c9f4af5e6 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -708,6 +708,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan ExperimentalCPUManagerReconcilePeriod: s.CPUManagerReconcilePeriod.Duration, ExperimentalPodPidsLimit: s.PodPidsLimit, EnforceCPULimits: s.CPUCFSQuota, + CPUCFSQuotaPeriod: s.CPUCFSQuotaPeriod.Duration, }, s.FailSwapOn, devicePluginEnabled, diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index a1f764150d2..9fdefcf71d5 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -155,6 +155,12 @@ const ( // Alternative container-level CPU affinity policies. CPUManager utilfeature.Feature = "CPUManager" + // owner: @szuecs + // alpha: v1.12 + // + // Enable nodes to change CPUCFSQuotaPeriod + CPUCFSQuotaPeriod utilfeature.Feature = "CustomCPUCFSQuotaPeriod" + // owner: @derekwaynecarr // beta: v1.10 // @@ -408,6 +414,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS ExpandInUsePersistentVolumes: {Default: false, PreRelease: utilfeature.Alpha}, AttachVolumeLimit: {Default: false, PreRelease: utilfeature.Alpha}, CPUManager: {Default: true, PreRelease: utilfeature.Beta}, + CPUCFSQuotaPeriod: {Default: false, PreRelease: utilfeature.Alpha}, ServiceNodeExclusion: {Default: false, PreRelease: utilfeature.Alpha}, MountContainers: {Default: false, PreRelease: utilfeature.Alpha}, VolumeScheduling: {Default: true, PreRelease: utilfeature.Beta}, diff --git a/pkg/kubelet/apis/config/helpers_test.go b/pkg/kubelet/apis/config/helpers_test.go index 1421f106b01..412975b98bf 100644 --- a/pkg/kubelet/apis/config/helpers_test.go +++ b/pkg/kubelet/apis/config/helpers_test.go @@ -145,6 +145,7 @@ var ( "Authorization.Webhook.CacheAuthorizedTTL.Duration", "Authorization.Webhook.CacheUnauthorizedTTL.Duration", "CPUCFSQuota", + "CPUCFSQuotaPeriod.Duration", "CPUManagerPolicy", "CPUManagerReconcilePeriod.Duration", "QOSReserved[*]", diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index 2f7540f4a2d..ab232157493 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -220,6 +220,8 @@ type KubeletConfiguration struct { // cpuCFSQuota enables CPU CFS quota enforcement for containers that // specify CPU limits CPUCFSQuota bool + // CPUCFSQuotaPeriod sets the CPU CFS quota period value, cpu.cfs_period_us, defaults to 100ms + CPUCFSQuotaPeriod metav1.Duration // maxOpenFiles is Number of files that can be opened by Kubelet process. MaxOpenFiles int64 // contentType is contentType of requests sent to apiserver. diff --git a/pkg/kubelet/apis/config/v1beta1/BUILD b/pkg/kubelet/apis/config/v1beta1/BUILD index 67bc052b9db..8eaa13c44ac 100644 --- a/pkg/kubelet/apis/config/v1beta1/BUILD +++ b/pkg/kubelet/apis/config/v1beta1/BUILD @@ -18,6 +18,7 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1", deps = [ + "//pkg/features:go_default_library", "//pkg/kubelet/apis/config:go_default_library", "//pkg/kubelet/qos:go_default_library", "//pkg/kubelet/types:go_default_library", diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index 004f90af8d4..4d939029fbf 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -21,6 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/qos" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/master/ports" @@ -154,6 +155,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.CPUCFSQuota == nil { obj.CPUCFSQuota = utilpointer.BoolPtr(true) } + if obj.CPUCFSQuotaPeriod == nil && obj.FeatureGates[string(features.CPUCFSQuotaPeriod)] { + obj.CPUCFSQuotaPeriod = &metav1.Duration{Duration: 100 * time.Millisecond} + } if obj.MaxOpenFiles == 0 { obj.MaxOpenFiles = 1000000 } diff --git a/pkg/kubelet/apis/config/v1beta1/types.go b/pkg/kubelet/apis/config/v1beta1/types.go index 91321f58d35..281298f029d 100644 --- a/pkg/kubelet/apis/config/v1beta1/types.go +++ b/pkg/kubelet/apis/config/v1beta1/types.go @@ -466,6 +466,13 @@ type KubeletConfiguration struct { // Default: true // +optional CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` + // CPUCFSQuotaPeriod is the CPU CFS quota period value, cpu.cfs_period_us. + // Dynamic Kubelet Config (beta): If dynamically updating this field, consider that + // limits set for containers will result in different cpu.cfs_quota settings. This + // will trigger container restarts on the node being reconfigured. + // Default: "100ms" + // +optional + CPUCFSQuotaPeriod *metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty"` // maxOpenFiles is Number of files that can be opened by Kubelet process. // Dynamic Kubelet Config (beta): If dynamically updating this field, consider that // it may impact the ability of the Kubelet to interact with the node's filesystem. diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go index 13281ad3135..55cb4f4bed7 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go @@ -280,6 +280,9 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in if err := v1.Convert_Pointer_bool_To_bool(&in.CPUCFSQuota, &out.CPUCFSQuota, s); err != nil { return err } + if err := v1.Convert_Pointer_v1_Duration_To_v1_Duration(&in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod, s); err != nil { + return err + } out.MaxOpenFiles = in.MaxOpenFiles out.ContentType = in.ContentType if err := v1.Convert_Pointer_int32_To_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil { @@ -406,6 +409,9 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in if err := v1.Convert_bool_To_Pointer_bool(&in.CPUCFSQuota, &out.CPUCFSQuota, s); err != nil { return err } + if err := v1.Convert_v1_Duration_To_Pointer_v1_Duration(&in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod, s); err != nil { + return err + } out.MaxOpenFiles = in.MaxOpenFiles out.ContentType = in.ContentType if err := v1.Convert_int32_To_Pointer_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil { diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.deepcopy.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.deepcopy.go index 2bf1a355b19..37be9cb1843 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package v1beta1 import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -178,6 +179,11 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = new(bool) **out = **in } + if in.CPUCFSQuotaPeriod != nil { + in, out := &in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod + *out = new(v1.Duration) + **out = **in + } if in.KubeAPIQPS != nil { in, out := &in.KubeAPIQPS, &out.KubeAPIQPS *out = new(int32) diff --git a/pkg/kubelet/apis/config/validation/BUILD b/pkg/kubelet/apis/config/validation/BUILD index 4264132db7a..f7348901d4e 100644 --- a/pkg/kubelet/apis/config/validation/BUILD +++ b/pkg/kubelet/apis/config/validation/BUILD @@ -43,6 +43,7 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/kubelet/apis/config:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", ], ) diff --git a/pkg/kubelet/apis/config/validation/validation.go b/pkg/kubelet/apis/config/validation/validation.go index d1d3a8b9ea0..9a39898fcfe 100644 --- a/pkg/kubelet/apis/config/validation/validation.go +++ b/pkg/kubelet/apis/config/validation/validation.go @@ -18,6 +18,7 @@ package validation import ( "fmt" + "time" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilvalidation "k8s.io/apimachinery/pkg/util/validation" @@ -54,6 +55,9 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error if kc.HealthzPort != 0 && utilvalidation.IsValidPortNum(int(kc.HealthzPort)) != nil { allErrors = append(allErrors, fmt.Errorf("invalid configuration: HealthzPort (--healthz-port) %v must be between 1 and 65535, inclusive", kc.HealthzPort)) } + if localFeatureGate.Enabled(features.CPUCFSQuotaPeriod) && utilvalidation.IsInRange(int(kc.CPUCFSQuotaPeriod.Duration), int(1*time.Microsecond), int(time.Second)) != nil { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: CPUCFSQuotaPeriod (--cpu-cfs-quota-period) %v must be between 1usec and 1sec, inclusive", kc.CPUCFSQuotaPeriod)) + } if utilvalidation.IsInRange(int(kc.ImageGCHighThresholdPercent), 0, 100) != nil { allErrors = append(allErrors, fmt.Errorf("invalid configuration: ImageGCHighThresholdPercent (--image-gc-high-threshold) %v must be between 0 and 100, inclusive", kc.ImageGCHighThresholdPercent)) } diff --git a/pkg/kubelet/apis/config/validation/validation_test.go b/pkg/kubelet/apis/config/validation/validation_test.go index 924015a64bf..27c3a961424 100644 --- a/pkg/kubelet/apis/config/validation/validation_test.go +++ b/pkg/kubelet/apis/config/validation/validation_test.go @@ -18,7 +18,9 @@ package validation import ( "testing" + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" ) @@ -48,6 +50,7 @@ func TestValidateKubeletConfiguration(t *testing.T) { RegistryPullQPS: 5, HairpinMode: kubeletconfig.PromiscuousBridge, NodeLeaseDurationSeconds: 1, + CPUCFSQuotaPeriod: metav1.Duration{Duration: 100 * time.Millisecond}, } if allErrors := ValidateKubeletConfiguration(successCase); allErrors != nil { t.Errorf("expect no errors, got %v", allErrors) @@ -77,6 +80,7 @@ func TestValidateKubeletConfiguration(t *testing.T) { RegistryPullQPS: -10, HairpinMode: "foo", NodeLeaseDurationSeconds: -1, + CPUCFSQuotaPeriod: metav1.Duration{Duration: 0}, } const numErrs = 23 if allErrors := ValidateKubeletConfiguration(errorCase); len(allErrors.(utilerrors.Aggregate).Errors()) != numErrs { diff --git a/pkg/kubelet/apis/config/zz_generated.deepcopy.go b/pkg/kubelet/apis/config/zz_generated.deepcopy.go index 3eddbbf3c09..266a862943f 100644 --- a/pkg/kubelet/apis/config/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/config/zz_generated.deepcopy.go @@ -123,6 +123,7 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { } } out.RuntimeRequestTimeout = in.RuntimeRequestTimeout + out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod if in.EvictionHard != nil { in, out := &in.EvictionHard, &out.EvictionHard *out = make(map[string]string, len(*in)) diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index 132c08b8b1f..6800caf05fd 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -134,11 +134,14 @@ go_test( embed = [":go_default_library"], deps = select({ "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/features:go_default_library", "//pkg/kubelet/eviction/api:go_default_library", "//pkg/util/mount:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", ], diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 60382616a13..8d467bf2054 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -114,6 +114,7 @@ type NodeConfig struct { ExperimentalCPUManagerReconcilePeriod time.Duration ExperimentalPodPidsLimit int64 EnforceCPULimits bool + CPUCFSQuotaPeriod time.Duration } type NodeAllocatableConfig struct { diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index ce01468e9bd..2f460ba8cba 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -307,6 +307,7 @@ func (cm *containerManagerImpl) NewPodContainerManager() PodContainerManager { cgroupManager: cm.cgroupManager, podPidsLimit: cm.ExperimentalPodPidsLimit, enforceCPULimits: cm.EnforceCPULimits, + cpuCFSQuotaPeriod: uint64(cm.CPUCFSQuotaPeriod / time.Microsecond), } } return &podContainerManagerNoop{ diff --git a/pkg/kubelet/cm/helpers_linux.go b/pkg/kubelet/cm/helpers_linux.go index d04128edd71..64b12854971 100644 --- a/pkg/kubelet/cm/helpers_linux.go +++ b/pkg/kubelet/cm/helpers_linux.go @@ -27,9 +27,11 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/v1/resource" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" + kubefeatures "k8s.io/kubernetes/pkg/features" ) const ( @@ -44,28 +46,29 @@ const ( ) // MilliCPUToQuota converts milliCPU to CFS quota and period values. -func MilliCPUToQuota(milliCPU int64) (quota int64, period uint64) { +func MilliCPUToQuota(milliCPU int64, period int64) (quota int64) { // CFS quota is measured in two values: - // - cfs_period_us=100ms (the amount of time to measure usage across) + // - cfs_period_us=100ms (the amount of time to measure usage across given by period) // - cfs_quota=20ms (the amount of cpu time allowed to be used across a period) // so in the above example, you are limited to 20% of a single CPU // for multi-cpu environments, you just scale equivalent amounts + // see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details if milliCPU == 0 { return } - // we set the period to 100ms by default - period = QuotaPeriod + if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUCFSQuotaPeriod) { + period = QuotaPeriod + } // we then convert your milliCPU to a value normalized over a period - quota = (milliCPU * QuotaPeriod) / MilliCPUToCPU + quota = (milliCPU * period) / MilliCPUToCPU // quota needs to be a minimum of 1ms. if quota < MinQuotaPeriod { quota = MinQuotaPeriod } - return } @@ -103,7 +106,7 @@ func HugePageLimits(resourceList v1.ResourceList) map[int64]int64 { } // ResourceConfigForPod takes the input pod and outputs the cgroup resource config. -func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool) *ResourceConfig { +func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool, cpuPeriod uint64) *ResourceConfig { // sum requests and limits. reqs, limits := resource.PodRequestsAndLimits(pod) @@ -122,7 +125,7 @@ func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool) *ResourceConfig { // convert to CFS values cpuShares := MilliCPUToShares(cpuRequests) - cpuQuota, cpuPeriod := MilliCPUToQuota(cpuLimits) + cpuQuota := MilliCPUToQuota(cpuLimits, int64(cpuPeriod)) // track if limits were applied for each resource. memoryLimitsDeclared := true diff --git a/pkg/kubelet/cm/helpers_linux_test.go b/pkg/kubelet/cm/helpers_linux_test.go index 24d1b58488a..e82b800bdb1 100644 --- a/pkg/kubelet/cm/helpers_linux_test.go +++ b/pkg/kubelet/cm/helpers_linux_test.go @@ -20,11 +20,15 @@ package cm import ( "reflect" + "strconv" "testing" + "time" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "strconv" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + pkgfeatures "k8s.io/kubernetes/pkg/features" ) // getResourceList returns a ResourceList with the @@ -49,14 +53,18 @@ func getResourceRequirements(requests, limits v1.ResourceList) v1.ResourceRequir } func TestResourceConfigForPod(t *testing.T) { + defaultQuotaPeriod := uint64(100 * time.Millisecond / time.Microsecond) + tunedQuotaPeriod := uint64(5 * time.Millisecond / time.Microsecond) + minShares := uint64(MinShares) burstableShares := MilliCPUToShares(100) memoryQuantity := resource.MustParse("200Mi") burstableMemory := memoryQuantity.Value() burstablePartialShares := MilliCPUToShares(200) - burstableQuota, burstablePeriod := MilliCPUToQuota(200) + burstableQuota := MilliCPUToQuota(200, int64(defaultQuotaPeriod)) guaranteedShares := MilliCPUToShares(100) - guaranteedQuota, guaranteedPeriod := MilliCPUToQuota(100) + guaranteedQuota := MilliCPUToQuota(100, int64(defaultQuotaPeriod)) + guaranteedTunedQuota := MilliCPUToQuota(100, int64(tunedQuotaPeriod)) memoryQuantity = resource.MustParse("100Mi") cpuNoLimit := int64(-1) guaranteedMemory := memoryQuantity.Value() @@ -64,6 +72,7 @@ func TestResourceConfigForPod(t *testing.T) { pod *v1.Pod expected *ResourceConfig enforceCPULimits bool + quotaPeriod uint64 }{ "besteffort": { pod: &v1.Pod{ @@ -76,6 +85,7 @@ func TestResourceConfigForPod(t *testing.T) { }, }, enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, expected: &ResourceConfig{CpuShares: &minShares}, }, "burstable-no-limits": { @@ -89,6 +99,7 @@ func TestResourceConfigForPod(t *testing.T) { }, }, enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, expected: &ResourceConfig{CpuShares: &burstableShares}, }, "burstable-with-limits": { @@ -102,7 +113,8 @@ func TestResourceConfigForPod(t *testing.T) { }, }, enforceCPULimits: true, - expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &burstableQuota, CpuPeriod: &burstablePeriod, Memory: &burstableMemory}, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &burstableQuota, CpuPeriod: &defaultQuotaPeriod, Memory: &burstableMemory}, }, "burstable-with-limits-no-cpu-enforcement": { pod: &v1.Pod{ @@ -115,7 +127,8 @@ func TestResourceConfigForPod(t *testing.T) { }, }, enforceCPULimits: false, - expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &cpuNoLimit, CpuPeriod: &burstablePeriod, Memory: &burstableMemory}, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &cpuNoLimit, CpuPeriod: &defaultQuotaPeriod, Memory: &burstableMemory}, }, "burstable-partial-limits": { pod: &v1.Pod{ @@ -131,6 +144,52 @@ func TestResourceConfigForPod(t *testing.T) { }, }, enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstablePartialShares}, + }, + "burstable-with-limits-with-tuned-quota": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: tunedQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &burstableQuota, CpuPeriod: &tunedQuotaPeriod, Memory: &burstableMemory}, + }, + "burstable-with-limits-no-cpu-enforcement-with-tuned-quota": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")), + }, + }, + }, + }, + enforceCPULimits: false, + quotaPeriod: tunedQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &cpuNoLimit, CpuPeriod: &tunedQuotaPeriod, Memory: &burstableMemory}, + }, + "burstable-partial-limits-with-tuned-quota": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")), + }, + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("", "")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: tunedQuotaPeriod, expected: &ResourceConfig{CpuShares: &burstablePartialShares}, }, "guaranteed": { @@ -144,7 +203,8 @@ func TestResourceConfigForPod(t *testing.T) { }, }, enforceCPULimits: true, - expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &guaranteedPeriod, Memory: &guaranteedMemory}, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory}, }, "guaranteed-no-cpu-enforcement": { pod: &v1.Pod{ @@ -157,11 +217,264 @@ func TestResourceConfigForPod(t *testing.T) { }, }, enforceCPULimits: false, - expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &guaranteedPeriod, Memory: &guaranteedMemory}, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory}, + }, + "guaranteed-with-tuned-quota": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: tunedQuotaPeriod, + expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedTunedQuota, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory}, + }, + "guaranteed-no-cpu-enforcement-with-tuned-quota": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")), + }, + }, + }, + }, + enforceCPULimits: false, + quotaPeriod: tunedQuotaPeriod, + expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory}, }, } + for testName, testCase := range testCases { - actual := ResourceConfigForPod(testCase.pod, testCase.enforceCPULimits) + + actual := ResourceConfigForPod(testCase.pod, testCase.enforceCPULimits, testCase.quotaPeriod) + + if !reflect.DeepEqual(actual.CpuPeriod, testCase.expected.CpuPeriod) { + t.Errorf("unexpected result, test: %v, cpu period not as expected", testName) + } + if !reflect.DeepEqual(actual.CpuQuota, testCase.expected.CpuQuota) { + t.Errorf("unexpected result, test: %v, cpu quota not as expected", testName) + } + if !reflect.DeepEqual(actual.CpuShares, testCase.expected.CpuShares) { + t.Errorf("unexpected result, test: %v, cpu shares not as expected", testName) + } + if !reflect.DeepEqual(actual.Memory, testCase.expected.Memory) { + t.Errorf("unexpected result, test: %v, memory not as expected", testName) + } + } +} + +func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) { + defaultQuotaPeriod := uint64(100 * time.Millisecond / time.Microsecond) + tunedQuotaPeriod := uint64(5 * time.Millisecond / time.Microsecond) + tunedQuota := int64(1 * time.Millisecond / time.Microsecond) + + utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUCFSQuotaPeriod, true) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUCFSQuotaPeriod, false) + + minShares := uint64(MinShares) + burstableShares := MilliCPUToShares(100) + memoryQuantity := resource.MustParse("200Mi") + burstableMemory := memoryQuantity.Value() + burstablePartialShares := MilliCPUToShares(200) + burstableQuota := MilliCPUToQuota(200, int64(defaultQuotaPeriod)) + guaranteedShares := MilliCPUToShares(100) + guaranteedQuota := MilliCPUToQuota(100, int64(defaultQuotaPeriod)) + guaranteedTunedQuota := MilliCPUToQuota(100, int64(tunedQuotaPeriod)) + memoryQuantity = resource.MustParse("100Mi") + cpuNoLimit := int64(-1) + guaranteedMemory := memoryQuantity.Value() + testCases := map[string]struct { + pod *v1.Pod + expected *ResourceConfig + enforceCPULimits bool + quotaPeriod uint64 + }{ + "besteffort": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("", ""), getResourceList("", "")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &minShares}, + }, + "burstable-no-limits": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("", "")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstableShares}, + }, + "burstable-with-limits": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &burstableQuota, CpuPeriod: &defaultQuotaPeriod, Memory: &burstableMemory}, + }, + "burstable-with-limits-no-cpu-enforcement": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")), + }, + }, + }, + }, + enforceCPULimits: false, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &cpuNoLimit, CpuPeriod: &defaultQuotaPeriod, Memory: &burstableMemory}, + }, + "burstable-partial-limits": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")), + }, + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("", "")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstablePartialShares}, + }, + "burstable-with-limits-with-tuned-quota": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: tunedQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &tunedQuota, CpuPeriod: &tunedQuotaPeriod, Memory: &burstableMemory}, + }, + "burstable-with-limits-no-cpu-enforcement-with-tuned-quota": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")), + }, + }, + }, + }, + enforceCPULimits: false, + quotaPeriod: tunedQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &cpuNoLimit, CpuPeriod: &tunedQuotaPeriod, Memory: &burstableMemory}, + }, + "burstable-partial-limits-with-tuned-quota": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")), + }, + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("", "")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: tunedQuotaPeriod, + expected: &ResourceConfig{CpuShares: &burstablePartialShares}, + }, + "guaranteed": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory}, + }, + "guaranteed-no-cpu-enforcement": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")), + }, + }, + }, + }, + enforceCPULimits: false, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory}, + }, + "guaranteed-with-tuned-quota": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")), + }, + }, + }, + }, + enforceCPULimits: true, + quotaPeriod: tunedQuotaPeriod, + expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedTunedQuota, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory}, + }, + "guaranteed-no-cpu-enforcement-with-tuned-quota": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")), + }, + }, + }, + }, + enforceCPULimits: false, + quotaPeriod: tunedQuotaPeriod, + expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory}, + }, + } + + for testName, testCase := range testCases { + + actual := ResourceConfigForPod(testCase.pod, testCase.enforceCPULimits, testCase.quotaPeriod) + if !reflect.DeepEqual(actual.CpuPeriod, testCase.expected.CpuPeriod) { t.Errorf("unexpected result, test: %v, cpu period not as expected", testName) } @@ -225,9 +538,9 @@ func TestMilliCPUToQuota(t *testing.T) { }, } for _, testCase := range testCases { - quota, period := MilliCPUToQuota(testCase.input) - if quota != testCase.quota || period != testCase.period { - t.Errorf("Input %v, expected quota %v period %v, but got quota %v period %v", testCase.input, testCase.quota, testCase.period, quota, period) + quota := MilliCPUToQuota(testCase.input, int64(testCase.period)) + if quota != testCase.quota { + t.Errorf("Input %v and %v, expected quota %v, but got quota %v", testCase.input, testCase.period, testCase.quota, quota) } } } diff --git a/pkg/kubelet/cm/helpers_unsupported.go b/pkg/kubelet/cm/helpers_unsupported.go index ee3ed91d557..82ee3b69344 100644 --- a/pkg/kubelet/cm/helpers_unsupported.go +++ b/pkg/kubelet/cm/helpers_unsupported.go @@ -28,13 +28,12 @@ const ( SharesPerCPU = 0 MilliCPUToCPU = 0 - QuotaPeriod = 0 MinQuotaPeriod = 0 ) -// MilliCPUToQuota converts milliCPU to CFS quota and period values. -func MilliCPUToQuota(milliCPU int64) (int64, int64) { - return 0, 0 +// MilliCPUToQuota converts milliCPU and period to CFS quota values. +func MilliCPUToQuota(milliCPU, period int64) int64 { + return 0 } // MilliCPUToShares converts the milliCPU to CFS shares. @@ -43,7 +42,7 @@ func MilliCPUToShares(milliCPU int64) int64 { } // ResourceConfigForPod takes the input pod and outputs the cgroup resource config. -func ResourceConfigForPod(pod *v1.Pod, enforceCPULimit bool) *ResourceConfig { +func ResourceConfigForPod(pod *v1.Pod, enforceCPULimit bool, cpuPeriod uint64) *ResourceConfig { return nil } diff --git a/pkg/kubelet/cm/pod_container_manager_linux.go b/pkg/kubelet/cm/pod_container_manager_linux.go index 703437160dc..d0c3a829947 100644 --- a/pkg/kubelet/cm/pod_container_manager_linux.go +++ b/pkg/kubelet/cm/pod_container_manager_linux.go @@ -51,6 +51,9 @@ type podContainerManagerImpl struct { podPidsLimit int64 // enforceCPULimits controls whether cfs quota is enforced or not enforceCPULimits bool + // cpuCFSQuotaPeriod is the cfs period value, cfs_period_us, setting per + // node for all containers in usec + cpuCFSQuotaPeriod uint64 } // Make sure that podContainerManagerImpl implements the PodContainerManager interface @@ -81,7 +84,7 @@ func (m *podContainerManagerImpl) EnsureExists(pod *v1.Pod) error { // Create the pod container containerConfig := &CgroupConfig{ Name: podContainerName, - ResourceParameters: ResourceConfigForPod(pod, m.enforceCPULimits), + ResourceParameters: ResourceConfigForPod(pod, m.enforceCPULimits, m.cpuCFSQuotaPeriod), } if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportPodPidsLimit) && m.podPidsLimit > 0 { containerConfig.ResourceParameters.PodPidsLimit = &m.podPidsLimit diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index f136b70e8a1..556157a4a49 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -668,6 +668,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, float32(kubeCfg.RegistryPullQPS), int(kubeCfg.RegistryBurst), kubeCfg.CPUCFSQuota, + kubeCfg.CPUCFSQuotaPeriod, runtimeService, imageService, kubeDeps.ContainerManager.InternalContainerLifecycle(), diff --git a/pkg/kubelet/kuberuntime/BUILD b/pkg/kubelet/kuberuntime/BUILD index df054cba861..f64e1a80853 100644 --- a/pkg/kubelet/kuberuntime/BUILD +++ b/pkg/kubelet/kuberuntime/BUILD @@ -83,6 +83,7 @@ go_library( go_test( name = "go_default_test", srcs = [ + "helpers_linux_test.go", "helpers_test.go", "instrumented_services_test.go", "kuberuntime_container_linux_test.go", diff --git a/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go b/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go index a630588e2e9..6eb64ee7b02 100644 --- a/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go @@ -21,6 +21,7 @@ import ( "time" cadvisorapi "github.com/google/cadvisor/info/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/flowcontrol" @@ -74,6 +75,7 @@ func NewFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS kubeRuntimeManager := &kubeGenericRuntimeManager{ recorder: recorder, cpuCFSQuota: false, + cpuCFSQuotaPeriod: metav1.Duration{Duration: time.Microsecond * 100}, livenessManager: proberesults.NewManager(), containerRefManager: kubecontainer.NewRefManager(), machineInfo: machineInfo, diff --git a/pkg/kubelet/kuberuntime/helpers_linux.go b/pkg/kubelet/kuberuntime/helpers_linux.go index 15abb4a0889..b6a4f4f85c9 100644 --- a/pkg/kubelet/kuberuntime/helpers_linux.go +++ b/pkg/kubelet/kuberuntime/helpers_linux.go @@ -18,6 +18,11 @@ limitations under the License. package kuberuntime +import ( + utilfeature "k8s.io/apiserver/pkg/util/feature" + kubefeatures "k8s.io/kubernetes/pkg/features" +) + const ( // Taken from lmctfy https://github.com/google/lmctfy/blob/master/lmctfy/controllers/cpu_controller.cc minShares = 2 @@ -25,7 +30,7 @@ const ( milliCPUToCPU = 1000 // 100000 is equivalent to 100ms - quotaPeriod = 100 * minQuotaPeriod + quotaPeriod = 100000 minQuotaPeriod = 1000 ) @@ -44,21 +49,22 @@ func milliCPUToShares(milliCPU int64) int64 { } // milliCPUToQuota converts milliCPU to CFS quota and period values -func milliCPUToQuota(milliCPU int64) (quota int64, period int64) { +func milliCPUToQuota(milliCPU int64, period int64) (quota int64) { // CFS quota is measured in two values: // - cfs_period_us=100ms (the amount of time to measure usage across) // - cfs_quota=20ms (the amount of cpu time allowed to be used across a period) // so in the above example, you are limited to 20% of a single CPU // for multi-cpu environments, you just scale equivalent amounts + // see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details if milliCPU == 0 { return } - - // we set the period to 100ms by default - period = quotaPeriod + if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUCFSQuotaPeriod) { + period = quotaPeriod + } // we then convert your milliCPU to a value normalized over a period - quota = (milliCPU * quotaPeriod) / milliCPUToCPU + quota = (milliCPU * period) / milliCPUToCPU // quota needs to be a minimum of 1ms. if quota < minQuotaPeriod { diff --git a/pkg/kubelet/kuberuntime/helpers_linux_test.go b/pkg/kubelet/kuberuntime/helpers_linux_test.go new file mode 100644 index 00000000000..73615c61cc2 --- /dev/null +++ b/pkg/kubelet/kuberuntime/helpers_linux_test.go @@ -0,0 +1,204 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kuberuntime + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/kubernetes/pkg/features" +) + +func TestMilliCPUToQuota(t *testing.T) { + for _, testCase := range []struct { + msg string + input int64 + expected int64 + period uint64 + }{ + { + msg: "all-zero", + input: int64(0), + expected: int64(0), + period: uint64(0), + }, + { + msg: "5 input default quota and period", + input: int64(5), + expected: int64(1000), + period: uint64(100000), + }, + { + msg: "9 input default quota and period", + input: int64(9), + expected: int64(1000), + period: uint64(100000), + }, + { + msg: "10 input default quota and period", + input: int64(10), + expected: int64(1000), + period: uint64(100000), + }, + { + msg: "200 input 20k quota and default period", + input: int64(200), + expected: int64(20000), + period: uint64(100000), + }, + { + msg: "500 input 50k quota and default period", + input: int64(500), + expected: int64(50000), + period: uint64(100000), + }, + { + msg: "1k input 100k quota and default period", + input: int64(1000), + expected: int64(100000), + period: uint64(100000), + }, + { + msg: "1500 input 150k quota and default period", + input: int64(1500), + expected: int64(150000), + period: uint64(100000), + }} { + t.Run(testCase.msg, func(t *testing.T) { + quota := milliCPUToQuota(testCase.input, int64(testCase.period)) + if quota != testCase.expected { + t.Errorf("Input %v and %v, expected quota %v, but got quota %v", testCase.input, testCase.period, testCase.expected, quota) + } + }) + } +} + +func TestMilliCPUToQuotaWithCustomCPUCFSQuotaPeriod(t *testing.T) { + utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CPUCFSQuotaPeriod, true) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CPUCFSQuotaPeriod, false) + + for _, testCase := range []struct { + msg string + input int64 + expected int64 + period uint64 + }{ + { + msg: "all-zero", + input: int64(0), + expected: int64(0), + period: uint64(0), + }, + { + msg: "5 input default quota and period", + input: int64(5), + expected: minQuotaPeriod, + period: uint64(100000), + }, + { + msg: "9 input default quota and period", + input: int64(9), + expected: minQuotaPeriod, + period: uint64(100000), + }, + { + msg: "10 input default quota and period", + input: int64(10), + expected: minQuotaPeriod, + period: uint64(100000), + }, + { + msg: "200 input 20k quota and default period", + input: int64(200), + expected: int64(20000), + period: uint64(100000), + }, + { + msg: "500 input 50k quota and default period", + input: int64(500), + expected: int64(50000), + period: uint64(100000), + }, + { + msg: "1k input 100k quota and default period", + input: int64(1000), + expected: int64(100000), + period: uint64(100000), + }, + { + msg: "1500 input 150k quota and default period", + input: int64(1500), + expected: int64(150000), + period: uint64(100000), + }, + { + msg: "5 input 10k period and default quota expected", + input: int64(5), + period: uint64(10000), + expected: minQuotaPeriod, + }, + { + msg: "5 input 5k period and default quota expected", + input: int64(5), + period: uint64(5000), + expected: minQuotaPeriod, + }, + { + msg: "9 input 10k period and default quota expected", + input: int64(9), + period: uint64(10000), + expected: minQuotaPeriod, + }, + { + msg: "10 input 200k period and 2000 quota expected", + input: int64(10), + period: uint64(200000), + expected: int64(2000), + }, + { + msg: "200 input 200k period and 40k quota", + input: int64(200), + period: uint64(200000), + expected: int64(40000), + }, + { + msg: "500 input 20k period and 20k expected quota", + input: int64(500), + period: uint64(20000), + expected: int64(10000), + }, + { + msg: "1000 input 10k period and 10k expected quota", + input: int64(1000), + period: uint64(10000), + expected: int64(10000), + }, + { + msg: "1500 input 5000 period and 7500 expected quota", + input: int64(1500), + period: uint64(5000), + expected: int64(7500), + }} { + t.Run(testCase.msg, func(t *testing.T) { + quota := milliCPUToQuota(testCase.input, int64(testCase.period)) + if quota != testCase.expected { + t.Errorf("Input %v and %v, expected quota %v, but got quota %v", testCase.input, testCase.period, testCase.expected, quota) + } + }) + } +} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go index cdac02ffb53..c5da7e3171d 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go @@ -19,6 +19,8 @@ limitations under the License. package kuberuntime import ( + "time" + "k8s.io/api/core/v1" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/qos" @@ -65,7 +67,8 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerConfig(container *v1.C if m.cpuCFSQuota { // if cpuLimit.Amount is nil, then the appropriate default value is returned // to allow full usage of cpu resource. - cpuQuota, cpuPeriod := milliCPUToQuota(cpuLimit.MilliValue()) + cpuPeriod := int64(m.cpuCFSQuotaPeriod.Duration / time.Microsecond) + cpuQuota := milliCPUToQuota(cpuLimit.MilliValue(), cpuPeriod) lc.Resources.CpuQuota = cpuQuota lc.Resources.CpuPeriod = cpuPeriod } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index 6024999de9b..e8fa8a327aa 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -98,6 +98,9 @@ type kubeGenericRuntimeManager struct { // If true, enforce container cpu limits with CFS quota support cpuCFSQuota bool + // CPUCFSQuotaPeriod sets the CPU CFS quota period value, cpu.cfs_period_us, defaults to 100ms + cpuCFSQuotaPeriod metav1.Duration + // wrapped image puller. imagePuller images.ImageManager @@ -146,6 +149,7 @@ func NewKubeGenericRuntimeManager( imagePullQPS float32, imagePullBurst int, cpuCFSQuota bool, + cpuCFSQuotaPeriod metav1.Duration, runtimeService internalapi.RuntimeService, imageService internalapi.ImageManagerService, internalLifecycle cm.InternalContainerLifecycle, @@ -154,6 +158,7 @@ func NewKubeGenericRuntimeManager( kubeRuntimeManager := &kubeGenericRuntimeManager{ recorder: recorder, cpuCFSQuota: cpuCFSQuota, + cpuCFSQuotaPeriod: cpuCFSQuotaPeriod, seccompProfileRoot: seccompProfileRoot, livenessManager: livenessManager, containerRefManager: containerRefManager,