diff --git a/pkg/kubelet/cm/helpers_linux.go b/pkg/kubelet/cm/helpers_linux.go index 5500621c9ab..9ea190aba71 100644 --- a/pkg/kubelet/cm/helpers_linux.go +++ b/pkg/kubelet/cm/helpers_linux.go @@ -118,16 +118,20 @@ func HugePageLimits(resourceList v1.ResourceList) map[int64]int64 { // ResourceConfigForPod takes the input pod and outputs the cgroup resource config. func ResourceConfigForPod(allocatedPod *v1.Pod, enforceCPULimits bool, cpuPeriod uint64, enforceMemoryQoS bool) *ResourceConfig { + podLevelResourcesEnabled := utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources) + // sum requests and limits. reqs := resource.PodRequests(allocatedPod, resource.PodResourcesOptions{ - // pod is already configured to the allocated resources, and we explicitly don't want to use - // the actual resources if we're instantiating a resize. - UseStatusResources: false, + // SkipPodLevelResources is set to false when PodLevelResources feature is enabled. + SkipPodLevelResources: !podLevelResourcesEnabled, + UseStatusResources: false, }) // track if limits were applied for each resource. memoryLimitsDeclared := true cpuLimitsDeclared := true limits := resource.PodLimits(allocatedPod, resource.PodResourcesOptions{ + // SkipPodLevelResources is set to false when PodLevelResources feature is enabled. + SkipPodLevelResources: !podLevelResourcesEnabled, ContainerFn: func(res v1.ResourceList, containerType resource.ContainerType) { if res.Cpu().IsZero() { cpuLimitsDeclared = false @@ -137,6 +141,16 @@ func ResourceConfigForPod(allocatedPod *v1.Pod, enforceCPULimits bool, cpuPeriod } }, }) + + if podLevelResourcesEnabled && resource.IsPodLevelResourcesSet(allocatedPod) { + if !allocatedPod.Spec.Resources.Limits.Cpu().IsZero() { + cpuLimitsDeclared = true + } + + if !allocatedPod.Spec.Resources.Limits.Memory().IsZero() { + memoryLimitsDeclared = true + } + } // map hugepage pagesize (bytes) to limits (bytes) hugePageLimits := HugePageLimits(reqs) diff --git a/pkg/kubelet/cm/helpers_linux_test.go b/pkg/kubelet/cm/helpers_linux_test.go index 1399a2acf0c..92bab254975 100644 --- a/pkg/kubelet/cm/helpers_linux_test.go +++ b/pkg/kubelet/cm/helpers_linux_test.go @@ -70,10 +70,11 @@ func TestResourceConfigForPod(t *testing.T) { cpuNoLimit := int64(-1) guaranteedMemory := memoryQuantity.Value() testCases := map[string]struct { - pod *v1.Pod - expected *ResourceConfig - enforceCPULimits bool - quotaPeriod uint64 // in microseconds + pod *v1.Pod + expected *ResourceConfig + enforceCPULimits bool + quotaPeriod uint64 // in microseconds + podLevelResourcesEnabled bool }{ "besteffort": { pod: &v1.Pod{ @@ -274,12 +275,126 @@ func TestResourceConfigForPod(t *testing.T) { quotaPeriod: tunedQuotaPeriod, expected: &ResourceConfig{CPUShares: &burstablePartialShares}, }, + "burstable-with-pod-level-requests": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: getResourceList("100m", "100Mi"), + }, + Containers: []v1.Container{ + { + Name: "Container with no resources", + }, + }, + }, + }, + podLevelResourcesEnabled: true, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CPUShares: &burstableShares}, + }, + "burstable-with-pod-and-container-level-requests": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: getResourceList("100m", "100Mi"), + }, + Containers: []v1.Container{ + { + Name: "Container with resources", + Resources: getResourceRequirements(getResourceList("10m", "50Mi"), getResourceList("", "")), + }, + }, + }, + }, + podLevelResourcesEnabled: true, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CPUShares: &burstableShares}, + }, + "burstable-with-pod-level-resources": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: getResourceList("100m", "100Mi"), + Limits: getResourceList("200m", "200Mi"), + }, + Containers: []v1.Container{ + { + Name: "Container with no resources", + }, + }, + }, + }, + podLevelResourcesEnabled: true, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CPUShares: &burstableShares, CPUQuota: &burstableQuota, CPUPeriod: &defaultQuotaPeriod, Memory: &burstableMemory}, + }, + "burstable-with-pod-and-container-level-resources": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: getResourceList("100m", "100Mi"), + Limits: getResourceList("200m", "200Mi"), + }, + Containers: []v1.Container{ + { + Name: "Container with resources", + Resources: getResourceRequirements(getResourceList("10m", "50Mi"), getResourceList("50m", "100Mi")), + }, + }, + }, + }, + podLevelResourcesEnabled: true, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CPUShares: &burstableShares, CPUQuota: &burstableQuota, CPUPeriod: &defaultQuotaPeriod, Memory: &burstableMemory}, + }, + "guaranteed-with-pod-level-resources": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: getResourceList("100m", "100Mi"), + Limits: getResourceList("100m", "100Mi"), + }, + Containers: []v1.Container{ + { + Name: "Container with no resources", + }, + }, + }, + }, + podLevelResourcesEnabled: true, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CPUShares: &guaranteedShares, CPUQuota: &guaranteedQuota, CPUPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory}, + }, + "guaranteed-with-pod-and-container-level-resources": { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: getResourceList("100m", "100Mi"), + Limits: getResourceList("100m", "100Mi"), + }, + Containers: []v1.Container{ + { + Name: "Container with resources", + Resources: getResourceRequirements(getResourceList("10m", "50Mi"), getResourceList("50m", "100Mi")), + }, + }, + }, + }, + podLevelResourcesEnabled: true, + enforceCPULimits: true, + quotaPeriod: defaultQuotaPeriod, + expected: &ResourceConfig{CPUShares: &guaranteedShares, CPUQuota: &guaranteedQuota, CPUPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory}, + }, } for testName, testCase := range testCases { - + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.PodLevelResources, testCase.podLevelResourcesEnabled) actual := ResourceConfigForPod(testCase.pod, testCase.enforceCPULimits, testCase.quotaPeriod, false) - if !reflect.DeepEqual(actual.CPUPeriod, testCase.expected.CPUPeriod) { t.Errorf("unexpected result, test: %v, cpu period not as expected. Expected: %v, Actual:%v", testName, *testCase.expected.CPUPeriod, *actual.CPUPeriod) } @@ -287,7 +402,7 @@ func TestResourceConfigForPod(t *testing.T) { t.Errorf("unexpected result, test: %v, cpu quota not as expected. Expected: %v, Actual:%v", testName, *testCase.expected.CPUQuota, *actual.CPUQuota) } if !reflect.DeepEqual(actual.CPUShares, testCase.expected.CPUShares) { - t.Errorf("unexpected result, test: %v, cpu shares not as expected. Expected: %v, Actual:%v", testName, *testCase.expected.CPUShares, &actual.CPUShares) + t.Errorf("unexpected result, test: %v, cpu shares not as expected. Expected: %v, Actual:%v", testName, *testCase.expected.CPUShares, *actual.CPUShares) } if !reflect.DeepEqual(actual.Memory, testCase.expected.Memory) { t.Errorf("unexpected result, test: %v, memory not as expected. Expected: %v, Actual:%v", testName, *testCase.expected.Memory, *actual.Memory) diff --git a/pkg/kubelet/cm/qos_container_manager_linux.go b/pkg/kubelet/cm/qos_container_manager_linux.go index 441aa952b65..0f88e10ff69 100644 --- a/pkg/kubelet/cm/qos_container_manager_linux.go +++ b/pkg/kubelet/cm/qos_container_manager_linux.go @@ -179,7 +179,11 @@ func (m *qosContainerManagerImpl) setCPUCgroupConfig(configs map[v1.PodQOSClass] // we only care about the burstable qos tier continue } - req := resource.PodRequests(pod, resource.PodResourcesOptions{Reuse: reuseReqs}) + req := resource.PodRequests(pod, resource.PodResourcesOptions{ + Reuse: reuseReqs, + // SkipPodLevelResources is set to false when PodLevelResources feature is enabled. + SkipPodLevelResources: !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources), + }) if request, found := req[v1.ResourceCPU]; found { burstablePodCPURequest += request.MilliValue() } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go index 61dba0e736c..34856be8c73 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go @@ -35,6 +35,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" utilfeature "k8s.io/apiserver/pkg/util/feature" + resourcehelper "k8s.io/component-helpers/resource" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/klog/v2" @@ -95,6 +96,34 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerConfig(container *v1.C return lc, nil } +// getCPULimit returns the memory limit for the container to be used to calculate +// Linux Container Resources. +func getCPULimit(pod *v1.Pod, container *v1.Container) *resource.Quantity { + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) { + // When container-level CPU limit is not set, the pod-level + // limit is used in the calculation for components relying on linux resource limits + // to be set. + if container.Resources.Limits.Cpu().IsZero() { + return pod.Spec.Resources.Limits.Cpu() + } + } + return container.Resources.Limits.Cpu() +} + +// getMemoryLimit returns the memory limit for the container to be used to calculate +// Linux Container Resources. +func getMemoryLimit(pod *v1.Pod, container *v1.Container) *resource.Quantity { + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) { + // When container-level memory limit is not set, the pod-level + // limit is used in the calculation for components relying on linux resource limits + // to be set. + if container.Resources.Limits.Memory().IsZero() { + return pod.Spec.Resources.Limits.Memory() + } + } + return container.Resources.Limits.Memory() +} + // generateLinuxContainerResources generates linux container resources config for runtime func (m *kubeGenericRuntimeManager) generateLinuxContainerResources(pod *v1.Pod, container *v1.Container, enforceMemoryQoS bool) *runtimeapi.LinuxContainerResources { // set linux container resources @@ -102,7 +131,10 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerResources(pod *v1.Pod, if _, cpuRequestExists := container.Resources.Requests[v1.ResourceCPU]; cpuRequestExists { cpuRequest = container.Resources.Requests.Cpu() } - lcr := m.calculateLinuxResources(cpuRequest, container.Resources.Limits.Cpu(), container.Resources.Limits.Memory()) + + memoryLimit := getMemoryLimit(pod, container) + cpuLimit := getCPULimit(pod, container) + lcr := m.calculateLinuxResources(cpuRequest, cpuLimit, memoryLimit) lcr.OomScoreAdj = int64(qos.GetContainerOOMScoreAdjust(pod, container, int64(m.machineInfo.MemoryCapacity))) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go index b9425b61efa..69960b72d28 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go @@ -167,13 +167,14 @@ func TestGenerateLinuxContainerConfigResources(t *testing.T) { assert.NoError(t, err) tests := []struct { - name string - podResources v1.ResourceRequirements - expected *runtimeapi.LinuxContainerResources + name string + containerResources v1.ResourceRequirements + podResources *v1.ResourceRequirements + expected *runtimeapi.LinuxContainerResources }{ { name: "Request 128M/1C, Limit 256M/3C", - podResources: v1.ResourceRequirements{ + containerResources: v1.ResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceMemory: resource.MustParse("128Mi"), v1.ResourceCPU: resource.MustParse("1"), @@ -192,7 +193,7 @@ func TestGenerateLinuxContainerConfigResources(t *testing.T) { }, { name: "Request 128M/2C, No Limit", - podResources: v1.ResourceRequirements{ + containerResources: v1.ResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceMemory: resource.MustParse("128Mi"), v1.ResourceCPU: resource.MustParse("2"), @@ -205,6 +206,27 @@ func TestGenerateLinuxContainerConfigResources(t *testing.T) { MemoryLimitInBytes: 0, }, }, + { + name: "Container Level Request 128M/1C, Pod Level Limit 256M/3C", + containerResources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("128Mi"), + v1.ResourceCPU: resource.MustParse("1"), + }, + }, + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("256Mi"), + v1.ResourceCPU: resource.MustParse("3"), + }, + }, + expected: &runtimeapi.LinuxContainerResources{ + CpuPeriod: 100000, + CpuQuota: 300000, + CpuShares: 1024, + MemoryLimitInBytes: 256 * 1024 * 1024, + }, + }, } for _, test := range tests { @@ -222,12 +244,17 @@ func TestGenerateLinuxContainerConfigResources(t *testing.T) { ImagePullPolicy: v1.PullIfNotPresent, Command: []string{"testCommand"}, WorkingDir: "testWorkingDir", - Resources: test.podResources, + Resources: test.containerResources, }, }, }, } + if test.podResources != nil { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, true) + pod.Spec.Resources = test.podResources + } + linuxConfig, err := m.generateLinuxContainerConfig(&pod.Spec.Containers[0], pod, new(int64), "", nil, false) assert.NoError(t, err) assert.Equal(t, test.expected.CpuPeriod, linuxConfig.GetResources().CpuPeriod, test.name) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_sandbox_linux.go b/pkg/kubelet/kuberuntime/kuberuntime_sandbox_linux.go index 8415103737a..6345e3af0a0 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_sandbox_linux.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_sandbox_linux.go @@ -22,7 +22,9 @@ package kuberuntime import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + utilfeature "k8s.io/apiserver/pkg/util/feature" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + "k8s.io/kubernetes/pkg/features" resourcehelper "k8s.io/component-helpers/resource" ) @@ -44,6 +46,8 @@ func (m *kubeGenericRuntimeManager) convertOverheadToLinuxResources(pod *v1.Pod) func (m *kubeGenericRuntimeManager) calculateSandboxResources(pod *v1.Pod) *runtimeapi.LinuxContainerResources { opts := resourcehelper.PodResourcesOptions{ ExcludeOverhead: true, + // SkipPodLevelResources is set to false when PodLevelResources feature is enabled. + SkipPodLevelResources: !utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources), } req := resourcehelper.PodRequests(pod, opts) lim := resourcehelper.PodLimits(pod, opts)