From 80ee072b85fb8616b5209cfcafc291ccbf868657 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 20 Jun 2019 14:50:09 -0700 Subject: [PATCH] pod-overhead: utilize pod overhead for cgroup sizing, eviction handling Pod and burstable QoS cgroups should take overhead of running a sandbox into account if the PodOverhead feature is enabled. These helper functions are utilized by Kubelet for sizing the pod and burstable QoS cgroups. Pod overhead is added to resource requests, regardless of the initial request values. A particular resource pod overhead is only added to a resource limit if a non-zero limit already existed. This commit updates eviction handling to also take Pod Overhead into account (if the feature is enabled). Signed-off-by: Eric Ernst --- pkg/api/v1/resource/BUILD | 6 ++ pkg/api/v1/resource/helpers.go | 45 ++++++++-- pkg/api/v1/resource/helpers_test.go | 125 ++++++++++++++++++++++++++-- 3 files changed, 163 insertions(+), 13 deletions(-) diff --git a/pkg/api/v1/resource/BUILD b/pkg/api/v1/resource/BUILD index 7cbc50774db..dca77a4d625 100644 --- a/pkg/api/v1/resource/BUILD +++ b/pkg/api/v1/resource/BUILD @@ -11,8 +11,12 @@ go_test( srcs = ["helpers_test.go"], embed = [":go_default_library"], deps = [ + "//pkg/features:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", ], ) @@ -22,8 +26,10 @@ go_library( srcs = ["helpers.go"], importpath = "k8s.io/kubernetes/pkg/api/v1/resource", deps = [ + "//pkg/features:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", ], ) diff --git a/pkg/api/v1/resource/helpers.go b/pkg/api/v1/resource/helpers.go index 1c18d56af0c..60a7478da00 100644 --- a/pkg/api/v1/resource/helpers.go +++ b/pkg/api/v1/resource/helpers.go @@ -28,8 +28,8 @@ import ( ) // addResourceList adds the resources in newList to list -func addResourceList(list, new v1.ResourceList) { - for name, quantity := range new { +func addResourceList(list, newList v1.ResourceList) { + for name, quantity := range newList { if value, ok := list[name]; !ok { list[name] = *quantity.Copy() } else { @@ -55,7 +55,9 @@ func maxResourceList(list, new v1.ResourceList) { } // PodRequestsAndLimits returns a dictionary of all defined resources summed up for all -// containers of the pod. +// containers of the pod. If PodOverhead feature is enabled, pod overhead is added to the +// total container resource requests and to the total container limits which have a +// non-zero quantity. func PodRequestsAndLimits(pod *v1.Pod) (reqs, limits v1.ResourceList) { reqs, limits = v1.ResourceList{}, v1.ResourceList{} for _, container := range pod.Spec.Containers { @@ -67,13 +69,35 @@ func PodRequestsAndLimits(pod *v1.Pod) (reqs, limits v1.ResourceList) { maxResourceList(reqs, container.Resources.Requests) maxResourceList(limits, container.Resources.Limits) } + + // if PodOverhead feature is supported, add overhead for running a pod + // to the sum of reqeuests and to non-zero limits: + if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { + addResourceList(reqs, pod.Spec.Overhead) + + for name, quantity := range pod.Spec.Overhead { + if value, ok := limits[name]; ok && !value.IsZero() { + value.Add(quantity) + limits[name] = value + } + } + } + return } // GetResourceRequestQuantity finds and returns the request quantity for a specific resource. func GetResourceRequestQuantity(pod *v1.Pod, resourceName v1.ResourceName) resource.Quantity { + requestQuantity := resource.Quantity{} - requestQuantity := resource.Quantity{Format: resource.BinarySI} + switch resourceName { + case v1.ResourceCPU: + requestQuantity = resource.Quantity{Format: resource.DecimalSI} + case v1.ResourceMemory, v1.ResourceStorage, v1.ResourceEphemeralStorage: + requestQuantity = resource.Quantity{Format: resource.BinarySI} + default: + requestQuantity = resource.Quantity{Format: resource.DecimalSI} + } if resourceName == v1.ResourceEphemeralStorage && !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) { // if the local storage capacity isolation feature gate is disabled, pods request 0 disk @@ -89,11 +113,19 @@ func GetResourceRequestQuantity(pod *v1.Pod, resourceName v1.ResourceName) resou for _, container := range pod.Spec.InitContainers { if rQuantity, ok := container.Resources.Requests[resourceName]; ok { if requestQuantity.Cmp(rQuantity) < 0 { - requestQuantity = rQuantity + requestQuantity = rQuantity.DeepCopy() } } } + // if PodOverhead feature is supported, add overhead for running a pod + // to the total requests if the resource total is non-zero + if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { + if podOverhead, ok := pod.Spec.Overhead[resourceName]; ok && !requestQuantity.IsZero() { + requestQuantity.Add(podOverhead) + } + } + return requestQuantity } @@ -107,10 +139,9 @@ func GetResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 { if resource == v1.ResourceCPU { return requestQuantity.MilliValue() - } else { - return requestQuantity.Value() } + return requestQuantity.Value() } // ExtractResourceValueByContainerName extracts the value of a resource diff --git a/pkg/api/v1/resource/helpers_test.go b/pkg/api/v1/resource/helpers_test.go index 5dc3d4776ee..0c3835d77d8 100644 --- a/pkg/api/v1/resource/helpers_test.go +++ b/pkg/api/v1/resource/helpers_test.go @@ -22,7 +22,11 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" ) func TestResourceHelpers(t *testing.T) { @@ -64,27 +68,53 @@ func TestDefaultResourceHelpers(t *testing.T) { } func TestGetResourceRequest(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodOverhead, true)() + cases := []struct { pod *v1.Pod - res v1.ResourceName + cName string + resourceName v1.ResourceName expectedValue int64 - expectedError error }{ { pod: getPod("foo", podResources{cpuRequest: "9"}), - res: v1.ResourceCPU, + resourceName: v1.ResourceCPU, expectedValue: 9000, }, { pod: getPod("foo", podResources{memoryRequest: "90Mi"}), - res: v1.ResourceMemory, + resourceName: v1.ResourceMemory, expectedValue: 94371840, }, + { + cName: "just-overhead for cpu", + pod: getPod("foo", podResources{cpuOverhead: "5", memoryOverhead: "5"}), + resourceName: v1.ResourceCPU, + expectedValue: 0, + }, + { + cName: "just-overhead for memory", + pod: getPod("foo", podResources{memoryOverhead: "5"}), + resourceName: v1.ResourceMemory, + expectedValue: 0, + }, + { + cName: "cpu overhead and req", + pod: getPod("foo", podResources{cpuRequest: "2", cpuOverhead: "5", memoryOverhead: "5"}), + resourceName: v1.ResourceCPU, + expectedValue: 7000, + }, + { + cName: "mem overhead and req", + pod: getPod("foo", podResources{cpuRequest: "2", memoryRequest: "1024", cpuOverhead: "5", memoryOverhead: "5"}), + resourceName: v1.ResourceMemory, + expectedValue: 1029, + }, } as := assert.New(t) for idx, tc := range cases { - actual := GetResourceRequest(tc.pod, tc.res) - as.Equal(actual, tc.expectedValue, "expected test case [%d] to return %q; got %q instead", idx, tc.expectedValue, actual) + actual := GetResourceRequest(tc.pod, tc.resourceName) + as.Equal(actual, tc.expectedValue, "expected test case [%d] %v: to return %q; got %q instead", idx, tc.cName, tc.expectedValue, actual) } } @@ -242,6 +272,78 @@ func TestExtractResourceValue(t *testing.T) { } } +func TestPodRequestsAndLimits(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodOverhead, true)() + + cases := []struct { + pod *v1.Pod + cName string + expectedRequests v1.ResourceList + expectedLimits v1.ResourceList + }{ + { + cName: "just-limit-no-overhead", + pod: getPod("foo", podResources{cpuLimit: "9"}), + expectedRequests: v1.ResourceList{}, + expectedLimits: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("9"), + }, + }, + { + cName: "just-overhead", + pod: getPod("foo", podResources{cpuOverhead: "5", memoryOverhead: "5"}), + expectedRequests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("5"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("5"), + }, + expectedLimits: v1.ResourceList{}, + }, + { + cName: "req-and-overhead", + pod: getPod("foo", podResources{cpuRequest: "1", memoryRequest: "10", cpuOverhead: "5", memoryOverhead: "5"}), + expectedRequests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("6"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("15"), + }, + expectedLimits: v1.ResourceList{}, + }, + { + cName: "all-req-lim-and-overhead", + pod: getPod("foo", podResources{cpuRequest: "1", cpuLimit: "2", memoryRequest: "10", memoryLimit: "12", cpuOverhead: "5", memoryOverhead: "5"}), + expectedRequests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("6"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("15"), + }, + expectedLimits: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("7"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("17"), + }, + }, + { + cName: "req-some-lim-and-overhead", + pod: getPod("foo", podResources{cpuRequest: "1", cpuLimit: "2", memoryRequest: "10", cpuOverhead: "5", memoryOverhead: "5"}), + expectedRequests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("6"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("15"), + }, + expectedLimits: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("7"), + }, + }, + } + for idx, tc := range cases { + resRequests, resLimits := PodRequestsAndLimits(tc.pod) + + if !equality.Semantic.DeepEqual(tc.expectedRequests, resRequests) { + t.Errorf("test case failure[%d]: %v, requests:\n expected:\t%v\ngot\t\t%v", idx, tc.cName, tc.expectedRequests, resRequests) + } + + if !equality.Semantic.DeepEqual(tc.expectedLimits, resLimits) { + t.Errorf("test case failure[%d]: %v, limits:\n expected:\t%v\ngot\t\t%v", idx, tc.cName, tc.expectedLimits, resLimits) + } + } +} + type podResources struct { cpuRequest, cpuLimit, memoryRequest, memoryLimit, cpuOverhead, memoryOverhead string } @@ -251,6 +353,9 @@ func getPod(cname string, resources podResources) *v1.Pod { Limits: make(v1.ResourceList), Requests: make(v1.ResourceList), } + + overhead := make(v1.ResourceList) + if resources.cpuLimit != "" { r.Limits[v1.ResourceCPU] = resource.MustParse(resources.cpuLimit) } @@ -263,6 +368,13 @@ func getPod(cname string, resources podResources) *v1.Pod { if resources.memoryRequest != "" { r.Requests[v1.ResourceMemory] = resource.MustParse(resources.memoryRequest) } + if resources.cpuOverhead != "" { + overhead[v1.ResourceCPU] = resource.MustParse(resources.cpuOverhead) + } + if resources.memoryOverhead != "" { + overhead[v1.ResourceMemory] = resource.MustParse(resources.memoryOverhead) + } + return &v1.Pod{ Spec: v1.PodSpec{ Containers: []v1.Container{ @@ -277,6 +389,7 @@ func getPod(cname string, resources podResources) *v1.Pod { Resources: r, }, }, + Overhead: overhead, }, } }