github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-10 21:50:05 +00:00
Code Issues Projects Releases Wiki Activity

dedupe pod resource request calculation

Browse Source
This commit is contained in:
Todd Neal
2023-03-08 20:52:21 -06:00
parent c67953a2d0
commit 4096c9209c
18 changed files with 980 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
pkg/api/v1/resource/helpers.go
View File

@@ -24,54 +24,127 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
)
// PodRequestsAndLimits returns a dictionary of all defined resources summed up for all
// containers of the pod. Pod overhead is added to the
// total container resource requests and to the total container limits which have a
// non-zero quantity.
func PodRequestsAndLimits(pod *v1.Pod) (reqs, limits v1.ResourceList) {
return PodRequestsAndLimitsReuse(pod, nil, nil)
// PodResourcesOptions controls the behavior of PodRequests and PodLimits.
type PodResourcesOptions struct {
// Reuse, if provided will be reused to accumulate resources and returned by the PodRequests or PodLimits
// functions. All existing values in Reuse will be lost.
Reuse v1.ResourceList
// InPlacePodVerticalScalingEnabled indicates that the in-place pod vertical scaling feature gate is enabled.
InPlacePodVerticalScalingEnabled bool
// ExcludeOverhead controls if pod overhead is excluded from the calculation.
ExcludeOverhead bool
// ContainerFn is called with the effective resources required for each container within the pod.
ContainerFn func(res v1.ResourceList, containerType podutil.ContainerType)
// NonMissingContainerRequests if provided will replace any missing container level requests for the specified resources
// with the given values. If the requests for those resources are explicitly set, even if zero, they will not be modified.
NonMissingContainerRequests v1.ResourceList
}
// PodRequestsAndLimitsWithoutOverhead will create a dictionary of all defined resources summed up for all
// containers of the pod.
func PodRequestsAndLimitsWithoutOverhead(pod *v1.Pod) (reqs, limits v1.ResourceList) {
reqs = make(v1.ResourceList, 4)
limits = make(v1.ResourceList, 4)
podRequestsAndLimitsWithoutOverhead(pod, reqs, limits)
// PodRequests computes the pod requests per the PodResourcesOptions supplied. If PodResourcesOptions is nil, then
// the requests are returned including pod overhead. The computation is part of the API and must be reviewed
// as an API change.
func PodRequests(pod *v1.Pod, opts PodResourcesOptions) v1.ResourceList {
// attempt to reuse the maps if passed, or allocate otherwise
reqs := reuseOrClearResourceList(opts.Reuse)
return reqs, limits
}
var containerStatuses map[string]*v1.ContainerStatus
if opts.InPlacePodVerticalScalingEnabled {
containerStatuses = map[string]*v1.ContainerStatus{}
for i := range pod.Status.ContainerStatuses {
containerStatuses[pod.Status.ContainerStatuses[i].Name] = &pod.Status.ContainerStatuses[i]
}
}
func podRequestsAndLimitsWithoutOverhead(pod *v1.Pod, reqs, limits v1.ResourceList) {
for _, container := range pod.Spec.Containers {
addResourceList(reqs, container.Resources.Requests)
containerReqs := container.Resources.Requests
if opts.InPlacePodVerticalScalingEnabled {
cs, found := containerStatuses[container.Name]
if found {
if pod.Status.Resize == v1.PodResizeStatusInfeasible {
containerReqs = cs.ResourcesAllocated
} else {
containerReqs = max(container.Resources.Requests, cs.ResourcesAllocated)
}
}
}
if len(opts.NonMissingContainerRequests) > 0 {
containerReqs = applyNonMissing(containerReqs, opts.NonMissingContainerRequests)
}
if opts.ContainerFn != nil {
opts.ContainerFn(containerReqs, podutil.Containers)
}
addResourceList(reqs, containerReqs)
}
// init containers define the minimum of any resource
// Note: In-place resize is not allowed for InitContainers, so no need to check for ResizeStatus value
for _, container := range pod.Spec.InitContainers {
containerReqs := container.Resources.Requests
if len(opts.NonMissingContainerRequests) > 0 {
containerReqs = applyNonMissing(containerReqs, opts.NonMissingContainerRequests)
}
if opts.ContainerFn != nil {
opts.ContainerFn(containerReqs, podutil.InitContainers)
}
maxResourceList(reqs, containerReqs)
}
// Add overhead for running a pod to the sum of requests if requested:
if !opts.ExcludeOverhead && pod.Spec.Overhead != nil {
addResourceList(reqs, pod.Spec.Overhead)
}
return reqs
}
// applyNonMissing will return a copy of the given resource list with any missing values replaced by the nonMissing values
func applyNonMissing(reqs v1.ResourceList, nonMissing v1.ResourceList) v1.ResourceList {
cp := v1.ResourceList{}
for k, v := range reqs {
cp[k] = v.DeepCopy()
}
for k, v := range nonMissing {
if _, found := reqs[k]; !found {
rk := cp[k]
rk.Add(v)
cp[k] = rk
}
}
return cp
}
// PodLimits computes the pod limits per the PodResourcesOptions supplied. If PodResourcesOptions is nil, then
// the limits are returned including pod overhead for any non-zero limits. The computation is part of the API and must be reviewed
// as an API change.
func PodLimits(pod *v1.Pod, opts PodResourcesOptions) v1.ResourceList {
// attempt to reuse the maps if passed, or allocate otherwise
limits := reuseOrClearResourceList(opts.Reuse)
for _, container := range pod.Spec.Containers {
if opts.ContainerFn != nil {
opts.ContainerFn(container.Resources.Limits, podutil.Containers)
}
addResourceList(limits, container.Resources.Limits)
}
// init containers define the minimum of any resource
for _, container := range pod.Spec.InitContainers {
maxResourceList(reqs, container.Resources.Requests)
if opts.ContainerFn != nil {
opts.ContainerFn(container.Resources.Limits, podutil.InitContainers)
}
maxResourceList(limits, container.Resources.Limits)
}
}
// PodRequestsAndLimitsReuse returns a dictionary of all defined resources summed up for all
// containers of the pod. Pod overhead is added to the
// total container resource requests and to the total container limits which have a
// non-zero quantity. The caller may avoid allocations of resource lists by passing
// a requests and limits list to the function, which will be cleared before use.
func PodRequestsAndLimitsReuse(pod *v1.Pod, reuseReqs, reuseLimits v1.ResourceList) (reqs, limits v1.ResourceList) {
// attempt to reuse the maps if passed, or allocate otherwise
reqs, limits = reuseOrClearResourceList(reuseReqs), reuseOrClearResourceList(reuseLimits)
podRequestsAndLimitsWithoutOverhead(pod, reqs, limits)
// Add overhead for running a pod
// to the sum of requests and to non-zero limits:
if pod.Spec.Overhead != nil {
addResourceList(reqs, pod.Spec.Overhead)
// Add overhead to non-zero limits if requested:
if !opts.ExcludeOverhead && pod.Spec.Overhead != nil {
for name, quantity := range pod.Spec.Overhead {
if value, ok := limits[name]; ok && !value.IsZero() {
value.Add(quantity)
@@ -80,19 +153,7 @@ func PodRequestsAndLimitsReuse(pod *v1.Pod, reuseReqs, reuseLimits v1.ResourceLi
}
}
return
}
// reuseOrClearResourceList is a helper for avoiding excessive allocations of
// resource lists within the inner loop of resource calculations.
func reuseOrClearResourceList(reuse v1.ResourceList) v1.ResourceList {
if reuse == nil {
return make(v1.ResourceList, 4)
}
for k := range reuse {
delete(reuse, k)
}
return reuse
return limits
}
// addResourceList adds the resources in newList to list.
@@ -116,6 +177,39 @@ func maxResourceList(list, newList v1.ResourceList) {
}
}
// max returns the result of max(a, b) for each named resource and is only used if we can't
// accumulate into an existing resource list
func max(a v1.ResourceList, b v1.ResourceList) v1.ResourceList {
result := v1.ResourceList{}
for key, value := range a {
if other, found := b[key]; found {
if value.Cmp(other) <= 0 {
result[key] = other.DeepCopy()
continue
}
}
result[key] = value.DeepCopy()
}
for key, value := range b {
if _, found := result[key]; !found {
result[key] = value.DeepCopy()
}
}
return result
}
// reuseOrClearResourceList is a helper for avoiding excessive allocations of
// resource lists within the inner loop of resource calculations.
func reuseOrClearResourceList(reuse v1.ResourceList) v1.ResourceList {
if reuse == nil {
return make(v1.ResourceList, 4)
}
for k := range reuse {
delete(reuse, k)
}
return reuse
}
// GetResourceRequestQuantity finds and returns the request quantity for a specific resource.
func GetResourceRequestQuantity(pod *v1.Pod, resourceName v1.ResourceName) resource.Quantity {
requestQuantity := resource.Quantity{}

447
pkg/api/v1/resource/helpers_test.go
View File

@@ -325,7 +325,8 @@ func TestPodRequestsAndLimits(t *testing.T) {
},
}
for idx, tc := range cases {
resRequests, resLimits := PodRequestsAndLimits(tc.pod)
resRequests := PodRequests(tc.pod, PodResourcesOptions{})
resLimits := PodLimits(tc.pod, PodResourcesOptions{})
if !equality.Semantic.DeepEqual(tc.expectedRequests, resRequests) {
t.Errorf("test case failure[%d]: %v, requests:\n expected:\t%v\ngot\t\t%v", idx, tc.cName, tc.expectedRequests, resRequests)
@@ -511,7 +512,8 @@ func TestPodRequestsAndLimitsWithoutOverhead(t *testing.T) {
},
}
for idx, tc := range cases {
resRequests, resLimits := PodRequestsAndLimitsWithoutOverhead(tc.pod)
resRequests := PodRequests(tc.pod, PodResourcesOptions{ExcludeOverhead: true})
resLimits := PodLimits(tc.pod, PodResourcesOptions{ExcludeOverhead: true})
if !equality.Semantic.DeepEqual(tc.expectedRequests, resRequests) {
t.Errorf("test case failure[%d]: %v, requests:\n expected:\t%v\ngot\t\t%v", idx, tc.name, tc.expectedRequests, resRequests)
@@ -572,3 +574,444 @@ func getPod(cname string, resources podResources) *v1.Pod {
},
}
}
func TestPodResourceRequests(t *testing.T) {
testCases := []struct {
description string
options PodResourcesOptions
overhead v1.ResourceList
podResizeStatus v1.PodResizeStatus
initContainers []v1.Container
containers []v1.Container
containerStatus []v1.ContainerStatus
expectedRequests v1.ResourceList
}{
{
description: "nil options, larger init container",
expectedRequests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4"),
},
initContainers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4"),
},
},
},
},
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1"),
},
},
},
},
},
{
description: "nil options, larger containers",
expectedRequests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5"),
},
initContainers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
},
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("3"),
},
},
},
},
},
{
description: "pod overhead excluded",
expectedRequests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5"),
},
options: PodResourcesOptions{
ExcludeOverhead: true,
},
overhead: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1"),
},
initContainers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
},
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("3"),
},
},
},
},
},
{
description: "pod overhead included",
expectedRequests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("6"),
v1.ResourceMemory: resource.MustParse("1Gi"),
},
overhead: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1"),
v1.ResourceMemory: resource.MustParse("1Gi"),
},
initContainers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
},
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("3"),
},
},
},
},
},
{
description: "resized, infeasible",
expectedRequests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
podResizeStatus: v1.PodResizeStatusInfeasible,
options: PodResourcesOptions{InPlacePodVerticalScalingEnabled: true},
containers: []v1.Container{
{
Name: "container-1",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4"),
},
},
},
},
containerStatus: []v1.ContainerStatus{
{
Name: "container-1",
ResourcesAllocated: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
},
{
description: "resized, no resize status",
expectedRequests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4"),
},
options: PodResourcesOptions{InPlacePodVerticalScalingEnabled: true},
containers: []v1.Container{
{
Name: "container-1",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4"),
},
},
},
},
containerStatus: []v1.ContainerStatus{
{
Name: "container-1",
ResourcesAllocated: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
},
{
description: "resized, infeasible, feature gate disabled",
expectedRequests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4"),
},
podResizeStatus: v1.PodResizeStatusInfeasible,
options: PodResourcesOptions{InPlacePodVerticalScalingEnabled: false},
containers: []v1.Container{
{
Name: "container-1",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4"),
},
},
},
},
containerStatus: []v1.ContainerStatus{
{
Name: "container-1",
ResourcesAllocated: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
},
}
for _, tc := range testCases {
p := &v1.Pod{
Spec: v1.PodSpec{
Containers: tc.containers,
InitContainers: tc.initContainers,
Overhead: tc.overhead,
},
Status: v1.PodStatus{
ContainerStatuses: tc.containerStatus,
Resize: tc.podResizeStatus,
},
}
request := PodRequests(p, tc.options)
if !resourcesEqual(tc.expectedRequests, request) {
t.Errorf("[%s] expected requests = %v, got %v", tc.description, tc.expectedRequests, request)
}
}
}
func TestPodResourceRequestsReuse(t *testing.T) {
expectedRequests := v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1"),
}
p := &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: expectedRequests,
},
},
},
},
}
opts := PodResourcesOptions{
Reuse: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("25"),
},
}
requests := PodRequests(p, opts)
if !resourcesEqual(expectedRequests, requests) {
t.Errorf("expected requests = %v, got %v", expectedRequests, requests)
}
// should re-use the maps we passed in
if !resourcesEqual(expectedRequests, opts.Reuse) {
t.Errorf("expected to re-use the requests")
}
}
func TestPodResourceLimits(t *testing.T) {
testCases := []struct {
description string
options PodResourcesOptions
overhead v1.ResourceList
initContainers []v1.Container
containers []v1.Container
expectedLimits v1.ResourceList
}{
{
description: "nil options, larger init container",
expectedLimits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4"),
},
initContainers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4"),
},
},
},
},
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1"),
},
},
},
},
},
{
description: "nil options, larger containers",
expectedLimits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5"),
},
initContainers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
},
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("3"),
},
},
},
},
},
{
description: "pod overhead excluded",
expectedLimits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5"),
},
options: PodResourcesOptions{
ExcludeOverhead: true,
},
overhead: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1"),
},
initContainers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
},
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("3"),
},
},
},
},
},
{
description: "pod overhead included",
overhead: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1"),
v1.ResourceMemory: resource.MustParse("1Gi"),
},
expectedLimits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("6"),
// overhead is only added to non-zero limits, so there will be no expected memory limit
},
initContainers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
},
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
},
{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("3"),
},
},
},
},
},
}
for _, tc := range testCases {
p := &v1.Pod{
Spec: v1.PodSpec{
Containers: tc.containers,
InitContainers: tc.initContainers,
Overhead: tc.overhead,
},
}
limits := PodLimits(p, tc.options)
if !resourcesEqual(tc.expectedLimits, limits) {
t.Errorf("[%s] expected limits = %v, got %v", tc.description, tc.expectedLimits, limits)
}
}
}
func resourcesEqual(lhs, rhs v1.ResourceList) bool {
if len(lhs) != len(rhs) {
return false
}
for name, lhsv := range lhs {
rhsv, ok := rhs[name]
if !ok {
return false
}
if !lhsv.Equal(rhsv) {
return false
}
}
return true
}