Merge pull request #128771 from tallclair/min-quota

[FG:InPlacePodVerticalScaling] Equate CPU limits below the minimum effective limit (10m)
This commit is contained in:
Kubernetes Prow Robot 2024-11-13 03:48:53 +00:00 committed by GitHub
commit 0926587bf0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 236 additions and 118 deletions

View File

@ -50,6 +50,10 @@ const (
// defined here:
// https://github.com/torvalds/linux/blob/cac03ac368fabff0122853de2422d4e17a32de08/kernel/sched/core.c#L10546
MinQuotaPeriod = 1000
// From the inverse of the conversion in MilliCPUToQuota:
// MinQuotaPeriod * MilliCPUToCPU / QuotaPeriod
MinMilliCPULimit = 10
)
// MilliCPUToQuota converts milliCPU to CFS quota and period values.

View File

@ -20,7 +20,7 @@ limitations under the License.
package cm
import (
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
)
@ -33,6 +33,7 @@ const (
QuotaPeriod = 0
MinQuotaPeriod = 0
MinMilliCPULimit = 0
)
// MilliCPUToQuota converts milliCPU and period to CFS quota values.

View File

@ -1806,7 +1806,9 @@ func allocatedResourcesMatchStatus(allocatedPod *v1.Pod, podStatus *kubecontaine
if !cpuLim.IsZero() {
return false
}
} else if !cpuLim.Equal(*cs.Resources.CPULimit) {
} else if !cpuLim.Equal(*cs.Resources.CPULimit) &&
(cpuLim.MilliValue() > cm.MinMilliCPULimit || cs.Resources.CPULimit.MilliValue() > cm.MinMilliCPULimit) {
// If both allocated & status CPU limits are at or below the minimum limit, then they are considered equal.
return false
}
}
@ -2152,7 +2154,12 @@ func (kl *Kubelet) convertToAPIContainerStatuses(pod *v1.Pod, podStatus *kubecon
resources := alloc
if resources.Limits != nil {
if cStatus.Resources != nil && cStatus.Resources.CPULimit != nil {
// If both the allocated & actual resources are at or below the minimum effective limit, preserve the
// allocated value in the API to avoid confusion and simplify comparisons.
if cStatus.Resources.CPULimit.MilliValue() > cm.MinMilliCPULimit ||
resources.Limits.Cpu().MilliValue() > cm.MinMilliCPULimit {
resources.Limits[v1.ResourceCPU] = cStatus.Resources.CPULimit.DeepCopy()
}
} else {
preserveOldResourcesValue(v1.ResourceCPU, oldStatus.Resources.Limits, resources.Limits)
}

View File

@ -4797,22 +4797,33 @@ func TestConvertToAPIContainerStatusesForResources(t *testing.T) {
},
},
"BurstableQoSPod with below min CPU": {
Resources: v1.ResourceRequirements{Requests: v1.ResourceList{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("100M"),
v1.ResourceCPU: resource.MustParse("1m"),
}},
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5m"),
},
},
ActualResources: &kubecontainer.ContainerResources{
CPURequest: resource.NewMilliQuantity(2, resource.DecimalSI),
CPULimit: resource.NewMilliQuantity(10, resource.DecimalSI),
},
OldStatus: v1.ContainerStatus{
Name: testContainerName,
Image: "img",
ImageID: "img1234",
State: v1.ContainerState{Running: &v1.ContainerStateRunning{}},
Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{
Resources: &v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("100M"),
v1.ResourceCPU: resource.MustParse("1m"),
}},
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5m"),
},
},
},
Expected: v1.ContainerStatus{
Name: testContainerName,
@ -4824,10 +4835,15 @@ func TestConvertToAPIContainerStatusesForResources(t *testing.T) {
v1.ResourceMemory: resource.MustParse("100M"),
v1.ResourceCPU: resource.MustParse("1m"),
},
Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{
Resources: &v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("100M"),
v1.ResourceCPU: resource.MustParse("1m"),
}},
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5m"),
},
},
},
},
"GuaranteedQoSPod with CPU and memory CRI status, with ephemeral storage": {
@ -6790,6 +6806,36 @@ func TestAllocatedResourcesMatchStatus(t *testing.T) {
CPURequest: resource.NewMilliQuantity(2, resource.DecimalSI),
},
expectMatch: true,
}, {
name: "burstable: min cpu limit",
allocatedResources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
},
},
statusResources: &kubecontainer.ContainerResources{
CPURequest: resource.NewMilliQuantity(10, resource.DecimalSI),
CPULimit: resource.NewMilliQuantity(10, resource.DecimalSI),
},
expectMatch: true,
}, {
name: "burstable: below min cpu limit",
allocatedResources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5m"),
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5m"),
},
},
statusResources: &kubecontainer.ContainerResources{
CPURequest: resource.NewMilliQuantity(5, resource.DecimalSI),
CPULimit: resource.NewMilliQuantity(10, resource.DecimalSI),
},
expectMatch: true,
}, {
name: "best effort",
allocatedResources: v1.ResourceRequirements{},

View File

@ -2675,8 +2675,11 @@ func TestHandlePodResourcesResize(t *testing.T) {
name string
originalRequests v1.ResourceList
newRequests v1.ResourceList
newRequestsAllocated bool // Whether the new requests have already been allocated (but not actuated)
expectedAllocations v1.ResourceList
originalLimits v1.ResourceList
newLimits v1.ResourceList
newResourcesAllocated bool // Whether the new requests have already been allocated (but not actuated)
expectedAllocatedReqs v1.ResourceList
expectedAllocatedLims v1.ResourceList
expectedResize v1.PodResizeStatus
expectBackoffReset bool
goos string
@ -2685,7 +2688,7 @@ func TestHandlePodResourcesResize(t *testing.T) {
name: "Request CPU and memory decrease - expect InProgress",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu500m, v1.ResourceMemory: mem500M},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu500m, v1.ResourceMemory: mem500M},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu500m, v1.ResourceMemory: mem500M},
expectedResize: v1.PodResizeStatusInProgress,
expectBackoffReset: true,
},
@ -2693,7 +2696,7 @@ func TestHandlePodResourcesResize(t *testing.T) {
name: "Request CPU increase, memory decrease - expect InProgress",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu1500m, v1.ResourceMemory: mem500M},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu1500m, v1.ResourceMemory: mem500M},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu1500m, v1.ResourceMemory: mem500M},
expectedResize: v1.PodResizeStatusInProgress,
expectBackoffReset: true,
},
@ -2701,7 +2704,7 @@ func TestHandlePodResourcesResize(t *testing.T) {
name: "Request CPU decrease, memory increase - expect InProgress",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu500m, v1.ResourceMemory: mem1500M},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu500m, v1.ResourceMemory: mem1500M},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu500m, v1.ResourceMemory: mem1500M},
expectedResize: v1.PodResizeStatusInProgress,
expectBackoffReset: true,
},
@ -2709,50 +2712,50 @@ func TestHandlePodResourcesResize(t *testing.T) {
name: "Request CPU and memory increase beyond current capacity - expect Deferred",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu2500m, v1.ResourceMemory: mem2500M},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedResize: v1.PodResizeStatusDeferred,
},
{
name: "Request CPU decrease and memory increase beyond current capacity - expect Deferred",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu500m, v1.ResourceMemory: mem2500M},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedResize: v1.PodResizeStatusDeferred,
},
{
name: "Request memory increase beyond node capacity - expect Infeasible",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem4500M},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedResize: v1.PodResizeStatusInfeasible,
},
{
name: "Request CPU increase beyond node capacity - expect Infeasible",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu5000m, v1.ResourceMemory: mem1000M},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedResize: v1.PodResizeStatusInfeasible,
},
{
name: "CPU increase in progress - expect InProgress",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu1500m, v1.ResourceMemory: mem1000M},
newRequestsAllocated: true,
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu1500m, v1.ResourceMemory: mem1000M},
newResourcesAllocated: true,
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu1500m, v1.ResourceMemory: mem1000M},
expectedResize: v1.PodResizeStatusInProgress,
},
{
name: "No resize",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedResize: "",
},
{
name: "windows node, expect Infeasible",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu500m, v1.ResourceMemory: mem500M},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu1000m, v1.ResourceMemory: mem1000M},
expectedResize: v1.PodResizeStatusInfeasible,
goos: "windows",
},
@ -2760,7 +2763,7 @@ func TestHandlePodResourcesResize(t *testing.T) {
name: "Increase CPU from min shares",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu2m},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu1000m},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu1000m},
expectedResize: v1.PodResizeStatusInProgress,
expectBackoffReset: true,
},
@ -2768,7 +2771,7 @@ func TestHandlePodResourcesResize(t *testing.T) {
name: "Decrease CPU to min shares",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1000m},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu2m},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu2m},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu2m},
expectedResize: v1.PodResizeStatusInProgress,
expectBackoffReset: true,
},
@ -2776,7 +2779,7 @@ func TestHandlePodResourcesResize(t *testing.T) {
name: "Equivalent min CPU shares",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu1m},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu2m},
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu2m},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu2m},
expectedResize: "",
// Even though the resize isn't being actuated, we still clear the container backoff
// since the allocation is changing.
@ -2786,8 +2789,54 @@ func TestHandlePodResourcesResize(t *testing.T) {
name: "Equivalent min CPU shares - already allocated",
originalRequests: v1.ResourceList{v1.ResourceCPU: cpu2m},
newRequests: v1.ResourceList{v1.ResourceCPU: cpu1m},
newRequestsAllocated: true,
expectedAllocations: v1.ResourceList{v1.ResourceCPU: cpu1m},
newResourcesAllocated: true,
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: cpu1m},
expectedResize: "",
},
{
name: "Increase CPU from min limit",
originalRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")},
originalLimits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")},
newRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")}, // Unchanged
newLimits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("20m")},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")},
expectedAllocatedLims: v1.ResourceList{v1.ResourceCPU: resource.MustParse("20m")},
expectedResize: v1.PodResizeStatusInProgress,
expectBackoffReset: true,
},
{
name: "Decrease CPU to min limit",
originalRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")},
originalLimits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("20m")},
newRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")}, // Unchanged
newLimits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")},
expectedAllocatedLims: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")},
expectedResize: v1.PodResizeStatusInProgress,
expectBackoffReset: true,
},
{
name: "Equivalent min CPU limit",
originalRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m")},
originalLimits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m")},
newRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m")}, // Unchanged
newLimits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m")},
expectedAllocatedLims: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")},
expectedResize: "",
// Even though the resize isn't being actuated, we still clear the container backoff
// since the allocation is changing.
expectBackoffReset: true,
},
{
name: "Equivalent min CPU limit - already allocated",
originalRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m")},
originalLimits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10m")},
newRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m")}, // Unchanged
newLimits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m")},
expectedAllocatedReqs: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m")},
expectedAllocatedLims: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m")},
newResourcesAllocated: true,
expectedResize: "",
},
}
@ -2803,12 +2852,14 @@ func TestHandlePodResourcesResize(t *testing.T) {
originalPod := testPod1.DeepCopy()
originalPod.Spec.Containers[0].Resources.Requests = tt.originalRequests
originalPod.Spec.Containers[0].Resources.Limits = tt.originalLimits
kubelet.podManager.UpdatePod(originalPod)
newPod := originalPod.DeepCopy()
newPod.Spec.Containers[0].Resources.Requests = tt.newRequests
newPod.Spec.Containers[0].Resources.Limits = tt.newLimits
if !tt.newRequestsAllocated {
if !tt.newResourcesAllocated {
require.NoError(t, kubelet.statusManager.SetPodAllocation(originalPod))
} else {
require.NoError(t, kubelet.statusManager.SetPodAllocation(newPod))
@ -2839,11 +2890,13 @@ func TestHandlePodResourcesResize(t *testing.T) {
updatedPod, err := kubelet.handlePodResourcesResize(newPod, podStatus)
require.NoError(t, err)
assert.Equal(t, tt.expectedAllocations, updatedPod.Spec.Containers[0].Resources.Requests, "updated pod spec resources")
assert.Equal(t, tt.expectedAllocatedReqs, updatedPod.Spec.Containers[0].Resources.Requests, "updated pod spec requests")
assert.Equal(t, tt.expectedAllocatedLims, updatedPod.Spec.Containers[0].Resources.Limits, "updated pod spec limits")
alloc, found := kubelet.statusManager.GetContainerResourceAllocation(string(newPod.UID), newPod.Spec.Containers[0].Name)
require.True(t, found, "container allocation")
assert.Equal(t, tt.expectedAllocations, alloc.Requests, "stored container allocation")
assert.Equal(t, tt.expectedAllocatedReqs, alloc.Requests, "stored container request allocation")
assert.Equal(t, tt.expectedAllocatedLims, alloc.Limits, "stored container limit allocation")
resizeStatus := kubelet.statusManager.GetPodResizeStatus(newPod.UID)
assert.Equal(t, tt.expectedResize, resizeStatus)

View File

@ -606,7 +606,12 @@ func (m *kubeGenericRuntimeManager) computePodResizeAction(pod *v1.Pod, containe
// then consider these equal.
desiredResources.cpuRequest = currentResources.cpuRequest
}
// Special case for minimum CPU limit
if desiredResources.cpuLimit <= cm.MinMilliCPULimit && currentResources.cpuLimit <= cm.MinMilliCPULimit {
// If both desired & current CPU limit are at or below the minimum effective limit,
// then consider these equal.
desiredResources.cpuLimit = currentResources.cpuLimit
}
if currentResources == desiredResources {
// No resize required.
return true
@ -848,14 +853,13 @@ func (m *kubeGenericRuntimeManager) updatePodContainerResources(pod *v1.Pod, res
actualRequest := nonNilQuantity(status.Resources.CPURequest)
desiredLimit := container.Resources.Limits.Cpu()
desiredRequest := container.Resources.Requests.Cpu()
if !actualLimit.Equal(*desiredLimit) {
return false // limits don't match
} else if actualRequest.Equal(*desiredRequest) {
return true // requests & limits both match
}
// Consider limits equal if both are at or below the effective minimum limit.
equalLimits := actualLimit.Equal(*desiredLimit) || (actualLimit.MilliValue() <= cm.MinMilliCPULimit &&
desiredLimit.MilliValue() <= cm.MinMilliCPULimit)
// Consider requests equal if both are at or below MinShares.
return actualRequest.MilliValue() <= cm.MinShares &&
desiredRequest.MilliValue() <= cm.MinShares
equalRequests := actualRequest.Equal(*desiredRequest) || (actualRequest.MilliValue() <= cm.MinShares &&
desiredRequest.MilliValue() <= cm.MinShares)
return equalLimits && equalRequests
default:
return true // Shouldn't happen.
}

View File

@ -2215,6 +2215,7 @@ func TestComputePodActionsForPodResize(t *testing.T) {
cpu1m := resource.MustParse("1m")
cpu2m := resource.MustParse("2m")
cpu10m := resource.MustParse("10m")
cpu100m := resource.MustParse("100m")
cpu200m := resource.MustParse("200m")
mem100M := resource.MustParse("100Mi")
@ -2406,10 +2407,12 @@ func TestComputePodActionsForPodResize(t *testing.T) {
c := &pod.Spec.Containers[1]
c.Resources = v1.ResourceRequirements{
Requests: v1.ResourceList{v1.ResourceCPU: cpu1m},
Limits: v1.ResourceList{v1.ResourceCPU: cpu1m},
}
if cStatus := status.FindContainerStatusByName(c.Name); cStatus != nil {
cStatus.Resources = &kubecontainer.ContainerResources{
CPURequest: ptr.To(cpu2m.DeepCopy()),
CPULimit: ptr.To(cpu10m.DeepCopy()),
}
}
},

View File

@ -584,20 +584,20 @@ func doPodResizeTests(f *framework.Framework) {
},
},
{
name: "Burstable QoS pod, one container with cpu requests - resize with equivalent request",
name: "Burstable QoS pod, one container with cpu requests and limits - resize with equivalents",
containers: []e2epod.ResizableContainerInfo{
{
Name: "c1",
Resources: &e2epod.ContainerResources{CPUReq: "2m"},
Resources: &e2epod.ContainerResources{CPUReq: "2m", CPULim: "10m"},
},
},
patchString: `{"spec":{"containers":[
{"name":"c1", "resources":{"requests":{"cpu":"1m"}}}
{"name":"c1", "resources":{"requests":{"cpu":"1m"},"limits":{"cpu":"5m"}}}
]}}`,
expected: []e2epod.ResizableContainerInfo{
{
Name: "c1",
Resources: &e2epod.ContainerResources{CPUReq: "1m"},
Resources: &e2epod.ContainerResources{CPUReq: "1m", CPULim: "5m"},
},
},
},