Files
kubernetes/pkg/kubelet/allocation/allocation_manager.go
2025-09-03 15:53:45 -07:00

730 lines
26 KiB
Go

/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package allocation
import (
"context"
"fmt"
"path/filepath"
"slices"
"sync"
"time"
v1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/tools/record"
resourcehelper "k8s.io/component-helpers/resource"
"k8s.io/klog/v2"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/pkg/api/v1/resource"
v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/allocation/state"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
"k8s.io/kubernetes/pkg/kubelet/cm/memorymanager"
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
"k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/pkg/kubelet/status"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/pkg/kubelet/util/format"
)
// podStatusManagerStateFile is the file name where status manager stores its state
const (
allocatedPodsStateFile = "allocated_pods_state"
initialRetryDelay = 30 * time.Second
retryDelay = 3 * time.Minute
TriggerReasonPodResized = "pod_resized"
TriggerReasonPodUpdated = "pod_updated"
TriggerReasonPodsAdded = "pods_added"
TriggerReasonPodsRemoved = "pods_removed"
triggerReasonPeriodic = "periodic_retry"
)
// AllocationManager tracks pod resource allocations.
type Manager interface {
// GetContainerResourceAllocation returns the AllocatedResources value for the container
GetContainerResourceAllocation(podUID types.UID, containerName string) (v1.ResourceRequirements, bool)
// UpdatePodFromAllocation overwrites the pod spec with the allocation.
// This function does a deep copy only if updates are needed.
// Returns the updated (or original) pod, and whether there was an allocation stored.
UpdatePodFromAllocation(pod *v1.Pod) (*v1.Pod, bool)
// SetAllocatedResources checkpoints the resources allocated to a pod's containers.
SetAllocatedResources(allocatedPod *v1.Pod) error
// AddPodAdmitHandlers adds the admit handlers to the allocation manager.
// TODO: See if we can remove this and just add them in the allocation manager constructor.
AddPodAdmitHandlers(handlers lifecycle.PodAdmitHandlers)
// SetContainerRuntime sets the allocation manager's container runtime.
// TODO: See if we can remove this and just add it in the allocation manager constructor.
SetContainerRuntime(runtime kubecontainer.Runtime)
// AddPod checks if a pod can be admitted. If so, it admits the pod and updates the allocation.
// The function returns a boolean value indicating whether the pod
// can be admitted, a brief single-word reason and a message explaining why
// the pod cannot be admitted.
// allocatedPods should represent the pods that have already been admitted, along with their
// admitted (allocated) resources.
AddPod(activePods []*v1.Pod, pod *v1.Pod) (ok bool, reason, message string)
// RemovePod removes any stored state for the given pod UID.
RemovePod(uid types.UID)
// RemoveOrphanedPods removes the stored state for any pods not included in the set of remaining pods.
RemoveOrphanedPods(remainingPods sets.Set[types.UID])
// Run starts the allocation manager. This is currently only used to handle periodic retry of
// pending resizes.
Run(ctx context.Context)
// PushPendingResize queues a pod with a pending resize request for later reevaluation.
PushPendingResize(uid types.UID)
// HasPendingResizes returns whether there are currently any pending resizes.
HasPendingResizes() bool
// RetryPendingResizes retries all pending resizes.
RetryPendingResizes(trigger string)
}
type manager struct {
allocated state.State
admitHandlers lifecycle.PodAdmitHandlers
containerRuntime kubecontainer.Runtime
statusManager status.Manager
sourcesReady config.SourcesReady
nodeConfig cm.NodeConfig
nodeAllocatableAbsolute v1.ResourceList
ticker *time.Ticker
triggerPodSync func(pod *v1.Pod)
getActivePods func() []*v1.Pod
getPodByUID func(types.UID) (*v1.Pod, bool)
allocationMutex sync.Mutex
podsWithPendingResizes []types.UID
recorder record.EventRecorder
}
func NewManager(checkpointDirectory string,
nodeConfig cm.NodeConfig,
nodeAllocatableAbsolute v1.ResourceList,
statusManager status.Manager,
triggerPodSync func(pod *v1.Pod),
getActivePods func() []*v1.Pod,
getPodByUID func(types.UID) (*v1.Pod, bool),
sourcesReady config.SourcesReady,
recorder record.EventRecorder,
) Manager {
return &manager{
allocated: newStateImpl(checkpointDirectory, allocatedPodsStateFile),
statusManager: statusManager,
admitHandlers: lifecycle.PodAdmitHandlers{},
sourcesReady: sourcesReady,
nodeConfig: nodeConfig,
nodeAllocatableAbsolute: nodeAllocatableAbsolute,
ticker: time.NewTicker(initialRetryDelay),
triggerPodSync: triggerPodSync,
getActivePods: getActivePods,
getPodByUID: getPodByUID,
recorder: recorder,
}
}
func newStateImpl(checkpointDirectory, checkpointName string) state.State {
if !utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) {
return state.NewNoopStateCheckpoint()
}
stateImpl, err := state.NewStateCheckpoint(checkpointDirectory, checkpointName)
if err != nil {
// This is a critical, non-recoverable failure.
klog.ErrorS(err, "Failed to initialize allocation checkpoint manager",
"checkpointPath", filepath.Join(checkpointDirectory, checkpointName))
panic(err)
}
return stateImpl
}
// NewInMemoryManager returns an allocation manager that doesn't persist state.
// For testing purposes only!
func NewInMemoryManager(nodeConfig cm.NodeConfig,
nodeAllocatableAbsolute v1.ResourceList,
statusManager status.Manager,
triggerPodSync func(pod *v1.Pod),
getActivePods func() []*v1.Pod,
getPodByUID func(types.UID) (*v1.Pod, bool),
sourcesReady config.SourcesReady,
) Manager {
return &manager{
allocated: state.NewStateMemory(nil),
statusManager: statusManager,
admitHandlers: lifecycle.PodAdmitHandlers{},
sourcesReady: sourcesReady,
nodeConfig: nodeConfig,
nodeAllocatableAbsolute: nodeAllocatableAbsolute,
ticker: time.NewTicker(initialRetryDelay),
triggerPodSync: triggerPodSync,
getActivePods: getActivePods,
getPodByUID: getPodByUID,
}
}
func (m *manager) Run(ctx context.Context) {
// Start a goroutine to periodically check for pending resizes and process them if needed.
go func() {
for {
select {
case <-m.ticker.C:
successfulResizes := m.retryPendingResizes(triggerReasonPeriodic)
for _, po := range successfulResizes {
klog.InfoS("Successfully retried resize after timeout", "pod", klog.KObj(po))
}
case <-ctx.Done():
m.ticker.Stop()
return
}
}
}()
}
func (m *manager) RetryPendingResizes(trigger string) {
m.retryPendingResizes(trigger)
}
func (m *manager) retryPendingResizes(trigger string) []*v1.Pod {
m.allocationMutex.Lock()
defer m.allocationMutex.Unlock()
if !m.sourcesReady.AllReady() {
klog.V(4).InfoS("Skipping evaluation of pending resizes; sources are not ready")
m.ticker.Reset(initialRetryDelay)
return nil
}
m.ticker.Reset(retryDelay)
var newPendingResizes []types.UID
var successfulResizes []*v1.Pod
// Retry all pending resizes.
for _, uid := range m.podsWithPendingResizes {
pod, found := m.getPodByUID(uid)
if !found {
klog.V(4).InfoS("Pod not found; removing from pending resizes", "podUID", uid)
continue
}
oldResizeStatus := m.statusManager.GetPodResizeConditions(uid)
isDeferred := m.statusManager.IsPodResizeDeferred(uid)
resizeAllocated, err := m.handlePodResourcesResize(pod)
switch {
case err != nil:
klog.ErrorS(err, "Failed to handle pod resources resize", "pod", klog.KObj(pod))
newPendingResizes = append(newPendingResizes, uid)
case m.statusManager.IsPodResizeDeferred(uid):
klog.V(4).InfoS("Pod resize is deferred; will reevaluate later", "pod", klog.KObj(pod))
newPendingResizes = append(newPendingResizes, uid)
case m.statusManager.IsPodResizeInfeasible(uid):
klog.V(4).InfoS("Pod resize is infeasible", "pod", klog.KObj(pod))
default:
klog.V(4).InfoS("Pod resize successfully allocated", "pod", klog.KObj(pod))
successfulResizes = append(successfulResizes, pod)
if isDeferred {
metrics.PodDeferredAcceptedResizes.WithLabelValues(trigger).Inc()
}
}
// If the pod resize status has changed, we need to update the pod status.
newResizeStatus := m.statusManager.GetPodResizeConditions(uid)
if resizeAllocated || !apiequality.Semantic.DeepEqual(oldResizeStatus, newResizeStatus) {
m.triggerPodSync(pod)
}
}
m.podsWithPendingResizes = newPendingResizes
return successfulResizes
}
func (m *manager) PushPendingResize(uid types.UID) {
m.allocationMutex.Lock()
defer m.allocationMutex.Unlock()
for _, p := range m.podsWithPendingResizes {
if p == uid {
// Pod is already in the pending resizes queue.
return
}
}
// Add the pod to the pending resizes list and sort by priority.
m.podsWithPendingResizes = append(m.podsWithPendingResizes, uid)
m.sortPendingResizes()
}
// sortPendingResizes sorts the list of pending resizes:
// - First, prioritizing resizes that do not increase requests.
// - Second, based on the pod's PriorityClass.
// - Third, based on the pod's QoS class.
// - Last, prioritizing resizes that have been in the deferred state the longest.
func (m *manager) sortPendingResizes() {
var pendingPods []*v1.Pod
for _, uid := range m.podsWithPendingResizes {
pod, found := m.getPodByUID(uid)
if !found {
klog.V(4).InfoS("Pod not found; removing from pending resizes", "podUID", uid)
continue
}
pendingPods = append(pendingPods, pod)
}
slices.SortFunc(pendingPods, func(firstPod, secondPod *v1.Pod) int {
// First, resizes that don't increase requests will be prioritized.
// These resizes are expected to always succeed.
firstPodIncreasing := m.isResizeIncreasingRequests(firstPod)
secondPodIncreasing := m.isResizeIncreasingRequests(secondPod)
if !firstPodIncreasing {
return -1
}
if !secondPodIncreasing {
return 1
}
// Second, pods with a higher PriorityClass will be prioritized.
firstPodPriority := int32(0)
if firstPod.Spec.Priority != nil {
firstPodPriority = *firstPod.Spec.Priority
}
secondPodPriority := int32(0)
if secondPod.Spec.Priority != nil {
secondPodPriority = *secondPod.Spec.Priority
}
if firstPodPriority > secondPodPriority {
return -1
}
if secondPodPriority > firstPodPriority {
return 1
}
// Third, pods with a higher QoS class will be prioritized, where guaranteed > burstable.
// Best effort pods don't have resource requests or limits, so we don't need to consider them here.
firstPodQOS := v1qos.GetPodQOS(firstPod)
secondPodQOS := v1qos.GetPodQOS(secondPod)
if firstPodQOS == v1.PodQOSGuaranteed && secondPodQOS != v1.PodQOSGuaranteed {
return -1
}
if secondPodQOS == v1.PodQOSGuaranteed && firstPodQOS != v1.PodQOSGuaranteed {
return 1
}
// If all else is the same, resize requests that have been pending longer will be
// evaluated first.
var firstPodLastTransitionTime *metav1.Time
firstPodResizeConditions := m.statusManager.GetPodResizeConditions(firstPod.UID)
for _, c := range firstPodResizeConditions {
if c.Type == v1.PodResizePending {
firstPodLastTransitionTime = &c.LastTransitionTime
}
}
var secondPodLastTransitionTime *metav1.Time
secondPodResizeConditions := m.statusManager.GetPodResizeConditions(secondPod.UID)
for _, c := range secondPodResizeConditions {
if c.Type == v1.PodResizePending {
secondPodLastTransitionTime = &c.LastTransitionTime
}
}
if firstPodLastTransitionTime == nil {
return 1
}
if secondPodLastTransitionTime == nil {
return -1
}
if firstPodLastTransitionTime.Before(secondPodLastTransitionTime) {
return -1
}
return 1
})
m.podsWithPendingResizes = make([]types.UID, len(pendingPods))
for i, pod := range pendingPods {
m.podsWithPendingResizes[i] = pod.UID
}
}
// isResizeIncreasingRequests returns true if any of the resource requests are increasing.
func (m *manager) isResizeIncreasingRequests(pod *v1.Pod) bool {
allocatedPod, updated := m.UpdatePodFromAllocation(pod)
if !updated {
return false
}
opts := resourcehelper.PodResourcesOptions{
SkipPodLevelResources: !utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources),
}
oldRequest := resourcehelper.PodRequests(allocatedPod, opts)
newRequest := resourcehelper.PodRequests(pod, opts)
return newRequest.Memory().Cmp(*oldRequest.Memory()) > 0 ||
newRequest.Cpu().Cmp(*oldRequest.Cpu()) > 0
}
func (m *manager) HasPendingResizes() bool {
m.allocationMutex.Lock()
defer m.allocationMutex.Unlock()
return len(m.podsWithPendingResizes) > 0
}
// GetContainerResourceAllocation returns the last checkpointed AllocatedResources values
// If checkpoint manager has not been initialized, it returns nil, false
func (m *manager) GetContainerResourceAllocation(podUID types.UID, containerName string) (v1.ResourceRequirements, bool) {
return m.allocated.GetContainerResources(podUID, containerName)
}
// UpdatePodFromAllocation overwrites the pod spec with the allocation.
// This function does a deep copy only if updates are needed.
func (m *manager) UpdatePodFromAllocation(pod *v1.Pod) (*v1.Pod, bool) {
if pod == nil {
return pod, false
}
allocated, ok := m.allocated.GetPodResourceInfo(pod.UID)
if !ok {
return pod, false
}
return updatePodFromAllocation(pod, allocated)
}
func updatePodFromAllocation(pod *v1.Pod, allocated state.PodResourceInfo) (*v1.Pod, bool) {
if pod == nil {
return pod, false
}
updated := false
containerAlloc := func(c v1.Container) (v1.ResourceRequirements, bool) {
if cAlloc, ok := allocated.ContainerResources[c.Name]; ok {
if !apiequality.Semantic.DeepEqual(c.Resources, cAlloc) {
// Allocation differs from pod spec, retrieve the allocation
if !updated {
// If this is the first update to be performed, copy the pod
pod = pod.DeepCopy()
updated = true
}
return cAlloc, true
}
}
return v1.ResourceRequirements{}, false
}
for i, c := range pod.Spec.Containers {
if cAlloc, found := containerAlloc(c); found {
// Allocation differs from pod spec, update
pod.Spec.Containers[i].Resources = cAlloc
}
}
for i, c := range pod.Spec.InitContainers {
if cAlloc, found := containerAlloc(c); found {
// Allocation differs from pod spec, update
pod.Spec.InitContainers[i].Resources = cAlloc
}
}
return pod, updated
}
// SetAllocatedResources checkpoints the resources allocated to a pod's containers
func (m *manager) SetAllocatedResources(pod *v1.Pod) error {
return m.allocated.SetPodResourceInfo(pod.UID, allocationFromPod(pod))
}
func allocationFromPod(pod *v1.Pod) state.PodResourceInfo {
var podAlloc state.PodResourceInfo
podAlloc.ContainerResources = make(map[string]v1.ResourceRequirements)
for _, container := range pod.Spec.Containers {
alloc := *container.Resources.DeepCopy()
podAlloc.ContainerResources[container.Name] = alloc
}
for _, container := range pod.Spec.InitContainers {
if podutil.IsRestartableInitContainer(&container) {
alloc := *container.Resources.DeepCopy()
podAlloc.ContainerResources[container.Name] = alloc
}
}
return podAlloc
}
func (m *manager) AddPodAdmitHandlers(handlers lifecycle.PodAdmitHandlers) {
for _, a := range handlers {
m.admitHandlers.AddPodAdmitHandler(a)
}
}
func (m *manager) SetContainerRuntime(runtime kubecontainer.Runtime) {
m.containerRuntime = runtime
}
func (m *manager) AddPod(activePods []*v1.Pod, pod *v1.Pod) (bool, string, string) {
m.allocationMutex.Lock()
defer m.allocationMutex.Unlock()
if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) {
// To handle kubelet restarts, test pod admissibility using AllocatedResources values
// (for cpu & memory) from checkpoint store. If found, that is the source of truth.
pod, _ = m.UpdatePodFromAllocation(pod)
}
// Check if we can admit the pod; if so, update the allocation.
allocatedPods := m.getAllocatedPods(activePods)
ok, reason, message := m.canAdmitPod(allocatedPods, pod)
if ok && utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) {
// Checkpoint the resource values at which the Pod has been admitted or resized.
if err := m.SetAllocatedResources(pod); err != nil {
// TODO(vinaykul,InPlacePodVerticalScaling): Can we recover from this in some way? Investigate
klog.ErrorS(err, "SetPodAllocation failed", "pod", klog.KObj(pod))
}
}
return ok, reason, message
}
func (m *manager) RemovePod(uid types.UID) {
if err := m.allocated.RemovePod(uid); err != nil {
// If the deletion fails, it will be retried by RemoveOrphanedPods, so we can safely ignore the error.
klog.V(3).ErrorS(err, "Failed to delete pod allocation", "podUID", uid)
}
}
func (m *manager) RemoveOrphanedPods(remainingPods sets.Set[types.UID]) {
m.allocated.RemoveOrphanedPods(remainingPods)
}
func (m *manager) handlePodResourcesResize(pod *v1.Pod) (bool, error) {
allocatedPod, updated := m.UpdatePodFromAllocation(pod)
if !updated {
// Desired resources == allocated resources. Pod allocation does not need to be updated.
m.statusManager.ClearPodResizePendingCondition(pod.UID)
return false, nil
} else if resizable, msg, reason := IsInPlacePodVerticalScalingAllowed(pod); !resizable {
// If there is a pending resize but the resize is not allowed, always use the allocated resources.
metrics.PodInfeasibleResizes.WithLabelValues(reason).Inc()
m.statusManager.SetPodResizePendingCondition(pod.UID, v1.PodReasonInfeasible, msg, pod.Generation)
return false, nil
} else if resizeNotAllowed, msg := disallowResizeForSwappableContainers(m.containerRuntime, pod, allocatedPod); resizeNotAllowed {
// If this resize involve swap recalculation, set as infeasible, as IPPR with swap is not supported for beta.
metrics.PodInfeasibleResizes.WithLabelValues("swap_limitation").Inc()
m.statusManager.SetPodResizePendingCondition(pod.UID, v1.PodReasonInfeasible, msg, pod.Generation)
return false, nil
}
// Desired resources != allocated resources. Can we update the allocation to the desired resources?
fit, reason, message := m.canResizePod(m.getAllocatedPods(m.getActivePods()), pod)
if fit {
// Update pod resource allocation checkpoint
if err := m.SetAllocatedResources(pod); err != nil {
return false, err
}
m.statusManager.ClearPodResizePendingCondition(pod.UID)
// Clear any errors that may have been surfaced from a previous resize and update the
// generation of the resize in-progress condition.
m.statusManager.ClearPodResizeInProgressCondition(pod.UID)
m.statusManager.SetPodResizeInProgressCondition(pod.UID, "", "", pod.Generation)
return true, nil
}
if reason != "" {
m.statusManager.SetPodResizePendingCondition(pod.UID, reason, message, pod.Generation)
}
return false, nil
}
func disallowResizeForSwappableContainers(runtime kubecontainer.Runtime, desiredPod, allocatedPod *v1.Pod) (bool, string) {
if desiredPod == nil || allocatedPod == nil {
return false, ""
}
restartableMemoryResizePolicy := func(resizePolicies []v1.ContainerResizePolicy) bool {
for _, policy := range resizePolicies {
if policy.ResourceName == v1.ResourceMemory {
return policy.RestartPolicy == v1.RestartContainer
}
}
return false
}
allocatedContainers := make(map[string]v1.Container)
for _, container := range append(allocatedPod.Spec.Containers, allocatedPod.Spec.InitContainers...) {
allocatedContainers[container.Name] = container
}
for _, desiredContainer := range append(desiredPod.Spec.Containers, desiredPod.Spec.InitContainers...) {
allocatedContainer, ok := allocatedContainers[desiredContainer.Name]
if !ok {
continue
}
origMemRequest := desiredContainer.Resources.Requests[v1.ResourceMemory]
newMemRequest := allocatedContainer.Resources.Requests[v1.ResourceMemory]
if !origMemRequest.Equal(newMemRequest) && !restartableMemoryResizePolicy(allocatedContainer.ResizePolicy) {
aSwapBehavior := runtime.GetContainerSwapBehavior(desiredPod, &desiredContainer)
bSwapBehavior := runtime.GetContainerSwapBehavior(allocatedPod, &allocatedContainer)
if aSwapBehavior != kubetypes.NoSwap || bSwapBehavior != kubetypes.NoSwap {
return true, "In-place resize of containers with swap is not supported."
}
}
}
return false, ""
}
// canAdmitPod determines if a pod can be admitted, and gives a reason if it
// cannot. "pod" is new pod, while "pods" are all admitted pods
// The function returns a boolean value indicating whether the pod
// can be admitted, a brief single-word reason and a message explaining why
// the pod cannot be admitted.
// allocatedPods should represent the pods that have already been admitted, along with their
// admitted (allocated) resources.
func (m *manager) canAdmitPod(allocatedPods []*v1.Pod, pod *v1.Pod) (bool, string, string) {
// Filter out the pod being evaluated.
allocatedPods = slices.DeleteFunc(allocatedPods, func(p *v1.Pod) bool { return p.UID == pod.UID })
// If any handler rejects, the pod is rejected.
attrs := &lifecycle.PodAdmitAttributes{Pod: pod, OtherPods: allocatedPods}
for _, podAdmitHandler := range m.admitHandlers {
if result := podAdmitHandler.Admit(attrs); !result.Admit {
klog.InfoS("Pod admission denied", "podUID", attrs.Pod.UID, "pod", klog.KObj(attrs.Pod), "reason", result.Reason, "message", result.Message)
return false, result.Reason, result.Message
}
}
return true, "", ""
}
// canResizePod determines if the requested resize is currently feasible.
// pod should hold the desired (pre-allocated) spec.
// Returns true if the resize can proceed; returns a reason and message
// otherwise.
func (m *manager) canResizePod(allocatedPods []*v1.Pod, pod *v1.Pod) (bool, string, string) {
// TODO: Move this logic into a PodAdmitHandler by introducing an operation field to
// lifecycle.PodAdmitAttributes, and combine canResizePod with canAdmitPod.
if v1qos.GetPodQOS(pod) == v1.PodQOSGuaranteed {
if !utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScalingExclusiveCPUs) &&
m.nodeConfig.CPUManagerPolicy == string(cpumanager.PolicyStatic) &&
m.guaranteedPodResourceResizeRequired(pod, v1.ResourceCPU) {
msg := fmt.Sprintf("Resize is infeasible for Guaranteed Pods alongside CPU Manager policy \"%s\"", string(cpumanager.PolicyStatic))
klog.V(3).InfoS(msg, "pod", format.Pod(pod))
metrics.PodInfeasibleResizes.WithLabelValues("guaranteed_pod_cpu_manager_static_policy").Inc()
return false, v1.PodReasonInfeasible, msg
}
if utilfeature.DefaultFeatureGate.Enabled(features.MemoryManager) &&
!utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScalingExclusiveMemory) &&
m.nodeConfig.MemoryManagerPolicy == string(memorymanager.PolicyTypeStatic) &&
m.guaranteedPodResourceResizeRequired(pod, v1.ResourceMemory) {
msg := fmt.Sprintf("Resize is infeasible for Guaranteed Pods alongside Memory Manager policy \"%s\"", string(memorymanager.PolicyTypeStatic))
klog.V(3).InfoS(msg, "pod", format.Pod(pod))
metrics.PodInfeasibleResizes.WithLabelValues("guaranteed_pod_memory_manager_static_policy").Inc()
return false, v1.PodReasonInfeasible, msg
}
}
cpuAvailable := m.nodeAllocatableAbsolute.Cpu().MilliValue()
memAvailable := m.nodeAllocatableAbsolute.Memory().Value()
cpuRequests := resource.GetResourceRequest(pod, v1.ResourceCPU)
memRequests := resource.GetResourceRequest(pod, v1.ResourceMemory)
if cpuRequests > cpuAvailable || memRequests > memAvailable {
var msg string
if memRequests > memAvailable {
msg = fmt.Sprintf("memory, requested: %d, capacity: %d", memRequests, memAvailable)
} else {
msg = fmt.Sprintf("cpu, requested: %d, capacity: %d", cpuRequests, cpuAvailable)
}
msg = "Node didn't have enough capacity: " + msg
klog.V(3).InfoS(msg, "pod", klog.KObj(pod))
metrics.PodInfeasibleResizes.WithLabelValues("insufficient_node_allocatable").Inc()
return false, v1.PodReasonInfeasible, msg
}
if ok, failReason, failMessage := m.canAdmitPod(allocatedPods, pod); !ok {
// Log reason and return.
klog.V(3).InfoS("Resize cannot be accommodated", "pod", klog.KObj(pod), "reason", failReason, "message", failMessage)
return false, v1.PodReasonDeferred, failMessage
}
return true, "", ""
}
func (m *manager) guaranteedPodResourceResizeRequired(pod *v1.Pod, resourceName v1.ResourceName) bool {
for container, containerType := range podutil.ContainerIter(&pod.Spec, podutil.InitContainers|podutil.Containers) {
if !IsResizableContainer(container, containerType) {
continue
}
requestedResources := container.Resources
allocatedresources, _ := m.GetContainerResourceAllocation(pod.UID, container.Name)
// For Guaranteed pods, requests must equal limits, so checking requests is sufficient.
if !requestedResources.Requests[resourceName].Equal(allocatedresources.Requests[resourceName]) {
return true
}
}
return false
}
func (m *manager) getAllocatedPods(activePods []*v1.Pod) []*v1.Pod {
if !utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) {
return activePods
}
allocatedPods := make([]*v1.Pod, len(activePods))
for i, pod := range activePods {
allocatedPods[i], _ = m.UpdatePodFromAllocation(pod)
}
return allocatedPods
}
func IsResizableContainer(container *v1.Container, containerType podutil.ContainerType) bool {
switch containerType {
case podutil.InitContainers:
return podutil.IsRestartableInitContainer(container)
case podutil.Containers:
return true
default:
return false
}
}