diff --git a/pkg/kubelet/eviction/eviction_manager.go b/pkg/kubelet/eviction/eviction_manager.go index cb981dbb7e8..99ba3ae0f56 100644 --- a/pkg/kubelet/eviction/eviction_manager.go +++ b/pkg/kubelet/eviction/eviction_manager.go @@ -181,10 +181,10 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd } // Start starts the control loop to observe and response to low compute resources. -func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration) { +func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, terminatedPodFunc TerminatedPodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration) { thresholdHandler := func(message string) { klog.InfoS(message) - m.synchronize(diskInfoProvider, podFunc) + m.synchronize(diskInfoProvider, podFunc, terminatedPodFunc) } klog.InfoS("Eviction manager: starting control loop") if m.config.KernelMemcgNotification || runtime.GOOS == "windows" { @@ -203,7 +203,7 @@ func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePod // start the eviction manager monitoring go func() { for { - evictedPods, err := m.synchronize(diskInfoProvider, podFunc) + evictedPods, err := m.synchronize(diskInfoProvider, podFunc, terminatedPodFunc) if evictedPods != nil && err == nil { klog.InfoS("Eviction manager: pods evicted, waiting for pod to be cleaned up", "pods", klog.KObjSlice(evictedPods)) m.waitForPodsCleanup(podCleanedUpFunc, evictedPods) @@ -240,7 +240,7 @@ func (m *managerImpl) IsUnderPIDPressure() bool { // synchronize is the main control loop that enforces eviction thresholds. // Returns the pod that was killed, or nil if no pod was killed. -func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc) ([]*v1.Pod, error) { +func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, terminatedPodFunc TerminatedPodsFunc) ([]*v1.Pod, error) { ctx := context.Background() // if we have nothing to do, just return thresholds := m.config.Thresholds @@ -378,6 +378,11 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act // record an event about the resources we are now attempting to reclaim via eviction m.recorder.Eventf(m.nodeRef, v1.EventTypeWarning, "EvictionThresholdMet", "Attempting to reclaim %s", resourceToReclaim) + // Evict terminated pods, so their resources can be reclaimed + if evictedPods := m.evictTerminatedPods(thresholds, statsFunc, resourceToReclaim, terminatedPodFunc, observations, thresholdToReclaim); len(evictedPods) != 0 { + klog.InfoS("Eviction manager: evicted", "terminated pods", klog.KObjSlice(evictedPods)) + } + // check if there are node-level resources we can reclaim to reduce pressure before evicting end-user pods. if m.reclaimNodeLevelResources(ctx, thresholdToReclaim.Signal, resourceToReclaim) { klog.InfoS("Eviction manager: able to reduce resource pressure without evicting pods.", "resourceName", resourceToReclaim) @@ -440,6 +445,36 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act return nil, nil } +func (m *managerImpl) evictTerminatedPods(thresholds []evictionapi.Threshold, statsFunc statsFunc, resourceToReclaim v1.ResourceName, terminatedPodFunc TerminatedPodsFunc, observations signalObservations, thresholdToReclaim evictionapi.Threshold) []*v1.Pod { + evictedPods := []*v1.Pod{} + if resourceToReclaim == v1.ResourceName(v1.NodeDiskPressure) { + return evictedPods + } + + terminatedPods := terminatedPodFunc() + + gracePeriodOverride := int64(immediateEvictionGracePeriodSeconds) + + for i := range terminatedPods { + pod := terminatedPods[i] + klog.InfoS("Eviction manager: evicted terminated pod", "pod", pod.Name) + + message, annotations := evictionMessage(resourceToReclaim, pod, statsFunc, thresholds, observations) + condition := &v1.PodCondition{ + Type: v1.DisruptionTarget, + Status: v1.ConditionTrue, + Reason: v1.PodReasonTerminationByKubelet, + Message: message, + } + if m.evictPod(pod, gracePeriodOverride, message, annotations, condition) { + metrics.Evictions.WithLabelValues(string(thresholdToReclaim.Signal)).Inc() + evictedPods = append(evictedPods, pod) + } + } + + return evictedPods +} + func (m *managerImpl) waitForPodsCleanup(podCleanedUpFunc PodCleanedUpFunc, pods []*v1.Pod) { timeout := m.clock.NewTimer(podCleanupTimeout) defer timeout.Stop() diff --git a/pkg/kubelet/eviction/types.go b/pkg/kubelet/eviction/types.go index 83cdfcdf853..da8a6062ba7 100644 --- a/pkg/kubelet/eviction/types.go +++ b/pkg/kubelet/eviction/types.go @@ -59,7 +59,7 @@ type Config struct { // Manager evaluates when an eviction threshold for node stability has been met on the node. type Manager interface { // Start starts the control loop to monitor eviction thresholds at specified interval. - Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration) + Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, terminatedPodFunc TerminatedPodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration) // IsUnderMemoryPressure returns true if the node is under memory pressure. IsUnderMemoryPressure() bool @@ -107,6 +107,9 @@ type MirrorPodFunc func(*v1.Pod) (*v1.Pod, bool) // ActivePodsFunc returns pods bound to the kubelet that are active (i.e. non-terminal state) type ActivePodsFunc func() []*v1.Pod +// TerminatedPodsFunc return pods bound to the kubelet that are inactive (i.e. terminal state) +type TerminatedPodsFunc func() []*v1.Pod + // PodCleanedUpFunc returns true if all resources associated with a pod have been reclaimed. type PodCleanedUpFunc func(*v1.Pod) bool diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 600a006f054..3d581feaa77 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -1705,7 +1705,7 @@ func (kl *Kubelet) initializeRuntimeDependentModules() { } // eviction manager must start after cadvisor because it needs to know if the container runtime has a dedicated imagefs // Eviction decisions are based on the allocated (rather than desired) pod resources. - kl.evictionManager.Start(kl.StatsProvider, kl.getAllocatedPods, kl.PodIsFinished, evictionMonitoringPeriod) + kl.evictionManager.Start(kl.StatsProvider, kl.getAllocatedPods, kl.getTerminatedPods, kl.PodIsFinished, evictionMonitoringPeriod) // container log manager must start after container runtime is up to retrieve information from container runtime // and inform container to reopen log file after log rotation. diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 86eeb0bd26d..dacfa0282f6 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -219,6 +219,14 @@ func (kl *Kubelet) getAllocatedPods() []*v1.Pod { return allocatedPods } +// Returns all terminated pods. +func (kl *Kubelet) getTerminatedPods() []*v1.Pod { + allPods := kl.podManager.GetPods() + + terminatedPods := kl.filterOutActivePods(allPods) + return terminatedPods +} + // makeBlockVolumes maps the raw block devices specified in the path of the container // Experimental func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVolumes kubecontainer.VolumeMap, blkutil volumepathhandler.BlockVolumePathHandler) ([]kubecontainer.DeviceInfo, error) { @@ -1144,6 +1152,21 @@ func (kl *Kubelet) filterOutInactivePods(pods []*v1.Pod) []*v1.Pod { return filteredPods } +// filterOutActivePods returns pods that are in a terminal phase +func (kl *Kubelet) filterOutActivePods(pods []*v1.Pod) []*v1.Pod { + filteredPods := make([]*v1.Pod, 0, len(pods)) + for _, p := range pods { + // skip pod if it is not in a terminal phase + if !(p.Status.Phase == v1.PodFailed || p.Status.Phase == v1.PodSucceeded) { + continue + } + + klog.InfoS("Eviction manager - Test: filterOutActivePods", "terminated pod", p) + filteredPods = append(filteredPods, p) + } + return filteredPods +} + // isAdmittedPodTerminal returns true if the provided config source pod is in // a terminal phase, or if the Kubelet has already indicated the pod has reached // a terminal phase but the config source has not accepted it yet. This method