Evict terminated pods on disk pressure

Before evicting running pods, terminal phase pods will be evicted first

pkg/kubelet/eviction/eviction_manager.go

@@ -181,10 +181,10 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd
 }
 
 // Start starts the control loop to observe and response to low compute resources.
-func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration) {
+func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, terminatedPodFunc TerminatedPodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration) {
 	thresholdHandler := func(message string) {
 		klog.InfoS(message)
-		m.synchronize(diskInfoProvider, podFunc)
+		m.synchronize(diskInfoProvider, podFunc, terminatedPodFunc)
 	}
 	klog.InfoS("Eviction manager: starting control loop")
 	if m.config.KernelMemcgNotification || runtime.GOOS == "windows" {
@@ -203,7 +203,7 @@ func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePod
 	// start the eviction manager monitoring
 	go func() {
 		for {
-			evictedPods, err := m.synchronize(diskInfoProvider, podFunc)
+			evictedPods, err := m.synchronize(diskInfoProvider, podFunc, terminatedPodFunc)
 			if evictedPods != nil && err == nil {
 				klog.InfoS("Eviction manager: pods evicted, waiting for pod to be cleaned up", "pods", klog.KObjSlice(evictedPods))
 				m.waitForPodsCleanup(podCleanedUpFunc, evictedPods)
@@ -240,7 +240,7 @@ func (m *managerImpl) IsUnderPIDPressure() bool {
 
 // synchronize is the main control loop that enforces eviction thresholds.
 // Returns the pod that was killed, or nil if no pod was killed.
-func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc) ([]*v1.Pod, error) {
+func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, terminatedPodFunc TerminatedPodsFunc) ([]*v1.Pod, error) {
 	ctx := context.Background()
 	// if we have nothing to do, just return
 	thresholds := m.config.Thresholds
@@ -378,6 +378,11 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 	// record an event about the resources we are now attempting to reclaim via eviction
 	m.recorder.Eventf(m.nodeRef, v1.EventTypeWarning, "EvictionThresholdMet", "Attempting to reclaim %s", resourceToReclaim)
 
+	// Evict terminated pods, so their resources can be reclaimed
+	if evictedPods := m.evictTerminatedPods(thresholds, statsFunc, resourceToReclaim, terminatedPodFunc, observations, thresholdToReclaim); len(evictedPods) != 0 {
+		klog.InfoS("Eviction manager: evicted", "terminated pods", klog.KObjSlice(evictedPods))
+	}
+
 	// check if there are node-level resources we can reclaim to reduce pressure before evicting end-user pods.
 	if m.reclaimNodeLevelResources(ctx, thresholdToReclaim.Signal, resourceToReclaim) {
 		klog.InfoS("Eviction manager: able to reduce resource pressure without evicting pods.", "resourceName", resourceToReclaim)
@@ -440,6 +445,36 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 	return nil, nil
 }
 
+func (m *managerImpl) evictTerminatedPods(thresholds []evictionapi.Threshold, statsFunc statsFunc, resourceToReclaim v1.ResourceName, terminatedPodFunc TerminatedPodsFunc, observations signalObservations, thresholdToReclaim evictionapi.Threshold) []*v1.Pod {
+	evictedPods := []*v1.Pod{}
+	if resourceToReclaim == v1.ResourceName(v1.NodeDiskPressure) {
+		return evictedPods
+	}
+
+	terminatedPods := terminatedPodFunc()
+
+	gracePeriodOverride := int64(immediateEvictionGracePeriodSeconds)
+
+	for i := range terminatedPods {
+		pod := terminatedPods[i]
+		klog.InfoS("Eviction manager: evicted terminated pod", "pod", pod.Name)
+
+		message, annotations := evictionMessage(resourceToReclaim, pod, statsFunc, thresholds, observations)
+		condition := &v1.PodCondition{
+			Type:    v1.DisruptionTarget,
+			Status:  v1.ConditionTrue,
+			Reason:  v1.PodReasonTerminationByKubelet,
+			Message: message,
+		}
+		if m.evictPod(pod, gracePeriodOverride, message, annotations, condition) {
+			metrics.Evictions.WithLabelValues(string(thresholdToReclaim.Signal)).Inc()
+			evictedPods = append(evictedPods, pod)
+		}
+	}
+
+	return evictedPods
+}
+
 func (m *managerImpl) waitForPodsCleanup(podCleanedUpFunc PodCleanedUpFunc, pods []*v1.Pod) {
 	timeout := m.clock.NewTimer(podCleanupTimeout)
 	defer timeout.Stop()

pkg/kubelet/eviction/types.go

@@ -59,7 +59,7 @@ type Config struct {
 // Manager evaluates when an eviction threshold for node stability has been met on the node.
 type Manager interface {
 	// Start starts the control loop to monitor eviction thresholds at specified interval.
-	Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration)
+	Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, terminatedPodFunc TerminatedPodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration)
 
 	// IsUnderMemoryPressure returns true if the node is under memory pressure.
 	IsUnderMemoryPressure() bool
@@ -107,6 +107,9 @@ type MirrorPodFunc func(*v1.Pod) (*v1.Pod, bool)
 // ActivePodsFunc returns pods bound to the kubelet that are active (i.e. non-terminal state)
 type ActivePodsFunc func() []*v1.Pod
 
+// TerminatedPodsFunc return pods bound to the kubelet that are inactive (i.e. terminal state)
+type TerminatedPodsFunc func() []*v1.Pod
+
 // PodCleanedUpFunc returns true if all resources associated with a pod have been reclaimed.
 type PodCleanedUpFunc func(*v1.Pod) bool
 

pkg/kubelet/kubelet.go

@@ -1705,7 +1705,7 @@ func (kl *Kubelet) initializeRuntimeDependentModules() {
 	}
 	// eviction manager must start after cadvisor because it needs to know if the container runtime has a dedicated imagefs
 	// Eviction decisions are based on the allocated (rather than desired) pod resources.
-	kl.evictionManager.Start(kl.StatsProvider, kl.getAllocatedPods, kl.PodIsFinished, evictionMonitoringPeriod)
+	kl.evictionManager.Start(kl.StatsProvider, kl.getAllocatedPods, kl.getTerminatedPods, kl.PodIsFinished, evictionMonitoringPeriod)
 
 	// container log manager must start after container runtime is up to retrieve information from container runtime
 	// and inform container to reopen log file after log rotation.

pkg/kubelet/kubelet_pods.go

@@ -219,6 +219,14 @@ func (kl *Kubelet) getAllocatedPods() []*v1.Pod {
 	return allocatedPods
 }
 
+// Returns all terminated pods.
+func (kl *Kubelet) getTerminatedPods() []*v1.Pod {
+	allPods := kl.podManager.GetPods()
+
+	terminatedPods := kl.filterOutActivePods(allPods)
+	return terminatedPods
+}
+
 // makeBlockVolumes maps the raw block devices specified in the path of the container
 // Experimental
 func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVolumes kubecontainer.VolumeMap, blkutil volumepathhandler.BlockVolumePathHandler) ([]kubecontainer.DeviceInfo, error) {
@@ -1144,6 +1152,21 @@ func (kl *Kubelet) filterOutInactivePods(pods []*v1.Pod) []*v1.Pod {
 	return filteredPods
 }
 
+// filterOutActivePods returns pods that are in a terminal phase
+func (kl *Kubelet) filterOutActivePods(pods []*v1.Pod) []*v1.Pod {
+	filteredPods := make([]*v1.Pod, 0, len(pods))
+	for _, p := range pods {
+		// skip pod if it is not in a terminal phase
+		if !(p.Status.Phase == v1.PodFailed || p.Status.Phase == v1.PodSucceeded) {
+			continue
+		}
+
+		klog.InfoS("Eviction manager - Test: filterOutActivePods", "terminated pod", p)
+		filteredPods = append(filteredPods, p)
+	}
+	return filteredPods
+}
+
 // isAdmittedPodTerminal returns true if the provided config source pod is in
 // a terminal phase, or if the Kubelet has already indicated the pod has reached
 // a terminal phase but the config source has not accepted it yet. This method