From af9e0be163a89a4cca8f602ed7ff60353ef866bb Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Thu, 6 Feb 2020 17:14:59 -0500 Subject: [PATCH] kubelet: Record kubelet_evictions when limits are hit The pod, container, and emptyDir volumes can all trigger evictions when their limits are breached. To ensure that administrators can alert on these type of evictions, update kubelet_evictions to include the following signal types: * ephemeralcontainerfs.limit - container ephemeral storage breaches its limit * ephemeralpodfs.limit - pod ephemeral storage breaches its limit * emptydirfs.limit - pod emptyDir storage breaches its limit --- pkg/kubelet/eviction/eviction_manager.go | 28 ++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pkg/kubelet/eviction/eviction_manager.go b/pkg/kubelet/eviction/eviction_manager.go index 00a1823eb8e..54d6dfc2a59 100644 --- a/pkg/kubelet/eviction/eviction_manager.go +++ b/pkg/kubelet/eviction/eviction_manager.go @@ -47,6 +47,15 @@ const ( podCleanupPollFreq = time.Second ) +const ( + // signalEphemeralContainerFsLimit is amount of storage available on filesystem requested by the container + signalEphemeralContainerFsLimit string = "ephemeralcontainerfs.limit" + // signalEphemeralPodFsLimit is amount of storage available on filesystem requested by the pod + signalEphemeralPodFsLimit string = "ephemeralpodfs.limit" + // signalEmptyDirFsLimit is amount of storage available on filesystem requested by an emptyDir + signalEmptyDirFsLimit string = "emptydirfs.limit" +) + // managerImpl implements Manager type managerImpl struct { // used to track time @@ -480,7 +489,11 @@ func (m *managerImpl) emptyDirLimitEviction(podStats statsapi.PodStats, pod *v1. used := podVolumeUsed[pod.Spec.Volumes[i].Name] if used != nil && size != nil && size.Sign() == 1 && used.Cmp(*size) > 0 { // the emptyDir usage exceeds the size limit, evict the pod - return m.evictPod(pod, 0, fmt.Sprintf(emptyDirMessageFmt, pod.Spec.Volumes[i].Name, size.String()), nil) + if m.evictPod(pod, 0, fmt.Sprintf(emptyDirMessageFmt, pod.Spec.Volumes[i].Name, size.String()), nil) { + metrics.Evictions.WithLabelValues(signalEmptyDirFsLimit).Inc() + return true + } + return false } } } @@ -512,7 +525,11 @@ func (m *managerImpl) podEphemeralStorageLimitEviction(podStats statsapi.PodStat podEphemeralStorageLimit := podLimits[v1.ResourceEphemeralStorage] if podEphemeralStorageTotalUsage.Cmp(podEphemeralStorageLimit) > 0 { // the total usage of pod exceeds the total size limit of containers, evict the pod - return m.evictPod(pod, 0, fmt.Sprintf(podEphemeralStorageMessageFmt, podEphemeralStorageLimit.String()), nil) + if m.evictPod(pod, 0, fmt.Sprintf(podEphemeralStorageMessageFmt, podEphemeralStorageLimit.String()), nil) { + metrics.Evictions.WithLabelValues(signalEphemeralPodFsLimit).Inc() + return true + } + return false } return false } @@ -534,8 +551,11 @@ func (m *managerImpl) containerEphemeralStorageLimitEviction(podStats statsapi.P if ephemeralStorageThreshold, ok := thresholdsMap[containerStat.Name]; ok { if ephemeralStorageThreshold.Cmp(*containerUsed) < 0 { - return m.evictPod(pod, 0, fmt.Sprintf(containerEphemeralStorageMessageFmt, containerStat.Name, ephemeralStorageThreshold.String()), nil) - + if m.evictPod(pod, 0, fmt.Sprintf(containerEphemeralStorageMessageFmt, containerStat.Name, ephemeralStorageThreshold.String()), nil) { + metrics.Evictions.WithLabelValues(signalEphemeralContainerFsLimit).Inc() + return true + } + return false } } }