From 62177fd36d7626592fef43bb9e09406ef9e35f19 Mon Sep 17 00:00:00 2001 From: ZhangKe10140699 Date: Mon, 1 Aug 2022 17:15:38 +0800 Subject: [PATCH] make eviction message more clear --- pkg/kubelet/eviction/eviction_manager.go | 2 +- pkg/kubelet/eviction/eviction_manager_test.go | 8 +++--- pkg/kubelet/eviction/helpers.go | 27 ++++++++++++++++--- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/pkg/kubelet/eviction/eviction_manager.go b/pkg/kubelet/eviction/eviction_manager.go index 8dfebe84ace..308ab482c90 100644 --- a/pkg/kubelet/eviction/eviction_manager.go +++ b/pkg/kubelet/eviction/eviction_manager.go @@ -388,7 +388,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act if !isHardEvictionThreshold(thresholdToReclaim) { gracePeriodOverride = m.config.MaxPodGracePeriodSeconds } - message, annotations := evictionMessage(resourceToReclaim, pod, statsFunc) + message, annotations := evictionMessage(resourceToReclaim, pod, statsFunc, thresholds, observations) var condition *v1.PodCondition if utilfeature.DefaultFeatureGate.Enabled(features.PodDisruptionConditions) { condition = &v1.PodCondition{ diff --git a/pkg/kubelet/eviction/eviction_manager_test.go b/pkg/kubelet/eviction/eviction_manager_test.go index 1bd8d66c683..47e39fb2d82 100644 --- a/pkg/kubelet/eviction/eviction_manager_test.go +++ b/pkg/kubelet/eviction/eviction_manager_test.go @@ -195,7 +195,7 @@ func TestMemoryPressure_VerifyPodStatus(t *testing.T) { wantPodStatus: v1.PodStatus{ Phase: v1.PodFailed, Reason: "Evicted", - Message: "The node was low on resource: memory. ", + Message: "The node was low on resource: memory. Threshold quantity: 2Gi, available: 1500Mi. ", }, }, } @@ -272,7 +272,7 @@ func TestMemoryPressure_VerifyPodStatus(t *testing.T) { Type: "DisruptionTarget", Status: "True", Reason: "TerminationByKubelet", - Message: "The node was low on resource: memory. ", + Message: "The node was low on resource: memory. Threshold quantity: 2Gi, available: 1500Mi. ", }) } @@ -296,7 +296,7 @@ func TestDiskPressureNodeFs_VerifyPodStatus(t *testing.T) { wantPodStatus: v1.PodStatus{ Phase: v1.PodFailed, Reason: "Evicted", - Message: "The node was low on resource: ephemeral-storage. ", + Message: "The node was low on resource: ephemeral-storage. Threshold quantity: 2Gi, available: 1536Mi. ", }, }, } @@ -373,7 +373,7 @@ func TestDiskPressureNodeFs_VerifyPodStatus(t *testing.T) { Type: "DisruptionTarget", Status: "True", Reason: "TerminationByKubelet", - Message: "The node was low on resource: ephemeral-storage. ", + Message: "The node was low on resource: ephemeral-storage. Threshold quantity: 2Gi, available: 1536Mi. ", }) } diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index bf5d6fe7c1a..be451070602 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -43,7 +43,7 @@ const ( // nodeConditionMessageFmt is the message for evictions due to resource pressure. nodeConditionMessageFmt = "The node had condition: %v. " // containerMessageFmt provides additional information for containers exceeding requests - containerMessageFmt = "Container %s was using %s, which exceeds its request of %s. " + containerMessageFmt = "Container %s was using %s, request is %s, has larger consumption of %v. " // containerEphemeralStorageMessageFmt provides additional information for containers which have exceeded their ES limit containerEphemeralStorageMessageFmt = "Container %s exceeded its local ephemeral storage limit %q. " // podEphemeralStorageMessageFmt provides additional information for pods which have exceeded their ES limit @@ -60,6 +60,8 @@ const ( OffendingContainersUsageKey = "offending_containers_usage" // StarvedResourceKey is the key for the starved resource in eviction event annotations StarvedResourceKey = "starved_resource" + // thresholdMetMessageFmt is the message for evictions due to resource pressure. + thresholdMetMessageFmt = "Threshold quantity: %v, available: %v. " ) var ( @@ -999,9 +1001,13 @@ func buildSignalToNodeReclaimFuncs(imageGC ImageGC, containerGC ContainerGC, wit } // evictionMessage constructs a useful message about why an eviction occurred, and annotations to provide metadata about the eviction -func evictionMessage(resourceToReclaim v1.ResourceName, pod *v1.Pod, stats statsFunc) (message string, annotations map[string]string) { +func evictionMessage(resourceToReclaim v1.ResourceName, pod *v1.Pod, stats statsFunc, thresholds []evictionapi.Threshold, observations signalObservations) (message string, annotations map[string]string) { annotations = make(map[string]string) message = fmt.Sprintf(nodeLowMessageFmt, resourceToReclaim) + quantity, available := getThresholdMetInfo(resourceToReclaim, thresholds, observations) + if quantity != nil && available != nil { + message += fmt.Sprintf(thresholdMetMessageFmt, quantity, available) + } containers := []string{} containerUsage := []string{} podStats, ok := stats(pod) @@ -1024,7 +1030,7 @@ func evictionMessage(resourceToReclaim v1.ResourceName, pod *v1.Pod, stats stats } } if usage != nil && usage.Cmp(requests) > 0 { - message += fmt.Sprintf(containerMessageFmt, container.Name, usage.String(), requests.String()) + message += fmt.Sprintf(containerMessageFmt, container.Name, usage.String(), requests.String(), resourceToReclaim) containers = append(containers, container.Name) containerUsage = append(containerUsage, usage.String()) } @@ -1036,3 +1042,18 @@ func evictionMessage(resourceToReclaim v1.ResourceName, pod *v1.Pod, stats stats annotations[StarvedResourceKey] = string(resourceToReclaim) return } + +// getThresholdMetInfo get the threshold quantity and available for the resource resourceToReclaim +func getThresholdMetInfo(resourceToReclaim v1.ResourceName, thresholds []evictionapi.Threshold, observations signalObservations) (quantity *resource.Quantity, available *resource.Quantity) { + for i := range thresholds { + threshold := thresholds[i] + if signalToResource[threshold.Signal] == resourceToReclaim { + observed, found := observations[threshold.Signal] + if found { + quantity := evictionapi.GetThresholdQuantity(threshold.Value, observed.capacity) + return quantity, observed.available + } + } + } + return nil, nil +}