Merge pull request #111597 from 249043822/evictionmessage

make eviction message more clear

pkg/kubelet/eviction/eviction_manager.go

@@ -388,7 +388,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 		if !isHardEvictionThreshold(thresholdToReclaim) {
 			gracePeriodOverride = m.config.MaxPodGracePeriodSeconds
 		}
-		message, annotations := evictionMessage(resourceToReclaim, pod, statsFunc)
+		message, annotations := evictionMessage(resourceToReclaim, pod, statsFunc, thresholds, observations)
 		var condition *v1.PodCondition
 		if utilfeature.DefaultFeatureGate.Enabled(features.PodDisruptionConditions) {
 			condition = &v1.PodCondition{

pkg/kubelet/eviction/eviction_manager_test.go

@@ -195,7 +195,7 @@ func TestMemoryPressure_VerifyPodStatus(t *testing.T) {
 			wantPodStatus: v1.PodStatus{
 				Phase:   v1.PodFailed,
 				Reason:  "Evicted",
-				Message: "The node was low on resource: memory. ",
+				Message: "The node was low on resource: memory. Threshold quantity: 2Gi, available: 1500Mi. ",
 			},
 		},
 	}
@@ -272,7 +272,7 @@ func TestMemoryPressure_VerifyPodStatus(t *testing.T) {
 						Type:    "DisruptionTarget",
 						Status:  "True",
 						Reason:  "TerminationByKubelet",
-						Message: "The node was low on resource: memory. ",
+						Message: "The node was low on resource: memory. Threshold quantity: 2Gi, available: 1500Mi. ",
 					})
 				}
 
@@ -296,7 +296,7 @@ func TestDiskPressureNodeFs_VerifyPodStatus(t *testing.T) {
 			wantPodStatus: v1.PodStatus{
 				Phase:   v1.PodFailed,
 				Reason:  "Evicted",
-				Message: "The node was low on resource: ephemeral-storage. ",
+				Message: "The node was low on resource: ephemeral-storage. Threshold quantity: 2Gi, available: 1536Mi. ",
 			},
 		},
 	}
@@ -373,7 +373,7 @@ func TestDiskPressureNodeFs_VerifyPodStatus(t *testing.T) {
 						Type:    "DisruptionTarget",
 						Status:  "True",
 						Reason:  "TerminationByKubelet",
-						Message: "The node was low on resource: ephemeral-storage. ",
+						Message: "The node was low on resource: ephemeral-storage. Threshold quantity: 2Gi, available: 1536Mi. ",
 					})
 				}
 

pkg/kubelet/eviction/helpers.go

@@ -43,7 +43,7 @@ const (
 	// nodeConditionMessageFmt is the message for evictions due to resource pressure.
 	nodeConditionMessageFmt = "The node had condition: %v. "
 	// containerMessageFmt provides additional information for containers exceeding requests
-	containerMessageFmt = "Container %s was using %s, which exceeds its request of %s. "
+	containerMessageFmt = "Container %s was using %s, request is %s, has larger consumption of %v. "
 	// containerEphemeralStorageMessageFmt provides additional information for containers which have exceeded their ES limit
 	containerEphemeralStorageMessageFmt = "Container %s exceeded its local ephemeral storage limit %q. "
 	// podEphemeralStorageMessageFmt provides additional information for pods which have exceeded their ES limit
@@ -60,6 +60,8 @@ const (
 	OffendingContainersUsageKey = "offending_containers_usage"
 	// StarvedResourceKey is the key for the starved resource in eviction event annotations
 	StarvedResourceKey = "starved_resource"
+	// thresholdMetMessageFmt is the message for evictions due to resource pressure.
+	thresholdMetMessageFmt = "Threshold quantity: %v, available: %v. "
 )
 
 var (
@@ -999,9 +1001,13 @@ func buildSignalToNodeReclaimFuncs(imageGC ImageGC, containerGC ContainerGC, wit
 }
 
 // evictionMessage constructs a useful message about why an eviction occurred, and annotations to provide metadata about the eviction
-func evictionMessage(resourceToReclaim v1.ResourceName, pod *v1.Pod, stats statsFunc) (message string, annotations map[string]string) {
+func evictionMessage(resourceToReclaim v1.ResourceName, pod *v1.Pod, stats statsFunc, thresholds []evictionapi.Threshold, observations signalObservations) (message string, annotations map[string]string) {
 	annotations = make(map[string]string)
 	message = fmt.Sprintf(nodeLowMessageFmt, resourceToReclaim)
+	quantity, available := getThresholdMetInfo(resourceToReclaim, thresholds, observations)
+	if quantity != nil && available != nil {
+		message += fmt.Sprintf(thresholdMetMessageFmt, quantity, available)
+	}
 	containers := []string{}
 	containerUsage := []string{}
 	podStats, ok := stats(pod)
@@ -1024,7 +1030,7 @@ func evictionMessage(resourceToReclaim v1.ResourceName, pod *v1.Pod, stats stats
 					}
 				}
 				if usage != nil && usage.Cmp(requests) > 0 {
-					message += fmt.Sprintf(containerMessageFmt, container.Name, usage.String(), requests.String())
+					message += fmt.Sprintf(containerMessageFmt, container.Name, usage.String(), requests.String(), resourceToReclaim)
 					containers = append(containers, container.Name)
 					containerUsage = append(containerUsage, usage.String())
 				}
@@ -1036,3 +1042,18 @@ func evictionMessage(resourceToReclaim v1.ResourceName, pod *v1.Pod, stats stats
 	annotations[StarvedResourceKey] = string(resourceToReclaim)
 	return
 }
+
+// getThresholdMetInfo get the threshold quantity and available for the resource resourceToReclaim
+func getThresholdMetInfo(resourceToReclaim v1.ResourceName, thresholds []evictionapi.Threshold, observations signalObservations) (quantity *resource.Quantity, available *resource.Quantity) {
+	for i := range thresholds {
+		threshold := thresholds[i]
+		if signalToResource[threshold.Signal] == resourceToReclaim {
+			observed, found := observations[threshold.Signal]
+			if found {
+				quantity := evictionapi.GetThresholdQuantity(threshold.Value, observed.capacity)
+				return quantity, observed.available
+			}
+		}
+	}
+	return nil, nil
+}