From 49c947ba15a195cd4e3b0d692f774d834b05e68f Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Fri, 6 Oct 2023 15:25:26 -0400 Subject: [PATCH] metrics: add and use ImageGarbageCollectedTotal to help find MaxAge thresholds and detect image addition/removal thrashing Signed-off-by: Peter Hunt --- pkg/kubelet/images/image_gc_manager.go | 2 ++ pkg/kubelet/metrics/metrics.go | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/pkg/kubelet/images/image_gc_manager.go b/pkg/kubelet/images/image_gc_manager.go index 1be95d6e421..8df2abaf9f6 100644 --- a/pkg/kubelet/images/image_gc_manager.go +++ b/pkg/kubelet/images/image_gc_manager.go @@ -36,6 +36,7 @@ import ( statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/events" + "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/util/sliceutils" ) @@ -429,6 +430,7 @@ func (im *realImageGCManager) freeImage(ctx context.Context, image evictionInfo) return err } delete(im.imageRecords, image.id) + metrics.ImageGarbageCollectedTotal.Inc() return err } diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index cd80e63c5f7..2b90bfd850c 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -116,6 +116,9 @@ const ( orphanPodCleanedVolumesKey = "orphan_pod_cleaned_volumes" orphanPodCleanedVolumesErrorsKey = "orphan_pod_cleaned_volumes_errors" + // Metric for tracking garbage collected images + ImageGarbageCollectedTotalKey = "image_garbage_collected_total" + // Values used in metric labels Container = "container" InitContainer = "init_container" @@ -786,6 +789,15 @@ var ( StabilityLevel: metrics.ALPHA, }, ) + + ImageGarbageCollectedTotal = metrics.NewCounter( + &metrics.CounterOpts{ + Subsystem: KubeletSubsystem, + Name: ImageGarbageCollectedTotalKey, + Help: "Total number of images garbage collected by the kubelet, whether through disk usage or image age.", + StabilityLevel: metrics.ALPHA, + }, + ) ) var registerMetrics sync.Once