metrics: add and use ImageGarbageCollectedTotal

to help find MaxAge thresholds and detect image addition/removal thrashing

Signed-off-by: Peter Hunt <pehunt@redhat.com>

pkg/kubelet/images/image_gc_manager.go

@@ -36,6 +36,7 @@ import (
 	statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/events"
+	"k8s.io/kubernetes/pkg/kubelet/metrics"
 	"k8s.io/kubernetes/pkg/kubelet/util/sliceutils"
 )
 
@@ -429,6 +430,7 @@ func (im *realImageGCManager) freeImage(ctx context.Context, image evictionInfo)
 		return err
 	}
 	delete(im.imageRecords, image.id)
+	metrics.ImageGarbageCollectedTotal.Inc()
 	return err
 }
 

pkg/kubelet/metrics/metrics.go

@@ -116,6 +116,9 @@ const (
 	orphanPodCleanedVolumesKey       = "orphan_pod_cleaned_volumes"
 	orphanPodCleanedVolumesErrorsKey = "orphan_pod_cleaned_volumes_errors"
 
+	// Metric for tracking garbage collected images
+	ImageGarbageCollectedTotalKey = "image_garbage_collected_total"
+
 	// Values used in metric labels
 	Container          = "container"
 	InitContainer      = "init_container"
@@ -786,6 +789,15 @@ var (
 			StabilityLevel: metrics.ALPHA,
 		},
 	)
+
+	ImageGarbageCollectedTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Subsystem:      KubeletSubsystem,
+			Name:           ImageGarbageCollectedTotalKey,
+			Help:           "Total number of images garbage collected by the kubelet, whether through disk usage or image age.",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
 )
 
 var registerMetrics sync.Once