metrics: add and use ImageGarbageCollectedTotal

to help find MaxAge thresholds and detect image addition/removal thrashing

Signed-off-by: Peter Hunt <pehunt@redhat.com>
This commit is contained in:
Peter Hunt 2023-10-06 15:25:26 -04:00
parent d8a5cd59c0
commit 49c947ba15
2 changed files with 14 additions and 0 deletions

View File

@ -36,6 +36,7 @@ import (
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/events"
"k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/pkg/kubelet/util/sliceutils" "k8s.io/kubernetes/pkg/kubelet/util/sliceutils"
) )
@ -429,6 +430,7 @@ func (im *realImageGCManager) freeImage(ctx context.Context, image evictionInfo)
return err return err
} }
delete(im.imageRecords, image.id) delete(im.imageRecords, image.id)
metrics.ImageGarbageCollectedTotal.Inc()
return err return err
} }

View File

@ -116,6 +116,9 @@ const (
orphanPodCleanedVolumesKey = "orphan_pod_cleaned_volumes" orphanPodCleanedVolumesKey = "orphan_pod_cleaned_volumes"
orphanPodCleanedVolumesErrorsKey = "orphan_pod_cleaned_volumes_errors" orphanPodCleanedVolumesErrorsKey = "orphan_pod_cleaned_volumes_errors"
// Metric for tracking garbage collected images
ImageGarbageCollectedTotalKey = "image_garbage_collected_total"
// Values used in metric labels // Values used in metric labels
Container = "container" Container = "container"
InitContainer = "init_container" InitContainer = "init_container"
@ -786,6 +789,15 @@ var (
StabilityLevel: metrics.ALPHA, StabilityLevel: metrics.ALPHA,
}, },
) )
ImageGarbageCollectedTotal = metrics.NewCounter(
&metrics.CounterOpts{
Subsystem: KubeletSubsystem,
Name: ImageGarbageCollectedTotalKey,
Help: "Total number of images garbage collected by the kubelet, whether through disk usage or image age.",
StabilityLevel: metrics.ALPHA,
},
)
) )
var registerMetrics sync.Once var registerMetrics sync.Once