From c5d8354e0e65255a310e7c5c609a23418b9f5021 Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Thu, 23 Dec 2021 16:18:32 +0800 Subject: [PATCH] add "kubelet_volume_stat_cal_duration_seconds_bucket" VolumeStatCalDuration metrics for fsquato monitoring benchmark --- pkg/kubelet/server/metrics/metrics.go | 14 ++++++++++++++ pkg/kubelet/server/stats/volume_stat_calculator.go | 9 ++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pkg/kubelet/server/metrics/metrics.go b/pkg/kubelet/server/metrics/metrics.go index d287de225db..45429c8e454 100644 --- a/pkg/kubelet/server/metrics/metrics.go +++ b/pkg/kubelet/server/metrics/metrics.go @@ -64,6 +64,19 @@ var ( }, []string{"method", "path", "server_type", "long_running"}, ) + // VolumeStatCalDuration tracks the duration in seconds to calculate volume stats. + // this metric is mainly for comparison between fsquota monitoring and `du` for disk usage. + VolumeStatCalDuration = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Subsystem: kubeletSubsystem, + Name: "volume_stat_cal_duration_seconds", + Help: "Duration in seconds to calculate volume stats", + // Use DefBuckets for now, will customize the buckets if necessary. + Buckets: metrics.DefBuckets, + StabilityLevel: metrics.ALPHA, + }, + []string{}, + ) ) var registerMetrics sync.Once @@ -74,6 +87,7 @@ func Register() { legacyregistry.MustRegister(HTTPRequests) legacyregistry.MustRegister(HTTPRequestsDuration) legacyregistry.MustRegister(HTTPInflightRequests) + legacyregistry.MustRegister(VolumeStatCalDuration) }) } diff --git a/pkg/kubelet/server/stats/volume_stat_calculator.go b/pkg/kubelet/server/stats/volume_stat_calculator.go index 63e9b1e7c39..7be21ff77fc 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator.go @@ -30,6 +30,7 @@ import ( "k8s.io/klog/v2" stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/features" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" ) @@ -133,7 +134,13 @@ func (s *volumeStatCalculator) calcAndStoreStats() { var ephemeralStats []stats.VolumeStats var persistentStats []stats.VolumeStats for name, v := range metricVolumes { - metric, err := v.GetMetrics() + metric, err := func() (*volume.Metrics, error) { + startTime := time.Now() + defer func() { + servermetrics.VolumeStatCalDuration.WithLabelValues().Observe(servermetrics.SinceInSeconds(startTime)) + }() + return v.GetMetrics() + }() if err != nil { // Expected for Volumes that don't support Metrics if !volume.IsNotSupported(err) {