From c5d8354e0e65255a310e7c5c609a23418b9f5021 Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Thu, 23 Dec 2021 16:18:32 +0800 Subject: [PATCH 1/3] add "kubelet_volume_stat_cal_duration_seconds_bucket" VolumeStatCalDuration metrics for fsquato monitoring benchmark --- pkg/kubelet/server/metrics/metrics.go | 14 ++++++++++++++ pkg/kubelet/server/stats/volume_stat_calculator.go | 9 ++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pkg/kubelet/server/metrics/metrics.go b/pkg/kubelet/server/metrics/metrics.go index d287de225db..45429c8e454 100644 --- a/pkg/kubelet/server/metrics/metrics.go +++ b/pkg/kubelet/server/metrics/metrics.go @@ -64,6 +64,19 @@ var ( }, []string{"method", "path", "server_type", "long_running"}, ) + // VolumeStatCalDuration tracks the duration in seconds to calculate volume stats. + // this metric is mainly for comparison between fsquota monitoring and `du` for disk usage. + VolumeStatCalDuration = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Subsystem: kubeletSubsystem, + Name: "volume_stat_cal_duration_seconds", + Help: "Duration in seconds to calculate volume stats", + // Use DefBuckets for now, will customize the buckets if necessary. + Buckets: metrics.DefBuckets, + StabilityLevel: metrics.ALPHA, + }, + []string{}, + ) ) var registerMetrics sync.Once @@ -74,6 +87,7 @@ func Register() { legacyregistry.MustRegister(HTTPRequests) legacyregistry.MustRegister(HTTPRequestsDuration) legacyregistry.MustRegister(HTTPInflightRequests) + legacyregistry.MustRegister(VolumeStatCalDuration) }) } diff --git a/pkg/kubelet/server/stats/volume_stat_calculator.go b/pkg/kubelet/server/stats/volume_stat_calculator.go index 63e9b1e7c39..7be21ff77fc 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator.go @@ -30,6 +30,7 @@ import ( "k8s.io/klog/v2" stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/features" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" ) @@ -133,7 +134,13 @@ func (s *volumeStatCalculator) calcAndStoreStats() { var ephemeralStats []stats.VolumeStats var persistentStats []stats.VolumeStats for name, v := range metricVolumes { - metric, err := v.GetMetrics() + metric, err := func() (*volume.Metrics, error) { + startTime := time.Now() + defer func() { + servermetrics.VolumeStatCalDuration.WithLabelValues().Observe(servermetrics.SinceInSeconds(startTime)) + }() + return v.GetMetrics() + }() if err != nil { // Expected for Volumes that don't support Metrics if !volume.IsNotSupported(err) { From 6611c36372366e4a20c9a8dd3ffac6815ce6bdb6 Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Thu, 6 Jan 2022 09:49:01 +0800 Subject: [PATCH 2/3] add volume type and seperated histogram for volume stat collection --- pkg/kubelet/server/metrics/metrics.go | 14 +++++++++----- pkg/kubelet/server/stats/volume_stat_calculator.go | 9 +-------- pkg/volume/csi/csi_metrics.go | 3 +++ pkg/volume/metrics_block.go | 5 +++++ pkg/volume/metrics_du.go | 6 ++++++ pkg/volume/metrics_statfs.go | 6 ++++++ 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/pkg/kubelet/server/metrics/metrics.go b/pkg/kubelet/server/metrics/metrics.go index 45429c8e454..e9a2b5fca76 100644 --- a/pkg/kubelet/server/metrics/metrics.go +++ b/pkg/kubelet/server/metrics/metrics.go @@ -68,14 +68,13 @@ var ( // this metric is mainly for comparison between fsquota monitoring and `du` for disk usage. VolumeStatCalDuration = metrics.NewHistogramVec( &metrics.HistogramOpts{ - Subsystem: kubeletSubsystem, - Name: "volume_stat_cal_duration_seconds", - Help: "Duration in seconds to calculate volume stats", - // Use DefBuckets for now, will customize the buckets if necessary. + Subsystem: kubeletSubsystem, + Name: "volume_metric_collection_duration_seconds", + Help: "Duration in seconds to calculate volume stats", Buckets: metrics.DefBuckets, StabilityLevel: metrics.ALPHA, }, - []string{}, + []string{"metric_source"}, ) ) @@ -95,3 +94,8 @@ func Register() { func SinceInSeconds(start time.Time) float64 { return time.Since(start).Seconds() } + +// CollectVolumeStatCalDuration collects the duration in seconds to calculate volume stats. +func CollectVolumeStatCalDuration(metricSource string, start time.Time) { + VolumeStatCalDuration.WithLabelValues(metricSource).Observe(SinceInSeconds(start)) +} diff --git a/pkg/kubelet/server/stats/volume_stat_calculator.go b/pkg/kubelet/server/stats/volume_stat_calculator.go index 7be21ff77fc..63e9b1e7c39 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator.go @@ -30,7 +30,6 @@ import ( "k8s.io/klog/v2" stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/features" - servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" ) @@ -134,13 +133,7 @@ func (s *volumeStatCalculator) calcAndStoreStats() { var ephemeralStats []stats.VolumeStats var persistentStats []stats.VolumeStats for name, v := range metricVolumes { - metric, err := func() (*volume.Metrics, error) { - startTime := time.Now() - defer func() { - servermetrics.VolumeStatCalDuration.WithLabelValues().Observe(servermetrics.SinceInSeconds(startTime)) - }() - return v.GetMetrics() - }() + metric, err := v.GetMetrics() if err != nil { // Expected for Volumes that don't support Metrics if !volume.IsNotSupported(err) { diff --git a/pkg/volume/csi/csi_metrics.go b/pkg/volume/csi/csi_metrics.go index 96fcff66a87..7315222e613 100644 --- a/pkg/volume/csi/csi_metrics.go +++ b/pkg/volume/csi/csi_metrics.go @@ -23,6 +23,7 @@ import ( "google.golang.org/grpc" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" ) @@ -51,6 +52,8 @@ func NewMetricsCsi(volumeID string, targetPath string, driverName csiDriverName) } func (mc *metricsCsi) GetMetrics() (*volume.Metrics, error) { + startTime := time.Now() + defer servermetrics.CollectVolumeStatCalDuration(string(mc.csiClientGetter.driverName), startTime) currentTime := metav1.Now() ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) defer cancel() diff --git a/pkg/volume/metrics_block.go b/pkg/volume/metrics_block.go index e0145ae91af..e370afca24d 100644 --- a/pkg/volume/metrics_block.go +++ b/pkg/volume/metrics_block.go @@ -21,9 +21,11 @@ import ( "io" "os" "runtime" + "time" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" ) var _ MetricsProvider = &metricsBlock{} @@ -49,6 +51,9 @@ func NewMetricsBlock(device string) MetricsProvider { // tools. Storage systems may have more information that they can provide by // going through specialized APIs. func (mb *metricsBlock) GetMetrics() (*Metrics, error) { + startTime := time.Now() + defer servermetrics.CollectVolumeStatCalDuration("block", startTime) + // TODO: Windows does not yet support VolumeMode=Block if runtime.GOOS == "windows" { return nil, NewNotImplementedError("Windows does not support Block volumes") diff --git a/pkg/volume/metrics_du.go b/pkg/volume/metrics_du.go index f080fac2b33..6c7a4af5757 100644 --- a/pkg/volume/metrics_du.go +++ b/pkg/volume/metrics_du.go @@ -17,8 +17,11 @@ limitations under the License. package volume import ( + "time" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume/util/fs" ) @@ -41,6 +44,9 @@ func NewMetricsDu(path string) MetricsProvider { // and gathering filesystem info for the Volume path. // See MetricsProvider.GetMetrics func (md *metricsDu) GetMetrics() (*Metrics, error) { + startTime := time.Now() + defer servermetrics.CollectVolumeStatCalDuration("du", startTime) + metrics := &Metrics{Time: metav1.Now()} if md.path == "" { return metrics, NewNoPathDefinedError() diff --git a/pkg/volume/metrics_statfs.go b/pkg/volume/metrics_statfs.go index 29151457e07..81a6748b19a 100644 --- a/pkg/volume/metrics_statfs.go +++ b/pkg/volume/metrics_statfs.go @@ -17,8 +17,11 @@ limitations under the License. package volume import ( + "time" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume/util/fs" ) @@ -40,6 +43,9 @@ func NewMetricsStatFS(path string) MetricsProvider { // GetMetrics calculates the volume usage and device free space by executing "du" // and gathering filesystem info for the Volume path. func (md *metricsStatFS) GetMetrics() (*Metrics, error) { + startTime := time.Now() + defer servermetrics.CollectVolumeStatCalDuration("statfs", startTime) + metrics := &Metrics{Time: metav1.Now()} if md.path == "" { return metrics, NewNoPathDefinedError() From 3967f03bb1c3dc86340c031bc0641cff6b532a1f Mon Sep 17 00:00:00 2001 From: -e Date: Thu, 20 Jan 2022 10:56:02 +0800 Subject: [PATCH 3/3] set metric_source to du or fsquota accordingly --- pkg/volume/metrics_du.go | 6 ------ pkg/volume/util/fs/fs.go | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/volume/metrics_du.go b/pkg/volume/metrics_du.go index 6c7a4af5757..f080fac2b33 100644 --- a/pkg/volume/metrics_du.go +++ b/pkg/volume/metrics_du.go @@ -17,11 +17,8 @@ limitations under the License. package volume import ( - "time" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume/util/fs" ) @@ -44,9 +41,6 @@ func NewMetricsDu(path string) MetricsProvider { // and gathering filesystem info for the Volume path. // See MetricsProvider.GetMetrics func (md *metricsDu) GetMetrics() (*Metrics, error) { - startTime := time.Now() - defer servermetrics.CollectVolumeStatCalDuration("du", startTime) - metrics := &Metrics{Time: metav1.Now()} if md.path == "" { return metrics, NewNoPathDefinedError() diff --git a/pkg/volume/util/fs/fs.go b/pkg/volume/util/fs/fs.go index acfc4278536..64301fd5de1 100644 --- a/pkg/volume/util/fs/fs.go +++ b/pkg/volume/util/fs/fs.go @@ -24,9 +24,11 @@ import ( "os" "path/filepath" "syscall" + "time" "golang.org/x/sys/unix" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume/util/fsquota" ) @@ -71,9 +73,13 @@ func DiskUsage(path string) (UsageInfo, error) { // First check whether the quota system knows about this directory // A nil quantity or error means that the path does not support quotas // or xfs_quota tool is missing and we should use other mechanisms. + startTime := time.Now() consumption, _ := fsquota.GetConsumption(path) if consumption != nil { usage.Bytes = consumption.Value() + defer servermetrics.CollectVolumeStatCalDuration("fsquota", startTime) + } else { + defer servermetrics.CollectVolumeStatCalDuration("du", startTime) } inodes, _ := fsquota.GetInodes(path)