diff --git a/pkg/kubelet/server/metrics/metrics.go b/pkg/kubelet/server/metrics/metrics.go index d287de225db..e9a2b5fca76 100644 --- a/pkg/kubelet/server/metrics/metrics.go +++ b/pkg/kubelet/server/metrics/metrics.go @@ -64,6 +64,18 @@ var ( }, []string{"method", "path", "server_type", "long_running"}, ) + // VolumeStatCalDuration tracks the duration in seconds to calculate volume stats. + // this metric is mainly for comparison between fsquota monitoring and `du` for disk usage. + VolumeStatCalDuration = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Subsystem: kubeletSubsystem, + Name: "volume_metric_collection_duration_seconds", + Help: "Duration in seconds to calculate volume stats", + Buckets: metrics.DefBuckets, + StabilityLevel: metrics.ALPHA, + }, + []string{"metric_source"}, + ) ) var registerMetrics sync.Once @@ -74,6 +86,7 @@ func Register() { legacyregistry.MustRegister(HTTPRequests) legacyregistry.MustRegister(HTTPRequestsDuration) legacyregistry.MustRegister(HTTPInflightRequests) + legacyregistry.MustRegister(VolumeStatCalDuration) }) } @@ -81,3 +94,8 @@ func Register() { func SinceInSeconds(start time.Time) float64 { return time.Since(start).Seconds() } + +// CollectVolumeStatCalDuration collects the duration in seconds to calculate volume stats. +func CollectVolumeStatCalDuration(metricSource string, start time.Time) { + VolumeStatCalDuration.WithLabelValues(metricSource).Observe(SinceInSeconds(start)) +} diff --git a/pkg/volume/csi/csi_metrics.go b/pkg/volume/csi/csi_metrics.go index 96fcff66a87..7315222e613 100644 --- a/pkg/volume/csi/csi_metrics.go +++ b/pkg/volume/csi/csi_metrics.go @@ -23,6 +23,7 @@ import ( "google.golang.org/grpc" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" ) @@ -51,6 +52,8 @@ func NewMetricsCsi(volumeID string, targetPath string, driverName csiDriverName) } func (mc *metricsCsi) GetMetrics() (*volume.Metrics, error) { + startTime := time.Now() + defer servermetrics.CollectVolumeStatCalDuration(string(mc.csiClientGetter.driverName), startTime) currentTime := metav1.Now() ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) defer cancel() diff --git a/pkg/volume/metrics_block.go b/pkg/volume/metrics_block.go index e0145ae91af..e370afca24d 100644 --- a/pkg/volume/metrics_block.go +++ b/pkg/volume/metrics_block.go @@ -21,9 +21,11 @@ import ( "io" "os" "runtime" + "time" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" ) var _ MetricsProvider = &metricsBlock{} @@ -49,6 +51,9 @@ func NewMetricsBlock(device string) MetricsProvider { // tools. Storage systems may have more information that they can provide by // going through specialized APIs. func (mb *metricsBlock) GetMetrics() (*Metrics, error) { + startTime := time.Now() + defer servermetrics.CollectVolumeStatCalDuration("block", startTime) + // TODO: Windows does not yet support VolumeMode=Block if runtime.GOOS == "windows" { return nil, NewNotImplementedError("Windows does not support Block volumes") diff --git a/pkg/volume/metrics_statfs.go b/pkg/volume/metrics_statfs.go index 29151457e07..81a6748b19a 100644 --- a/pkg/volume/metrics_statfs.go +++ b/pkg/volume/metrics_statfs.go @@ -17,8 +17,11 @@ limitations under the License. package volume import ( + "time" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume/util/fs" ) @@ -40,6 +43,9 @@ func NewMetricsStatFS(path string) MetricsProvider { // GetMetrics calculates the volume usage and device free space by executing "du" // and gathering filesystem info for the Volume path. func (md *metricsStatFS) GetMetrics() (*Metrics, error) { + startTime := time.Now() + defer servermetrics.CollectVolumeStatCalDuration("statfs", startTime) + metrics := &Metrics{Time: metav1.Now()} if md.path == "" { return metrics, NewNoPathDefinedError() diff --git a/pkg/volume/util/fs/fs.go b/pkg/volume/util/fs/fs.go index acfc4278536..64301fd5de1 100644 --- a/pkg/volume/util/fs/fs.go +++ b/pkg/volume/util/fs/fs.go @@ -24,9 +24,11 @@ import ( "os" "path/filepath" "syscall" + "time" "golang.org/x/sys/unix" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/volume/util/fsquota" ) @@ -71,9 +73,13 @@ func DiskUsage(path string) (UsageInfo, error) { // First check whether the quota system knows about this directory // A nil quantity or error means that the path does not support quotas // or xfs_quota tool is missing and we should use other mechanisms. + startTime := time.Now() consumption, _ := fsquota.GetConsumption(path) if consumption != nil { usage.Bytes = consumption.Value() + defer servermetrics.CollectVolumeStatCalDuration("fsquota", startTime) + } else { + defer servermetrics.CollectVolumeStatCalDuration("du", startTime) } inodes, _ := fsquota.GetInodes(path)