Merge pull request #107201 from pacoxu/add-metrics-volume-stats-cal

add VolumeStatCalDuration metrics for fsquato monitoring benchmark

pkg/kubelet/server/metrics/metrics.go

@@ -64,6 +64,18 @@ var (
 		},
 		[]string{"method", "path", "server_type", "long_running"},
 	)
+	// VolumeStatCalDuration tracks the duration in seconds to calculate volume stats.
+	// this metric is mainly for comparison between fsquota monitoring and `du` for disk usage.
+	VolumeStatCalDuration = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Subsystem:      kubeletSubsystem,
+			Name:           "volume_metric_collection_duration_seconds",
+			Help:           "Duration in seconds to calculate volume stats",
+			Buckets:        metrics.DefBuckets,
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"metric_source"},
+	)
 )
 
 var registerMetrics sync.Once
@@ -74,6 +86,7 @@ func Register() {
 		legacyregistry.MustRegister(HTTPRequests)
 		legacyregistry.MustRegister(HTTPRequestsDuration)
 		legacyregistry.MustRegister(HTTPInflightRequests)
+		legacyregistry.MustRegister(VolumeStatCalDuration)
 	})
 }
 
@@ -81,3 +94,8 @@ func Register() {
 func SinceInSeconds(start time.Time) float64 {
 	return time.Since(start).Seconds()
 }
+
+// CollectVolumeStatCalDuration collects the duration in seconds to calculate volume stats.
+func CollectVolumeStatCalDuration(metricSource string, start time.Time) {
+	VolumeStatCalDuration.WithLabelValues(metricSource).Observe(SinceInSeconds(start))
+}

pkg/volume/csi/csi_metrics.go

@@ -23,6 +23,7 @@ import (
 
 	"google.golang.org/grpc"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 	"k8s.io/kubernetes/pkg/volume"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
 )
@@ -51,6 +52,8 @@ func NewMetricsCsi(volumeID string, targetPath string, driverName csiDriverName)
 }
 
 func (mc *metricsCsi) GetMetrics() (*volume.Metrics, error) {
+	startTime := time.Now()
+	defer servermetrics.CollectVolumeStatCalDuration(string(mc.csiClientGetter.driverName), startTime)
 	currentTime := metav1.Now()
 	ctx, cancel := context.WithTimeout(context.Background(), csiTimeout)
 	defer cancel()

pkg/volume/metrics_block.go

@@ -21,9 +21,11 @@ import (
 	"io"
 	"os"
 	"runtime"
+	"time"
 
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 )
 
 var _ MetricsProvider = &metricsBlock{}
@@ -49,6 +51,9 @@ func NewMetricsBlock(device string) MetricsProvider {
 // tools. Storage systems may have more information that they can provide by
 // going through specialized APIs.
 func (mb *metricsBlock) GetMetrics() (*Metrics, error) {
+	startTime := time.Now()
+	defer servermetrics.CollectVolumeStatCalDuration("block", startTime)
+
 	// TODO: Windows does not yet support VolumeMode=Block
 	if runtime.GOOS == "windows" {
 		return nil, NewNotImplementedError("Windows does not support Block volumes")

pkg/volume/metrics_statfs.go

@@ -17,8 +17,11 @@ limitations under the License.
 package volume
 
 import (
+	"time"
+
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 	"k8s.io/kubernetes/pkg/volume/util/fs"
 )
 
@@ -40,6 +43,9 @@ func NewMetricsStatFS(path string) MetricsProvider {
 // GetMetrics calculates the volume usage and device free space by executing "du"
 // and gathering filesystem info for the Volume path.
 func (md *metricsStatFS) GetMetrics() (*Metrics, error) {
+	startTime := time.Now()
+	defer servermetrics.CollectVolumeStatCalDuration("statfs", startTime)
+
 	metrics := &Metrics{Time: metav1.Now()}
 	if md.path == "" {
 		return metrics, NewNoPathDefinedError()

pkg/volume/util/fs/fs.go

@@ -24,9 +24,11 @@ import (
 	"os"
 	"path/filepath"
 	"syscall"
+	"time"
 
 	"golang.org/x/sys/unix"
 
+	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 	"k8s.io/kubernetes/pkg/volume/util/fsquota"
 )
 
@@ -71,9 +73,13 @@ func DiskUsage(path string) (UsageInfo, error) {
 	// First check whether the quota system knows about this directory
 	// A nil quantity or error means that the path does not support quotas
 	// or xfs_quota tool is missing and we should use other mechanisms.
+	startTime := time.Now()
 	consumption, _ := fsquota.GetConsumption(path)
 	if consumption != nil {
 		usage.Bytes = consumption.Value()
+		defer servermetrics.CollectVolumeStatCalDuration("fsquota", startTime)
+	} else {
+		defer servermetrics.CollectVolumeStatCalDuration("du", startTime)
 	}
 
 	inodes, _ := fsquota.GetInodes(path)