Merge pull request #108296 from aojea/client_go_size_metrics

client-go: add request and response size metrics

staging/src/k8s.io/client-go/rest/request.go

@@ -901,6 +901,11 @@ func (r *Request) request(ctx context.Context, fn func(*http.Request, *http.Resp
 		}
 		resp, err := client.Do(req)
 		updateURLMetrics(ctx, r, resp, err)
+		// The value -1 or a value of 0 with a non-nil Body indicates that the length is unknown.
+		// https://pkg.go.dev/net/http#Request
+		if req.ContentLength >= 0 && !(req.Body != nil && req.ContentLength == 0) {
+			metrics.RequestSize.Observe(ctx, r.verb, r.URL().Host, float64(req.ContentLength))
+		}
 		if err != nil {
 			r.backoff.UpdateBackoff(r.URL(), err, 0)
 		} else {
@@ -963,6 +968,9 @@ func (r *Request) Do(ctx context.Context) Result {
 	if err != nil {
 		return Result{err: err}
 	}
+	if result.err == nil || len(result.body) > 0 {
+		metrics.ResponseSize.Observe(ctx, r.verb, r.URL().Host, float64(len(result.body)))
+	}
 	return result
 }
 
@@ -979,6 +987,9 @@ func (r *Request) DoRaw(ctx context.Context) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	if result.err == nil || len(result.body) > 0 {
+		metrics.ResponseSize.Observe(ctx, r.verb, r.URL().Host, float64(len(result.body)))
+	}
 	return result.body, result.err
 }
 

staging/src/k8s.io/client-go/tools/metrics/metrics.go

@@ -42,6 +42,11 @@ type LatencyMetric interface {
 	Observe(ctx context.Context, verb string, u url.URL, latency time.Duration)
 }
 
+// SizeMetric observes client response size partitioned by verb and host.
+type SizeMetric interface {
+	Observe(ctx context.Context, verb string, host string, size float64)
+}
+
 // ResultMetric counts response codes partitioned by method and host.
 type ResultMetric interface {
 	Increment(ctx context.Context, code string, method string, host string)
@@ -60,6 +65,10 @@ var (
 	ClientCertRotationAge DurationMetric = noopDuration{}
 	// RequestLatency is the latency metric that rest clients will update.
 	RequestLatency LatencyMetric = noopLatency{}
+	// RequestSize is the request size metric that rest clients will update.
+	RequestSize SizeMetric = noopSize{}
+	// ResponseSize is the response size metric that rest clients will update.
+	ResponseSize SizeMetric = noopSize{}
 	// RateLimiterLatency is the client side rate limiter latency metric.
 	RateLimiterLatency LatencyMetric = noopLatency{}
 	// RequestResult is the result metric that rest clients will update.
@@ -74,6 +83,8 @@ type RegisterOpts struct {
 	ClientCertExpiry      ExpiryMetric
 	ClientCertRotationAge DurationMetric
 	RequestLatency        LatencyMetric
+	RequestSize           SizeMetric
+	ResponseSize          SizeMetric
 	RateLimiterLatency    LatencyMetric
 	RequestResult         ResultMetric
 	ExecPluginCalls       CallsMetric
@@ -92,6 +103,12 @@ func Register(opts RegisterOpts) {
 		if opts.RequestLatency != nil {
 			RequestLatency = opts.RequestLatency
 		}
+		if opts.RequestSize != nil {
+			RequestSize = opts.RequestSize
+		}
+		if opts.ResponseSize != nil {
+			ResponseSize = opts.ResponseSize
+		}
 		if opts.RateLimiterLatency != nil {
 			RateLimiterLatency = opts.RateLimiterLatency
 		}
@@ -116,6 +133,10 @@ type noopLatency struct{}
 
 func (noopLatency) Observe(context.Context, string, url.URL, time.Duration) {}
 
+type noopSize struct{}
+
+func (noopSize) Observe(context.Context, string, string, float64) {}
+
 type noopResult struct{}
 
 func (noopResult) Increment(context.Context, string, string, string) {}

staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go

@@ -33,9 +33,32 @@ var (
 	// "verb", and "host" labels. It is used for the rest client latency metrics.
 	requestLatency = k8smetrics.NewHistogramVec(
 		&k8smetrics.HistogramOpts{
-			Name:    "rest_client_request_duration_seconds",
-			Help:    "Request latency in seconds. Broken down by verb, and host.",
-			Buckets: []float64{0.005, 0.025, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 15.0, 30.0, 60.0},
+			Name:           "rest_client_request_duration_seconds",
+			Help:           "Request latency in seconds. Broken down by verb, and host.",
+			StabilityLevel: k8smetrics.ALPHA,
+			Buckets:        []float64{0.005, 0.025, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 15.0, 30.0, 60.0},
+		},
+		[]string{"verb", "host"},
+	)
+
+	requestSize = k8smetrics.NewHistogramVec(
+		&k8smetrics.HistogramOpts{
+			Name:           "rest_client_request_size_bytes",
+			Help:           "Request size in bytes. Broken down by verb and host.",
+			StabilityLevel: k8smetrics.ALPHA,
+			// 64 bytes to 16MB
+			Buckets: []float64{64, 256, 512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216},
+		},
+		[]string{"verb", "host"},
+	)
+
+	responseSize = k8smetrics.NewHistogramVec(
+		&k8smetrics.HistogramOpts{
+			Name:           "rest_client_response_size_bytes",
+			Help:           "Response size in bytes. Broken down by verb and host.",
+			StabilityLevel: k8smetrics.ALPHA,
+			// 64 bytes to 16MB
+			Buckets: []float64{64, 256, 512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216},
 		},
 		[]string{"verb", "host"},
 	)
@@ -121,6 +144,8 @@ var (
 func init() {
 
 	legacyregistry.MustRegister(requestLatency)
+	legacyregistry.MustRegister(requestSize)
+	legacyregistry.MustRegister(responseSize)
 	legacyregistry.MustRegister(rateLimiterLatency)
 	legacyregistry.MustRegister(requestResult)
 	legacyregistry.RawMustRegister(execPluginCertTTL)
@@ -129,6 +154,8 @@ func init() {
 		ClientCertExpiry:      execPluginCertTTLAdapter,
 		ClientCertRotationAge: &rotationAdapter{m: execPluginCertRotation},
 		RequestLatency:        &latencyAdapter{m: requestLatency},
+		RequestSize:           &sizeAdapter{m: requestSize},
+		ResponseSize:          &sizeAdapter{m: responseSize},
 		RateLimiterLatency:    &latencyAdapter{m: rateLimiterLatency},
 		RequestResult:         &resultAdapter{requestResult},
 		ExecPluginCalls:       &callsAdapter{m: execPluginCalls},
@@ -143,6 +170,14 @@ func (l *latencyAdapter) Observe(ctx context.Context, verb string, u url.URL, la
 	l.m.WithContext(ctx).WithLabelValues(verb, u.Host).Observe(latency.Seconds())
 }
 
+type sizeAdapter struct {
+	m *k8smetrics.HistogramVec
+}
+
+func (s *sizeAdapter) Observe(ctx context.Context, verb string, host string, size float64) {
+	s.m.WithContext(ctx).WithLabelValues(verb, host).Observe(size)
+}
+
 type resultAdapter struct {
 	m *k8smetrics.CounterVec
 }