client-go: add request and response size metrics

Get metrics for the request and response  size, so we can correlate latency
and size on a request, otherwise we could get confused because we don't know if the
network is slow or just the request size huge.

Kubernetes-commit: 64d9d0585f6dbc9266f31b6d0f795d6c0421495e

rest/request.go

@@ -901,6 +901,11 @@ func (r *Request) request(ctx context.Context, fn func(*http.Request, *http.Resp
 		}
 		resp, err := client.Do(req)
 		updateURLMetrics(ctx, r, resp, err)
+		// The value -1 or a value of 0 with a non-nil Body indicates that the length is unknown.
+		// https://pkg.go.dev/net/http#Request
+		if req.ContentLength >= 0 && !(req.Body != nil && req.ContentLength == 0) {
+			metrics.RequestSize.Observe(ctx, r.verb, r.URL().Host, float64(req.ContentLength))
+		}
 		if err != nil {
 			r.backoff.UpdateBackoff(r.URL(), err, 0)
 		} else {
@@ -963,6 +968,9 @@ func (r *Request) Do(ctx context.Context) Result {
 	if err != nil {
 		return Result{err: err}
 	}
+	if result.err == nil || len(result.body) > 0 {
+		metrics.ResponseSize.Observe(ctx, r.verb, r.URL().Host, float64(len(result.body)))
+	}
 	return result
 }
 
@@ -979,6 +987,9 @@ func (r *Request) DoRaw(ctx context.Context) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	if result.err == nil || len(result.body) > 0 {
+		metrics.ResponseSize.Observe(ctx, r.verb, r.URL().Host, float64(len(result.body)))
+	}
 	return result.body, result.err
 }
 

tools/metrics/metrics.go

@@ -42,6 +42,11 @@ type LatencyMetric interface {
 	Observe(ctx context.Context, verb string, u url.URL, latency time.Duration)
 }
 
+// SizeMetric observes client response size partitioned by verb and host.
+type SizeMetric interface {
+	Observe(ctx context.Context, verb string, host string, size float64)
+}
+
 // ResultMetric counts response codes partitioned by method and host.
 type ResultMetric interface {
 	Increment(ctx context.Context, code string, method string, host string)
@@ -60,6 +65,10 @@ var (
 	ClientCertRotationAge DurationMetric = noopDuration{}
 	// RequestLatency is the latency metric that rest clients will update.
 	RequestLatency LatencyMetric = noopLatency{}
+	// RequestSize is the request size metric that rest clients will update.
+	RequestSize SizeMetric = noopSize{}
+	// ResponseSize is the response size metric that rest clients will update.
+	ResponseSize SizeMetric = noopSize{}
 	// RateLimiterLatency is the client side rate limiter latency metric.
 	RateLimiterLatency LatencyMetric = noopLatency{}
 	// RequestResult is the result metric that rest clients will update.
@@ -74,6 +83,8 @@ type RegisterOpts struct {
 	ClientCertExpiry      ExpiryMetric
 	ClientCertRotationAge DurationMetric
 	RequestLatency        LatencyMetric
+	RequestSize           SizeMetric
+	ResponseSize          SizeMetric
 	RateLimiterLatency    LatencyMetric
 	RequestResult         ResultMetric
 	ExecPluginCalls       CallsMetric
@@ -92,6 +103,12 @@ func Register(opts RegisterOpts) {
 		if opts.RequestLatency != nil {
 			RequestLatency = opts.RequestLatency
 		}
+		if opts.RequestSize != nil {
+			RequestSize = opts.RequestSize
+		}
+		if opts.ResponseSize != nil {
+			ResponseSize = opts.ResponseSize
+		}
 		if opts.RateLimiterLatency != nil {
 			RateLimiterLatency = opts.RateLimiterLatency
 		}
@@ -116,6 +133,10 @@ type noopLatency struct{}
 
 func (noopLatency) Observe(context.Context, string, url.URL, time.Duration) {}
 
+type noopSize struct{}
+
+func (noopSize) Observe(context.Context, string, string, float64) {}
+
 type noopResult struct{}
 
 func (noopResult) Increment(context.Context, string, string, string) {}