client-go: add request and response size metrics

Get metrics for the request and response  size, so we can correlate latency
and size on a request, otherwise we could get confused because we don't know if the
network is slow or just the request size huge.

Kubernetes-commit: 64d9d0585f6dbc9266f31b6d0f795d6c0421495e

rest/request.go

@@ -901,6 +901,11 @@ func (r *Request) request(ctx context.Context, fn func(*http.Request, *http.Resp
 		}
 		resp, err := client.Do(req)
 		updateURLMetrics(ctx, r, resp, err)
+		// The value -1 or a value of 0 with a non-nil Body indicates that the length is unknown.
+		// https://pkg.go.dev/net/http#Request
+		if req.ContentLength >= 0 && !(req.Body != nil && req.ContentLength == 0) {
+			metrics.RequestSize.Observe(ctx, r.verb, r.URL().Host, float64(req.ContentLength))
+		}
 		if err != nil {
 			r.backoff.UpdateBackoff(r.URL(), err, 0)
 		} else {
@@ -963,6 +968,9 @@ func (r *Request) Do(ctx context.Context) Result {
 	if err != nil {
 		return Result{err: err}
 	}
+	if result.err == nil || len(result.body) > 0 {
+		metrics.ResponseSize.Observe(ctx, r.verb, r.URL().Host, float64(len(result.body)))
+	}
 	return result
 }
 
@@ -979,6 +987,9 @@ func (r *Request) DoRaw(ctx context.Context) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	if result.err == nil || len(result.body) > 0 {
+		metrics.ResponseSize.Observe(ctx, r.verb, r.URL().Host, float64(len(result.body)))
+	}
 	return result.body, result.err
 }