client-go: add DNS resolver latency metrics (#115357)

* client-go: add DNS resolver latency metrics

* client-go: add locking to DNS latency metrics

* client-go: add locking for whole DNSStart and DNSDone

Signed-off-by: Vu Dinh <vudinh@outlook.com>

* Fix a mismatched ctx on the request

Signed-off-by: Vu Dinh <vudinh@outlook.com>

* Clean up request code and fix comments

Signed-off-by: Vu Dinh <vudinh@outlook.com>

---------

Signed-off-by: Vu Dinh <vudinh@outlook.com>
Co-authored-by: Vu Dinh <vudinh@outlook.com>

Kubernetes-commit: 1c7e87cff27aa009488a9d55342220e223d5c146

rest/request.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"mime"
 	"net/http"
+	"net/http/httptrace"
 	"net/url"
 	"os"
 	"path"
@@ -925,15 +926,38 @@ func (r *Request) newHTTPRequest(ctx context.Context) (*http.Request, error) {
 	}
 
 	url := r.URL().String()
-	req, err := http.NewRequest(r.verb, url, body)
+	req, err := http.NewRequestWithContext(httptrace.WithClientTrace(ctx, newDNSMetricsTrace(ctx)), r.verb, url, body)
 	if err != nil {
 		return nil, err
 	}
-	req = req.WithContext(ctx)
 	req.Header = r.headers
 	return req, nil
 }
 
+// newDNSMetricsTrace returns an HTTP trace that tracks time spent on DNS lookups per host.
+// This metric is available in client as "rest_client_dns_resolution_duration_seconds".
+func newDNSMetricsTrace(ctx context.Context) *httptrace.ClientTrace {
+	type dnsMetric struct {
+		start time.Time
+		host  string
+		sync.Mutex
+	}
+	dns := &dnsMetric{}
+	return &httptrace.ClientTrace{
+		DNSStart: func(info httptrace.DNSStartInfo) {
+			dns.Lock()
+			defer dns.Unlock()
+			dns.start = time.Now()
+			dns.host = info.Host
+		},
+		DNSDone: func(info httptrace.DNSDoneInfo) {
+			dns.Lock()
+			defer dns.Unlock()
+			metrics.ResolverLatency.Observe(ctx, dns.host, time.Since(dns.start))
+		},
+	}
+}
+
 // request connects to the server and invokes the provided function when a server response is
 // received. It handles retry behavior and up front validation of requests. It will invoke
 // fn at most once. It will return an error if a problem occurred prior to connecting to the