endpoints/metrics: define watchListLatencies metric and associated functions

staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go

@@ -18,6 +18,7 @@ package metrics
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -26,8 +27,12 @@ import (
 	"time"
 
 	restful "github.com/emicklei/go-restful/v3"
+
+	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	utilsets "k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/authentication/user"
@@ -280,6 +285,17 @@ var (
 		[]string{"code_path"},
 	)
 
+	watchListLatencies = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Subsystem:      APIServerComponent,
+			Name:           "watch_list_duration_seconds",
+			Help:           "Response latency distribution in seconds for watch list requests broken by group, version, resource and scope.",
+			Buckets:        []float64{0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 2, 4, 6, 8, 10, 15, 20, 30, 45, 60},
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"group", "version", "resource", "scope"},
+	)
+
 	metrics = []resettableCollector{
 		deprecatedRequestGauge,
 		requestCounter,
@@ -300,6 +316,7 @@ var (
 		requestAbortsTotal,
 		requestPostTimeoutTotal,
 		requestTimestampComparisonDuration,
+		watchListLatencies,
 	}
 
 	// these are the valid request methods which we report in our metrics. Any other request methods
@@ -511,6 +528,18 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp
 	fn()
 }
 
+// RecordWatchListLatency simply records response latency for watch list requests.
+func RecordWatchListLatency(ctx context.Context, gvr schema.GroupVersionResource, metricsScope string) {
+	requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(ctx)
+	if !ok {
+		utilruntime.HandleError(fmt.Errorf("unable to measure watchlist latency because no received ts found in the ctx, gvr: %s", gvr))
+		return
+	}
+	elapsedSeconds := time.Since(requestReceivedTimestamp).Seconds()
+
+	watchListLatencies.WithContext(ctx).WithLabelValues(gvr.Group, gvr.Version, gvr.Resource, metricsScope).Observe(elapsedSeconds)
+}
+
 // MonitorRequest handles standard transformations for client and the reported verb and then invokes Monitor to record
 // a request. verb must be uppercase to be backwards compatible with existing monitoring tooling.
 func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, httpCode, respSize int, elapsed time.Duration) {
@@ -621,6 +650,26 @@ func CleanScope(requestInfo *request.RequestInfo) string {
 	return ""
 }
 
+// CleanListScope computes the request scope for metrics.
+//
+// Note that normally we would use CleanScope for computation.
+// But due to the same reasons mentioned in determineRequestNamespaceAndName we cannot.
+func CleanListScope(ctx context.Context, opts *metainternalversion.ListOptions) string {
+	namespace, name := determineRequestNamespaceAndName(ctx, opts)
+	if len(name) > 0 {
+		return "resource"
+	}
+	if len(namespace) > 0 {
+		return "namespace"
+	}
+	if requestInfo, ok := request.RequestInfoFrom(ctx); ok {
+		if requestInfo.IsResourceRequest {
+			return "cluster"
+		}
+	}
+	return ""
+}
+
 // CanonicalVerb distinguishes LISTs from GETs (and HEADs). It assumes verb is
 // UPPERCASE.
 func CanonicalVerb(verb string, scope string) string {
@@ -655,6 +704,30 @@ func CleanVerb(verb string, request *http.Request, requestInfo *request.RequestI
 	return reportedVerb
 }
 
+// determineRequestNamespaceAndName computes name and namespace for the given requests
+//
+// note that the logic of this function was copy&pasted from cacher.go
+// after an unsuccessful attempt of moving it to RequestInfo
+//
+// see: https://github.com/kubernetes/kubernetes/pull/120520
+func determineRequestNamespaceAndName(ctx context.Context, opts *metainternalversion.ListOptions) (namespace, name string) {
+	if requestNamespace, ok := request.NamespaceFrom(ctx); ok && len(requestNamespace) > 0 {
+		namespace = requestNamespace
+	} else if opts != nil && opts.FieldSelector != nil {
+		if selectorNamespace, ok := opts.FieldSelector.RequiresExactMatch("metadata.namespace"); ok {
+			namespace = selectorNamespace
+		}
+	}
+	if requestInfo, ok := request.RequestInfoFrom(ctx); ok && requestInfo != nil && len(requestInfo.Name) > 0 {
+		name = requestInfo.Name
+	} else if opts != nil && opts.FieldSelector != nil {
+		if selectorName, ok := opts.FieldSelector.RequiresExactMatch("metadata.name"); ok {
+			name = selectorName
+		}
+	}
+	return
+}
+
 // cleanVerb additionally ensures that unknown verbs don't clog up the metrics.
 func cleanVerb(verb, suggestedVerb string, request *http.Request, requestInfo *request.RequestInfo) string {
 	// CanonicalVerb (being an input for this function) doesn't handle correctly the

staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics_test.go

@@ -17,11 +17,14 @@ limitations under the License.
 package metrics
 
 import (
+	"context"
 	"net/http"
 	"net/url"
 	"strings"
 	"testing"
 
+	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/endpoints/responsewriter"
 	"k8s.io/component-base/metrics/legacyregistry"
@@ -466,3 +469,61 @@ func TestRecordDroppedRequests(t *testing.T) {
 		})
 	}
 }
+
+func TestCleanListScope(t *testing.T) {
+	scenarios := []struct {
+		name          string
+		ctx           context.Context
+		opts          *metainternalversion.ListOptions
+		expectedScope string
+	}{
+		{
+			name: "empty scope",
+		},
+		{
+			name: "empty scope with empty request info",
+			ctx:  request.WithRequestInfo(context.TODO(), &request.RequestInfo{}),
+		},
+		{
+			name:          "namespace from ctx",
+			ctx:           request.WithNamespace(context.TODO(), "foo"),
+			expectedScope: "namespace",
+		},
+		{
+			name: "namespace from field selector",
+			opts: &metainternalversion.ListOptions{
+				FieldSelector: fields.ParseSelectorOrDie("metadata.namespace=foo"),
+			},
+			expectedScope: "namespace",
+		},
+		{
+			name:          "name from request info",
+			ctx:           request.WithRequestInfo(context.TODO(), &request.RequestInfo{Name: "bar"}),
+			expectedScope: "resource",
+		},
+		{
+			name: "name from field selector",
+			opts: &metainternalversion.ListOptions{
+				FieldSelector: fields.ParseSelectorOrDie("metadata.name=bar"),
+			},
+			expectedScope: "resource",
+		},
+		{
+			name:          "cluster scope request",
+			ctx:           request.WithRequestInfo(context.TODO(), &request.RequestInfo{IsResourceRequest: true}),
+			expectedScope: "cluster",
+		},
+	}
+
+	for _, scenario := range scenarios {
+		t.Run(scenario.name, func(t *testing.T) {
+			if scenario.ctx == nil {
+				scenario.ctx = context.TODO()
+			}
+			actualScope := CleanListScope(scenario.ctx, scenario.opts)
+			if actualScope != scenario.expectedScope {
+				t.Errorf("unexpected scope = %s, expected = %s", actualScope, scenario.expectedScope)
+			}
+		})
+	}
+}