diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 2ad59dfe937..38b0d647d6a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -248,7 +248,7 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } if handler != nil { - handler = metrics.InstrumentHandlerFunc(verb, resource, subresource, scope, handler) + handler = metrics.InstrumentHandlerFunc(verb, requestInfo.APIGroup, requestInfo.APIVersion, resource, subresource, scope, handler) handler(w, req) return } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go index f387403dce0..f0b20cc77a7 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go @@ -176,6 +176,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag return nil, err } + group, version := a.group.GroupVersion.Group, a.group.GroupVersion.Version + fqKindToRegister, err := GetResourceKind(a.group.GroupVersion, storage, a.group.Typer) if err != nil { return nil, err @@ -571,9 +573,9 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if needOverride { // need change the reported verb - handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), resource, subresource, requestScope, handler) + handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), group, version, resource, subresource, requestScope, handler) } else { - handler = metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, handler) + handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, handler) } if a.enableAPIResponseCompression { @@ -607,7 +609,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if isSubresource { doc = "list " + subresource + " of objects of kind " + kind } - handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, false, a.minRequestTimeout)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, false, a.minRequestTimeout)) if a.enableAPIResponseCompression { handler = genericfilters.RestfulWithCompression(handler) } @@ -642,7 +644,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if isSubresource { doc = "replace " + subresource + " of the specified " + kind } - handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulUpdateResource(updater, reqScope, admit)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulUpdateResource(updater, reqScope, admit)) route := ws.PUT(action.Path).To(handler). Doc(doc). Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). @@ -669,7 +671,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag string(types.MergePatchType), string(types.StrategicMergePatchType), } - handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulPatchResource(patcher, reqScope, admit, supportedTypes)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulPatchResource(patcher, reqScope, admit, supportedTypes)) route := ws.PATCH(action.Path).To(handler). Doc(doc). Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). @@ -691,7 +693,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag } else { handler = restfulCreateResource(creater, reqScope, admit) } - handler = metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, handler) + handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, handler) article := getArticleForNoun(kind, " ") doc := "create" + article + kind if isSubresource { @@ -720,7 +722,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if isSubresource { doc = "delete " + subresource + " of" + article + kind } - handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulDeleteResource(gracefulDeleter, isGracefulDeleter, reqScope, admit)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulDeleteResource(gracefulDeleter, isGracefulDeleter, reqScope, admit)) route := ws.DELETE(action.Path).To(handler). Doc(doc). Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). @@ -743,7 +745,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if isSubresource { doc = "delete collection of " + subresource + " of a " + kind } - handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulDeleteCollection(collectionDeleter, isCollectionDeleter, reqScope, admit)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulDeleteCollection(collectionDeleter, isCollectionDeleter, reqScope, admit)) route := ws.DELETE(action.Path).To(handler). Doc(doc). Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). @@ -763,7 +765,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag doc = "watch changes to " + subresource + " of an object of kind " + kind } doc += ". deprecated: use the 'watch' parameter with a list operation instead, filtered to a single item with the 'fieldSelector' parameter." - handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout)) route := ws.GET(action.Path).To(handler). Doc(doc). Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). @@ -783,7 +785,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag doc = "watch individual changes to a list of " + subresource + " of " + kind } doc += ". deprecated: use the 'watch' parameter with a list operation instead." - handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout)) route := ws.GET(action.Path).To(handler). Doc(doc). Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). @@ -806,7 +808,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if isSubresource { doc = "connect " + method + " requests to " + subresource + " of " + kind } - handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulConnectResource(connecter, reqScope, admit, path, isSubresource)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulConnectResource(connecter, reqScope, admit, path, isSubresource)) route := ws.Method(method).Path(action.Path). To(handler). Doc(doc). diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go index ae16d43bb1e..83c012c6164 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go @@ -46,35 +46,35 @@ var ( requestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "apiserver_request_count", - Help: "Counter of apiserver requests broken out for each verb, API resource, client, and HTTP response contentType and code.", + Help: "Counter of apiserver requests broken out for each verb, group, version, resource, scope, client, and HTTP response contentType and code.", }, - []string{"verb", "resource", "subresource", "scope", "client", "contentType", "code"}, + []string{"verb", "group", "version", "resource", "subresource", "scope", "client", "contentType", "code"}, ) longRunningRequestGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "apiserver_longrunning_gauge", - Help: "Gauge of all active long-running apiserver requests broken out by verb, API resource, and scope. Not all requests are tracked this way.", + Help: "Gauge of all active long-running apiserver requests broken out by verb, group, version, resource, and scope. Not all requests are tracked this way.", }, - []string{"verb", "resource", "subresource", "scope"}, + []string{"verb", "group", "version", "resource", "subresource", "scope"}, ) requestLatencies = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "apiserver_request_latencies", - Help: "Response latency distribution in microseconds for each verb, resource and subresource.", + Help: "Response latency distribution in microseconds for each verb, group, version, resource, subresource, and scope.", // Use buckets ranging from 125 ms to 8 seconds. Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7), }, - []string{"verb", "resource", "subresource", "scope"}, + []string{"verb", "group", "version", "resource", "subresource", "scope"}, ) requestLatenciesSummary = prometheus.NewSummaryVec( prometheus.SummaryOpts{ Name: "apiserver_request_latencies_summary", - Help: "Response latency summary in microseconds for each verb, resource and subresource.", + Help: "Response latency summary in microseconds for each verb, group, version, resource, subresource, and scope.", // Make the sliding window of 5h. // TODO: The value for this should be based on our SLI definition (medium term). MaxAge: 5 * time.Hour, }, - []string{"verb", "resource", "subresource", "scope"}, + []string{"verb", "group", "version", "resource", "subresource", "scope"}, ) responseSizes = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -83,7 +83,7 @@ var ( // Use buckets ranging from 1000 bytes (1KB) to 10^9 bytes (1GB). Buckets: prometheus.ExponentialBuckets(1000, 10.0, 7), }, - []string{"verb", "resource", "subresource", "scope"}, + []string{"verb", "group", "version", "resource", "subresource", "scope"}, ) // DroppedRequests is a number of requests dropped with 'Try again later' response" DroppedRequests = prometheus.NewCounterVec( @@ -163,9 +163,9 @@ func Record(req *http.Request, requestInfo *request.RequestInfo, contentType str } scope := CleanScope(requestInfo) if requestInfo.IsResourceRequest { - MonitorRequest(req, strings.ToUpper(requestInfo.Verb), requestInfo.Resource, requestInfo.Subresource, scope, contentType, code, responseSizeInBytes, elapsed) + MonitorRequest(req, strings.ToUpper(requestInfo.Verb), requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, contentType, code, responseSizeInBytes, elapsed) } else { - MonitorRequest(req, strings.ToUpper(requestInfo.Verb), "", requestInfo.Path, scope, contentType, code, responseSizeInBytes, elapsed) + MonitorRequest(req, strings.ToUpper(requestInfo.Verb), "", "", "", requestInfo.Path, scope, contentType, code, responseSizeInBytes, elapsed) } } @@ -179,9 +179,9 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, fn f scope := CleanScope(requestInfo) reportedVerb := cleanVerb(strings.ToUpper(requestInfo.Verb), req) if requestInfo.IsResourceRequest { - g = longRunningRequestGauge.WithLabelValues(reportedVerb, requestInfo.Resource, requestInfo.Subresource, scope) + g = longRunningRequestGauge.WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope) } else { - g = longRunningRequestGauge.WithLabelValues(reportedVerb, "", requestInfo.Path, scope) + g = longRunningRequestGauge.WithLabelValues(reportedVerb, "", "", "", requestInfo.Path, scope) } g.Inc() defer g.Dec() @@ -190,22 +190,22 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, fn f // MonitorRequest handles standard transformations for client and the reported verb and then invokes Monitor to record // a request. verb must be uppercase to be backwards compatible with existing monitoring tooling. -func MonitorRequest(req *http.Request, verb, resource, subresource, scope, contentType string, httpCode, respSize int, elapsed time.Duration) { +func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, contentType string, httpCode, respSize int, elapsed time.Duration) { reportedVerb := cleanVerb(verb, req) client := cleanUserAgent(utilnet.GetHTTPClient(req)) elapsedMicroseconds := float64(elapsed / time.Microsecond) - requestCounter.WithLabelValues(reportedVerb, resource, subresource, scope, client, contentType, codeToString(httpCode)).Inc() - requestLatencies.WithLabelValues(reportedVerb, resource, subresource, scope).Observe(elapsedMicroseconds) - requestLatenciesSummary.WithLabelValues(reportedVerb, resource, subresource, scope).Observe(elapsedMicroseconds) + requestCounter.WithLabelValues(reportedVerb, group, version, resource, subresource, scope, client, contentType, codeToString(httpCode)).Inc() + requestLatencies.WithLabelValues(reportedVerb, group, version, resource, subresource, scope).Observe(elapsedMicroseconds) + requestLatenciesSummary.WithLabelValues(reportedVerb, group, version, resource, subresource, scope).Observe(elapsedMicroseconds) // We are only interested in response sizes of read requests. if verb == "GET" || verb == "LIST" { - responseSizes.WithLabelValues(reportedVerb, resource, subresource, scope).Observe(float64(respSize)) + responseSizes.WithLabelValues(reportedVerb, group, version, resource, subresource, scope).Observe(float64(respSize)) } } // InstrumentRouteFunc works like Prometheus' InstrumentHandlerFunc but wraps // the go-restful RouteFunction instead of a HandlerFunc plus some Kubernetes endpoint specific information. -func InstrumentRouteFunc(verb, resource, subresource, scope string, routeFunc restful.RouteFunction) restful.RouteFunction { +func InstrumentRouteFunc(verb, group, version, resource, subresource, scope string, routeFunc restful.RouteFunction) restful.RouteFunction { return restful.RouteFunction(func(request *restful.Request, response *restful.Response) { now := time.Now() @@ -224,12 +224,12 @@ func InstrumentRouteFunc(verb, resource, subresource, scope string, routeFunc re routeFunc(request, response) - MonitorRequest(request.Request, verb, resource, subresource, scope, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now)) + MonitorRequest(request.Request, verb, group, version, resource, subresource, scope, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now)) }) } // InstrumentHandlerFunc works like Prometheus' InstrumentHandlerFunc but adds some Kubernetes endpoint specific information. -func InstrumentHandlerFunc(verb, resource, subresource, scope string, handler http.HandlerFunc) http.HandlerFunc { +func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope string, handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { now := time.Now() @@ -246,7 +246,7 @@ func InstrumentHandlerFunc(verb, resource, subresource, scope string, handler ht handler(w, req) - MonitorRequest(req, verb, resource, subresource, scope, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now)) + MonitorRequest(req, verb, group, version, resource, subresource, scope, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now)) } }