diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 9bee9ba855f..b974340ed0b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -360,7 +360,7 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } if handlerFunc != nil { - handlerFunc = metrics.InstrumentHandlerFunc(verb, requestInfo.APIGroup, requestInfo.APIVersion, resource, subresource, scope, metrics.APIServerComponent, handlerFunc) + handlerFunc = metrics.InstrumentHandlerFunc(verb, requestInfo.APIGroup, requestInfo.APIVersion, resource, subresource, scope, metrics.APIServerComponent, false, "", handlerFunc) handler := genericfilters.WithWaitGroup(handlerFunc, longRunningFilter, crdInfo.waitGroup) handler.ServeHTTP(w, req) return diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go index e9f8f4f7669..53c45bc4d25 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go @@ -629,8 +629,9 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag var ( enableWarningHeaders = utilfeature.DefaultFeatureGate.Enabled(features.WarningHeaders) - warnings []string - deprecated bool + warnings []string + deprecated bool + removedRelease string ) { @@ -639,6 +640,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag currentMajor, currentMinor, _ := deprecation.MajorMinor(versioninfo.Get()) deprecated = deprecation.IsDeprecated(versionedPtrWithGVK, currentMajor, currentMinor) if deprecated { + removedRelease = deprecation.RemovedRelease(versionedPtrWithGVK) warnings = append(warnings, deprecation.WarningMessage(versionedPtrWithGVK)) } } @@ -654,9 +656,9 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if needOverride { // need change the reported verb - handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler) + handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler) } else { - handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler) + handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler) } if enableWarningHeaders { handler = utilwarning.AddWarningsHandler(handler, warnings) @@ -690,7 +692,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if isSubresource { doc = "list " + subresource + " of objects of kind " + kind } - handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulListResource(lister, watcher, reqScope, false, a.minRequestTimeout)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, false, a.minRequestTimeout)) if enableWarningHeaders { handler = utilwarning.AddWarningsHandler(handler, warnings) } @@ -725,7 +727,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if isSubresource { doc = "replace " + subresource + " of the specified " + kind } - handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulUpdateResource(updater, reqScope, admit)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulUpdateResource(updater, reqScope, admit)) if enableWarningHeaders { handler = utilwarning.AddWarningsHandler(handler, warnings) } @@ -758,7 +760,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) { supportedTypes = append(supportedTypes, string(types.ApplyPatchType)) } - handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulPatchResource(patcher, reqScope, admit, supportedTypes)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulPatchResource(patcher, reqScope, admit, supportedTypes)) if enableWarningHeaders { handler = utilwarning.AddWarningsHandler(handler, warnings) } @@ -783,7 +785,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag } else { handler = restfulCreateResource(creater, reqScope, admit) } - handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler) + handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler) if enableWarningHeaders { handler = utilwarning.AddWarningsHandler(handler, warnings) } @@ -819,7 +821,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if deleteReturnsDeletedObject { deleteReturnType = producedObject } - handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulDeleteResource(gracefulDeleter, isGracefulDeleter, reqScope, admit)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulDeleteResource(gracefulDeleter, isGracefulDeleter, reqScope, admit)) if enableWarningHeaders { handler = utilwarning.AddWarningsHandler(handler, warnings) } @@ -845,7 +847,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if isSubresource { doc = "delete collection of " + subresource + " of a " + kind } - handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulDeleteCollection(collectionDeleter, isCollectionDeleter, reqScope, admit)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulDeleteCollection(collectionDeleter, isCollectionDeleter, reqScope, admit)) if enableWarningHeaders { handler = utilwarning.AddWarningsHandler(handler, warnings) } @@ -875,7 +877,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag doc = "watch changes to " + subresource + " of an object of kind " + kind } doc += ". deprecated: use the 'watch' parameter with a list operation instead, filtered to a single item with the 'fieldSelector' parameter." - handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout)) if enableWarningHeaders { handler = utilwarning.AddWarningsHandler(handler, warnings) } @@ -898,7 +900,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag doc = "watch individual changes to a list of " + subresource + " of " + kind } doc += ". deprecated: use the 'watch' parameter with a list operation instead." - handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout)) if enableWarningHeaders { handler = utilwarning.AddWarningsHandler(handler, warnings) } @@ -924,7 +926,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if isSubresource { doc = "connect " + method + " requests to " + subresource + " of " + kind } - handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulConnectResource(connecter, reqScope, admit, path, isSubresource)) + handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulConnectResource(connecter, reqScope, admit, path, isSubresource)) if enableWarningHeaders { handler = utilwarning.AddWarningsHandler(handler, warnings) } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go index 6f1a836e347..f2e04c21ea1 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go @@ -60,6 +60,15 @@ const ( * the metric stability policy. */ var ( + deprecatedRequestGauge = compbasemetrics.NewGaugeVec( + &compbasemetrics.GaugeOpts{ + Name: "apiserver_requested_deprecated_apis", + Help: "Gauge of deprecated APIs that have been requested, broken out by API group, version, resource, subresource, and removed_release.", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"group", "version", "resource", "subresource", "removed_release"}, + ) + // TODO(a-robinson): Add unit tests for the handling of these metrics once // the upstream library supports it. requestCounter = compbasemetrics.NewCounterVec( @@ -162,6 +171,7 @@ var ( kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl\.exe))`) metrics = []resettableCollector{ + deprecatedRequestGauge, requestCounter, longRunningRequestGauge, requestLatencies, @@ -288,12 +298,15 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp // MonitorRequest handles standard transformations for client and the reported verb and then invokes Monitor to record // a request. verb must be uppercase to be backwards compatible with existing monitoring tooling. -func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, component, contentType string, httpCode, respSize int, elapsed time.Duration) { +func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, contentType string, httpCode, respSize int, elapsed time.Duration) { reportedVerb := cleanVerb(verb, req) dryRun := cleanDryRun(req.URL) elapsedSeconds := elapsed.Seconds() cleanContentType := cleanContentType(contentType) requestCounter.WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component, cleanContentType, codeToString(httpCode)).Inc() + if deprecated { + deprecatedRequestGauge.WithLabelValues(group, version, resource, subresource, removedRelease).Set(1) + } requestLatencies.WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component).Observe(elapsedSeconds) // We are only interested in response sizes of read requests. if verb == "GET" || verb == "LIST" { @@ -303,7 +316,7 @@ func MonitorRequest(req *http.Request, verb, group, version, resource, subresour // InstrumentRouteFunc works like Prometheus' InstrumentHandlerFunc but wraps // the go-restful RouteFunction instead of a HandlerFunc plus some Kubernetes endpoint specific information. -func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, component string, routeFunc restful.RouteFunction) restful.RouteFunction { +func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, routeFunc restful.RouteFunction) restful.RouteFunction { return restful.RouteFunction(func(request *restful.Request, response *restful.Response) { now := time.Now() @@ -322,12 +335,12 @@ func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, com routeFunc(request, response) - MonitorRequest(request.Request, verb, group, version, resource, subresource, scope, component, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now)) + MonitorRequest(request.Request, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now)) }) } // InstrumentHandlerFunc works like Prometheus' InstrumentHandlerFunc but adds some Kubernetes endpoint specific information. -func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, component string, handler http.HandlerFunc) http.HandlerFunc { +func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { now := time.Now() @@ -344,7 +357,7 @@ func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, c handler(w, req) - MonitorRequest(req, verb, group, version, resource, subresource, scope, component, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now)) + MonitorRequest(req, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now)) } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go index cf7b7387cf8..e558098def6 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go +++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go @@ -131,6 +131,8 @@ func InstallPathHandler(mux mux, path string, checks ...HealthChecker) { /* subresource = */ path, /* scope = */ "", /* component = */ "", + /* deprecated */ false, + /* removedRelease */ "", handleRootHealthz(checks...))) for _, check := range checks { mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check)) diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go index 5de14267556..2dad5f7bd9f 100644 --- a/test/integration/metrics/metrics_test.go +++ b/test/integration/metrics/metrics_test.go @@ -95,11 +95,17 @@ func TestApiserverMetrics(t *testing.T) { t.Fatalf("unexpected error getting pods: %v", err) } + // Make a request to a deprecated API to ensure there's at least one data point + if _, err := client.RbacV1beta1().Roles(metav1.NamespaceDefault).List(context.TODO(), metav1.ListOptions{}); err != nil { + t.Fatalf("unexpected error getting rbac roles: %v", err) + } + metrics, err := scrapeMetrics(s) if err != nil { t.Fatal(err) } checkForExpectedMetrics(t, metrics, []string{ + "apiserver_requested_deprecated_apis", "apiserver_request_total", "apiserver_request_duration_seconds_sum", "etcd_request_duration_seconds_sum",