From f03290c3902fedde6e4693f26df93e23577d82ee Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Mon, 6 Jul 2020 14:29:18 -0400 Subject: [PATCH 1/3] Move feature gate check to crdInfo construction time --- .../pkg/apiserver/customresource_handler.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index f51928c4530..65745eef55f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -329,10 +329,8 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - if utilfeature.DefaultFeatureGate.Enabled(features.WarningHeaders) { - for _, w := range crdInfo.warnings[requestInfo.APIVersion] { - warning.AddWarning(req.Context(), "", w) - } + for _, w := range crdInfo.warnings[requestInfo.APIVersion] { + warning.AddWarning(req.Context(), "", w) } verb := strings.ToUpper(requestInfo.Verb) @@ -883,10 +881,12 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd statusScopes[v.Name] = &statusScope if v.Deprecated { - if v.DeprecationWarning != nil { - warnings[v.Name] = append(warnings[v.Name], *v.DeprecationWarning) - } else { - warnings[v.Name] = append(warnings[v.Name], defaultDeprecationWarning(v.Name, crd.Spec)) + if utilfeature.DefaultFeatureGate.Enabled(features.WarningHeaders) { + if v.DeprecationWarning != nil { + warnings[v.Name] = append(warnings[v.Name], *v.DeprecationWarning) + } else { + warnings[v.Name] = append(warnings[v.Name], defaultDeprecationWarning(v.Name, crd.Spec)) + } } } } From a17e29724540ce798b02246d99da5d236c849a79 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Mon, 6 Jul 2020 14:32:55 -0400 Subject: [PATCH 2/3] Add deprecated metric for requests to deprecated custom resource versions --- .../pkg/apiserver/customresource_handler.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 65745eef55f..36cc1982163 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -142,6 +142,9 @@ type crdInfo struct { spec *apiextensionsv1.CustomResourceDefinitionSpec acceptedNames *apiextensionsv1.CustomResourceDefinitionNames + // Deprecated per version + deprecated map[string]bool + // Warnings per version warnings map[string][]string @@ -329,6 +332,7 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } + deprecated := crdInfo.deprecated[requestInfo.APIVersion] for _, w := range crdInfo.warnings[requestInfo.APIVersion] { warning.AddWarning(req.Context(), "", w) } @@ -370,7 +374,7 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } if handlerFunc != nil { - handlerFunc = metrics.InstrumentHandlerFunc(verb, requestInfo.APIGroup, requestInfo.APIVersion, resource, subresource, scope, metrics.APIServerComponent, false, "", handlerFunc) + handlerFunc = metrics.InstrumentHandlerFunc(verb, requestInfo.APIGroup, requestInfo.APIVersion, resource, subresource, scope, metrics.APIServerComponent, deprecated, "", handlerFunc) handler := genericfilters.WithWaitGroup(handlerFunc, longRunningFilter, crdInfo.waitGroup) handler.ServeHTTP(w, req) return @@ -620,6 +624,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd storages := map[string]customresource.CustomResourceStorage{} statusScopes := map[string]*handlers.RequestScope{} scaleScopes := map[string]*handlers.RequestScope{} + deprecated := map[string]bool{} warnings := map[string][]string{} equivalentResourceRegistry := runtime.NewEquivalentResourceRegistry() @@ -881,6 +886,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd statusScopes[v.Name] = &statusScope if v.Deprecated { + deprecated[v.Name] = true if utilfeature.DefaultFeatureGate.Enabled(features.WarningHeaders) { if v.DeprecationWarning != nil { warnings[v.Name] = append(warnings[v.Name], *v.DeprecationWarning) @@ -898,6 +904,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd requestScopes: requestScopes, scaleRequestScopes: scaleScopes, statusRequestScopes: statusScopes, + deprecated: deprecated, warnings: warnings, storageVersion: storageVersion, waitGroup: &utilwaitgroup.SafeWaitGroup{}, From 9550f5080fde480c0712a0870c56b4e2415f8637 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Mon, 6 Jul 2020 15:03:24 -0400 Subject: [PATCH 3/3] Add audit annotation for requests to deprecated API endpoints --- .../k8s.io/apiserver/pkg/endpoints/metrics/BUILD | 1 + .../apiserver/pkg/endpoints/metrics/metrics.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/BUILD index fee49f24acb..952f0a2bc3d 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/BUILD @@ -15,6 +15,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/validation:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go index f2e04c21ea1..26b08307fa4 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/apimachinery/pkg/types" utilsets "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -223,6 +224,16 @@ const ( MutatingKind = "mutating" ) +const ( + // deprecatedAnnotationKey is a key for an audit annotation set to + // "true" on requests made to deprecated API versions + deprecatedAnnotationKey = "k8s.io/deprecated" + // removedReleaseAnnotationKey is a key for an audit annotation set to + // the target removal release, in "." format, + // on requests made to deprecated API versions with a target removal release + removedReleaseAnnotationKey = "k8s.io/removed-release" +) + var registerMetrics sync.Once // Register all metrics. @@ -306,6 +317,10 @@ func MonitorRequest(req *http.Request, verb, group, version, resource, subresour requestCounter.WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component, cleanContentType, codeToString(httpCode)).Inc() if deprecated { deprecatedRequestGauge.WithLabelValues(group, version, resource, subresource, removedRelease).Set(1) + audit.AddAuditAnnotation(req.Context(), deprecatedAnnotationKey, "true") + if len(removedRelease) > 0 { + audit.AddAuditAnnotation(req.Context(), removedReleaseAnnotationKey, removedRelease) + } } requestLatencies.WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component).Observe(elapsedSeconds) // We are only interested in response sizes of read requests.