From de020ecb1cb10cc63c56e2693515a9c59a4b0af4 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 22 Aug 2019 15:36:39 +0200 Subject: [PATCH 1/3] aggregator/apiextensions: v(2) log why OpenAPI spec is regenerated --- .../pkg/controller/openapi/controller.go | 2 ++ .../pkg/controllers/openapi/aggregator/BUILD | 1 + .../controllers/openapi/aggregator/aggregator.go | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go index 4d54b411f3d..9ff5d4df7c0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go @@ -168,6 +168,7 @@ func (c *Controller) sync(name string) error { return nil } delete(c.crdSpecs, name) + klog.V(2).Infof("Updating CRD OpenAPI spec because %s was removed", name) return c.updateSpecLocked() } @@ -183,6 +184,7 @@ func (c *Controller) sync(name string) error { // update specs of this CRD c.crdSpecs[crd.Name] = newSpecs + klog.V(2).Infof("Updating CRD OpenAPI spec because %s changed", name) return c.updateSpecLocked() } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/BUILD index df9ed4ded05..12f8444658a 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/BUILD @@ -17,6 +17,7 @@ go_library( "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/aggregator:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go index 5edc223a6a6..8cf12d9aae6 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go @@ -25,9 +25,10 @@ import ( restful "github.com/emicklei/go-restful" "github.com/go-openapi/spec" + "k8s.io/klog" "k8s.io/apiserver/pkg/server" - "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "k8s.io/kube-openapi/pkg/aggregator" "k8s.io/kube-openapi/pkg/builder" "k8s.io/kube-openapi/pkg/common" @@ -104,6 +105,10 @@ func BuildAndRegisterAggregator(downloader *Downloader, delegationTarget server. } // Build initial spec to serve. + klog.V(2).Infof("Building initial OpenAPI spec") + defer func(start time.Time) { + klog.V(2).Infof("Finished initial OpenAPI spec generation after %v", time.Now().Sub(start)) + }(time.Now()) specToServe, err := s.buildOpenAPISpec() if err != nil { return nil, err @@ -209,6 +214,10 @@ func (s *specAggregator) tryUpdatingServiceSpecs(specInfo *openAPISpecInfo) erro if existedBefore && origSpecInfo != nil && origSpecInfo.etag == specInfo.etag { return nil } + klog.V(2).Infof("Updating OpenAPI spec because %s is updated", specInfo.apiService.Name) + defer func(start time.Time) { + klog.V(2).Infof("Finished OpenAPI spec generation after %v", time.Now().Sub(start)) + }(time.Now()) if err := s.updateOpenAPISpec(); err != nil { if existedBefore { s.openAPISpecs[specInfo.apiService.Name] = origSpecInfo @@ -228,6 +237,10 @@ func (s *specAggregator) tryDeleteServiceSpecs(apiServiceName string) error { return nil } delete(s.openAPISpecs, apiServiceName) + klog.V(2).Infof("Updating OpenAPI spec because %s is removed", apiServiceName) + defer func(start time.Time) { + klog.V(2).Infof("Finished OpenAPI spec generation after %v", time.Now().Sub(start)) + }(time.Now()) if err := s.updateOpenAPISpec(); err != nil { s.openAPISpecs[apiServiceName] = orgSpecInfo return err From 63dbb234b89bbdf244efdfc059570c6a44275572 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Aug 2019 13:55:30 +0200 Subject: [PATCH 2/3] apiextension: add metric for openapi regeneration --- .../pkg/controller/openapi/BUILD | 7 +++- .../pkg/controller/openapi/controller.go | 8 +++- .../pkg/controller/openapi/metrics.go | 37 +++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/metrics.go diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD index cd6e61f53f9..f6f2340b10d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["controller.go"], + srcs = [ + "controller.go", + "metrics.go", + ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/controller/openapi", importpath = "k8s.io/apiextensions-apiserver/pkg/controller/openapi", visibility = ["//visibility:public"], @@ -17,6 +20,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/handler:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go index 9ff5d4df7c0..52d719dffcb 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go @@ -169,11 +169,12 @@ func (c *Controller) sync(name string) error { } delete(c.crdSpecs, name) klog.V(2).Infof("Updating CRD OpenAPI spec because %s was removed", name) + regenerationCounter.With(map[string]string{"crd": name, "reason": "remove"}) return c.updateSpecLocked() } // compute CRD spec and see whether it changed - oldSpecs := c.crdSpecs[crd.Name] + oldSpecs, updated := c.crdSpecs[crd.Name] newSpecs, changed, err := buildVersionSpecs(crd, oldSpecs) if err != nil { return err @@ -185,6 +186,11 @@ func (c *Controller) sync(name string) error { // update specs of this CRD c.crdSpecs[crd.Name] = newSpecs klog.V(2).Infof("Updating CRD OpenAPI spec because %s changed", name) + reason := "add" + if updated { + reason = "update" + } + regenerationCounter.With(map[string]string{"crd": name, "reason": reason}) return c.updateSpecLocked() } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/metrics.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/metrics.go new file mode 100644 index 00000000000..5bcfa8f2880 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/metrics.go @@ -0,0 +1,37 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +var ( + regenerationCounter = metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "apiextensions_openapi_v2_regeneration_count", + Help: "Counter of OpenAPI v2 spec regeneration count broken down by causing CRD name and reason.", + StabilityLevel: metrics.ALPHA, + }, + []string{"crd", "reason"}, + ) +) + +func init() { + legacyregistry.MustRegister(regenerationCounter) +} From beee72e28b5e740f3032cf953d2f4660bf9ae890 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Aug 2019 13:56:15 +0200 Subject: [PATCH 3/3] aggregator: add metric for openapi regeneration --- .../pkg/controllers/openapi/aggregator/BUILD | 3 ++ .../openapi/aggregator/aggregator.go | 25 ++++++++-- .../controllers/openapi/aggregator/metrics.go | 46 +++++++++++++++++++ vendor/modules.txt | 1 + 4 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/metrics.go diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/BUILD index 12f8444658a..741a93907d2 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/BUILD @@ -5,6 +5,7 @@ go_library( srcs = [ "aggregator.go", "downloader.go", + "metrics.go", "priority.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator", @@ -14,6 +15,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go index 8cf12d9aae6..db026594831 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go @@ -25,6 +25,7 @@ import ( restful "github.com/emicklei/go-restful" "github.com/go-openapi/spec" + "k8s.io/klog" "k8s.io/apiserver/pkg/server" @@ -107,7 +108,11 @@ func BuildAndRegisterAggregator(downloader *Downloader, delegationTarget server. // Build initial spec to serve. klog.V(2).Infof("Building initial OpenAPI spec") defer func(start time.Time) { - klog.V(2).Infof("Finished initial OpenAPI spec generation after %v", time.Now().Sub(start)) + duration := time.Now().Sub(start) + klog.V(2).Infof("Finished initial OpenAPI spec generation after %v", duration) + + regenerationCounter.With(map[string]string{"apiservice": "*", "reason": "startup"}) + regenerationDurationGauge.With(map[string]string{"reason": "startup"}).Set(duration.Seconds()) }(time.Now()) specToServe, err := s.buildOpenAPISpec() if err != nil { @@ -207,6 +212,7 @@ func (s *specAggregator) tryUpdatingServiceSpecs(specInfo *openAPISpecInfo) erro if specInfo == nil { return fmt.Errorf("invalid input: specInfo must be non-nil") } + _, updated := s.openAPISpecs[specInfo.apiService.Name] origSpecInfo, existedBefore := s.openAPISpecs[specInfo.apiService.Name] s.openAPISpecs[specInfo.apiService.Name] = specInfo @@ -216,7 +222,16 @@ func (s *specAggregator) tryUpdatingServiceSpecs(specInfo *openAPISpecInfo) erro } klog.V(2).Infof("Updating OpenAPI spec because %s is updated", specInfo.apiService.Name) defer func(start time.Time) { - klog.V(2).Infof("Finished OpenAPI spec generation after %v", time.Now().Sub(start)) + duration := time.Now().Sub(start) + klog.V(2).Infof("Finished OpenAPI spec generation after %v", duration) + + reason := "add" + if updated { + reason = "update" + } + + regenerationCounter.With(map[string]string{"apiservice": specInfo.apiService.Name, "reason": reason}) + regenerationDurationGauge.With(map[string]string{"reason": reason}).Set(duration.Seconds()) }(time.Now()) if err := s.updateOpenAPISpec(); err != nil { if existedBefore { @@ -239,7 +254,11 @@ func (s *specAggregator) tryDeleteServiceSpecs(apiServiceName string) error { delete(s.openAPISpecs, apiServiceName) klog.V(2).Infof("Updating OpenAPI spec because %s is removed", apiServiceName) defer func(start time.Time) { - klog.V(2).Infof("Finished OpenAPI spec generation after %v", time.Now().Sub(start)) + duration := time.Now().Sub(start) + klog.V(2).Infof("Finished OpenAPI spec generation after %v", duration) + + regenerationCounter.With(map[string]string{"apiservice": apiServiceName, "reason": "delete"}) + regenerationDurationGauge.With(map[string]string{"reason": "delete"}).Set(duration.Seconds()) }(time.Now()) if err := s.updateOpenAPISpec(); err != nil { s.openAPISpecs[apiServiceName] = orgSpecInfo diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/metrics.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/metrics.go new file mode 100644 index 00000000000..b0a6e54e43c --- /dev/null +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/metrics.go @@ -0,0 +1,46 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aggregator + +import ( + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +var ( + regenerationCounter = metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "aggregator_openapi_v2_regeneration_count", + Help: "Counter of OpenAPI v2 spec regeneration count broken down by causing APIService name and reason.", + StabilityLevel: metrics.ALPHA, + }, + []string{"apiservice", "reason"}, + ) + regenerationDurationGauge = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Name: "aggregator_openapi_v2_regeneration_duration", + Help: "Gauge of OpenAPI v2 spec regeneration duration in seconds.", + StabilityLevel: metrics.ALPHA, + }, + []string{"reason"}, + ) +) + +func init() { + legacyregistry.MustRegister(regenerationCounter) + legacyregistry.MustRegister(regenerationDurationGauge) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 4de00e10515..5baa6d4cc3d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1637,6 +1637,7 @@ k8s.io/component-base/featuregate k8s.io/component-base/featuregate/testing k8s.io/component-base/logs k8s.io/component-base/metrics +k8s.io/component-base/metrics/legacyregistry k8s.io/component-base/version # k8s.io/cri-api v0.0.0 => ./staging/src/k8s.io/cri-api k8s.io/cri-api/pkg/apis