From bbce0468d416989cc0904c9826b2fc491366a5a9 Mon Sep 17 00:00:00 2001 From: Shihang Zhang Date: Wed, 3 Feb 2021 10:59:54 -0800 Subject: [PATCH] add metrics for rootcacertpublisher controller --- .../certificates/rootcacertpublisher/BUILD | 15 ++- .../rootcacertpublisher/metrics.go | 72 ++++++++++++++ .../rootcacertpublisher/metrics_test.go | 97 +++++++++++++++++++ .../rootcacertpublisher/publisher.go | 9 +- 4 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 pkg/controller/certificates/rootcacertpublisher/metrics.go create mode 100644 pkg/controller/certificates/rootcacertpublisher/metrics_test.go diff --git a/pkg/controller/certificates/rootcacertpublisher/BUILD b/pkg/controller/certificates/rootcacertpublisher/BUILD index a1d5341f045..6cc14a97a87 100644 --- a/pkg/controller/certificates/rootcacertpublisher/BUILD +++ b/pkg/controller/certificates/rootcacertpublisher/BUILD @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["publisher.go"], + srcs = [ + "metrics.go", + "publisher.go", + ], importpath = "k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher", visibility = ["//visibility:public"], deps = [ @@ -16,6 +19,8 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//staging/src/k8s.io/component-base/metrics/prometheus/ratelimiter:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", ], @@ -23,15 +28,21 @@ go_library( go_test( name = "go_default_test", - srcs = ["publisher_test.go"], + srcs = [ + "metrics_test.go", + "publisher_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/controller:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", + "//staging/src/k8s.io/component-base/metrics/testutil:go_default_library", ], ) diff --git a/pkg/controller/certificates/rootcacertpublisher/metrics.go b/pkg/controller/certificates/rootcacertpublisher/metrics.go new file mode 100644 index 00000000000..83081d32e53 --- /dev/null +++ b/pkg/controller/certificates/rootcacertpublisher/metrics.go @@ -0,0 +1,72 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rootcacertpublisher + +import ( + "strconv" + "sync" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +// RootCACertPublisher - subsystem name used by root_ca_cert_publisher +const RootCACertPublisher = "root_ca_cert_publisher" + +var ( + syncCounter = metrics.NewCounterVec( + &metrics.CounterOpts{ + Subsystem: RootCACertPublisher, + Name: "sync_total", + Help: "Number of namespace syncs happened in root ca cert publisher.", + StabilityLevel: metrics.ALPHA, + }, + []string{"namespace", "code"}, + ) + syncLatency = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Subsystem: RootCACertPublisher, + Name: "sync_duration_seconds", + Help: "Number of namespace syncs happened in root ca cert publisher.", + Buckets: metrics.ExponentialBuckets(0.001, 2, 15), + StabilityLevel: metrics.ALPHA, + }, + []string{"namespace", "code"}, + ) +) + +func recordMetrics(start time.Time, ns string, err error) { + code := "500" + if err == nil { + code = "200" + } else if se, ok := err.(*apierrors.StatusError); ok && se.Status().Code != 0 { + code = strconv.Itoa(int(se.Status().Code)) + } + syncLatency.WithLabelValues(ns, code).Observe(time.Since(start).Seconds()) + syncCounter.WithLabelValues(ns, code).Inc() +} + +var once sync.Once + +func registerMetrics() { + once.Do(func() { + legacyregistry.MustRegister(syncCounter) + legacyregistry.MustRegister(syncLatency) + }) +} diff --git a/pkg/controller/certificates/rootcacertpublisher/metrics_test.go b/pkg/controller/certificates/rootcacertpublisher/metrics_test.go new file mode 100644 index 00000000000..3979b00b787 --- /dev/null +++ b/pkg/controller/certificates/rootcacertpublisher/metrics_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rootcacertpublisher + +import ( + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/component-base/metrics/testutil" +) + +func TestSyncCounter(t *testing.T) { + testCases := []struct { + desc string + err error + metrics []string + want string + }{ + { + desc: "nil error", + err: nil, + metrics: []string{ + "root_ca_cert_publisher_sync_total", + }, + want: ` +# HELP root_ca_cert_publisher_sync_total [ALPHA] Number of namespace syncs happened in root ca cert publisher. +# TYPE root_ca_cert_publisher_sync_total counter +root_ca_cert_publisher_sync_total{code="200",namespace="test-ns"} 1 + `, + }, + { + desc: "kube api error", + err: apierrors.NewNotFound(corev1.Resource("configmap"), "test-configmap"), + metrics: []string{ + "root_ca_cert_publisher_sync_total", + }, + want: ` +# HELP root_ca_cert_publisher_sync_total [ALPHA] Number of namespace syncs happened in root ca cert publisher. +# TYPE root_ca_cert_publisher_sync_total counter +root_ca_cert_publisher_sync_total{code="404",namespace="test-ns"} 1 + `, + }, + { + desc: "kube api error without code", + err: &apierrors.StatusError{}, + metrics: []string{ + "root_ca_cert_publisher_sync_total", + }, + want: ` +# HELP root_ca_cert_publisher_sync_total [ALPHA] Number of namespace syncs happened in root ca cert publisher. +# TYPE root_ca_cert_publisher_sync_total counter +root_ca_cert_publisher_sync_total{code="500",namespace="test-ns"} 1 + `, + }, + { + desc: "general error", + err: errors.New("test"), + metrics: []string{ + "root_ca_cert_publisher_sync_total", + }, + want: ` +# HELP root_ca_cert_publisher_sync_total [ALPHA] Number of namespace syncs happened in root ca cert publisher. +# TYPE root_ca_cert_publisher_sync_total counter +root_ca_cert_publisher_sync_total{code="500",namespace="test-ns"} 1 + `, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + recordMetrics(time.Now(), "test-ns", tc.err) + defer syncCounter.Reset() + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.want), tc.metrics...); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/pkg/controller/certificates/rootcacertpublisher/publisher.go b/pkg/controller/certificates/rootcacertpublisher/publisher.go index ee8925cb3f9..f0411517ef8 100644 --- a/pkg/controller/certificates/rootcacertpublisher/publisher.go +++ b/pkg/controller/certificates/rootcacertpublisher/publisher.go @@ -40,6 +40,10 @@ import ( // to access api-server const RootCACertConfigMapName = "kube-root-ca.crt" +func init() { + registerMetrics() +} + // NewPublisher construct a new controller which would manage the configmap // which stores certificates in each namespace. It will make sure certificate // configmap exists in each namespace. @@ -170,16 +174,17 @@ func (c *Publisher) processNextWorkItem() bool { return true } -func (c *Publisher) syncNamespace(ns string) error { +func (c *Publisher) syncNamespace(ns string) (err error) { startTime := time.Now() defer func() { + recordMetrics(startTime, ns, err) klog.V(4).Infof("Finished syncing namespace %q (%v)", ns, time.Since(startTime)) }() cm, err := c.cmLister.ConfigMaps(ns).Get(RootCACertConfigMapName) switch { case apierrors.IsNotFound(err): - _, err := c.client.CoreV1().ConfigMaps(ns).Create(context.TODO(), &v1.ConfigMap{ + _, err = c.client.CoreV1().ConfigMaps(ns).Create(context.TODO(), &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: RootCACertConfigMapName, },