From c291e6355c44e84c2e1d503d1d9bf3e8fab9e194 Mon Sep 17 00:00:00 2001 From: Nilekh Chaudhari <1626598+nilekhc@users.noreply.github.com> Date: Wed, 5 Jul 2023 22:28:15 +0000 Subject: [PATCH] feat: implements metrics for encryption config hot reload Signed-off-by: Nilekh Chaudhari <1626598+nilekhc@users.noreply.github.com> --- .../encryptionconfig/controller/controller.go | 10 +- .../encryptionconfig/metrics/metrics.go | 86 +++++++++++++++ .../encryptionconfig/metrics/metrics_test.go | 104 ++++++++++++++++++ vendor/modules.txt | 1 + 4 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go index b8c66826bf5..94782ccbacd 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/options/encryptionconfig" + "k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" ) @@ -163,16 +164,19 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex ctx, closeTransformers := context.WithCancel(serverCtx) defer func() { - // TODO: increment success metric when updatedEffectiveConfig=true - // TODO can work queue metrics help here? if !updatedEffectiveConfig { // avoid leaking if we're not using the newly constructed transformers (due to an error or them not being changed) closeTransformers() } + + if updatedEffectiveConfig && err == nil { + metrics.RecordEncryptionConfigAutomaticReloadSuccess() + } + if err != nil { - // TODO: increment failure metric + metrics.RecordEncryptionConfigAutomaticReloadFailure() utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err)) // add dummy item back to the queue to trigger file content processing. d.queue.AddRateLimited(key) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go new file mode 100644 index 00000000000..799b584cf7a --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go @@ -0,0 +1,86 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "sync" + + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +const ( + namespace = "apiserver" + subsystem = "encryption_config_controller" +) + +var ( + encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounter( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "automatic_reload_failures_total", + Help: "Total number of failed automatic reloads of encryption configuration.", + StabilityLevel: metrics.ALPHA, + }, + ) + + encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounter( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "automatic_reload_success_total", + Help: "Total number of successful automatic reloads of encryption configuration.", + StabilityLevel: metrics.ALPHA, + }, + ) + + encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "automatic_reload_last_timestamp_seconds", + Help: "Timestamp of the last successful or failed automatic reload of encryption configuration.", + StabilityLevel: metrics.ALPHA, + }, + []string{"status"}, + ) +) + +var registerMetrics sync.Once + +func RegisterMetrics() { + registerMetrics.Do(func() { + legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal) + legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal) + legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds) + }) +} + +func RecordEncryptionConfigAutomaticReloadFailure() { + encryptionConfigAutomaticReloadFailureTotal.Inc() + recordEncryptionConfigAutomaticReloadTimestamp("failure") +} + +func RecordEncryptionConfigAutomaticReloadSuccess() { + encryptionConfigAutomaticReloadSuccessTotal.Inc() + recordEncryptionConfigAutomaticReloadTimestamp("success") +} + +func recordEncryptionConfigAutomaticReloadTimestamp(result string) { + encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result).SetToCurrentTime() +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go new file mode 100644 index 00000000000..d6b34faeedd --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go @@ -0,0 +1,104 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "strings" + "testing" + + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/component-base/metrics/testutil" +) + +func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) { + expectedValue := ` + # HELP apiserver_encryption_config_controller_automatic_reload_failures_total [ALPHA] Total number of failed automatic reloads of encryption configuration. + # TYPE apiserver_encryption_config_controller_automatic_reload_failures_total counter + apiserver_encryption_config_controller_automatic_reload_failures_total 1 + ` + metrics := []string{ + namespace + "_" + subsystem + "_automatic_reload_failures_total", + } + + encryptionConfigAutomaticReloadFailureTotal.Reset() + RegisterMetrics() + + RecordEncryptionConfigAutomaticReloadFailure() + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { + t.Fatal(err) + } +} + +func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) { + expectedValue := ` + # HELP apiserver_encryption_config_controller_automatic_reload_success_total [ALPHA] Total number of successful automatic reloads of encryption configuration. + # TYPE apiserver_encryption_config_controller_automatic_reload_success_total counter + apiserver_encryption_config_controller_automatic_reload_success_total 1 + ` + metrics := []string{ + namespace + "_" + subsystem + "_automatic_reload_success_total", + } + + encryptionConfigAutomaticReloadSuccessTotal.Reset() + RegisterMetrics() + + RecordEncryptionConfigAutomaticReloadSuccess() + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { + t.Fatal(err) + } +} + +func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) { + testCases := []struct { + expectedValue string + resultLabel string + timestamp int64 + }{ + { + expectedValue: ` + # HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration. + # TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge + apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="failure"} 1.689101941e+09 + `, + resultLabel: "failure", + timestamp: 1689101941, + }, + { + expectedValue: ` + # HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration. + # TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge + apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="success"} 1.689101941e+09 + `, + resultLabel: "success", + timestamp: 1689101941, + }, + } + + metrics := []string{ + namespace + "_" + subsystem + "_automatic_reload_last_timestamp_seconds", + } + RegisterMetrics() + + for _, tc := range testCases { + encryptionConfigAutomaticReloadLastTimestampSeconds.Reset() + encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel).Set(float64(tc.timestamp)) + + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedValue), metrics...); err != nil { + t.Fatal(err) + } + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ba6fe9cc31f..0eff77c600a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1528,6 +1528,7 @@ k8s.io/apiserver/pkg/server/mux k8s.io/apiserver/pkg/server/options k8s.io/apiserver/pkg/server/options/encryptionconfig k8s.io/apiserver/pkg/server/options/encryptionconfig/controller +k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics k8s.io/apiserver/pkg/server/resourceconfig k8s.io/apiserver/pkg/server/routes k8s.io/apiserver/pkg/server/storage