diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD index decb48beb59..9c4716b41ff 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD @@ -8,8 +8,17 @@ load( go_test( name = "go_default_test", - srcs = ["transformer_test.go"], + srcs = [ + "metrics_test.go", + "transformer_test.go", + ], embed = [":go_default_library"], + deps = [ + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus/testutil:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + ], ) go_library( @@ -20,7 +29,10 @@ go_library( ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/storage/value", importpath = "k8s.io/apiserver/pkg/storage/value", - deps = ["//vendor/github.com/prometheus/client_golang/prometheus:go_default_library"], + deps = [ + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + ], ) filegroup( diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go index c9dc66a8f35..f827bffaf92 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,6 +20,8 @@ import ( "sync" "time" + "google.golang.org/grpc/status" + "github.com/prometheus/client_golang/prometheus" ) @@ -53,12 +55,23 @@ var ( }, []string{"transformation_type"}, ) - transformerFailuresTotal = prometheus.NewCounterVec( + + transformerOperationsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "transformation_operations_total", + Help: "Total number of transformations.", + }, + []string{"transformation_type", "status"}, + ) + + deprecatedTransformerFailuresTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: subsystem, Name: "transformation_failures_total", - Help: "Total number of failed transformation operations.", + Help: "(Deprecated) Total number of failed transformation operations.", }, []string{"transformation_type"}, ) @@ -106,7 +119,8 @@ func RegisterMetrics() { registerMetrics.Do(func() { prometheus.MustRegister(transformerLatencies) prometheus.MustRegister(deprecatedTransformerLatencies) - prometheus.MustRegister(transformerFailuresTotal) + prometheus.MustRegister(transformerOperationsTotal) + prometheus.MustRegister(deprecatedTransformerFailuresTotal) prometheus.MustRegister(envelopeTransformationCacheMissTotal) prometheus.MustRegister(dataKeyGenerationLatencies) prometheus.MustRegister(deprecatedDataKeyGenerationLatencies) @@ -115,14 +129,17 @@ func RegisterMetrics() { } // RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations. +// Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead. func RecordTransformation(transformationType string, start time.Time, err error) { - if err != nil { - transformerFailuresTotal.WithLabelValues(transformationType).Inc() - return - } + transformerOperationsTotal.WithLabelValues(transformationType, status.Code(err).String()).Inc() - transformerLatencies.WithLabelValues(transformationType).Observe(sinceInSeconds(start)) - deprecatedTransformerLatencies.WithLabelValues(transformationType).Observe(sinceInMicroseconds(start)) + switch { + case err == nil: + transformerLatencies.WithLabelValues(transformationType).Observe(sinceInSeconds(start)) + deprecatedTransformerLatencies.WithLabelValues(transformationType).Observe(sinceInMicroseconds(start)) + default: + deprecatedTransformerFailuresTotal.WithLabelValues(transformationType).Inc() + } } // RecordCacheMiss records a miss on Key Encryption Key(KEK) - call to KMS was required to decrypt KEK. diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go new file mode 100644 index 00000000000..aba5454591b --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package value + +import ( + "errors" + "strings" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestTotals(t *testing.T) { + testCases := []struct { + desc string + metrics []string + error error + want string + }{ + { + desc: "non-status error", + metrics: []string{ + "apiserver_storage_transformation_operations_total", + "apiserver_storage_transformation_failures_total", + }, + error: errors.New("foo"), + want: ` + # HELP apiserver_storage_transformation_failures_total (Deprecated) Total number of failed transformation operations. + # TYPE apiserver_storage_transformation_failures_total counter + apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1 + # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # TYPE apiserver_storage_transformation_operations_total counter + apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="encrypt"} 1 +`, + }, + { + desc: "error is nil", + metrics: []string{ + "apiserver_storage_transformation_operations_total", + "apiserver_storage_transformation_failures_total", + }, + want: ` + # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # TYPE apiserver_storage_transformation_operations_total counter + apiserver_storage_transformation_operations_total{status="OK",transformation_type="encrypt"} 1 +`, + }, + { + desc: "status error from kms-plugin", + metrics: []string{ + "apiserver_storage_transformation_operations_total", + "apiserver_storage_transformation_failures_total", + }, + error: status.Error(codes.FailedPrecondition, "foo"), + want: ` + # HELP apiserver_storage_transformation_failures_total (Deprecated) Total number of failed transformation operations. + # TYPE apiserver_storage_transformation_failures_total counter + apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1 + # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # TYPE apiserver_storage_transformation_operations_total counter + apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="encrypt"} 1 +`, + }, + } + + RegisterMetrics() + + for _, tt := range testCases { + t.Run(tt.desc, func(t *testing.T) { + RecordTransformation("encrypt", time.Now(), tt.error) + defer transformerOperationsTotal.Reset() + defer deprecatedTransformerFailuresTotal.Reset() + if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { + t.Fatal(err) + } + }) + } +}