Merge pull request #70715 from immutableT/kube-apiserver-metrics

Add transformation_success_total and transformation_last_status metrics.

staging/src/k8s.io/apiserver/pkg/storage/value/BUILD

@@ -8,8 +8,17 @@ load(
 
 go_test(
     name = "go_default_test",
-    srcs = ["transformer_test.go"],
+    srcs = [
+        "metrics_test.go",
+        "transformer_test.go",
+    ],
     embed = [":go_default_library"],
+    deps = [
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus/testutil:go_default_library",
+        "//vendor/google.golang.org/grpc/codes:go_default_library",
+        "//vendor/google.golang.org/grpc/status:go_default_library",
+    ],
 )
 
 go_library(
@@ -20,7 +29,10 @@ go_library(
     ],
     importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/storage/value",
     importpath = "k8s.io/apiserver/pkg/storage/value",
-    deps = ["//vendor/github.com/prometheus/client_golang/prometheus:go_default_library"],
+    deps = [
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/google.golang.org/grpc/status:go_default_library",
+    ],
 )
 
 filegroup(

staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2017 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,6 +20,8 @@ import (
 	"sync"
 	"time"
 
+	"google.golang.org/grpc/status"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -53,12 +55,23 @@ var (
 		},
 		[]string{"transformation_type"},
 	)
-	transformerFailuresTotal = prometheus.NewCounterVec(
+
+	transformerOperationsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "transformation_operations_total",
+			Help:      "Total number of transformations.",
+		},
+		[]string{"transformation_type", "status"},
+	)
+
+	deprecatedTransformerFailuresTotal = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "transformation_failures_total",
-			Help:      "Total number of failed transformation operations.",
+			Help:      "(Deprecated) Total number of failed transformation operations.",
 		},
 		[]string{"transformation_type"},
 	)
@@ -106,7 +119,8 @@ func RegisterMetrics() {
 	registerMetrics.Do(func() {
 		prometheus.MustRegister(transformerLatencies)
 		prometheus.MustRegister(deprecatedTransformerLatencies)
-		prometheus.MustRegister(transformerFailuresTotal)
+		prometheus.MustRegister(transformerOperationsTotal)
+		prometheus.MustRegister(deprecatedTransformerFailuresTotal)
 		prometheus.MustRegister(envelopeTransformationCacheMissTotal)
 		prometheus.MustRegister(dataKeyGenerationLatencies)
 		prometheus.MustRegister(deprecatedDataKeyGenerationLatencies)
@@ -115,14 +129,17 @@ func RegisterMetrics() {
 }
 
 // RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations.
+// Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead.
 func RecordTransformation(transformationType string, start time.Time, err error) {
-	if err != nil {
-		transformerFailuresTotal.WithLabelValues(transformationType).Inc()
-		return
-	}
+	transformerOperationsTotal.WithLabelValues(transformationType, status.Code(err).String()).Inc()
 
-	transformerLatencies.WithLabelValues(transformationType).Observe(sinceInSeconds(start))
-	deprecatedTransformerLatencies.WithLabelValues(transformationType).Observe(sinceInMicroseconds(start))
+	switch {
+	case err == nil:
+		transformerLatencies.WithLabelValues(transformationType).Observe(sinceInSeconds(start))
+		deprecatedTransformerLatencies.WithLabelValues(transformationType).Observe(sinceInMicroseconds(start))
+	default:
+		deprecatedTransformerFailuresTotal.WithLabelValues(transformationType).Inc()
+	}
 }
 
 // RecordCacheMiss records a miss on Key Encryption Key(KEK) - call to KMS was required to decrypt KEK.

staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package value
+
+import (
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestTotals(t *testing.T) {
+	testCases := []struct {
+		desc    string
+		metrics []string
+		error   error
+		want    string
+	}{
+		{
+			desc: "non-status error",
+			metrics: []string{
+				"apiserver_storage_transformation_operations_total",
+				"apiserver_storage_transformation_failures_total",
+			},
+			error: errors.New("foo"),
+			want: `
+  # HELP apiserver_storage_transformation_failures_total (Deprecated) Total number of failed transformation operations.
+  # TYPE apiserver_storage_transformation_failures_total counter
+  apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1
+	# HELP apiserver_storage_transformation_operations_total Total number of transformations.
+  # TYPE apiserver_storage_transformation_operations_total counter
+  apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="encrypt"} 1
+`,
+		},
+		{
+			desc: "error is nil",
+			metrics: []string{
+				"apiserver_storage_transformation_operations_total",
+				"apiserver_storage_transformation_failures_total",
+			},
+			want: `
+	# HELP apiserver_storage_transformation_operations_total Total number of transformations.
+  # TYPE apiserver_storage_transformation_operations_total counter
+  apiserver_storage_transformation_operations_total{status="OK",transformation_type="encrypt"} 1
+`,
+		},
+		{
+			desc: "status error from kms-plugin",
+			metrics: []string{
+				"apiserver_storage_transformation_operations_total",
+				"apiserver_storage_transformation_failures_total",
+			},
+			error: status.Error(codes.FailedPrecondition, "foo"),
+			want: `
+  # HELP apiserver_storage_transformation_failures_total (Deprecated) Total number of failed transformation operations.
+  # TYPE apiserver_storage_transformation_failures_total counter
+  apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1
+	# HELP apiserver_storage_transformation_operations_total Total number of transformations.
+  # TYPE apiserver_storage_transformation_operations_total counter
+  apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="encrypt"} 1
+`,
+		},
+	}
+
+	RegisterMetrics()
+
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			RecordTransformation("encrypt", time.Now(), tt.error)
+			defer transformerOperationsTotal.Reset()
+			defer deprecatedTransformerFailuresTotal.Reset()
+			if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}