Merge pull request #115649 from aramase/grpc-metrics

[KMSv2] Add metrics for grpc service

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -566,7 +566,7 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
 			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
 		}
 
-		envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
+		envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Name, config.Timeout.Duration)
 		if err != nil {
 			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
 		}

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go

@@ -104,7 +104,7 @@ func newMockErrorEnvelopeService(endpoint string, timeout time.Duration) (envelo
 }
 
 // The factory method to create mock envelope kmsv2 service.
-func newMockEnvelopeKMSv2Service(ctx context.Context, endpoint string, timeout time.Duration) (kmsservice.Service, error) {
+func newMockEnvelopeKMSv2Service(ctx context.Context, endpoint, providerName string, timeout time.Duration) (kmsservice.Service, error) {
 	return &testKMSv2EnvelopeService{nil}, nil
 }
 
@@ -679,7 +679,7 @@ func TestKMSPluginHealthzTTL(t *testing.T) {
 func TestKMSv2PluginHealthzTTL(t *testing.T) {
 	ctx := testContext(t)
 
-	service, _ := newMockEnvelopeKMSv2Service(ctx, "unix:///tmp/testprovider.sock", 3*time.Second)
+	service, _ := newMockEnvelopeKMSv2Service(ctx, "unix:///tmp/testprovider.sock", "providerName", 3*time.Second)
 	errService, _ := newMockErrorEnvelopeKMSv2Service("unix:///tmp/testprovider.sock", 3*time.Second)
 
 	testCases := []struct {

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service.go

@@ -27,6 +27,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/util"
 	"k8s.io/klog/v2"
 	kmsapi "k8s.io/kms/apis/v2alpha1"
@@ -46,7 +47,7 @@ type gRPCService struct {
 }
 
 // NewGRPCService returns an envelope.Service which use gRPC to communicate the remote KMS provider.
-func NewGRPCService(ctx context.Context, endpoint string, callTimeout time.Duration) (kmsservice.Service, error) {
+func NewGRPCService(ctx context.Context, endpoint, providerName string, callTimeout time.Duration) (kmsservice.Service, error) {
 	klog.V(4).Infof("Configure KMS provider with endpoint: %s", endpoint)
 
 	addr, err := util.ParseEndpoint(endpoint)
@@ -70,7 +71,9 @@ func NewGRPCService(ctx context.Context, endpoint string, callTimeout time.Durat
 					klog.V(4).Infof("Successfully dialed Unix socket %v", addr)
 				}
 				return c, err
-			}))
+			}),
+		grpc.WithChainUnaryInterceptor(recordMetricsInterceptor(providerName)),
+	)
 
 	if err != nil {
 		return nil, fmt.Errorf("failed to create connection to %s, error: %v", endpoint, err)
@@ -138,3 +141,13 @@ func (g *gRPCService) Status(ctx context.Context) (*kmsservice.StatusResponse, e
 	}
 	return &kmsservice.StatusResponse{Version: response.Version, Healthz: response.Healthz, KeyID: response.KeyId}, nil
 }
+
+func recordMetricsInterceptor(providerName string) grpc.UnaryClientInterceptor {
+	return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		start := time.Now()
+		respErr := invoker(ctx, method, req, reply, cc, opts...)
+		elapsed := time.Since(start)
+		metrics.RecordKMSOperationLatency(providerName, method, elapsed, respErr)
+		return respErr
+	}
+}

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service_unix_test.go

@@ -25,12 +25,18 @@ import (
 	"testing"
 	"time"
 
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
 	mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v2alpha1"
+	"k8s.io/component-base/metrics/testutil"
 
 	"k8s.io/apimachinery/pkg/util/uuid"
 	kmsservice "k8s.io/kms/service"
 )
 
+const (
+	testProviderName = "providerName"
+)
+
 type testSocket struct {
 	path     string
 	endpoint string
@@ -57,7 +63,7 @@ func TestKMSPluginLateStart(t *testing.T) {
 
 	ctx := testContext(t)
 
-	service, err := NewGRPCService(ctx, s.endpoint, callTimeout)
+	service, err := NewGRPCService(ctx, s.endpoint, testProviderName, callTimeout)
 	if err != nil {
 		t.Fatalf("failed to create envelope service, error: %v", err)
 	}
@@ -141,7 +147,7 @@ func TestTimeouts(t *testing.T) {
 				// Simulating late start of kube-apiserver - plugin is up before kube-apiserver, if requested by the testcase.
 				time.Sleep(tt.kubeAPIServerDelay)
 
-				service, err = NewGRPCService(ctx, socketName.endpoint, tt.callTimeout)
+				service, err = NewGRPCService(ctx, socketName.endpoint, testProviderName, tt.callTimeout)
 				if err != nil {
 					t.Fatalf("failed to create envelope service, error: %v", err)
 				}
@@ -211,7 +217,7 @@ func TestIntermittentConnectionLoss(t *testing.T) {
 	ctx := testContext(t)
 
 	//  connect to kms plugin
-	service, err := NewGRPCService(ctx, endpoint.endpoint, timeout)
+	service, err := NewGRPCService(ctx, endpoint.endpoint, testProviderName, timeout)
 	if err != nil {
 		t.Fatalf("failed to create envelope service, error: %v", err)
 	}
@@ -278,7 +284,7 @@ func TestGRPCService(t *testing.T) {
 	ctx := testContext(t)
 
 	// Create the gRPC client service.
-	service, err := NewGRPCService(ctx, endpoint.endpoint, 1*time.Second)
+	service, err := NewGRPCService(ctx, endpoint.endpoint, testProviderName, 1*time.Second)
 	if err != nil {
 		t.Fatalf("failed to create envelope service, error: %v", err)
 	}
@@ -321,7 +327,7 @@ func TestGRPCServiceConcurrentAccess(t *testing.T) {
 	ctx := testContext(t)
 
 	// Create the gRPC client service.
-	service, err := NewGRPCService(ctx, endpoint.endpoint, 15*time.Second)
+	service, err := NewGRPCService(ctx, endpoint.endpoint, testProviderName, 15*time.Second)
 	if err != nil {
 		t.Fatalf("failed to create envelope service, error: %v", err)
 	}
@@ -389,7 +395,7 @@ func TestInvalidConfiguration(t *testing.T) {
 
 	for _, testCase := range invalidConfigs {
 		t.Run(testCase.name, func(t *testing.T) {
-			_, err := NewGRPCService(ctx, testCase.endpoint, 1*time.Second)
+			_, err := NewGRPCService(ctx, testCase.endpoint, testProviderName, 1*time.Second)
 			if err == nil {
 				t.Fatalf("should fail to create envelope service for %s.", testCase.name)
 			}
@@ -397,6 +403,99 @@ func TestInvalidConfiguration(t *testing.T) {
 	}
 }
 
+func TestKMSOperationsMetric(t *testing.T) {
+	endpoint := newEndpoint()
+	f, err := mock.NewBase64Plugin(endpoint.path)
+	if err != nil {
+		t.Fatalf("failed to start test KMS provider server, error: %v", err)
+	}
+	if err := f.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
+
+	ctx := testContext(t)
+
+	service, err := NewGRPCService(ctx, endpoint.endpoint, testProviderName, 15*time.Second)
+	if err != nil {
+		t.Fatalf("failed to create envelope service, error: %v", err)
+	}
+	defer destroyService(service)
+	metrics.RegisterMetrics()
+
+	testCases := []struct {
+		name        string
+		operation   func()
+		labelValues []string
+		wantCount   uint64
+	}{
+		{
+			name: "encrypt",
+			operation: func() {
+				if _, err = service.Encrypt(ctx, "1", []byte("test data")); err != nil {
+					t.Fatalf("failed when execute encrypt, error: %v", err)
+				}
+			},
+			labelValues: []string{testProviderName, "/v2alpha1.KeyManagementService/Encrypt", "OK"},
+			wantCount:   1,
+		},
+		{
+			name: "decrypt",
+			operation: func() {
+				if _, err = service.Decrypt(ctx, "1", &kmsservice.DecryptRequest{Ciphertext: []byte("testdata"), KeyID: "1"}); err != nil {
+					t.Fatalf("failed when execute decrypt, error: %v", err)
+				}
+			},
+			labelValues: []string{testProviderName, "/v2alpha1.KeyManagementService/Decrypt", "OK"},
+			wantCount:   1,
+		},
+		{
+			name: "status",
+			operation: func() {
+				if _, err = service.Status(ctx); err != nil {
+					t.Fatalf("failed when execute status, error: %v", err)
+				}
+			},
+			labelValues: []string{testProviderName, "/v2alpha1.KeyManagementService/Status", "OK"},
+			wantCount:   1,
+		},
+		{
+			name: "multiple status calls",
+			operation: func() {
+				for i := 0; i < 10; i++ {
+					if _, err = service.Status(ctx); err != nil {
+						t.Fatalf("failed when execute status, error: %v", err)
+					}
+				}
+			},
+			labelValues: []string{testProviderName, "/v2alpha1.KeyManagementService/Status", "OK"},
+			wantCount:   10,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.operation()
+			defer metrics.KMSOperationsLatencyMetric.Reset()
+			sampleSum, err := testutil.GetHistogramMetricValue(metrics.KMSOperationsLatencyMetric.WithLabelValues(tt.labelValues...))
+			if err != nil {
+				t.Fatalf("failed to get metric value, error: %v", err)
+			}
+			// apiserver_envelope_encryption_kms_operations_latency_seconds_sum{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 0.000881432
+			if sampleSum == 0 {
+				t.Fatalf("expected metric value to be greater than 0, got %v", sampleSum)
+			}
+			count, err := testutil.GetHistogramMetricCount(metrics.KMSOperationsLatencyMetric.WithLabelValues(tt.labelValues...))
+			if err != nil {
+				t.Fatalf("failed to get metric count, error: %v", err)
+			}
+			// apiserver_envelope_encryption_kms_operations_latency_seconds_count{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1
+			if count != tt.wantCount {
+				t.Fatalf("expected metric count to be %v, got %v", tt.wantCount, count)
+			}
+		})
+	}
+}
+
 func testContext(t *testing.T) context.Context {
 	ctx, cancel := context.WithCancel(context.Background())
 	t.Cleanup(cancel)

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go

@@ -17,9 +17,12 @@ limitations under the License.
 package metrics
 
 import (
+	"errors"
 	"sync"
 	"time"
 
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
 )
@@ -67,6 +70,20 @@ var (
 		},
 		[]string{"transformation_type"},
 	)
+
+	KMSOperationsLatencyMetric = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "kms_operations_latency_seconds",
+			Help:           "KMS operation duration with gRPC error code status total.",
+			StabilityLevel: metrics.ALPHA,
+			// Use custom buckets to avoid the default buckets which are too small for KMS operations.
+			// Ranges from 1ms to 2m11s
+			Buckets: metrics.ExponentialBuckets(0.001, 2, 18),
+		},
+		[]string{"provider_name", "method_name", "grpc_status_code"},
+	)
 )
 
 var registerMetricsFunc sync.Once
@@ -75,6 +92,7 @@ func RegisterMetrics() {
 	registerMetricsFunc.Do(func() {
 		legacyregistry.MustRegister(dekCacheFillPercent)
 		legacyregistry.MustRegister(dekCacheInterArrivals)
+		legacyregistry.MustRegister(KMSOperationsLatencyMetric)
 	})
 }
 
@@ -104,3 +122,28 @@ func RecordArrival(transformationType string, start time.Time) {
 func RecordDekCacheFillPercent(percent float64) {
 	dekCacheFillPercent.Set(percent)
 }
+
+// RecordKMSOperationLatency records the latency of KMS operation.
+func RecordKMSOperationLatency(providerName, methodName string, duration time.Duration, err error) {
+	KMSOperationsLatencyMetric.WithLabelValues(providerName, methodName, getErrorCode(err)).Observe(duration.Seconds())
+}
+
+type gRPCError interface {
+	GRPCStatus() *status.Status
+}
+
+func getErrorCode(err error) string {
+	if err == nil {
+		return codes.OK.String()
+	}
+
+	// handle errors wrapped with fmt.Errorf and similar
+	var s gRPCError
+	if errors.As(err, &s) {
+		return s.GRPCStatus().Code().String()
+	}
+
+	// This is not gRPC error. The operation must have failed before gRPC
+	// method was called, otherwise we would get gRPC error.
+	return "unknown-non-grpc"
+}

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics_test.go

@@ -0,0 +1,176 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
+)
+
+func TestRecordKMSOperationLatency(t *testing.T) {
+	testCases := []struct {
+		name         string
+		methodName   string
+		duration     time.Duration
+		operationErr error
+		want         string
+	}{
+		{
+			name:         "operation success",
+			methodName:   "/v2alpha1.KeyManagementService/Encrypt",
+			duration:     1 * time.Second,
+			operationErr: nil,
+			want: `
+			# HELP apiserver_envelope_encryption_kms_operations_latency_seconds [ALPHA] KMS operation duration with gRPC error code status total.
+			# TYPE apiserver_envelope_encryption_kms_operations_latency_seconds histogram
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.001"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.002"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.004"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.008"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.016"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.032"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.064"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.128"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.256"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.512"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="1.024"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="2.048"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="4.096"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="8.192"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="16.384"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="32.768"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="65.536"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="131.072"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="+Inf"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_sum{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_count{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1
+			`,
+		},
+		{
+			name:         "operation error",
+			methodName:   "/v2alpha1.KeyManagementService/Encrypt",
+			duration:     1 * time.Second,
+			operationErr: status.Error(codes.Internal, "some error"),
+			want: `
+			# HELP apiserver_envelope_encryption_kms_operations_latency_seconds [ALPHA] KMS operation duration with gRPC error code status total.
+			# TYPE apiserver_envelope_encryption_kms_operations_latency_seconds histogram
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.001"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.002"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.004"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.008"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.016"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.032"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.064"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.128"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.256"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.512"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="1.024"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="2.048"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="4.096"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="8.192"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="16.384"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="32.768"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="65.536"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="131.072"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="+Inf"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_sum{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_count{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1
+			`,
+		},
+		{
+			name:         "wrapped not found error",
+			methodName:   "/v2alpha1.KeyManagementService/Encrypt",
+			duration:     1 * time.Second,
+			operationErr: fmt.Errorf("some low level thing failed: %w", status.Error(codes.NotFound, "some error")),
+			want: `
+			# HELP apiserver_envelope_encryption_kms_operations_latency_seconds [ALPHA] KMS operation duration with gRPC error code status total.
+			# TYPE apiserver_envelope_encryption_kms_operations_latency_seconds histogram
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.001"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.002"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.004"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.008"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.016"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.032"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.064"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.128"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.256"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.512"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="1.024"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="2.048"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="4.096"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="8.192"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="16.384"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="32.768"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="65.536"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="131.072"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="+Inf"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_sum{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_count{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1
+			`,
+		},
+		{
+			name:         "non gRPC error",
+			methodName:   "/v2alpha1.KeyManagementService/Encrypt",
+			duration:     1 * time.Second,
+			operationErr: fmt.Errorf("some bad thing happened"),
+			want: `
+			# HELP apiserver_envelope_encryption_kms_operations_latency_seconds [ALPHA] KMS operation duration with gRPC error code status total.
+			# TYPE apiserver_envelope_encryption_kms_operations_latency_seconds histogram
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.001"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.002"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.004"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.008"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.016"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.032"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.064"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.128"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.256"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.512"} 0
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="1.024"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="2.048"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="4.096"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="8.192"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="16.384"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="32.768"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="65.536"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="131.072"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="+Inf"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_sum{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1
+			apiserver_envelope_encryption_kms_operations_latency_seconds_count{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1
+			`,
+		},
+	}
+
+	RegisterMetrics()
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			RecordKMSOperationLatency("providerName", tt.methodName, tt.duration, tt.operationErr)
+			defer KMSOperationsLatencyMetric.Reset()
+			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), "apiserver_envelope_encryption_kms_operations_latency_seconds"); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}

test/integration/controlplane/transformation/kmsv2_transformation_test.go

@@ -414,9 +414,9 @@ func TestKMSv2SingleService(t *testing.T) {
 
 	var kmsv2Calls int
 	origEnvelopeKMSv2ServiceFactory := encryptionconfig.EnvelopeKMSv2ServiceFactory
-	encryptionconfig.EnvelopeKMSv2ServiceFactory = func(ctx context.Context, endpoint string, callTimeout time.Duration) (kmsv2svc.Service, error) {
+	encryptionconfig.EnvelopeKMSv2ServiceFactory = func(ctx context.Context, endpoint, providerName string, callTimeout time.Duration) (kmsv2svc.Service, error) {
 		kmsv2Calls++
-		return origEnvelopeKMSv2ServiceFactory(ctx, endpoint, callTimeout)
+		return origEnvelopeKMSv2ServiceFactory(ctx, endpoint, providerName, callTimeout)
 	}
 	t.Cleanup(func() {
 		encryptionconfig.EnvelopeKMSv2ServiceFactory = origEnvelopeKMSv2ServiceFactory