diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go index 5dc735a5866..e1ff774ea0f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go @@ -566,7 +566,7 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName) } - envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration) + envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Name, config.Timeout.Duration) if err != nil { return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go index 5e398d4b434..44221973ffa 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go @@ -104,7 +104,7 @@ func newMockErrorEnvelopeService(endpoint string, timeout time.Duration) (envelo } // The factory method to create mock envelope kmsv2 service. -func newMockEnvelopeKMSv2Service(ctx context.Context, endpoint string, timeout time.Duration) (kmsservice.Service, error) { +func newMockEnvelopeKMSv2Service(ctx context.Context, endpoint, providerName string, timeout time.Duration) (kmsservice.Service, error) { return &testKMSv2EnvelopeService{nil}, nil } @@ -679,7 +679,7 @@ func TestKMSPluginHealthzTTL(t *testing.T) { func TestKMSv2PluginHealthzTTL(t *testing.T) { ctx := testContext(t) - service, _ := newMockEnvelopeKMSv2Service(ctx, "unix:///tmp/testprovider.sock", 3*time.Second) + service, _ := newMockEnvelopeKMSv2Service(ctx, "unix:///tmp/testprovider.sock", "providerName", 3*time.Second) errService, _ := newMockErrorEnvelopeKMSv2Service("unix:///tmp/testprovider.sock", 3*time.Second) testCases := []struct { diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service.go index cd22fa8d932..ac4dea8b2c0 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service.go @@ -27,6 +27,7 @@ import ( "google.golang.org/grpc/credentials/insecure" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics" "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/util" "k8s.io/klog/v2" kmsapi "k8s.io/kms/apis/v2alpha1" @@ -46,7 +47,7 @@ type gRPCService struct { } // NewGRPCService returns an envelope.Service which use gRPC to communicate the remote KMS provider. -func NewGRPCService(ctx context.Context, endpoint string, callTimeout time.Duration) (kmsservice.Service, error) { +func NewGRPCService(ctx context.Context, endpoint, providerName string, callTimeout time.Duration) (kmsservice.Service, error) { klog.V(4).Infof("Configure KMS provider with endpoint: %s", endpoint) addr, err := util.ParseEndpoint(endpoint) @@ -70,7 +71,9 @@ func NewGRPCService(ctx context.Context, endpoint string, callTimeout time.Durat klog.V(4).Infof("Successfully dialed Unix socket %v", addr) } return c, err - })) + }), + grpc.WithChainUnaryInterceptor(recordMetricsInterceptor(providerName)), + ) if err != nil { return nil, fmt.Errorf("failed to create connection to %s, error: %v", endpoint, err) @@ -138,3 +141,13 @@ func (g *gRPCService) Status(ctx context.Context) (*kmsservice.StatusResponse, e } return &kmsservice.StatusResponse{Version: response.Version, Healthz: response.Healthz, KeyID: response.KeyId}, nil } + +func recordMetricsInterceptor(providerName string) grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + start := time.Now() + respErr := invoker(ctx, method, req, reply, cc, opts...) + elapsed := time.Since(start) + metrics.RecordKMSOperationLatency(providerName, method, elapsed, respErr) + return respErr + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service_unix_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service_unix_test.go index 7f2ac4f94d8..290e3d55324 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service_unix_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service_unix_test.go @@ -25,12 +25,18 @@ import ( "testing" "time" + "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics" mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v2alpha1" + "k8s.io/component-base/metrics/testutil" "k8s.io/apimachinery/pkg/util/uuid" kmsservice "k8s.io/kms/service" ) +const ( + testProviderName = "providerName" +) + type testSocket struct { path string endpoint string @@ -57,7 +63,7 @@ func TestKMSPluginLateStart(t *testing.T) { ctx := testContext(t) - service, err := NewGRPCService(ctx, s.endpoint, callTimeout) + service, err := NewGRPCService(ctx, s.endpoint, testProviderName, callTimeout) if err != nil { t.Fatalf("failed to create envelope service, error: %v", err) } @@ -141,7 +147,7 @@ func TestTimeouts(t *testing.T) { // Simulating late start of kube-apiserver - plugin is up before kube-apiserver, if requested by the testcase. time.Sleep(tt.kubeAPIServerDelay) - service, err = NewGRPCService(ctx, socketName.endpoint, tt.callTimeout) + service, err = NewGRPCService(ctx, socketName.endpoint, testProviderName, tt.callTimeout) if err != nil { t.Fatalf("failed to create envelope service, error: %v", err) } @@ -211,7 +217,7 @@ func TestIntermittentConnectionLoss(t *testing.T) { ctx := testContext(t) // connect to kms plugin - service, err := NewGRPCService(ctx, endpoint.endpoint, timeout) + service, err := NewGRPCService(ctx, endpoint.endpoint, testProviderName, timeout) if err != nil { t.Fatalf("failed to create envelope service, error: %v", err) } @@ -278,7 +284,7 @@ func TestGRPCService(t *testing.T) { ctx := testContext(t) // Create the gRPC client service. - service, err := NewGRPCService(ctx, endpoint.endpoint, 1*time.Second) + service, err := NewGRPCService(ctx, endpoint.endpoint, testProviderName, 1*time.Second) if err != nil { t.Fatalf("failed to create envelope service, error: %v", err) } @@ -321,7 +327,7 @@ func TestGRPCServiceConcurrentAccess(t *testing.T) { ctx := testContext(t) // Create the gRPC client service. - service, err := NewGRPCService(ctx, endpoint.endpoint, 15*time.Second) + service, err := NewGRPCService(ctx, endpoint.endpoint, testProviderName, 15*time.Second) if err != nil { t.Fatalf("failed to create envelope service, error: %v", err) } @@ -389,7 +395,7 @@ func TestInvalidConfiguration(t *testing.T) { for _, testCase := range invalidConfigs { t.Run(testCase.name, func(t *testing.T) { - _, err := NewGRPCService(ctx, testCase.endpoint, 1*time.Second) + _, err := NewGRPCService(ctx, testCase.endpoint, testProviderName, 1*time.Second) if err == nil { t.Fatalf("should fail to create envelope service for %s.", testCase.name) } @@ -397,6 +403,99 @@ func TestInvalidConfiguration(t *testing.T) { } } +func TestKMSOperationsMetric(t *testing.T) { + endpoint := newEndpoint() + f, err := mock.NewBase64Plugin(endpoint.path) + if err != nil { + t.Fatalf("failed to start test KMS provider server, error: %v", err) + } + if err := f.Start(); err != nil { + t.Fatalf("Failed to start kms-plugin, err: %v", err) + } + + ctx := testContext(t) + + service, err := NewGRPCService(ctx, endpoint.endpoint, testProviderName, 15*time.Second) + if err != nil { + t.Fatalf("failed to create envelope service, error: %v", err) + } + defer destroyService(service) + metrics.RegisterMetrics() + + testCases := []struct { + name string + operation func() + labelValues []string + wantCount uint64 + }{ + { + name: "encrypt", + operation: func() { + if _, err = service.Encrypt(ctx, "1", []byte("test data")); err != nil { + t.Fatalf("failed when execute encrypt, error: %v", err) + } + }, + labelValues: []string{testProviderName, "/v2alpha1.KeyManagementService/Encrypt", "OK"}, + wantCount: 1, + }, + { + name: "decrypt", + operation: func() { + if _, err = service.Decrypt(ctx, "1", &kmsservice.DecryptRequest{Ciphertext: []byte("testdata"), KeyID: "1"}); err != nil { + t.Fatalf("failed when execute decrypt, error: %v", err) + } + }, + labelValues: []string{testProviderName, "/v2alpha1.KeyManagementService/Decrypt", "OK"}, + wantCount: 1, + }, + { + name: "status", + operation: func() { + if _, err = service.Status(ctx); err != nil { + t.Fatalf("failed when execute status, error: %v", err) + } + }, + labelValues: []string{testProviderName, "/v2alpha1.KeyManagementService/Status", "OK"}, + wantCount: 1, + }, + { + name: "multiple status calls", + operation: func() { + for i := 0; i < 10; i++ { + if _, err = service.Status(ctx); err != nil { + t.Fatalf("failed when execute status, error: %v", err) + } + } + }, + labelValues: []string{testProviderName, "/v2alpha1.KeyManagementService/Status", "OK"}, + wantCount: 10, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + tt.operation() + defer metrics.KMSOperationsLatencyMetric.Reset() + sampleSum, err := testutil.GetHistogramMetricValue(metrics.KMSOperationsLatencyMetric.WithLabelValues(tt.labelValues...)) + if err != nil { + t.Fatalf("failed to get metric value, error: %v", err) + } + // apiserver_envelope_encryption_kms_operations_latency_seconds_sum{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 0.000881432 + if sampleSum == 0 { + t.Fatalf("expected metric value to be greater than 0, got %v", sampleSum) + } + count, err := testutil.GetHistogramMetricCount(metrics.KMSOperationsLatencyMetric.WithLabelValues(tt.labelValues...)) + if err != nil { + t.Fatalf("failed to get metric count, error: %v", err) + } + // apiserver_envelope_encryption_kms_operations_latency_seconds_count{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1 + if count != tt.wantCount { + t.Fatalf("expected metric count to be %v, got %v", tt.wantCount, count) + } + }) + } +} + func testContext(t *testing.T) context.Context { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go index ca0e0ad0f69..70f65d50116 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go @@ -17,9 +17,12 @@ limitations under the License. package metrics import ( + "errors" "sync" "time" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" ) @@ -67,6 +70,20 @@ var ( }, []string{"transformation_type"}, ) + + KMSOperationsLatencyMetric = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "kms_operations_latency_seconds", + Help: "KMS operation duration with gRPC error code status total.", + StabilityLevel: metrics.ALPHA, + // Use custom buckets to avoid the default buckets which are too small for KMS operations. + // Ranges from 1ms to 2m11s + Buckets: metrics.ExponentialBuckets(0.001, 2, 18), + }, + []string{"provider_name", "method_name", "grpc_status_code"}, + ) ) var registerMetricsFunc sync.Once @@ -75,6 +92,7 @@ func RegisterMetrics() { registerMetricsFunc.Do(func() { legacyregistry.MustRegister(dekCacheFillPercent) legacyregistry.MustRegister(dekCacheInterArrivals) + legacyregistry.MustRegister(KMSOperationsLatencyMetric) }) } @@ -104,3 +122,28 @@ func RecordArrival(transformationType string, start time.Time) { func RecordDekCacheFillPercent(percent float64) { dekCacheFillPercent.Set(percent) } + +// RecordKMSOperationLatency records the latency of KMS operation. +func RecordKMSOperationLatency(providerName, methodName string, duration time.Duration, err error) { + KMSOperationsLatencyMetric.WithLabelValues(providerName, methodName, getErrorCode(err)).Observe(duration.Seconds()) +} + +type gRPCError interface { + GRPCStatus() *status.Status +} + +func getErrorCode(err error) string { + if err == nil { + return codes.OK.String() + } + + // handle errors wrapped with fmt.Errorf and similar + var s gRPCError + if errors.As(err, &s) { + return s.GRPCStatus().Code().String() + } + + // This is not gRPC error. The operation must have failed before gRPC + // method was called, otherwise we would get gRPC error. + return "unknown-non-grpc" +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics_test.go new file mode 100644 index 00000000000..8eeb0dae539 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics_test.go @@ -0,0 +1,176 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/component-base/metrics/testutil" +) + +func TestRecordKMSOperationLatency(t *testing.T) { + testCases := []struct { + name string + methodName string + duration time.Duration + operationErr error + want string + }{ + { + name: "operation success", + methodName: "/v2alpha1.KeyManagementService/Encrypt", + duration: 1 * time.Second, + operationErr: nil, + want: ` + # HELP apiserver_envelope_encryption_kms_operations_latency_seconds [ALPHA] KMS operation duration with gRPC error code status total. + # TYPE apiserver_envelope_encryption_kms_operations_latency_seconds histogram + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.001"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.002"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.004"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.008"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.016"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.032"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.064"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.128"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.256"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.512"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="1.024"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="2.048"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="4.096"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="8.192"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="16.384"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="32.768"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="65.536"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="131.072"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="+Inf"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_sum{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_count{grpc_status_code="OK",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1 + `, + }, + { + name: "operation error", + methodName: "/v2alpha1.KeyManagementService/Encrypt", + duration: 1 * time.Second, + operationErr: status.Error(codes.Internal, "some error"), + want: ` + # HELP apiserver_envelope_encryption_kms_operations_latency_seconds [ALPHA] KMS operation duration with gRPC error code status total. + # TYPE apiserver_envelope_encryption_kms_operations_latency_seconds histogram + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.001"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.002"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.004"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.008"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.016"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.032"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.064"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.128"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.256"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.512"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="1.024"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="2.048"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="4.096"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="8.192"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="16.384"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="32.768"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="65.536"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="131.072"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="+Inf"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_sum{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_count{grpc_status_code="Internal",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1 + `, + }, + { + name: "wrapped not found error", + methodName: "/v2alpha1.KeyManagementService/Encrypt", + duration: 1 * time.Second, + operationErr: fmt.Errorf("some low level thing failed: %w", status.Error(codes.NotFound, "some error")), + want: ` + # HELP apiserver_envelope_encryption_kms_operations_latency_seconds [ALPHA] KMS operation duration with gRPC error code status total. + # TYPE apiserver_envelope_encryption_kms_operations_latency_seconds histogram + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.001"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.002"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.004"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.008"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.016"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.032"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.064"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.128"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.256"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.512"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="1.024"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="2.048"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="4.096"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="8.192"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="16.384"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="32.768"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="65.536"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="131.072"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="+Inf"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_sum{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_count{grpc_status_code="NotFound",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1 + `, + }, + { + name: "non gRPC error", + methodName: "/v2alpha1.KeyManagementService/Encrypt", + duration: 1 * time.Second, + operationErr: fmt.Errorf("some bad thing happened"), + want: ` + # HELP apiserver_envelope_encryption_kms_operations_latency_seconds [ALPHA] KMS operation duration with gRPC error code status total. + # TYPE apiserver_envelope_encryption_kms_operations_latency_seconds histogram + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.001"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.002"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.004"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.008"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.016"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.032"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.064"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.128"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.256"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="0.512"} 0 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="1.024"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="2.048"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="4.096"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="8.192"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="16.384"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="32.768"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="65.536"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="131.072"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_bucket{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName",le="+Inf"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_sum{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1 + apiserver_envelope_encryption_kms_operations_latency_seconds_count{grpc_status_code="unknown-non-grpc",method_name="/v2alpha1.KeyManagementService/Encrypt",provider_name="providerName"} 1 + `, + }, + } + + RegisterMetrics() + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + RecordKMSOperationLatency("providerName", tt.methodName, tt.duration, tt.operationErr) + defer KMSOperationsLatencyMetric.Reset() + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), "apiserver_envelope_encryption_kms_operations_latency_seconds"); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index 82ab59f3f3e..793715ede24 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -414,9 +414,9 @@ func TestKMSv2SingleService(t *testing.T) { var kmsv2Calls int origEnvelopeKMSv2ServiceFactory := encryptionconfig.EnvelopeKMSv2ServiceFactory - encryptionconfig.EnvelopeKMSv2ServiceFactory = func(ctx context.Context, endpoint string, callTimeout time.Duration) (kmsv2svc.Service, error) { + encryptionconfig.EnvelopeKMSv2ServiceFactory = func(ctx context.Context, endpoint, providerName string, callTimeout time.Duration) (kmsv2svc.Service, error) { kmsv2Calls++ - return origEnvelopeKMSv2ServiceFactory(ctx, endpoint, callTimeout) + return origEnvelopeKMSv2ServiceFactory(ctx, endpoint, providerName, callTimeout) } t.Cleanup(func() { encryptionconfig.EnvelopeKMSv2ServiceFactory = origEnvelopeKMSv2ServiceFactory