diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go index 35ec0136920..d2b59a29dca 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go @@ -61,10 +61,10 @@ var ( Namespace: namespace, Subsystem: subsystem, Name: "transformation_operations_total", - Help: "Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption", + Help: "Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.", StabilityLevel: metrics.ALPHA, }, - []string{"transformation_type", "transformer_prefix", "status"}, + []string{"resource", "transformation_type", "transformer_prefix", "status"}, ) envelopeTransformationCacheMissTotal = metrics.NewCounter( @@ -113,8 +113,8 @@ func RegisterMetrics() { // RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations. // Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead. -func RecordTransformation(transformationType, transformerPrefix string, elapsed time.Duration, err error) { - transformerOperationsTotal.WithLabelValues(transformationType, transformerPrefix, getErrorCode(err)).Inc() +func RecordTransformation(resource, transformationType, transformerPrefix string, elapsed time.Duration, err error) { + transformerOperationsTotal.WithLabelValues(resource, transformationType, transformerPrefix, getErrorCode(err)).Inc() if err == nil { transformerLatencies.WithLabelValues(transformationType, transformerPrefix).Observe(elapsed.Seconds()) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go index 9b77a5856d4..06ae526473c 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go @@ -27,6 +27,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/component-base/metrics/legacyregistry" "k8s.io/component-base/metrics/testutil" ) @@ -44,10 +45,11 @@ func TestTotals(t *testing.T) { wrappedErrTransformer := PrefixTransformer{Prefix: []byte("k8s:enc:kms:v1:"), Transformer: &testTransformer{err: wrappedErr}} testCases := []struct { - desc string - prefix Transformer - metrics []string - want string + desc string + prefix Transformer + metrics []string + want string + expectErr bool }{ { desc: "non-status error", @@ -56,11 +58,12 @@ func TestTotals(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 - apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 `, + expectErr: true, }, { desc: "ok", @@ -69,11 +72,12 @@ func TestTotals(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 - apiserver_storage_transformation_operations_total{status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 `, + expectErr: false, }, { desc: "failed precondition", @@ -82,11 +86,12 @@ func TestTotals(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 - apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="FailedPrecondition",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="FailedPrecondition",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 `, + expectErr: true, }, { desc: "internal", @@ -95,11 +100,12 @@ func TestTotals(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="Internal",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 - apiserver_storage_transformation_operations_total{status="Internal",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="Internal",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="Internal",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 `, + expectErr: true, }, { desc: "wrapped not found error", @@ -108,21 +114,29 @@ func TestTotals(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="NotFound",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 - apiserver_storage_transformation_operations_total{status="NotFound",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="NotFound",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="NotFound",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 `, + expectErr: true, }, } RegisterMetrics() transformerOperationsTotal.Reset() + reqCtx := request.WithRequestInfo(context.Background(), &request.RequestInfo{Resource: "test"}) for _, tt := range testCases { t.Run(tt.desc, func(t *testing.T) { - tt.prefix.TransformToStorage(context.Background(), []byte("value"), nil) - tt.prefix.TransformFromStorage(context.Background(), []byte("k8s:enc:kms:v1:value"), nil) + _, err := tt.prefix.TransformToStorage(reqCtx, []byte("value"), nil) + if (err != nil) != tt.expectErr { + t.Fatal(err) + } + _, _, err = tt.prefix.TransformFromStorage(reqCtx, []byte("k8s:enc:kms:v1:value"), nil) + if (err != nil) != tt.expectErr { + t.Fatal(err) + } defer transformerOperationsTotal.Reset() if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { t.Fatal(err) @@ -229,7 +243,7 @@ func TestLatency(t *testing.T) { for _, tt := range testCases { t.Run(tt.desc, func(t *testing.T) { - RecordTransformation(tt.transformationType, tt.prefix, tt.elapsed, nil) + RecordTransformation("", tt.transformationType, tt.prefix, tt.elapsed, nil) defer transformerLatencies.Reset() if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { t.Fatal(err) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go index c5e97ac2daa..eab90b3fa3b 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go @@ -105,6 +105,7 @@ func NewPrefixTransformers(err error, transformers ...PrefixTransformer) Transfo func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, bool, error) { start := time.Now() var errs []error + resource := getResourceFromContext(ctx) for i, transformer := range t.transformers { if bytes.HasPrefix(data, transformer.Prefix) { result, stale, err := transformer.Transformer.TransformFromStorage(ctx, data[len(transformer.Prefix):], dataCtx) @@ -116,9 +117,9 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by continue } if len(transformer.Prefix) == 0 { - RecordTransformation("from_storage", "identity", time.Since(start), err) + RecordTransformation(resource, "from_storage", "identity", time.Since(start), err) } else { - RecordTransformation("from_storage", string(transformer.Prefix), time.Since(start), err) + RecordTransformation(resource, "from_storage", string(transformer.Prefix), time.Since(start), err) } // It is valid to have overlapping prefixes when the same encryption provider @@ -163,7 +164,7 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by logTransformErr(ctx, err, "failed to decrypt data") return nil, false, err } - RecordTransformation("from_storage", "unknown", time.Since(start), t.err) + RecordTransformation(resource, "from_storage", "unknown", time.Since(start), t.err) return nil, false, t.err } @@ -171,8 +172,9 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by func (t *prefixTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, error) { start := time.Now() transformer := t.transformers[0] + resource := getResourceFromContext(ctx) result, err := transformer.Transformer.TransformToStorage(ctx, data, dataCtx) - RecordTransformation("to_storage", string(transformer.Prefix), time.Since(start), err) + RecordTransformation(resource, "to_storage", string(transformer.Prefix), time.Since(start), err) if err != nil { logTransformErr(ctx, err, "failed to encrypt data") return nil, err @@ -209,5 +211,11 @@ func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestIn if reqInfo, found := genericapirequest.RequestInfoFrom(ctx); found { return reqInfo } + klog.V(4).InfoSDepth(1, "no request info on context") return &genericapirequest.RequestInfo{} } + +func getResourceFromContext(ctx context.Context) string { + reqInfo := getRequestInfoFromContext(ctx) + return schema.GroupResource{Group: reqInfo.APIGroup, Resource: reqInfo.Resource}.String() +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go index 946bfc8aabe..52716adb890 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go @@ -73,8 +73,9 @@ func TestPrefixFrom(t *testing.T) { {[]byte("fails:value"), nil, false, transformErr, 2}, {[]byte("stale:value"), []byte("value3"), true, nil, 3}, } + reqCtx := genericapirequest.WithRequestInfo(context.Background(), &genericapirequest.RequestInfo{Resource: "test"}) for i, test := range testCases { - got, stale, err := p.TransformFromStorage(context.Background(), test.input, nil) + got, stale, err := p.TransformFromStorage(reqCtx, test.input, nil) if err != test.err || stale != test.stale || !bytes.Equal(got, test.expect) { t.Errorf("%d: unexpected out: %q %t %#v", i, string(got), stale, err) continue @@ -97,9 +98,10 @@ func TestPrefixTo(t *testing.T) { {[]PrefixTransformer{{Prefix: []byte("second:"), Transformer: &testTransformer{to: []byte("value2")}}}, []byte("second:value2"), nil}, {[]PrefixTransformer{{Prefix: []byte("fails:"), Transformer: &testTransformer{err: transformErr}}}, nil, transformErr}, } + reqCtx := genericapirequest.WithRequestInfo(context.Background(), &genericapirequest.RequestInfo{Resource: "test"}) for i, test := range testCases { p := NewPrefixTransformers(testErr, test.transformers...) - got, err := p.TransformToStorage(context.Background(), []byte("value"), nil) + got, err := p.TransformToStorage(reqCtx, []byte("value"), nil) if err != test.err || !bytes.Equal(got, test.expect) { t.Errorf("%d: unexpected out: %q %#v", i, string(got), err) continue @@ -119,12 +121,12 @@ func TestPrefixFromMetrics(t *testing.T) { otherTransformerErr := PrefixTransformer{Prefix: []byte("other:"), Transformer: &testTransformer{err: transformerErr}} testCases := []struct { - desc string - input []byte - prefix Transformer - metrics []string - want string - err error + desc string + input []byte + prefix Transformer + metrics []string + want string + expectErr bool }{ { desc: "identity prefix", @@ -134,11 +136,11 @@ func TestPrefixFromMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="identity"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="from_storage",transformer_prefix="identity"} 1 `, - err: nil, + expectErr: false, }, { desc: "other prefix (ok)", @@ -148,11 +150,11 @@ func TestPrefixFromMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="other:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="from_storage",transformer_prefix="other:"} 1 `, - err: nil, + expectErr: false, }, { desc: "other prefix (error)", @@ -162,11 +164,11 @@ func TestPrefixFromMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="other:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="other:"} 1 `, - err: nil, + expectErr: true, }, { desc: "unknown prefix", @@ -176,20 +178,23 @@ func TestPrefixFromMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="unknown"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="unknown"} 1 `, - err: nil, + expectErr: true, }, } RegisterMetrics() transformerOperationsTotal.Reset() - + reqCtx := genericapirequest.WithRequestInfo(context.Background(), &genericapirequest.RequestInfo{Resource: "test"}) for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - tc.prefix.TransformFromStorage(context.Background(), tc.input, nil) + _, _, err := tc.prefix.TransformFromStorage(reqCtx, tc.input, nil) + if (err != nil) != tc.expectErr { + t.Fatal(err) + } defer transformerOperationsTotal.Reset() if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.want), tc.metrics...); err != nil { t.Fatal(err) @@ -203,14 +208,15 @@ func TestPrefixToMetrics(t *testing.T) { transformerErr := fmt.Errorf("test error") otherTransformer := PrefixTransformer{Prefix: []byte("other:"), Transformer: &testTransformer{from: []byte("value1")}} otherTransformerErr := PrefixTransformer{Prefix: []byte("other:"), Transformer: &testTransformer{err: transformerErr}} - + reqCtx := genericapirequest.WithRequestInfo(context.Background(), &genericapirequest.RequestInfo{Resource: "test"}) testCases := []struct { - desc string - input []byte - prefix Transformer - metrics []string - want string - err error + desc string + input []byte + prefix Transformer + metrics []string + want string + expectErr bool + ctx context.Context }{ { desc: "ok", @@ -220,11 +226,42 @@ func TestPrefixToMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1 `, - err: nil, + expectErr: false, + ctx: reqCtx, + }, + { + desc: "missing request context", + input: []byte("value"), + prefix: NewPrefixTransformers(testErr, otherTransformer), + metrics: []string{ + "apiserver_storage_transformation_operations_total", + }, + want: ` + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. + # TYPE apiserver_storage_transformation_operations_total counter + apiserver_storage_transformation_operations_total{resource="",status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1 + `, + expectErr: false, + ctx: context.Background(), + }, + { + desc: "request context with api group", + input: []byte("value"), + prefix: NewPrefixTransformers(testErr, otherTransformer), + metrics: []string{ + "apiserver_storage_transformation_operations_total", + }, + want: ` + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. + # TYPE apiserver_storage_transformation_operations_total counter + apiserver_storage_transformation_operations_total{resource="test.testGroup",status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1 + `, + expectErr: false, + ctx: genericapirequest.WithRequestInfo(context.Background(), &genericapirequest.RequestInfo{APIGroup: "testGroup", Resource: "test"}), }, { desc: "error", @@ -234,20 +271,23 @@ func TestPrefixToMetrics(t *testing.T) { "apiserver_storage_transformation_operations_total", }, want: ` - # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption + # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource. # TYPE apiserver_storage_transformation_operations_total counter - apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="other:"} 1 + apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="other:"} 1 `, - err: nil, + expectErr: true, + ctx: reqCtx, }, } RegisterMetrics() transformerOperationsTotal.Reset() - for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - tc.prefix.TransformToStorage(context.Background(), tc.input, nil) + _, err := tc.prefix.TransformToStorage(tc.ctx, tc.input, nil) + if (err != nil) != tc.expectErr { + t.Fatal(err) + } defer transformerOperationsTotal.Reset() if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.want), tc.metrics...); err != nil { t.Fatal(err) @@ -285,7 +325,7 @@ func TestLogTransformErr(t *testing.T) { ctx: context.Background(), err: errors.New("decryption failed"), message: "failed to decrypt data", - expectedLog: "\"failed to decrypt data\" err=\"decryption failed\" group=\"\" version=\"\" resource=\"\" subresource=\"\" verb=\"\" namespace=\"\" name=\"\"\n", + expectedLog: "\"no request info on context\"\n\"failed to decrypt data\" err=\"decryption failed\" group=\"\" version=\"\" resource=\"\" subresource=\"\" verb=\"\" namespace=\"\" name=\"\"\n", }, } @@ -301,7 +341,7 @@ func TestLogTransformErr(t *testing.T) { logTransformErr(tt.ctx, tt.err, tt.message) // remove timestamp and goroutine id from log message to make it easier to compare - gotLog := regexp.MustCompile(`\w+ \d+:\d+:\d+\.\d+.*\d+.*transformer_test.go:\d+].`).ReplaceAllString(buf.String(), "") + gotLog := regexp.MustCompile(`\w+ \d+:\d+:\d+\.\d+.*\d+.*(transformer_test\.go|transformer\.go):\d+].`).ReplaceAllString(buf.String(), "") if gotLog != tt.expectedLog { t.Errorf("expected log message %q, got %q", tt.expectedLog, gotLog) diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index 12d09d8b2e8..94cdd0f60da 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -301,6 +301,8 @@ resources: `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`, `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`, `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`, + `apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v2:kms-provider:"} 2`, + `apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v2:kms-provider:"} 1`, } defer func() { body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx) @@ -310,7 +312,7 @@ resources: var gotMetricStrings []string trimFP := regexp.MustCompile(`(.*)(} \d+\.\d+.*)`) for _, line := range strings.Split(string(body), "\n") { - if strings.HasPrefix(line, "apiserver_envelope_") { + if strings.HasPrefix(line, "apiserver_envelope_") || strings.HasPrefix(line, "apiserver_storage_transformation_operations") { if strings.HasPrefix(line, "apiserver_envelope_encryption_dek_cache_fill_percent") { continue // this can be ignored as it is KMS v1 only }