github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-14 06:15:45 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #126512 from kmala/metrics

Browse Source 
add resource to the transformation metrics
This commit is contained in:
Kubernetes Prow Robot 2024-08-21 22:06:26 +01:00 committed by GitHub
commit 77737c3eb3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 134 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go
View File

@ -61,10 +61,10 @@ var (
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "transformation_operations_total", Name: "transformation_operations_total",
Help: "Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption", Help: "Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.",
StabilityLevel: metrics.ALPHA, StabilityLevel: metrics.ALPHA,
}, },
[]string{"transformation_type", "transformer_prefix", "status"}, []string{"resource", "transformation_type", "transformer_prefix", "status"},
) )
envelopeTransformationCacheMissTotal = metrics.NewCounter( envelopeTransformationCacheMissTotal = metrics.NewCounter(
@ -113,8 +113,8 @@ func RegisterMetrics() {
// RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations. // RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations.
// Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead. // Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead.
func RecordTransformation(transformationType, transformerPrefix string, elapsed time.Duration, err error) { func RecordTransformation(resource, transformationType, transformerPrefix string, elapsed time.Duration, err error) {
transformerOperationsTotal.WithLabelValues(transformationType, transformerPrefix, getErrorCode(err)).Inc() transformerOperationsTotal.WithLabelValues(resource, transformationType, transformerPrefix, getErrorCode(err)).Inc()
if err == nil { if err == nil {
transformerLatencies.WithLabelValues(transformationType, transformerPrefix).Observe(elapsed.Seconds()) transformerLatencies.WithLabelValues(transformationType, transformerPrefix).Observe(elapsed.Seconds())

58
staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go
View File

@ -27,6 +27,7 @@ import (
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/component-base/metrics/legacyregistry" "k8s.io/component-base/metrics/legacyregistry"
"k8s.io/component-base/metrics/testutil" "k8s.io/component-base/metrics/testutil"
) )
@ -44,10 +45,11 @@ func TestTotals(t *testing.T) {
wrappedErrTransformer := PrefixTransformer{Prefix: []byte("k8s:enc:kms:v1:"), Transformer: &testTransformer{err: wrappedErr}} wrappedErrTransformer := PrefixTransformer{Prefix: []byte("k8s:enc:kms:v1:"), Transformer: &testTransformer{err: wrappedErr}}
testCases := []struct { testCases := []struct {
desc string desc string
prefix Transformer prefix Transformer
metrics []string metrics []string
want string want string
expectErr bool
}{ }{
{ {
desc: "non-status error", desc: "non-status error",
@ -56,11 +58,12 @@ func TestTotals(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,
expectErr: true,
}, },
{ {
desc: "ok", desc: "ok",
@ -69,11 +72,12 @@ func TestTotals(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
apiserver_storage_transformation_operations_total{status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,
expectErr: false,
}, },
{ {
desc: "failed precondition", desc: "failed precondition",
@ -82,11 +86,12 @@ func TestTotals(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="FailedPrecondition",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="FailedPrecondition",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,
expectErr: true,
}, },
{ {
desc: "internal", desc: "internal",
@ -95,11 +100,12 @@ func TestTotals(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="Internal",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="Internal",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
apiserver_storage_transformation_operations_total{status="Internal",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="Internal",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,
expectErr: true,
}, },
{ {
desc: "wrapped not found error", desc: "wrapped not found error",
@ -108,21 +114,29 @@ func TestTotals(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="NotFound",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="NotFound",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
apiserver_storage_transformation_operations_total{status="NotFound",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="NotFound",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,
expectErr: true,
}, },
} }
RegisterMetrics() RegisterMetrics()
transformerOperationsTotal.Reset() transformerOperationsTotal.Reset()
reqCtx := request.WithRequestInfo(context.Background(), &request.RequestInfo{Resource: "test"})
for _, tt := range testCases { for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) { t.Run(tt.desc, func(t *testing.T) {
tt.prefix.TransformToStorage(context.Background(), []byte("value"), nil) _, err := tt.prefix.TransformToStorage(reqCtx, []byte("value"), nil)
tt.prefix.TransformFromStorage(context.Background(), []byte("k8s:enc:kms:v1:value"), nil) if (err != nil) != tt.expectErr {
t.Fatal(err)
}
_, _, err = tt.prefix.TransformFromStorage(reqCtx, []byte("k8s:enc:kms:v1:value"), nil)
if (err != nil) != tt.expectErr {
t.Fatal(err)
}
defer transformerOperationsTotal.Reset() defer transformerOperationsTotal.Reset()
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
t.Fatal(err) t.Fatal(err)
@ -229,7 +243,7 @@ func TestLatency(t *testing.T) {
for _, tt := range testCases { for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) { t.Run(tt.desc, func(t *testing.T) {
RecordTransformation(tt.transformationType, tt.prefix, tt.elapsed, nil) RecordTransformation("", tt.transformationType, tt.prefix, tt.elapsed, nil)
defer transformerLatencies.Reset() defer transformerLatencies.Reset()
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
t.Fatal(err) t.Fatal(err)

16
staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go
View File

@ -105,6 +105,7 @@ func NewPrefixTransformers(err error, transformers ...PrefixTransformer) Transfo
func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, bool, error) { func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, bool, error) {
start := time.Now() start := time.Now()
var errs []error var errs []error
resource := getResourceFromContext(ctx)
for i, transformer := range t.transformers { for i, transformer := range t.transformers {
if bytes.HasPrefix(data, transformer.Prefix) { if bytes.HasPrefix(data, transformer.Prefix) {
result, stale, err := transformer.Transformer.TransformFromStorage(ctx, data[len(transformer.Prefix):], dataCtx) result, stale, err := transformer.Transformer.TransformFromStorage(ctx, data[len(transformer.Prefix):], dataCtx)
@ -116,9 +117,9 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by
continue continue
} }
if len(transformer.Prefix) == 0 { if len(transformer.Prefix) == 0 {
RecordTransformation("from_storage", "identity", time.Since(start), err) RecordTransformation(resource, "from_storage", "identity", time.Since(start), err)
} else { } else {
RecordTransformation("from_storage", string(transformer.Prefix), time.Since(start), err) RecordTransformation(resource, "from_storage", string(transformer.Prefix), time.Since(start), err)
} }
// It is valid to have overlapping prefixes when the same encryption provider // It is valid to have overlapping prefixes when the same encryption provider
@ -163,7 +164,7 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by
logTransformErr(ctx, err, "failed to decrypt data") logTransformErr(ctx, err, "failed to decrypt data")
return nil, false, err return nil, false, err
} }
RecordTransformation("from_storage", "unknown", time.Since(start), t.err) RecordTransformation(resource, "from_storage", "unknown", time.Since(start), t.err)
return nil, false, t.err return nil, false, t.err
} }
@ -171,8 +172,9 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by
func (t *prefixTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, error) { func (t *prefixTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, error) {
start := time.Now() start := time.Now()
transformer := t.transformers[0] transformer := t.transformers[0]
resource := getResourceFromContext(ctx)
result, err := transformer.Transformer.TransformToStorage(ctx, data, dataCtx) result, err := transformer.Transformer.TransformToStorage(ctx, data, dataCtx)
RecordTransformation("to_storage", string(transformer.Prefix), time.Since(start), err) RecordTransformation(resource, "to_storage", string(transformer.Prefix), time.Since(start), err)
if err != nil { if err != nil {
logTransformErr(ctx, err, "failed to encrypt data") logTransformErr(ctx, err, "failed to encrypt data")
return nil, err return nil, err
@ -209,5 +211,11 @@ func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestIn
if reqInfo, found := genericapirequest.RequestInfoFrom(ctx); found { if reqInfo, found := genericapirequest.RequestInfoFrom(ctx); found {
return reqInfo return reqInfo
} }
klog.V(4).InfoSDepth(1, "no request info on context")
return &genericapirequest.RequestInfo{} return &genericapirequest.RequestInfo{}
} }
func getResourceFromContext(ctx context.Context) string {
reqInfo := getRequestInfoFromContext(ctx)
return schema.GroupResource{Group: reqInfo.APIGroup, Resource: reqInfo.Resource}.String()
}

118
staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go
View File

@ -73,8 +73,9 @@ func TestPrefixFrom(t *testing.T) {
{[]byte("fails:value"), nil, false, transformErr, 2}, {[]byte("fails:value"), nil, false, transformErr, 2},
{[]byte("stale:value"), []byte("value3"), true, nil, 3}, {[]byte("stale:value"), []byte("value3"), true, nil, 3},
} }
reqCtx := genericapirequest.WithRequestInfo(context.Background(), &genericapirequest.RequestInfo{Resource: "test"})
for i, test := range testCases { for i, test := range testCases {
got, stale, err := p.TransformFromStorage(context.Background(), test.input, nil) got, stale, err := p.TransformFromStorage(reqCtx, test.input, nil)
if err != test.err || stale != test.stale || !bytes.Equal(got, test.expect) { if err != test.err || stale != test.stale || !bytes.Equal(got, test.expect) {
t.Errorf("%d: unexpected out: %q %t %#v", i, string(got), stale, err) t.Errorf("%d: unexpected out: %q %t %#v", i, string(got), stale, err)
continue continue
@ -97,9 +98,10 @@ func TestPrefixTo(t *testing.T) {
{[]PrefixTransformer{{Prefix: []byte("second:"), Transformer: &testTransformer{to: []byte("value2")}}}, []byte("second:value2"), nil}, {[]PrefixTransformer{{Prefix: []byte("second:"), Transformer: &testTransformer{to: []byte("value2")}}}, []byte("second:value2"), nil},
{[]PrefixTransformer{{Prefix: []byte("fails:"), Transformer: &testTransformer{err: transformErr}}}, nil, transformErr}, {[]PrefixTransformer{{Prefix: []byte("fails:"), Transformer: &testTransformer{err: transformErr}}}, nil, transformErr},
} }
reqCtx := genericapirequest.WithRequestInfo(context.Background(), &genericapirequest.RequestInfo{Resource: "test"})
for i, test := range testCases { for i, test := range testCases {
p := NewPrefixTransformers(testErr, test.transformers...) p := NewPrefixTransformers(testErr, test.transformers...)
got, err := p.TransformToStorage(context.Background(), []byte("value"), nil) got, err := p.TransformToStorage(reqCtx, []byte("value"), nil)
if err != test.err || !bytes.Equal(got, test.expect) { if err != test.err || !bytes.Equal(got, test.expect) {
t.Errorf("%d: unexpected out: %q %#v", i, string(got), err) t.Errorf("%d: unexpected out: %q %#v", i, string(got), err)
continue continue
@ -119,12 +121,12 @@ func TestPrefixFromMetrics(t *testing.T) {
otherTransformerErr := PrefixTransformer{Prefix: []byte("other:"), Transformer: &testTransformer{err: transformerErr}} otherTransformerErr := PrefixTransformer{Prefix: []byte("other:"), Transformer: &testTransformer{err: transformerErr}}
testCases := []struct { testCases := []struct {
desc string desc string
input []byte input []byte
prefix Transformer prefix Transformer
metrics []string metrics []string
want string want string
err error expectErr bool
}{ }{
{ {
desc: "identity prefix", desc: "identity prefix",
@ -134,11 +136,11 @@ func TestPrefixFromMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="identity"} 1 apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="from_storage",transformer_prefix="identity"} 1
`, `,
err: nil, expectErr: false,
}, },
{ {
desc: "other prefix (ok)", desc: "other prefix (ok)",
@ -148,11 +150,11 @@ func TestPrefixFromMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="other:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="from_storage",transformer_prefix="other:"} 1
`, `,
err: nil, expectErr: false,
}, },
{ {
desc: "other prefix (error)", desc: "other prefix (error)",
@ -162,11 +164,11 @@ func TestPrefixFromMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="other:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="other:"} 1
`, `,
err: nil, expectErr: true,
}, },
{ {
desc: "unknown prefix", desc: "unknown prefix",
@ -176,20 +178,23 @@ func TestPrefixFromMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="unknown"} 1 apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="unknown"} 1
`, `,
err: nil, expectErr: true,
}, },
} }
RegisterMetrics() RegisterMetrics()
transformerOperationsTotal.Reset() transformerOperationsTotal.Reset()
reqCtx := genericapirequest.WithRequestInfo(context.Background(), &genericapirequest.RequestInfo{Resource: "test"})
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) { t.Run(tc.desc, func(t *testing.T) {
tc.prefix.TransformFromStorage(context.Background(), tc.input, nil) _, _, err := tc.prefix.TransformFromStorage(reqCtx, tc.input, nil)
if (err != nil) != tc.expectErr {
t.Fatal(err)
}
defer transformerOperationsTotal.Reset() defer transformerOperationsTotal.Reset()
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.want), tc.metrics...); err != nil { if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.want), tc.metrics...); err != nil {
t.Fatal(err) t.Fatal(err)
@ -203,14 +208,15 @@ func TestPrefixToMetrics(t *testing.T) {
transformerErr := fmt.Errorf("test error") transformerErr := fmt.Errorf("test error")
otherTransformer := PrefixTransformer{Prefix: []byte("other:"), Transformer: &testTransformer{from: []byte("value1")}} otherTransformer := PrefixTransformer{Prefix: []byte("other:"), Transformer: &testTransformer{from: []byte("value1")}}
otherTransformerErr := PrefixTransformer{Prefix: []byte("other:"), Transformer: &testTransformer{err: transformerErr}} otherTransformerErr := PrefixTransformer{Prefix: []byte("other:"), Transformer: &testTransformer{err: transformerErr}}
reqCtx := genericapirequest.WithRequestInfo(context.Background(), &genericapirequest.RequestInfo{Resource: "test"})
testCases := []struct { testCases := []struct {
desc string desc string
input []byte input []byte
prefix Transformer prefix Transformer
metrics []string metrics []string
want string want string
err error expectErr bool
ctx context.Context
}{ }{
{ {
desc: "ok", desc: "ok",
@ -220,11 +226,42 @@ func TestPrefixToMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1
`, `,
err: nil, expectErr: false,
ctx: reqCtx,
},
{
desc: "missing request context",
input: []byte("value"),
prefix: NewPrefixTransformers(testErr, otherTransformer),
metrics: []string{
"apiserver_storage_transformation_operations_total",
},
want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{resource="",status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1
`,
expectErr: false,
ctx: context.Background(),
},
{
desc: "request context with api group",
input: []byte("value"),
prefix: NewPrefixTransformers(testErr, otherTransformer),
metrics: []string{
"apiserver_storage_transformation_operations_total",
},
want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{resource="test.testGroup",status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1
`,
expectErr: false,
ctx: genericapirequest.WithRequestInfo(context.Background(), &genericapirequest.RequestInfo{APIGroup: "testGroup", Resource: "test"}),
}, },
{ {
desc: "error", desc: "error",
@ -234,20 +271,23 @@ func TestPrefixToMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="other:"} 1 apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="other:"} 1
`, `,
err: nil, expectErr: true,
ctx: reqCtx,
}, },
} }
RegisterMetrics() RegisterMetrics()
transformerOperationsTotal.Reset() transformerOperationsTotal.Reset()
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) { t.Run(tc.desc, func(t *testing.T) {
tc.prefix.TransformToStorage(context.Background(), tc.input, nil) _, err := tc.prefix.TransformToStorage(tc.ctx, tc.input, nil)
if (err != nil) != tc.expectErr {
t.Fatal(err)
}
defer transformerOperationsTotal.Reset() defer transformerOperationsTotal.Reset()
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.want), tc.metrics...); err != nil { if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.want), tc.metrics...); err != nil {
t.Fatal(err) t.Fatal(err)
@ -285,7 +325,7 @@ func TestLogTransformErr(t *testing.T) {
ctx: context.Background(), ctx: context.Background(),
err: errors.New("decryption failed"), err: errors.New("decryption failed"),
message: "failed to decrypt data", message: "failed to decrypt data",
expectedLog: "\"failed to decrypt data\" err=\"decryption failed\" group=\"\" version=\"\" resource=\"\" subresource=\"\" verb=\"\" namespace=\"\" name=\"\"\n", expectedLog: "\"no request info on context\"\n\"failed to decrypt data\" err=\"decryption failed\" group=\"\" version=\"\" resource=\"\" subresource=\"\" verb=\"\" namespace=\"\" name=\"\"\n",
}, },
} }
@ -301,7 +341,7 @@ func TestLogTransformErr(t *testing.T) {
logTransformErr(tt.ctx, tt.err, tt.message) logTransformErr(tt.ctx, tt.err, tt.message)
// remove timestamp and goroutine id from log message to make it easier to compare // remove timestamp and goroutine id from log message to make it easier to compare
gotLog := regexp.MustCompile(`\w+ \d+:\d+:\d+\.\d+.*\d+.*transformer_test.go:\d+].`).ReplaceAllString(buf.String(), "") gotLog := regexp.MustCompile(`\w+ \d+:\d+:\d+\.\d+.*\d+.*(transformer_test\.go|transformer\.go):\d+].`).ReplaceAllString(buf.String(), "")
if gotLog != tt.expectedLog { if gotLog != tt.expectedLog {
t.Errorf("expected log message %q, got %q", tt.expectedLog, gotLog) t.Errorf("expected log message %q, got %q", tt.expectedLog, gotLog)

4
test/integration/controlplane/transformation/kmsv2_transformation_test.go
View File

@ -301,6 +301,8 @@ resources:
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`, `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`,
`apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`, `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`,
`apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`, `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`,
`apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v2:kms-provider:"} 2`,
`apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v2:kms-provider:"} 1`,
} }
defer func() { defer func() {
body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx) body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)
@ -310,7 +312,7 @@ resources:
var gotMetricStrings []string var gotMetricStrings []string
trimFP := regexp.MustCompile(`(.*)(} \d+\.\d+.*)`) trimFP := regexp.MustCompile(`(.*)(} \d+\.\d+.*)`)
for _, line := range strings.Split(string(body), "\n") { for _, line := range strings.Split(string(body), "\n") {
if strings.HasPrefix(line, "apiserver_envelope_") { if strings.HasPrefix(line, "apiserver_envelope_") || strings.HasPrefix(line, "apiserver_storage_transformation_operations") {
if strings.HasPrefix(line, "apiserver_envelope_encryption_dek_cache_fill_percent") { if strings.HasPrefix(line, "apiserver_envelope_encryption_dek_cache_fill_percent") {
continue // this can be ignored as it is KMS v1 only continue // this can be ignored as it is KMS v1 only
} }