github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-29 22:46:12 +00:00
Code Issues Projects Releases Wiki Activity

change envelope transformer to return status error for better monitoring

Browse Source 
Change-Id: I8263c4673d5f57617acf315c7af6ebe5aacd9c7c
This commit is contained in:
Shihang Zhang 2019-09-10 13:12:31 -07:00
parent 4ce0a300d7
commit cba43530d7
2 changed files with 64 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go
View File

@ -94,13 +94,17 @@ func (t *envelopeTransformer) TransformFromStorage(data []byte, context value.Co
value.RecordCacheMiss() value.RecordCacheMiss()
key, err := t.envelopeService.Decrypt(encKey) key, err := t.envelopeService.Decrypt(encKey)
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while decrypting key: %q", err) // Do NOT wrap this err using fmt.Errorf() or similar functions
// because this gRPC status error has useful error code when
// record the metric.
return nil, false, err
} }
transformer, err = t.addTransformer(encKey, key) transformer, err = t.addTransformer(encKey, key)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }
} }
return transformer.TransformFromStorage(encData, context) return transformer.TransformFromStorage(encData, context)
} }
@ -113,6 +117,9 @@ func (t *envelopeTransformer) TransformToStorage(data []byte, context value.Cont
encKey, err := t.envelopeService.Encrypt(newKey) encKey, err := t.envelopeService.Encrypt(newKey)
if err != nil { if err != nil {
// Do NOT wrap this err using fmt.Errorf() or similar functions
// because this gRPC status error has useful error code when
// record the metric.
return nil, err return nil, err
} }

79
staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go
View File

@ -20,7 +20,6 @@ import (
"errors" "errors"
"strings" "strings"
"testing" "testing"
"time"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
@ -30,63 +29,97 @@ import (
) )
func TestTotals(t *testing.T) { func TestTotals(t *testing.T) {
nonStatusErr := errors.New("test error")
failedPreconditionErr := status.Error(codes.FailedPrecondition, "test error")
internalErr := status.Error(codes.Internal, "test error")
nonStatusErrTransformer := PrefixTransformer{Prefix: []byte("k8s:enc:kms:v1:"), Transformer: &testTransformer{err: nonStatusErr}}
failedPreconditionErrTransformer := PrefixTransformer{Prefix: []byte("k8s:enc:kms:v1:"), Transformer: &testTransformer{err: failedPreconditionErr}}
internalErrTransformer := PrefixTransformer{Prefix: []byte("k8s:enc:kms:v1:"), Transformer: &testTransformer{err: internalErr}}
okTransformer := PrefixTransformer{Prefix: []byte("k8s:enc:kms:v1:"), Transformer: &testTransformer{from: []byte("value")}}
testCases := []struct { testCases := []struct {
desc string desc string
prefix Transformer
metrics []string metrics []string
error error
want string want string
}{ }{
{ {
desc: "non-status error", desc: "non-status error",
prefix: NewPrefixTransformers(nil, nonStatusErrTransformer),
metrics: []string{ metrics: []string{
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
"apiserver_storage_transformation_failures_total", "apiserver_storage_transformation_failures_total",
}, },
error: errors.New("foo"),
want: ` want: `
# HELP apiserver_storage_transformation_failures_total [ALPHA] (Deprecated) Total number of failed transformation operations. # HELP apiserver_storage_transformation_failures_total [ALPHA] (Deprecated) Total number of failed transformation operations.
# TYPE apiserver_storage_transformation_failures_total counter # TYPE apiserver_storage_transformation_failures_total counter
apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1 apiserver_storage_transformation_failures_total{transformation_type="from_storage"} 1
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. apiserver_storage_transformation_failures_total{transformation_type="to_storage"} 1
# TYPE apiserver_storage_transformation_operations_total counter # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1 # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,
}, },
{ {
desc: "error is nil", desc: "ok",
prefix: NewPrefixTransformers(nil, okTransformer),
metrics: []string{ metrics: []string{
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
"apiserver_storage_transformation_failures_total", "apiserver_storage_transformation_failures_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="OK",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
apiserver_storage_transformation_operations_total{status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,
}, },
{ {
desc: "status error from kms-plugin", desc: "failed precondition",
prefix: NewPrefixTransformers(nil, failedPreconditionErrTransformer),
metrics: []string{ metrics: []string{
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
"apiserver_storage_transformation_failures_total", "apiserver_storage_transformation_failures_total",
}, },
error: status.Error(codes.FailedPrecondition, "foo"),
want: ` want: `
# HELP apiserver_storage_transformation_failures_total [ALPHA] (Deprecated) Total number of failed transformation operations. # HELP apiserver_storage_transformation_failures_total [ALPHA] (Deprecated) Total number of failed transformation operations.
# TYPE apiserver_storage_transformation_failures_total counter # TYPE apiserver_storage_transformation_failures_total counter
apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1 apiserver_storage_transformation_failures_total{transformation_type="from_storage"} 1
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. apiserver_storage_transformation_failures_total{transformation_type="to_storage"} 1
# TYPE apiserver_storage_transformation_operations_total counter # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1 # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
`,
},
{
desc: "internal",
prefix: NewPrefixTransformers(nil, internalErrTransformer),
metrics: []string{
"apiserver_storage_transformation_operations_total",
"apiserver_storage_transformation_failures_total",
},
want: `
# HELP apiserver_storage_transformation_failures_total [ALPHA] (Deprecated) Total number of failed transformation operations.
# TYPE apiserver_storage_transformation_failures_total counter
apiserver_storage_transformation_failures_total{transformation_type="from_storage"} 1
apiserver_storage_transformation_failures_total{transformation_type="to_storage"} 1
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="Internal",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
apiserver_storage_transformation_operations_total{status="Internal",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,
}, },
} }
RegisterMetrics() RegisterMetrics()
transformerOperationsTotal.Reset()
for _, tt := range testCases { for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) { t.Run(tt.desc, func(t *testing.T) {
RecordTransformation("encrypt", "k8s:enc:kms:v1:", time.Now(), tt.error) tt.prefix.TransformToStorage([]byte("value"), nil)
tt.prefix.TransformFromStorage([]byte("k8s:enc:kms:v1:value"), nil)
defer transformerOperationsTotal.Reset() defer transformerOperationsTotal.Reset()
defer deprecatedTransformerFailuresTotal.Reset() defer deprecatedTransformerFailuresTotal.Reset()
if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {