From e54864f53de75cd56c0fe94777e1d3de0c559c7f Mon Sep 17 00:00:00 2001 From: immutablet Date: Tue, 27 Feb 2018 17:24:27 -0800 Subject: [PATCH] Instrument transformer.go with latency metrics. --- .../k8s.io/apiserver/pkg/storage/value/BUILD | 6 +- .../apiserver/pkg/storage/value/metrics.go | 55 +++++++++++++++++++ .../pkg/storage/value/transformer.go | 7 +++ test/integration/master/BUILD | 1 + .../master/secrets_transformation_test.go | 1 + .../master/transformation_testcase.go | 40 ++++++++++++-- 6 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD index 2e132c760ec..541bb5dfefe 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD @@ -14,8 +14,12 @@ go_test( go_library( name = "go_default_library", - srcs = ["transformer.go"], + srcs = [ + "metrics.go", + "transformer.go", + ], importpath = "k8s.io/apiserver/pkg/storage/value", + deps = ["//vendor/github.com/prometheus/client_golang/prometheus:go_default_library"], ) filegroup( diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go new file mode 100644 index 00000000000..1cdb006a44c --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go @@ -0,0 +1,55 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package value + +import ( + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + valueSubsystem = "value" +) + +var ( + TransformerOperationalLatencies = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Subsystem: valueSubsystem, + Name: "storage_transformation_latency_microseconds", + Help: "Latency in microseconds of value transformation operations.", + }, + []string{"transformation_type"}, + ) +) + +var registerMetrics sync.Once + +func RegisterMetrics() { + registerMetrics.Do(func() { + prometheus.MustRegister(TransformerOperationalLatencies) + }) +} + +func RecordTransformation(transformationType string, start time.Time) { + TransformerOperationalLatencies.WithLabelValues(transformationType).Observe(sinceInMicroseconds(start)) +} + +func sinceInMicroseconds(start time.Time) float64 { + return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds()) +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go index 04af5d6db7b..8998147169e 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go @@ -21,8 +21,13 @@ import ( "bytes" "fmt" "sync" + "time" ) +func init() { + RegisterMetrics() +} + // Context is additional information that a storage transformation may need to verify the data at rest. type Context interface { // AuthenticatedData should return an array of bytes that describes the current value. If the value changes, @@ -80,12 +85,14 @@ func (t *MutableTransformer) Set(transformer Transformer) { } func (t *MutableTransformer) TransformFromStorage(data []byte, context Context) (out []byte, stale bool, err error) { + defer RecordTransformation("from_storage", time.Now()) t.lock.RLock() transformer := t.transformer t.lock.RUnlock() return transformer.TransformFromStorage(data, context) } func (t *MutableTransformer) TransformToStorage(data []byte, context Context) (out []byte, err error) { + defer RecordTransformation("to_storage", time.Now()) t.lock.RLock() transformer := t.transformer t.lock.RUnlock() diff --git a/test/integration/master/BUILD b/test/integration/master/BUILD index 8c9d588720f..c4976379710 100644 --- a/test/integration/master/BUILD +++ b/test/integration/master/BUILD @@ -180,6 +180,7 @@ go_library( "//test/integration/framework:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library", diff --git a/test/integration/master/secrets_transformation_test.go b/test/integration/master/secrets_transformation_test.go index 34a790c775f..a8d408435d5 100644 --- a/test/integration/master/secrets_transformation_test.go +++ b/test/integration/master/secrets_transformation_test.go @@ -124,6 +124,7 @@ func runBenchmark(b *testing.B, transformerConfig string) { b.StartTimer() test.benchmark(b) b.StopTimer() + test.printMetrics() } func unSealWithGCMTransformer(cipherText []byte, ctx value.Context, diff --git a/test/integration/master/transformation_testcase.go b/test/integration/master/transformation_testcase.go index 48013bcf50d..7a88c1a0a91 100644 --- a/test/integration/master/transformation_testcase.go +++ b/test/integration/master/transformation_testcase.go @@ -29,6 +29,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/ghodss/yaml" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,11 +43,12 @@ import ( ) const ( - secretKey = "api_key" - secretVal = "086a7ffc-0225-11e8-ba89-0ed5f89f718b" - encryptionConfigFileName = "encryption.conf" - testNamespace = "secret-encryption-test" - testSecret = "test-secret" + secretKey = "api_key" + secretVal = "086a7ffc-0225-11e8-ba89-0ed5f89f718b" + encryptionConfigFileName = "encryption.conf" + testNamespace = "secret-encryption-test" + testSecret = "test-secret" + latencySummaryMetricsFamily = "value_storage_transformation_latency_microseconds" ) type unSealSecret func(cipherText []byte, ctx value.Context, config encryptionconfig.ProviderConfig) ([]byte, error) @@ -237,3 +239,31 @@ func (e *transformTest) readRawRecordFromETCD(path string) (*clientv3.GetRespons return response, nil } + +func (e *transformTest) printMetrics() error { + metrics, err := prometheus.DefaultGatherer.Gather() + if err != nil { + return fmt.Errorf("failed to gather metrics: %s", err) + } + + metricsOfInterest := []string{latencySummaryMetricsFamily} + + for _, mf := range metrics { + if contains(metricsOfInterest, *mf.Name) { + for _, metric := range mf.GetMetric() { + e.logger.Logf("%v", metric) + } + } + } + + return nil +} + +func contains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +}