diff --git a/test/instrumentation/BUILD b/test/instrumentation/BUILD index 06b96716acd..767a10783c3 100644 --- a/test/instrumentation/BUILD +++ b/test/instrumentation/BUILD @@ -1,10 +1,21 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["main.go"], + srcs = [ + "decode_metric.go", + "error.go", + "find_stable_metric.go", + "main.go", + "metric.go", + ], importpath = "k8s.io/kubernetes/test/instrumentation", visibility = ["//visibility:private"], + deps = [ + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/gopkg.in/yaml.v2:go_default_library", + ], ) go_binary( @@ -26,3 +37,30 @@ filegroup( tags = ["automanaged"], visibility = ["//visibility:public"], ) + +genrule( + name = "list_stable_metrics", + srcs = [ + "//:all-srcs", + ], + outs = ["stable-metrics-list.yaml"], + cmd = "./$(locations :instrumentation) $(locations //:all-srcs) > $@", + message = "Listing all stable metrics.", + tools = [":instrumentation"], +) + +sh_test( + name = "verify_stable_metric", + srcs = ["verify-stable-metrics.sh"], + data = [ + "testdata/stable-metrics-list.yaml", + ":list_stable_metrics", + ], +) + +go_test( + name = "go_default_test", + srcs = ["main_test.go"], + data = glob(["testdata/**"]), + embed = [":go_default_library"], +) diff --git a/test/instrumentation/README.md b/test/instrumentation/README.md index 62b0336e4bb..87d4c90fac1 100644 --- a/test/instrumentation/README.md +++ b/test/instrumentation/README.md @@ -1,3 +1,13 @@ -This is a WIP directory for ensuring stability rules around kubernetes metrics. +This directory contains the regression test for controlling the list of stable metrics -Design [Metrics validation and verification](https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-validation-and-verification.md) +If you add or remove a stable metric, this test will fail and you will need +to update the golden list of tests stored in `testdata/`. Changes to that file +require review by sig-instrumentation. + +To update the list, run + +```console +./update-stable-metrics.sh +``` + +Add the changed file to your PR, then send for review. diff --git a/test/instrumentation/decode_metric.go b/test/instrumentation/decode_metric.go new file mode 100644 index 00000000000..fc724aa2ec4 --- /dev/null +++ b/test/instrumentation/decode_metric.go @@ -0,0 +1,239 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "go/ast" + "go/token" + "sort" + "strconv" + "strings" + + "k8s.io/component-base/metrics" +) + +func decodeMetricCalls(fs []*ast.CallExpr, metricsImportName string) ([]metric, []error) { + finder := metricDecoder{ + metricsImportName: metricsImportName, + } + ms := make([]metric, 0, len(fs)) + errors := []error{} + for _, f := range fs { + m, err := finder.decodeNewMetricCall(f) + if err != nil { + errors = append(errors, err) + continue + } + ms = append(ms, m) + } + return ms, errors +} + +type metricDecoder struct { + metricsImportName string +} + +func (c *metricDecoder) decodeNewMetricCall(fc *ast.CallExpr) (metric, error) { + var m metric + var err error + se, ok := fc.Fun.(*ast.SelectorExpr) + if !ok { + return m, newDecodeErrorf(fc, errNotDirectCall) + } + functionName := se.Sel.String() + functionImport, ok := se.X.(*ast.Ident) + if !ok { + return m, newDecodeErrorf(fc, errNotDirectCall) + } + if functionImport.String() != c.metricsImportName { + return m, newDecodeErrorf(fc, errNotDirectCall) + } + switch functionName { + case "NewCounter", "NewGauge", "NewHistogram": + m, err = c.decodeMetric(fc) + case "NewCounterVec", "NewGaugeVec", "NewHistogramVec": + m, err = c.decodeMetricVec(fc) + case "NewSummary", "NewSummaryVec": + return m, newDecodeErrorf(fc, errStableSummary) + default: + return m, newDecodeErrorf(fc, errNotDirectCall) + } + if err != nil { + return m, err + } + m.Type = getMetricType(functionName) + return m, nil +} + +func getMetricType(functionName string) string { + switch functionName { + case "NewCounter", "NewCounterVec": + return counterMetricType + case "NewGauge", "NewGaugeVec": + return gaugeMetricType + case "NewHistogram", "NewHistogramVec": + return histogramMetricType + default: + panic("getMetricType expects correct function name") + } +} + +func (c *metricDecoder) decodeMetric(call *ast.CallExpr) (metric, error) { + if len(call.Args) != 1 { + return metric{}, newDecodeErrorf(call, errInvalidNewMetricCall) + } + return c.decodeOpts(call.Args[0]) +} + +func (c *metricDecoder) decodeMetricVec(call *ast.CallExpr) (metric, error) { + if len(call.Args) != 2 { + return metric{}, newDecodeErrorf(call, errInvalidNewMetricCall) + } + m, err := c.decodeOpts(call.Args[0]) + if err != nil { + return m, err + } + labels, err := decodeLabels(call.Args[1]) + if err != nil { + return m, err + } + sort.Strings(labels) + m.Labels = labels + return m, nil +} + +func decodeLabels(expr ast.Expr) ([]string, error) { + cl, ok := expr.(*ast.CompositeLit) + if !ok { + return nil, newDecodeErrorf(expr, errInvalidNewMetricCall) + } + labels := make([]string, len(cl.Elts)) + for i, el := range cl.Elts { + bl, ok := el.(*ast.BasicLit) + if !ok { + return nil, newDecodeErrorf(bl, errLabels) + } + if bl.Kind != token.STRING { + return nil, newDecodeErrorf(bl, errLabels) + } + labels[i] = strings.Trim(bl.Value, `"`) + } + return labels, nil +} + +func (c *metricDecoder) decodeOpts(expr ast.Expr) (metric, error) { + m := metric{ + Labels: []string{}, + } + ue, ok := expr.(*ast.UnaryExpr) + if !ok { + return m, newDecodeErrorf(expr, errInvalidNewMetricCall) + } + cl, ok := ue.X.(*ast.CompositeLit) + if !ok { + return m, newDecodeErrorf(expr, errInvalidNewMetricCall) + } + + for _, expr := range cl.Elts { + kv, ok := expr.(*ast.KeyValueExpr) + if !ok { + return m, newDecodeErrorf(expr, errPositionalArguments) + } + key := fmt.Sprintf("%v", kv.Key) + + switch key { + case "Namespace", "Subsystem", "Name", "Help": + k, ok := kv.Value.(*ast.BasicLit) + if !ok { + return m, newDecodeErrorf(expr, errNonStringAttribute) + } + if k.Kind != token.STRING { + return m, newDecodeErrorf(expr, errNonStringAttribute) + } + value := strings.Trim(k.Value, `"`) + switch key { + case "Namespace": + m.Namespace = value + case "Subsystem": + m.Subsystem = value + case "Name": + m.Name = value + case "Help": + m.Help = value + } + case "Buckets": + buckets, err := decodeBuckets(kv) + if err != nil { + return m, err + } + sort.Float64s(buckets) + m.Buckets = buckets + case "StabilityLevel": + level, err := decodeStabilityLevel(kv.Value, c.metricsImportName) + if err != nil { + return m, err + } + m.StabilityLevel = string(*level) + default: + return m, newDecodeErrorf(expr, errFieldNotSupported, key) + } + } + return m, nil +} + +func decodeBuckets(kv *ast.KeyValueExpr) ([]float64, error) { + cl, ok := kv.Value.(*ast.CompositeLit) + if !ok { + return nil, newDecodeErrorf(kv, errBuckets) + } + buckets := make([]float64, len(cl.Elts)) + for i, elt := range cl.Elts { + bl, ok := elt.(*ast.BasicLit) + if !ok { + return nil, newDecodeErrorf(bl, errBuckets) + } + if bl.Kind != token.FLOAT && bl.Kind != token.INT { + return nil, newDecodeErrorf(bl, errBuckets) + } + value, err := strconv.ParseFloat(bl.Value, 64) + if err != nil { + return nil, err + } + buckets[i] = value + } + return buckets, nil +} + +func decodeStabilityLevel(expr ast.Expr, metricsFrameworkImportName string) (*metrics.StabilityLevel, error) { + se, ok := expr.(*ast.SelectorExpr) + if !ok { + return nil, newDecodeErrorf(expr, errStabilityLevel) + } + s, ok := se.X.(*ast.Ident) + if !ok { + return nil, newDecodeErrorf(expr, errStabilityLevel) + } + if s.String() != metricsFrameworkImportName { + return nil, newDecodeErrorf(expr, errStabilityLevel) + } + if se.Sel.Name != "ALPHA" && se.Sel.Name != "STABLE" { + return nil, newDecodeErrorf(expr, errStabilityLevel) + } + stability := metrics.StabilityLevel(se.Sel.Name) + return &stability, nil +} diff --git a/test/instrumentation/error.go b/test/instrumentation/error.go new file mode 100644 index 00000000000..a4fc1a32513 --- /dev/null +++ b/test/instrumentation/error.go @@ -0,0 +1,59 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "go/ast" + "go/token" +) + +const ( + errNotDirectCall = "Opts for STABLE metric was not directly passed to new metric function" + errPositionalArguments = "Positional arguments are not supported" + errStabilityLevel = "StabilityLevel should be passed STABLE, ALPHA or removed" + errStableSummary = "Stable summary metric is not supported" + errInvalidNewMetricCall = "Invalid new metric call, please ensure code compiles" + errNonStringAttribute = "Non string attribute it not supported" + errFieldNotSupported = "Field %s is not supported" + errBuckets = "Buckets were not set to list of floats" + errLabels = "Labels were not set to list of strings" + errImport = `Importing through "." metrics framework is not supported` +) + +type decodeError struct { + msg string + pos token.Pos +} + +func newDecodeErrorf(node ast.Node, format string, a ...interface{}) *decodeError { + return &decodeError{ + msg: fmt.Sprintf(format, a...), + pos: node.Pos(), + } +} + +var _ error = (*decodeError)(nil) + +func (e decodeError) Error() string { + return e.msg +} + +func (e decodeError) errorWithFileInformation(fileset *token.FileSet) error { + position := fileset.Position(e.pos) + return fmt.Errorf("%s:%d:%d: %s", position.Filename, position.Line, position.Column, e.msg) +} diff --git a/test/instrumentation/find_stable_metric.go b/test/instrumentation/find_stable_metric.go new file mode 100644 index 00000000000..46605024a39 --- /dev/null +++ b/test/instrumentation/find_stable_metric.go @@ -0,0 +1,123 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "go/ast" + + "k8s.io/component-base/metrics" +) + +var metricsOptionStructuresNames = []string{ + "KubeOpts", + "CounterOpts", + "GaugeOpts", + "HistogramOpts", + "SummaryOpts", +} + +func findStableMetricDeclaration(tree ast.Node, metricsImportName string) ([]*ast.CallExpr, []error) { + v := stableMetricFinder{ + metricsImportName: metricsImportName, + stableMetricsFunctionCalls: []*ast.CallExpr{}, + errors: []error{}, + } + ast.Walk(&v, tree) + return v.stableMetricsFunctionCalls, v.errors +} + +// Implements visitor pattern for ast.Node that collects all stable metric expressions +type stableMetricFinder struct { + metricsImportName string + currentFunctionCall *ast.CallExpr + stableMetricsFunctionCalls []*ast.CallExpr + errors []error +} + +var _ ast.Visitor = (*stableMetricFinder)(nil) + +func (f *stableMetricFinder) Visit(node ast.Node) (w ast.Visitor) { + switch opts := node.(type) { + case *ast.CallExpr: + f.currentFunctionCall = opts + case *ast.CompositeLit: + se, ok := opts.Type.(*ast.SelectorExpr) + if !ok { + return f + } + if !isMetricOps(se.Sel.Name) { + return f + } + id, ok := se.X.(*ast.Ident) + if !ok { + return f + } + if id.Name != f.metricsImportName { + return f + } + stabilityLevel, err := getStabilityLevel(opts, f.metricsImportName) + if err != nil { + f.errors = append(f.errors, err) + return nil + } + switch *stabilityLevel { + case metrics.STABLE: + if f.currentFunctionCall == nil { + f.errors = append(f.errors, newDecodeErrorf(opts, errNotDirectCall)) + return nil + } + f.stableMetricsFunctionCalls = append(f.stableMetricsFunctionCalls, f.currentFunctionCall) + f.currentFunctionCall = nil + case metrics.ALPHA: + return nil + } + default: + if f.currentFunctionCall == nil || node == nil || node.Pos() < f.currentFunctionCall.Rparen { + return f + } + f.currentFunctionCall = nil + } + return f +} + +func isMetricOps(name string) bool { + var found = false + for _, optsName := range metricsOptionStructuresNames { + if name != optsName { + found = true + break + } + } + return found +} + +func getStabilityLevel(opts *ast.CompositeLit, metricsFrameworkImportName string) (*metrics.StabilityLevel, error) { + for _, expr := range opts.Elts { + kv, ok := expr.(*ast.KeyValueExpr) + if !ok { + return nil, newDecodeErrorf(expr, errPositionalArguments) + } + key := fmt.Sprintf("%v", kv.Key) + if key != "StabilityLevel" { + continue + } + return decodeStabilityLevel(kv.Value, metricsFrameworkImportName) + } + stability := metrics.ALPHA + return &stability, nil +} diff --git a/test/instrumentation/main.go b/test/instrumentation/main.go index 0aa277281b8..118f51413a0 100644 --- a/test/instrumentation/main.go +++ b/test/instrumentation/main.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,6 +16,120 @@ limitations under the License. package main -func main() { +import ( + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" + "gopkg.in/yaml.v2" +) + +const ( + metricFrameworkPath = `"k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"` + // Should equal to final directory name of metricFrameworkPath + defaultFrameworkImportName = "metrics" +) + +func main() { + flag.Parse() + if len(flag.Args()) < 1 { + fmt.Fprintf(os.Stderr, "USAGE: %s [...]\n", os.Args[0]) + os.Exit(64) + } + + stableMetrics := []metric{} + errors := []error{} + + for _, arg := range flag.Args() { + ms, es := searchPathForStableMetrics(arg) + stableMetrics = append(stableMetrics, ms...) + errors = append(errors, es...) + } + for _, err := range errors { + fmt.Fprintf(os.Stderr, "%s\n", err) + } + if len(errors) != 0 { + os.Exit(1) + } + sort.Sort(byFQName(stableMetrics)) + data, err := yaml.Marshal(stableMetrics) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + fmt.Print(string(data)) +} + +func searchPathForStableMetrics(path string) ([]metric, []error) { + ms := []metric{} + errors := []error{} + err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if strings.HasPrefix(path, "vendor") { + return filepath.SkipDir + } + if !strings.HasSuffix(path, ".go") { + return nil + } + ms, es := searchFileForStableMetrics(path, nil) + errors = append(errors, es...) + ms = append(ms, ms...) + return nil + }) + if err != nil { + errors = append(errors, err) + } + return ms, errors +} + +// Pass either only filename of existing file or src including source code in any format and a filename that it comes from +func searchFileForStableMetrics(filename string, src interface{}) ([]metric, []error) { + fileset := token.NewFileSet() + tree, err := parser.ParseFile(fileset, filename, src, parser.AllErrors) + if err != nil { + return []metric{}, []error{err} + } + metricsImportName, err := getMetricsFrameworkImportName(tree) + if err != nil { + return []metric{}, addFileInformationToErrors([]error{err}, fileset) + } + if metricsImportName == "" { + return []metric{}, []error{} + } + + stableMetricsFunctionCalls, errors := findStableMetricDeclaration(tree, metricsImportName) + metrics, es := decodeMetricCalls(stableMetricsFunctionCalls, metricsImportName) + errors = append(errors, es...) + return metrics, addFileInformationToErrors(errors, fileset) +} + +func getMetricsFrameworkImportName(tree *ast.File) (string, error) { + var importName string + for _, im := range tree.Imports { + if im.Path.Value == metricFrameworkPath { + if im.Name == nil { + importName = defaultFrameworkImportName + } else { + if im.Name.Name == "." { + return "", newDecodeErrorf(im, errImport) + } + importName = im.Name.Name + } + } + } + return importName, nil +} + +func addFileInformationToErrors(es []error, fileset *token.FileSet) []error { + for i := range es { + if de, ok := es[i].(*decodeError); ok { + es[i] = de.errorWithFileInformation(fileset) + } + } + return es } diff --git a/test/instrumentation/main_test.go b/test/instrumentation/main_test.go new file mode 100644 index 00000000000..5b81a64af59 --- /dev/null +++ b/test/instrumentation/main_test.go @@ -0,0 +1,493 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "reflect" + "testing" +) + +const fakeFilename = "testdata/metric.go" + +func TestSkipMetrics(t *testing.T) { + for _, test := range []struct { + testName string + src string + }{ + { + testName: "Skip alpha metric with local variable", + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var name = "metric" +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + Name: name, + StabilityLevel: metrics.ALPHA, + }, + ) +`}, + { + testName: "Skip alpha metric created via function call", + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +func getName() string { + return "metric" +} +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + Name: getName(), + StabilityLevel: metrics.ALPHA, + }, + ) +`}, + { + testName: "Skip metric without stability set", + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + Name: "metric", + }, + ) +`}, + { + testName: "Skip functions of similar signature (not imported from framework path) with import rename", + src: ` +package test +import metrics "k8s.io/fake/path" +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + StabilityLevel: metrics.STABLE, + }, + ) +`}, + { + testName: "Skip functions of similar signature (not imported from framework path)", + src: ` +package test +import "k8s.io/fake/path/metrics" +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + StabilityLevel: metrics.STABLE, + }, + ) +`}, + { + testName: "Skip . package import of non metric framework", + src: ` +package test +import . "k8s.io/fake/path" +var _ = NewCounter( + &CounterOpts{ + StabilityLevel: STABLE, + }, + ) +`}, + } { + t.Run(test.testName, func(t *testing.T) { + metrics, errors := searchFileForStableMetrics(fakeFilename, test.src) + if len(metrics) != 0 { + t.Errorf("Didn't expect any stable metrics found, got: %d", len(metrics)) + } + if len(errors) != 0 { + t.Errorf("Didn't expect any errors found, got: %s", errors) + } + }) + } +} + +func TestStableMetric(t *testing.T) { + for _, test := range []struct { + testName string + src string + metric metric + }{ + { + testName: "Counter", + metric: metric{ + Name: "metric", + Namespace: "namespace", + Subsystem: "subsystem", + StabilityLevel: "STABLE", + Help: "help", + Type: counterMetricType, + }, + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + Name: "metric", + Subsystem: "subsystem", + Namespace: "namespace", + Help: "help", + StabilityLevel: metrics.STABLE, + }, +) +`}, + { + testName: "CounterVec", + metric: metric{ + Name: "metric", + Namespace: "namespace", + Subsystem: "subsystem", + Labels: []string{"label-1"}, + StabilityLevel: "STABLE", + Help: "help", + Type: counterMetricType, + }, + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "metric", + Namespace: "namespace", + Subsystem: "subsystem", + Help: "help", + StabilityLevel: metrics.STABLE, + }, + []string{"label-1"}, + ) +`}, + { + testName: "Gauge", + metric: metric{ + Name: "gauge", + Namespace: "namespace", + Subsystem: "subsystem", + StabilityLevel: "STABLE", + Help: "help", + Type: gaugeMetricType, + }, + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewGauge( + &metrics.GaugeOpts{ + Name: "gauge", + Namespace: "namespace", + Subsystem: "subsystem", + Help: "help", + StabilityLevel: metrics.STABLE, + }, + ) +`}, + { + testName: "GaugeVec", + metric: metric{ + Name: "gauge", + Namespace: "namespace", + Subsystem: "subsystem", + StabilityLevel: "STABLE", + Help: "help", + Type: gaugeMetricType, + Labels: []string{"label-1", "label-2"}, + }, + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Name: "gauge", + Namespace: "namespace", + Subsystem: "subsystem", + Help: "help", + StabilityLevel: metrics.STABLE, + }, + []string{"label-2", "label-1"}, + ) +`}, + { + testName: "Histogram", + metric: metric{ + Name: "histogram", + Namespace: "namespace", + Subsystem: "subsystem", + StabilityLevel: "STABLE", + Buckets: []float64{0.001, 0.01, 0.1, 1, 10, 100}, + Help: "help", + Type: histogramMetricType, + }, + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewHistogram( + &metrics.HistogramOpts{ + Name: "histogram", + Namespace: "namespace", + Subsystem: "subsystem", + StabilityLevel: metrics.STABLE, + Help: "help", + Buckets: []float64{0.001, 0.01, 0.1, 1, 10, 100}, + }, + ) +`}, + { + testName: "HistogramVec", + metric: metric{ + Name: "histogram", + Namespace: "namespace", + Subsystem: "subsystem", + StabilityLevel: "STABLE", + Buckets: []float64{0.001, 0.01, 0.1, 1, 10, 100}, + Help: "help", + Type: histogramMetricType, + Labels: []string{"label-1", "label-2"}, + }, + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Name: "histogram", + Namespace: "namespace", + Subsystem: "subsystem", + StabilityLevel: metrics.STABLE, + Help: "help", + Buckets: []float64{0.001, 0.01, 0.1, 1, 10, 100}, + }, + []string{"label-2", "label-1"}, + ) +`}, + { + testName: "Custom import", + metric: metric{ + Name: "metric", + StabilityLevel: "STABLE", + Type: counterMetricType, + }, + src: ` +package test +import custom "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = custom.NewCounter( + &custom.CounterOpts{ + Name: "metric", + StabilityLevel: custom.STABLE, + }, + ) +`}, + } { + t.Run(test.testName, func(t *testing.T) { + metrics, errors := searchFileForStableMetrics(fakeFilename, test.src) + if len(errors) != 0 { + t.Errorf("Unexpected errors: %s", errors) + } + if len(metrics) != 1 { + t.Fatalf("Unexpected number of metrics: got %d, want 1", len(metrics)) + } + if test.metric.Labels == nil { + test.metric.Labels = []string{} + } + if !reflect.DeepEqual(metrics[0], test.metric) { + t.Errorf("metric:\ngot %v\nwant %v", metrics[0], test.metric) + } + }) + } +} + +func TestIncorrectStableMetricDeclarations(t *testing.T) { + for _, test := range []struct { + testName string + src string + err error + }{ + { + testName: "Fail on stable summary metric (Summary is DEPRECATED)", + err: fmt.Errorf("testdata/metric.go:4:9: Stable summary metric is not supported"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewSummary( + &metrics.SummaryOpts{ + StabilityLevel: metrics.STABLE, + }, + ) +`}, + { + testName: "Fail on stable metric with attribute set to variable", + err: fmt.Errorf("testdata/metric.go:7:4: Non string attribute it not supported"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +const name = "metric" +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + Name: name, + StabilityLevel: metrics.STABLE, + }, + ) +`}, + { + testName: "Fail on stable metric with attribute set to local function return", + err: fmt.Errorf("testdata/metric.go:9:4: Non string attribute it not supported"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +func getName() string { + return "metric" +} +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + Name: getName(), + StabilityLevel: metrics.STABLE, + }, + ) +`}, + { + testName: "Fail on stable metric with attribute set to imported function return", + err: fmt.Errorf("testdata/metric.go:7:4: Non string attribute it not supported"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +import "k8s.io/kubernetes/utils" +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + Name: utils.getMetricName(), + StabilityLevel: metrics.STABLE, + }, + ) +`}, + { + testName: "Fail on metric with stability set to function return", + err: fmt.Errorf("testdata/metric.go:9:20: StabilityLevel should be passed STABLE, ALPHA or removed"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +func getMetricStability() metrics.StabilityLevel { + return metrics.STABLE +} +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + StabilityLevel: getMetricsStability(), + }, + ) +`}, + { + testName: "error for passing stability as string", + err: fmt.Errorf("testdata/metric.go:6:20: StabilityLevel should be passed STABLE, ALPHA or removed"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + StabilityLevel: "stable", + }, + ) +`}, + { + testName: "error for passing stability as unknown const", + err: fmt.Errorf("testdata/metric.go:6:20: StabilityLevel should be passed STABLE, ALPHA or removed"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + StabilityLevel: metrics.UNKNOWN, + }, + ) +`}, + { + testName: "error for passing stability as variable", + err: fmt.Errorf("testdata/metric.go:7:20: StabilityLevel should be passed STABLE, ALPHA or removed"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var stable = metrics.STABLE +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + StabilityLevel: stable, + }, + ) +`}, + { + testName: "error for stable metric created via function call", + err: fmt.Errorf("testdata/metric.go:6:10: Opts for STABLE metric was not directly passed to new metric function"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewCounter(getStableCounterOpts()) +func getStableCounterOpts() *metrics.CounterOpts { + return &metrics.CounterOpts{ + StabilityLevel: metrics.STABLE, + } +} +`}, + { + testName: "error . package import of metric framework", + err: fmt.Errorf(`testdata/metric.go:3:8: Importing through "." metrics framework is not supported`), + src: ` +package test +import . "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = NewCounter( + &CounterOpts{ + StabilityLevel: STABLE, + }, + ) +`}, + { + testName: "error stable metric opts passed to local function", + err: fmt.Errorf("testdata/metric.go:4:9: Opts for STABLE metric was not directly passed to new metric function"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = RegisterMetric( + &metrics.CounterOpts{ + StabilityLevel: metrics.STABLE, + }, + ) +`}, + { + testName: "error stable metric opts passed to imported function", + err: fmt.Errorf("testdata/metric.go:4:9: Opts for STABLE metric was not directly passed to new metric function"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = test.RegisterMetric( + &metrics.CounterOpts{ + StabilityLevel: metrics.STABLE, + }, + ) +`}, + { + testName: "error stable metric opts passed to imported function", + err: fmt.Errorf("testdata/metric.go:6:4: Positional arguments are not supported"), + src: ` +package test +import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics" +var _ = metrics.NewCounter( + &metrics.CounterOpts{ + "counter", + }, + ) +`}, + } { + t.Run(test.testName, func(t *testing.T) { + _, errors := searchFileForStableMetrics(fakeFilename, test.src) + if len(errors) != 1 { + t.Fatalf("Unexpected number of errors, got %d, want 1", len(errors)) + } + if !reflect.DeepEqual(errors[0], test.err) { + t.Errorf("error:\ngot %v\nwant %v", errors[0], test.err) + } + }) + } +} diff --git a/test/instrumentation/metric.go b/test/instrumentation/metric.go new file mode 100644 index 00000000000..f786c99a890 --- /dev/null +++ b/test/instrumentation/metric.go @@ -0,0 +1,53 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const ( + counterMetricType = "Counter" + gaugeMetricType = "Gauge" + histogramMetricType = "Histogram" +) + +type metric struct { + Name string `yaml:"name"` + Subsystem string `yaml:"subsystem,omitempty"` + Namespace string `yaml:"namespace,omitempty"` + Help string `yaml:"help,omitempty"` + Type string `yaml:"type,omitempty"` + DeprecatedVersion string `yaml:"deprecatedVersion,omitempty"` + StabilityLevel string `yaml:"stabilityLevel,omitempty"` + Labels []string `yaml:"labels,omitempty"` + Buckets []float64 `yaml:"buckets,omitempty"` +} + +func (m metric) buildFQName() string { + return prometheus.BuildFQName(m.Namespace, m.Subsystem, m.Name) +} + +type byFQName []metric + +func (ms byFQName) Len() int { return len(ms) } +func (ms byFQName) Less(i, j int) bool { + return ms[i].buildFQName() < ms[j].buildFQName() +} +func (ms byFQName) Swap(i, j int) { + ms[i], ms[j] = ms[j], ms[i] +} diff --git a/test/instrumentation/testdata/stable-metrics-list.yaml b/test/instrumentation/testdata/stable-metrics-list.yaml new file mode 100644 index 00000000000..fe51488c706 --- /dev/null +++ b/test/instrumentation/testdata/stable-metrics-list.yaml @@ -0,0 +1 @@ +[] diff --git a/test/instrumentation/update-stable-metrics.sh b/test/instrumentation/update-stable-metrics.sh new file mode 100755 index 00000000000..99d5be53aef --- /dev/null +++ b/test/instrumentation/update-stable-metrics.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. + +bazel build //test/instrumentation:list_stable_metrics +cp "$KUBE_ROOT/bazel-genfiles/test/instrumentation/stable-metrics-list.yaml" "$KUBE_ROOT/test/instrumentation/testdata/stable-metrics-list.yaml" diff --git a/test/instrumentation/verify-stable-metrics.sh b/test/instrumentation/verify-stable-metrics.sh new file mode 100755 index 00000000000..89d761b1f5e --- /dev/null +++ b/test/instrumentation/verify-stable-metrics.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -o errexit +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. + +# detect if run from bazel +if [ -z "${TEST_BINARY}" ]; then + bazel build //test/instrumentation:list_stable_metrics + OUTPUT_FILE="$KUBE_ROOT/bazel-genfiles/test/instrumentation/stable-metrics-list.yaml" +else + OUTPUT_FILE="$KUBE_ROOT/test/instrumentation/stable-metrics-list.yaml" +fi + +if diff -u "$KUBE_ROOT/test/instrumentation/testdata/stable-metrics-list.yaml" "$OUTPUT_FILE"; then + echo PASS + exit 0 +fi +echo 'Diffs in stable metrics detected, please run "test/instrumentation/update-stable-metrics.sh"' +exit 1