Implement stable metric validation and verification

Based on KEP:
https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-validation-and-verification.md

Add //test/instrumentation:stable_metric_test that compares metrics
in source code to those available in
"test/instrumentation/testdata/stable-metrics-list.yaml".

test/instrumentation/BUILD

@@ -1,10 +1,21 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["main.go"],
+    srcs = [
+        "decode_metric.go",
+        "error.go",
+        "find_stable_metric.go",
+        "main.go",
+        "metric.go",
+    ],
     importpath = "k8s.io/kubernetes/test/instrumentation",
     visibility = ["//visibility:private"],
+    deps = [
+        "//staging/src/k8s.io/component-base/metrics:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/gopkg.in/yaml.v2:go_default_library",
+    ],
 )
 
 go_binary(
@@ -26,3 +37,30 @@ filegroup(
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
 )
+
+genrule(
+    name = "list_stable_metrics",
+    srcs = [
+        "//:all-srcs",
+    ],
+    outs = ["stable-metrics-list.yaml"],
+    cmd = "./$(locations :instrumentation) $(locations //:all-srcs) > $@",
+    message = "Listing all stable metrics.",
+    tools = [":instrumentation"],
+)
+
+sh_test(
+    name = "verify_stable_metric",
+    srcs = ["verify-stable-metrics.sh"],
+    data = [
+        "testdata/stable-metrics-list.yaml",
+        ":list_stable_metrics",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["main_test.go"],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+)

test/instrumentation/README.md

@@ -1,3 +1,13 @@
-This is a WIP directory for ensuring stability rules around kubernetes metrics.
+This directory contains the regression test for controlling the list of stable metrics
 
-Design [Metrics validation and verification](https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-validation-and-verification.md)
+If you add or remove a stable metric, this test will fail and you will need
+to update the golden list of tests stored in `testdata/`.  Changes to that file
+require review by sig-instrumentation.
+
+To update the list, run
+
+```console
+./update-stable-metrics.sh
+```
+
+Add the changed file to your PR, then send for review.

test/instrumentation/decode_metric.go

@@ -0,0 +1,239 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	"go/token"
+	"sort"
+	"strconv"
+	"strings"
+
+	"k8s.io/component-base/metrics"
+)
+
+func decodeMetricCalls(fs []*ast.CallExpr, metricsImportName string) ([]metric, []error) {
+	finder := metricDecoder{
+		metricsImportName: metricsImportName,
+	}
+	ms := make([]metric, 0, len(fs))
+	errors := []error{}
+	for _, f := range fs {
+		m, err := finder.decodeNewMetricCall(f)
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+		ms = append(ms, m)
+	}
+	return ms, errors
+}
+
+type metricDecoder struct {
+	metricsImportName string
+}
+
+func (c *metricDecoder) decodeNewMetricCall(fc *ast.CallExpr) (metric, error) {
+	var m metric
+	var err error
+	se, ok := fc.Fun.(*ast.SelectorExpr)
+	if !ok {
+		return m, newDecodeErrorf(fc, errNotDirectCall)
+	}
+	functionName := se.Sel.String()
+	functionImport, ok := se.X.(*ast.Ident)
+	if !ok {
+		return m, newDecodeErrorf(fc, errNotDirectCall)
+	}
+	if functionImport.String() != c.metricsImportName {
+		return m, newDecodeErrorf(fc, errNotDirectCall)
+	}
+	switch functionName {
+	case "NewCounter", "NewGauge", "NewHistogram":
+		m, err = c.decodeMetric(fc)
+	case "NewCounterVec", "NewGaugeVec", "NewHistogramVec":
+		m, err = c.decodeMetricVec(fc)
+	case "NewSummary", "NewSummaryVec":
+		return m, newDecodeErrorf(fc, errStableSummary)
+	default:
+		return m, newDecodeErrorf(fc, errNotDirectCall)
+	}
+	if err != nil {
+		return m, err
+	}
+	m.Type = getMetricType(functionName)
+	return m, nil
+}
+
+func getMetricType(functionName string) string {
+	switch functionName {
+	case "NewCounter", "NewCounterVec":
+		return counterMetricType
+	case "NewGauge", "NewGaugeVec":
+		return gaugeMetricType
+	case "NewHistogram", "NewHistogramVec":
+		return histogramMetricType
+	default:
+		panic("getMetricType expects correct function name")
+	}
+}
+
+func (c *metricDecoder) decodeMetric(call *ast.CallExpr) (metric, error) {
+	if len(call.Args) != 1 {
+		return metric{}, newDecodeErrorf(call, errInvalidNewMetricCall)
+	}
+	return c.decodeOpts(call.Args[0])
+}
+
+func (c *metricDecoder) decodeMetricVec(call *ast.CallExpr) (metric, error) {
+	if len(call.Args) != 2 {
+		return metric{}, newDecodeErrorf(call, errInvalidNewMetricCall)
+	}
+	m, err := c.decodeOpts(call.Args[0])
+	if err != nil {
+		return m, err
+	}
+	labels, err := decodeLabels(call.Args[1])
+	if err != nil {
+		return m, err
+	}
+	sort.Strings(labels)
+	m.Labels = labels
+	return m, nil
+}
+
+func decodeLabels(expr ast.Expr) ([]string, error) {
+	cl, ok := expr.(*ast.CompositeLit)
+	if !ok {
+		return nil, newDecodeErrorf(expr, errInvalidNewMetricCall)
+	}
+	labels := make([]string, len(cl.Elts))
+	for i, el := range cl.Elts {
+		bl, ok := el.(*ast.BasicLit)
+		if !ok {
+			return nil, newDecodeErrorf(bl, errLabels)
+		}
+		if bl.Kind != token.STRING {
+			return nil, newDecodeErrorf(bl, errLabels)
+		}
+		labels[i] = strings.Trim(bl.Value, `"`)
+	}
+	return labels, nil
+}
+
+func (c *metricDecoder) decodeOpts(expr ast.Expr) (metric, error) {
+	m := metric{
+		Labels: []string{},
+	}
+	ue, ok := expr.(*ast.UnaryExpr)
+	if !ok {
+		return m, newDecodeErrorf(expr, errInvalidNewMetricCall)
+	}
+	cl, ok := ue.X.(*ast.CompositeLit)
+	if !ok {
+		return m, newDecodeErrorf(expr, errInvalidNewMetricCall)
+	}
+
+	for _, expr := range cl.Elts {
+		kv, ok := expr.(*ast.KeyValueExpr)
+		if !ok {
+			return m, newDecodeErrorf(expr, errPositionalArguments)
+		}
+		key := fmt.Sprintf("%v", kv.Key)
+
+		switch key {
+		case "Namespace", "Subsystem", "Name", "Help":
+			k, ok := kv.Value.(*ast.BasicLit)
+			if !ok {
+				return m, newDecodeErrorf(expr, errNonStringAttribute)
+			}
+			if k.Kind != token.STRING {
+				return m, newDecodeErrorf(expr, errNonStringAttribute)
+			}
+			value := strings.Trim(k.Value, `"`)
+			switch key {
+			case "Namespace":
+				m.Namespace = value
+			case "Subsystem":
+				m.Subsystem = value
+			case "Name":
+				m.Name = value
+			case "Help":
+				m.Help = value
+			}
+		case "Buckets":
+			buckets, err := decodeBuckets(kv)
+			if err != nil {
+				return m, err
+			}
+			sort.Float64s(buckets)
+			m.Buckets = buckets
+		case "StabilityLevel":
+			level, err := decodeStabilityLevel(kv.Value, c.metricsImportName)
+			if err != nil {
+				return m, err
+			}
+			m.StabilityLevel = string(*level)
+		default:
+			return m, newDecodeErrorf(expr, errFieldNotSupported, key)
+		}
+	}
+	return m, nil
+}
+
+func decodeBuckets(kv *ast.KeyValueExpr) ([]float64, error) {
+	cl, ok := kv.Value.(*ast.CompositeLit)
+	if !ok {
+		return nil, newDecodeErrorf(kv, errBuckets)
+	}
+	buckets := make([]float64, len(cl.Elts))
+	for i, elt := range cl.Elts {
+		bl, ok := elt.(*ast.BasicLit)
+		if !ok {
+			return nil, newDecodeErrorf(bl, errBuckets)
+		}
+		if bl.Kind != token.FLOAT && bl.Kind != token.INT {
+			return nil, newDecodeErrorf(bl, errBuckets)
+		}
+		value, err := strconv.ParseFloat(bl.Value, 64)
+		if err != nil {
+			return nil, err
+		}
+		buckets[i] = value
+	}
+	return buckets, nil
+}
+
+func decodeStabilityLevel(expr ast.Expr, metricsFrameworkImportName string) (*metrics.StabilityLevel, error) {
+	se, ok := expr.(*ast.SelectorExpr)
+	if !ok {
+		return nil, newDecodeErrorf(expr, errStabilityLevel)
+	}
+	s, ok := se.X.(*ast.Ident)
+	if !ok {
+		return nil, newDecodeErrorf(expr, errStabilityLevel)
+	}
+	if s.String() != metricsFrameworkImportName {
+		return nil, newDecodeErrorf(expr, errStabilityLevel)
+	}
+	if se.Sel.Name != "ALPHA" && se.Sel.Name != "STABLE" {
+		return nil, newDecodeErrorf(expr, errStabilityLevel)
+	}
+	stability := metrics.StabilityLevel(se.Sel.Name)
+	return &stability, nil
+}

test/instrumentation/error.go

@@ -0,0 +1,59 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	"go/token"
+)
+
+const (
+	errNotDirectCall        = "Opts for STABLE metric was not directly passed to new metric function"
+	errPositionalArguments  = "Positional arguments are not supported"
+	errStabilityLevel       = "StabilityLevel should be passed STABLE, ALPHA or removed"
+	errStableSummary        = "Stable summary metric is not supported"
+	errInvalidNewMetricCall = "Invalid new metric call, please ensure code compiles"
+	errNonStringAttribute   = "Non string attribute it not supported"
+	errFieldNotSupported    = "Field %s is not supported"
+	errBuckets              = "Buckets were not set to list of floats"
+	errLabels               = "Labels were not set to list of strings"
+	errImport               = `Importing through "." metrics framework is not supported`
+)
+
+type decodeError struct {
+	msg string
+	pos token.Pos
+}
+
+func newDecodeErrorf(node ast.Node, format string, a ...interface{}) *decodeError {
+	return &decodeError{
+		msg: fmt.Sprintf(format, a...),
+		pos: node.Pos(),
+	}
+}
+
+var _ error = (*decodeError)(nil)
+
+func (e decodeError) Error() string {
+	return e.msg
+}
+
+func (e decodeError) errorWithFileInformation(fileset *token.FileSet) error {
+	position := fileset.Position(e.pos)
+	return fmt.Errorf("%s:%d:%d: %s", position.Filename, position.Line, position.Column, e.msg)
+}

test/instrumentation/find_stable_metric.go

@@ -0,0 +1,123 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"go/ast"
+
+	"k8s.io/component-base/metrics"
+)
+
+var metricsOptionStructuresNames = []string{
+	"KubeOpts",
+	"CounterOpts",
+	"GaugeOpts",
+	"HistogramOpts",
+	"SummaryOpts",
+}
+
+func findStableMetricDeclaration(tree ast.Node, metricsImportName string) ([]*ast.CallExpr, []error) {
+	v := stableMetricFinder{
+		metricsImportName:          metricsImportName,
+		stableMetricsFunctionCalls: []*ast.CallExpr{},
+		errors:                     []error{},
+	}
+	ast.Walk(&v, tree)
+	return v.stableMetricsFunctionCalls, v.errors
+}
+
+// Implements visitor pattern for ast.Node that collects all stable metric expressions
+type stableMetricFinder struct {
+	metricsImportName          string
+	currentFunctionCall        *ast.CallExpr
+	stableMetricsFunctionCalls []*ast.CallExpr
+	errors                     []error
+}
+
+var _ ast.Visitor = (*stableMetricFinder)(nil)
+
+func (f *stableMetricFinder) Visit(node ast.Node) (w ast.Visitor) {
+	switch opts := node.(type) {
+	case *ast.CallExpr:
+		f.currentFunctionCall = opts
+	case *ast.CompositeLit:
+		se, ok := opts.Type.(*ast.SelectorExpr)
+		if !ok {
+			return f
+		}
+		if !isMetricOps(se.Sel.Name) {
+			return f
+		}
+		id, ok := se.X.(*ast.Ident)
+		if !ok {
+			return f
+		}
+		if id.Name != f.metricsImportName {
+			return f
+		}
+		stabilityLevel, err := getStabilityLevel(opts, f.metricsImportName)
+		if err != nil {
+			f.errors = append(f.errors, err)
+			return nil
+		}
+		switch *stabilityLevel {
+		case metrics.STABLE:
+			if f.currentFunctionCall == nil {
+				f.errors = append(f.errors, newDecodeErrorf(opts, errNotDirectCall))
+				return nil
+			}
+			f.stableMetricsFunctionCalls = append(f.stableMetricsFunctionCalls, f.currentFunctionCall)
+			f.currentFunctionCall = nil
+		case metrics.ALPHA:
+			return nil
+		}
+	default:
+		if f.currentFunctionCall == nil || node == nil || node.Pos() < f.currentFunctionCall.Rparen {
+			return f
+		}
+		f.currentFunctionCall = nil
+	}
+	return f
+}
+
+func isMetricOps(name string) bool {
+	var found = false
+	for _, optsName := range metricsOptionStructuresNames {
+		if name != optsName {
+			found = true
+			break
+		}
+	}
+	return found
+}
+
+func getStabilityLevel(opts *ast.CompositeLit, metricsFrameworkImportName string) (*metrics.StabilityLevel, error) {
+	for _, expr := range opts.Elts {
+		kv, ok := expr.(*ast.KeyValueExpr)
+		if !ok {
+			return nil, newDecodeErrorf(expr, errPositionalArguments)
+		}
+		key := fmt.Sprintf("%v", kv.Key)
+		if key != "StabilityLevel" {
+			continue
+		}
+		return decodeStabilityLevel(kv.Value, metricsFrameworkImportName)
+	}
+	stability := metrics.ALPHA
+	return &stability, nil
+}

test/instrumentation/main.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2017 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,6 +16,120 @@ limitations under the License.
 
 package main
 
-func main() {
+import (
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
 
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	metricFrameworkPath = `"k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"`
+	// Should equal to final directory name of metricFrameworkPath
+	defaultFrameworkImportName = "metrics"
+)
+
+func main() {
+	flag.Parse()
+	if len(flag.Args()) < 1 {
+		fmt.Fprintf(os.Stderr, "USAGE: %s <DIR or FILE> [...]\n", os.Args[0])
+		os.Exit(64)
+	}
+
+	stableMetrics := []metric{}
+	errors := []error{}
+
+	for _, arg := range flag.Args() {
+		ms, es := searchPathForStableMetrics(arg)
+		stableMetrics = append(stableMetrics, ms...)
+		errors = append(errors, es...)
+	}
+	for _, err := range errors {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+	}
+	if len(errors) != 0 {
+		os.Exit(1)
+	}
+	sort.Sort(byFQName(stableMetrics))
+	data, err := yaml.Marshal(stableMetrics)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+	fmt.Print(string(data))
+}
+
+func searchPathForStableMetrics(path string) ([]metric, []error) {
+	ms := []metric{}
+	errors := []error{}
+	err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+		if strings.HasPrefix(path, "vendor") {
+			return filepath.SkipDir
+		}
+		if !strings.HasSuffix(path, ".go") {
+			return nil
+		}
+		ms, es := searchFileForStableMetrics(path, nil)
+		errors = append(errors, es...)
+		ms = append(ms, ms...)
+		return nil
+	})
+	if err != nil {
+		errors = append(errors, err)
+	}
+	return ms, errors
+}
+
+// Pass either only filename of existing file or src including source code in any format and a filename that it comes from
+func searchFileForStableMetrics(filename string, src interface{}) ([]metric, []error) {
+	fileset := token.NewFileSet()
+	tree, err := parser.ParseFile(fileset, filename, src, parser.AllErrors)
+	if err != nil {
+		return []metric{}, []error{err}
+	}
+	metricsImportName, err := getMetricsFrameworkImportName(tree)
+	if err != nil {
+		return []metric{}, addFileInformationToErrors([]error{err}, fileset)
+	}
+	if metricsImportName == "" {
+		return []metric{}, []error{}
+	}
+
+	stableMetricsFunctionCalls, errors := findStableMetricDeclaration(tree, metricsImportName)
+	metrics, es := decodeMetricCalls(stableMetricsFunctionCalls, metricsImportName)
+	errors = append(errors, es...)
+	return metrics, addFileInformationToErrors(errors, fileset)
+}
+
+func getMetricsFrameworkImportName(tree *ast.File) (string, error) {
+	var importName string
+	for _, im := range tree.Imports {
+		if im.Path.Value == metricFrameworkPath {
+			if im.Name == nil {
+				importName = defaultFrameworkImportName
+			} else {
+				if im.Name.Name == "." {
+					return "", newDecodeErrorf(im, errImport)
+				}
+				importName = im.Name.Name
+			}
+		}
+	}
+	return importName, nil
+}
+
+func addFileInformationToErrors(es []error, fileset *token.FileSet) []error {
+	for i := range es {
+		if de, ok := es[i].(*decodeError); ok {
+			es[i] = de.errorWithFileInformation(fileset)
+		}
+	}
+	return es
 }

test/instrumentation/main_test.go

@@ -0,0 +1,493 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+const fakeFilename = "testdata/metric.go"
+
+func TestSkipMetrics(t *testing.T) {
+	for _, test := range []struct {
+		testName string
+		src      string
+	}{
+		{
+			testName: "Skip alpha metric with local variable",
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var name = "metric"
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Name:           name,
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+`},
+		{
+			testName: "Skip alpha metric created via function call",
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+func getName() string {
+	return "metric"
+}
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Name:           getName(),
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+`},
+		{
+			testName: "Skip metric without stability set",
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Name: "metric",
+		},
+	)
+`},
+		{
+			testName: "Skip functions of similar signature (not imported from framework path) with import rename",
+			src: `
+package test
+import metrics "k8s.io/fake/path"
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+`},
+		{
+			testName: "Skip functions of similar signature (not imported from framework path)",
+			src: `
+package test
+import "k8s.io/fake/path/metrics"
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+`},
+		{
+			testName: "Skip . package import of non metric framework",
+			src: `
+package test
+import . "k8s.io/fake/path"
+var _ = NewCounter(
+		&CounterOpts{
+			StabilityLevel: STABLE,
+		},
+	)
+`},
+	} {
+		t.Run(test.testName, func(t *testing.T) {
+			metrics, errors := searchFileForStableMetrics(fakeFilename, test.src)
+			if len(metrics) != 0 {
+				t.Errorf("Didn't expect any stable metrics found, got: %d", len(metrics))
+			}
+			if len(errors) != 0 {
+				t.Errorf("Didn't expect any errors found, got: %s", errors)
+			}
+		})
+	}
+}
+
+func TestStableMetric(t *testing.T) {
+	for _, test := range []struct {
+		testName string
+		src      string
+		metric   metric
+	}{
+		{
+			testName: "Counter",
+			metric: metric{
+				Name:           "metric",
+				Namespace:      "namespace",
+				Subsystem:      "subsystem",
+				StabilityLevel: "STABLE",
+				Help:           "help",
+				Type:           counterMetricType,
+			},
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewCounter(
+	&metrics.CounterOpts{
+		Name: "metric",
+		Subsystem: "subsystem",
+		Namespace: "namespace",
+		Help: "help",
+		StabilityLevel: metrics.STABLE,
+	},
+)
+`},
+		{
+			testName: "CounterVec",
+			metric: metric{
+				Name:           "metric",
+				Namespace:      "namespace",
+				Subsystem:      "subsystem",
+				Labels:         []string{"label-1"},
+				StabilityLevel: "STABLE",
+				Help:           "help",
+				Type:           counterMetricType,
+			},
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Name: "metric",
+			Namespace: "namespace",
+			Subsystem: "subsystem",
+			Help: "help",
+			StabilityLevel: metrics.STABLE,
+		},
+		[]string{"label-1"},
+	)
+`},
+		{
+			testName: "Gauge",
+			metric: metric{
+				Name:           "gauge",
+				Namespace:      "namespace",
+				Subsystem:      "subsystem",
+				StabilityLevel: "STABLE",
+				Help:           "help",
+				Type:           gaugeMetricType,
+			},
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewGauge(
+		&metrics.GaugeOpts{
+			Name: "gauge",
+			Namespace: "namespace",
+			Subsystem: "subsystem",
+			Help: "help",
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+`},
+		{
+			testName: "GaugeVec",
+			metric: metric{
+				Name:           "gauge",
+				Namespace:      "namespace",
+				Subsystem:      "subsystem",
+				StabilityLevel: "STABLE",
+				Help:           "help",
+				Type:           gaugeMetricType,
+				Labels:         []string{"label-1", "label-2"},
+			},
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Name: "gauge",
+			Namespace: "namespace",
+			Subsystem: "subsystem",
+			Help: "help",
+			StabilityLevel: metrics.STABLE,
+		},
+		[]string{"label-2", "label-1"},
+	)
+`},
+		{
+			testName: "Histogram",
+			metric: metric{
+				Name:           "histogram",
+				Namespace:      "namespace",
+				Subsystem:      "subsystem",
+				StabilityLevel: "STABLE",
+				Buckets:        []float64{0.001, 0.01, 0.1, 1, 10, 100},
+				Help:           "help",
+				Type:           histogramMetricType,
+			},
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewHistogram(
+		&metrics.HistogramOpts{
+			Name: "histogram",
+			Namespace: "namespace",
+			Subsystem: "subsystem",
+			StabilityLevel: metrics.STABLE,
+			Help: "help",
+			Buckets: []float64{0.001, 0.01, 0.1, 1, 10, 100},
+		},
+	)
+`},
+		{
+			testName: "HistogramVec",
+			metric: metric{
+				Name:           "histogram",
+				Namespace:      "namespace",
+				Subsystem:      "subsystem",
+				StabilityLevel: "STABLE",
+				Buckets:        []float64{0.001, 0.01, 0.1, 1, 10, 100},
+				Help:           "help",
+				Type:           histogramMetricType,
+				Labels:         []string{"label-1", "label-2"},
+			},
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Name: "histogram",
+			Namespace: "namespace",
+			Subsystem: "subsystem",
+			StabilityLevel: metrics.STABLE,
+			Help: "help",
+			Buckets: []float64{0.001, 0.01, 0.1, 1, 10, 100},
+		},
+		[]string{"label-2", "label-1"},
+	)
+`},
+		{
+			testName: "Custom import",
+			metric: metric{
+				Name:           "metric",
+				StabilityLevel: "STABLE",
+				Type:           counterMetricType,
+			},
+			src: `
+package test
+import custom "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = custom.NewCounter(
+		&custom.CounterOpts{
+			Name: "metric",
+			StabilityLevel: custom.STABLE,
+		},
+	)
+`},
+	} {
+		t.Run(test.testName, func(t *testing.T) {
+			metrics, errors := searchFileForStableMetrics(fakeFilename, test.src)
+			if len(errors) != 0 {
+				t.Errorf("Unexpected errors: %s", errors)
+			}
+			if len(metrics) != 1 {
+				t.Fatalf("Unexpected number of metrics: got %d, want 1", len(metrics))
+			}
+			if test.metric.Labels == nil {
+				test.metric.Labels = []string{}
+			}
+			if !reflect.DeepEqual(metrics[0], test.metric) {
+				t.Errorf("metric:\ngot  %v\nwant %v", metrics[0], test.metric)
+			}
+		})
+	}
+}
+
+func TestIncorrectStableMetricDeclarations(t *testing.T) {
+	for _, test := range []struct {
+		testName string
+		src      string
+		err      error
+	}{
+		{
+			testName: "Fail on stable summary metric (Summary is DEPRECATED)",
+			err:      fmt.Errorf("testdata/metric.go:4:9: Stable summary metric is not supported"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewSummary(
+		&metrics.SummaryOpts{
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+`},
+		{
+			testName: "Fail on stable metric with attribute set to variable",
+			err:      fmt.Errorf("testdata/metric.go:7:4: Non string attribute it not supported"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+const name = "metric"
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Name:           name,
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+`},
+		{
+			testName: "Fail on stable metric with attribute set to local function return",
+			err:      fmt.Errorf("testdata/metric.go:9:4: Non string attribute it not supported"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+func getName() string {
+	return "metric"
+}
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Name:           getName(),
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+`},
+		{
+			testName: "Fail on stable metric with attribute set to imported function return",
+			err:      fmt.Errorf("testdata/metric.go:7:4: Non string attribute it not supported"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+import "k8s.io/kubernetes/utils"
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Name:           utils.getMetricName(),
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+`},
+		{
+			testName: "Fail on metric with stability set to function return",
+			err:      fmt.Errorf("testdata/metric.go:9:20: StabilityLevel should be passed STABLE, ALPHA or removed"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+func getMetricStability() metrics.StabilityLevel {
+	return metrics.STABLE
+}
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			StabilityLevel: getMetricsStability(),
+		},
+	)
+`},
+		{
+			testName: "error for passing stability as string",
+			err:      fmt.Errorf("testdata/metric.go:6:20: StabilityLevel should be passed STABLE, ALPHA or removed"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			StabilityLevel: "stable",
+		},
+	)
+`},
+		{
+			testName: "error for passing stability as unknown const",
+			err:      fmt.Errorf("testdata/metric.go:6:20: StabilityLevel should be passed STABLE, ALPHA or removed"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			StabilityLevel: metrics.UNKNOWN,
+		},
+	)
+`},
+		{
+			testName: "error for passing stability as variable",
+			err:      fmt.Errorf("testdata/metric.go:7:20: StabilityLevel should be passed STABLE, ALPHA or removed"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var stable = metrics.STABLE
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			StabilityLevel: stable,
+		},
+	)
+`},
+		{
+			testName: "error for stable metric created via function call",
+			err:      fmt.Errorf("testdata/metric.go:6:10: Opts for STABLE metric was not directly passed to new metric function"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewCounter(getStableCounterOpts())
+func getStableCounterOpts() *metrics.CounterOpts {
+	return &metrics.CounterOpts{
+		StabilityLevel: metrics.STABLE,
+	}
+}
+`},
+		{
+			testName: "error . package import of metric framework",
+			err:      fmt.Errorf(`testdata/metric.go:3:8: Importing through "." metrics framework is not supported`),
+			src: `
+package test
+import . "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = NewCounter(
+		&CounterOpts{
+			StabilityLevel: STABLE,
+		},
+	)
+`},
+		{
+			testName: "error stable metric opts passed to local function",
+			err:      fmt.Errorf("testdata/metric.go:4:9: Opts for STABLE metric was not directly passed to new metric function"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = RegisterMetric(
+		&metrics.CounterOpts{
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+`},
+		{
+			testName: "error stable metric opts passed to imported function",
+			err:      fmt.Errorf("testdata/metric.go:4:9: Opts for STABLE metric was not directly passed to new metric function"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = test.RegisterMetric(
+		&metrics.CounterOpts{
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+`},
+		{
+			testName: "error stable metric opts passed to imported function",
+			err:      fmt.Errorf("testdata/metric.go:6:4: Positional arguments are not supported"),
+			src: `
+package test
+import "k8s.io/kubernetes/staging/src/k8s.io/component-base/metrics"
+var _ = metrics.NewCounter(
+		&metrics.CounterOpts{
+			"counter",
+		},
+	)
+`},
+	} {
+		t.Run(test.testName, func(t *testing.T) {
+			_, errors := searchFileForStableMetrics(fakeFilename, test.src)
+			if len(errors) != 1 {
+				t.Fatalf("Unexpected number of errors, got %d, want 1", len(errors))
+			}
+			if !reflect.DeepEqual(errors[0], test.err) {
+				t.Errorf("error:\ngot  %v\nwant %v", errors[0], test.err)
+			}
+		})
+	}
+}

test/instrumentation/metric.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	counterMetricType   = "Counter"
+	gaugeMetricType     = "Gauge"
+	histogramMetricType = "Histogram"
+)
+
+type metric struct {
+	Name              string    `yaml:"name"`
+	Subsystem         string    `yaml:"subsystem,omitempty"`
+	Namespace         string    `yaml:"namespace,omitempty"`
+	Help              string    `yaml:"help,omitempty"`
+	Type              string    `yaml:"type,omitempty"`
+	DeprecatedVersion string    `yaml:"deprecatedVersion,omitempty"`
+	StabilityLevel    string    `yaml:"stabilityLevel,omitempty"`
+	Labels            []string  `yaml:"labels,omitempty"`
+	Buckets           []float64 `yaml:"buckets,omitempty"`
+}
+
+func (m metric) buildFQName() string {
+	return prometheus.BuildFQName(m.Namespace, m.Subsystem, m.Name)
+}
+
+type byFQName []metric
+
+func (ms byFQName) Len() int { return len(ms) }
+func (ms byFQName) Less(i, j int) bool {
+	return ms[i].buildFQName() < ms[j].buildFQName()
+}
+func (ms byFQName) Swap(i, j int) {
+	ms[i], ms[j] = ms[j], ms[i]
+}

test/instrumentation/testdata/stable-metrics-list.yaml

@@ -0,0 +1 @@
+[]

test/instrumentation/update-stable-metrics.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
+
+bazel build //test/instrumentation:list_stable_metrics
+cp "$KUBE_ROOT/bazel-genfiles/test/instrumentation/stable-metrics-list.yaml" "$KUBE_ROOT/test/instrumentation/testdata/stable-metrics-list.yaml"

test/instrumentation/verify-stable-metrics.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -o errexit
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
+
+# detect if run from bazel
+if [ -z "${TEST_BINARY}" ]; then
+  bazel build //test/instrumentation:list_stable_metrics
+  OUTPUT_FILE="$KUBE_ROOT/bazel-genfiles/test/instrumentation/stable-metrics-list.yaml"
+else
+  OUTPUT_FILE="$KUBE_ROOT/test/instrumentation/stable-metrics-list.yaml"
+fi
+
+if diff -u "$KUBE_ROOT/test/instrumentation/testdata/stable-metrics-list.yaml" "$OUTPUT_FILE"; then
+  echo PASS
+  exit 0
+fi
+echo 'Diffs in stable metrics detected, please run "test/instrumentation/update-stable-metrics.sh"'
+exit 1