Merge pull request #89393 from Jefftree/bdd-linkage

Add CLI script for listing untested conformance behaviors

test/conformance/BUILD

@@ -9,6 +9,7 @@ go_library(
     importpath = "k8s.io/kubernetes/test/conformance",
     visibility = ["//visibility:private"],
     deps = [
+        "//test/conformance/behaviors:go_default_library",
         "//vendor/github.com/onsi/ginkgo/types:go_default_library",
         "//vendor/gopkg.in/yaml.v2:go_default_library",
     ],
@@ -32,7 +33,7 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//test/conformance/behaviors:all-srcs",
-        "//test/conformance/kubetestgen:all-srcs",
+        "//test/conformance/kubeconform:all-srcs",
     ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
@@ -77,6 +78,7 @@ go_test(
     srcs = ["walk_test.go"],
     data = glob(["testdata/**"]),
     embed = [":go_default_library"],
+    deps = ["//test/conformance/behaviors:go_default_library"],
 )
 
 genrule(

test/conformance/behaviors/types.go

@@ -37,3 +37,21 @@ type Behavior struct {
 	APIType     string `json:"apiType,omitempty"`
 	Description string `json:"description,omitempty"`
 }
+
+// ConformanceData describes the structure of the conformance.yaml file
+type ConformanceData struct {
+	// A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number.
+	URL string `yaml:"-"`
+	// Extracted from the "Testname:" comment before the test
+	TestName string
+	// CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]`
+	CodeName string
+	// Extracted from the "Description:" comment before the test
+	Description string
+	// Version when this test is added or modified ex: v1.12, v1.13
+	Release string
+	// File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes.
+	File string
+	// Behaviors is the list of conformance behaviors tested by a particular e2e test
+	Behaviors []string `yaml:"behaviors,omitempty"`
+}

test/conformance/kubeconform/BUILD

@@ -2,8 +2,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 
 go_library(
     name = "go_default_library",
-    srcs = ["kubetestgen.go"],
+    srcs = [
-    importpath = "k8s.io/kubernetes/test/conformance/kubetestgen",
+        "gen.go",
+        "kubeconform.go",
+        "link.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/conformance/kubeconform",
     visibility = ["//visibility:private"],
     deps = [
         "//test/conformance/behaviors:go_default_library",
@@ -29,7 +33,7 @@ filegroup(
 )
 
 go_binary(
-    name = "kubetestgen",
+    name = "kubeconform",
     data = ["//api/openapi-spec"],
     embed = [":go_default_library"],
     visibility = ["//visibility:public"],

test/conformance/kubeconform/gen.go

@@ -17,7 +17,6 @@ limitations under the License.
 package main
 
 import (
-	"flag"
 	"fmt"
 	"os"
 	"sort"
@@ -30,29 +29,10 @@ import (
 	"k8s.io/kubernetes/test/conformance/behaviors"
 )
 
-type options struct {
-	schemaPath   string
-	resource     string
-	area         string
-	behaviorsDir string
-}
-
-func parseFlags() *options {
-	o := &options{}
-	flag.StringVar(&o.schemaPath, "schema", "", "Path to the OpenAPI schema")
-	flag.StringVar(&o.resource, "resource", ".*", "Resource name")
-	flag.StringVar(&o.area, "area", "default", "Area name to use")
-	flag.StringVar(&o.behaviorsDir, "dir", "../behaviors/", "Path to the behaviors directory")
-	flag.Parse()
-	return o
-}
-
 var defMap map[string]analysis.SchemaRef
 
-func main() {
+func gen(o *options) {
 	defMap = make(map[string]analysis.SchemaRef)
-	o := parseFlags()
-
 	d, err := loads.JSONSpec(o.schemaPath)
 	if err != nil {
 		fmt.Printf("ERROR: %s\n", err.Error())

test/conformance/kubeconform/kubeconform.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+)
+
+type options struct {
+	// Flags only used for generating behaviors
+	schemaPath string
+	resource   string
+	area       string
+
+	// Flags only used for linking behaviors
+	testdata    string
+	listMissing bool
+
+	// Flags shared between CLI tools
+	behaviorsDir string
+}
+
+func parseFlags() *options {
+	o := &options{}
+
+	flag.StringVar(&o.schemaPath, "schema", "", "Path to the OpenAPI schema")
+	flag.StringVar(&o.resource, "resource", ".*", "Resource name")
+	flag.StringVar(&o.area, "area", "default", "Area name to use")
+
+	flag.StringVar(&o.testdata, "testdata", "../testdata/conformance.yaml", "YAML file containing test linkage data")
+	flag.BoolVar(&o.listMissing, "missing", true, "Only list behaviors missing tests")
+
+	flag.StringVar(&o.behaviorsDir, "dir", "../behaviors", "Path to the behaviors directory")
+
+	flag.Parse()
+	return o
+}
+
+func main() {
+	o := parseFlags()
+	action := flag.Arg(0)
+	if action == "gen" {
+		gen(o)
+	} else if action == "link" {
+		link(o)
+	} else {
+		fmt.Printf("Unknown argument %s\n", action)
+	}
+}

test/conformance/kubeconform/link.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	"gopkg.in/yaml.v2"
+
+	"k8s.io/kubernetes/test/conformance/behaviors"
+)
+
+func link(o *options) {
+	var behaviorFiles []string
+	behaviorsMapping := make(map[string][]string)
+	var conformanceDataList []behaviors.ConformanceData
+
+	err := filepath.Walk(o.behaviorsDir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				fmt.Printf("%v", err)
+			}
+			r, _ := regexp.Compile(".+.yaml$")
+			if r.MatchString(path) {
+				behaviorFiles = append(behaviorFiles, path)
+			}
+			return nil
+		})
+	if err != nil {
+		fmt.Printf("%v", err)
+		return
+	}
+	fmt.Printf("%v", behaviorFiles)
+
+	for _, behaviorFile := range behaviorFiles {
+		var suite behaviors.Suite
+
+		yamlFile, err := ioutil.ReadFile(behaviorFile)
+		if err != nil {
+			fmt.Printf("%v", err)
+			return
+		}
+		err = yaml.UnmarshalStrict(yamlFile, &suite)
+		if err != nil {
+			fmt.Printf("%v", err)
+			return
+		}
+
+		for _, behavior := range suite.Behaviors {
+			behaviorsMapping[behavior.ID] = nil
+		}
+	}
+
+	conformanceYaml, err := ioutil.ReadFile(o.testdata)
+	if err != nil {
+		fmt.Printf("%v", err)
+		return
+	}
+
+	err = yaml.Unmarshal(conformanceYaml, &conformanceDataList)
+	if err != nil {
+		fmt.Printf("%v", err)
+		return
+	}
+
+	for _, data := range conformanceDataList {
+		for _, behaviorID := range data.Behaviors {
+			if _, ok := behaviorsMapping[behaviorID]; !ok {
+				fmt.Printf("Error, cannot find behavior \"%s\"", behaviorID)
+				return
+			}
+			behaviorsMapping[behaviorID] = append(behaviorsMapping[behaviorID], data.CodeName)
+		}
+	}
+	printBehaviorsMapping(behaviorsMapping, o)
+}
+
+func printBehaviorsMapping(behaviorsMapping map[string][]string, o *options) {
+	for behaviorID, tests := range behaviorsMapping {
+		if o.listMissing {
+			if tests == nil {
+				fmt.Println(behaviorID)
+			} else {
+				fmt.Println(behaviorID)
+			}
+		}
+	}
+}

test/conformance/walk.go

@@ -34,6 +34,8 @@ import (
 	"text/template"
 
 	"github.com/onsi/ginkgo/types"
+
+	"k8s.io/kubernetes/test/conformance/behaviors"
 )
 
 var (
@@ -63,23 +65,6 @@ type frame struct {
 	Line int
 }
 
-type conformanceData struct {
-	// A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number.
-	URL string `yaml:"-"`
-	// Extracted from the "Testname:" comment before the test
-	TestName string
-	// CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]`
-	CodeName string
-	// Extracted from the "Description:" comment before the test
-	Description string
-	// Version when this test is added or modified ex: v1.12, v1.13
-	Release string
-	// File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes.
-	File string
-	// Behaviors is the list of conformance behaviors tested by a particular e2e test
-	Behaviors []string `yaml:"behaviors,omitempty"`
-}
-
 func main() {
 	flag.Parse()
 
@@ -95,7 +80,7 @@ func main() {
 
 	seenLines = map[string]struct{}{}
 	dec := json.NewDecoder(f)
-	testInfos := []*conformanceData{}
+	testInfos := []*behaviors.ConformanceData{}
 	for {
 		var spec *types.SpecSummary
 		if err := dec.Decode(&spec); err == io.EOF {
@@ -124,8 +109,8 @@ func isConformance(spec *types.SpecSummary) bool {
 	return strings.Contains(getTestName(spec), "[Conformance]")
 }
 
-func getTestInfo(spec *types.SpecSummary) *conformanceData {
+func getTestInfo(spec *types.SpecSummary) *behaviors.ConformanceData {
-	var c *conformanceData
+	var c *behaviors.ConformanceData
 	var err error
 	// The key to this working is that we don't need to parse every file or walk
 	// every componentCodeLocation. The last componentCodeLocation is going to typically start
@@ -155,7 +140,7 @@ func getTestName(spec *types.SpecSummary) string {
 	return strings.Join(spec.ComponentTexts[1:], " ")
 }
 
-func saveAllTestInfo(dataSet []*conformanceData) {
+func saveAllTestInfo(dataSet []*behaviors.ConformanceData) {
 	if *confDoc {
 		// Note: this assumes that you're running from the root of the kube src repo
 		templ, err := template.ParseFiles("./test/conformance/cf_header.md")
@@ -186,7 +171,7 @@ func saveAllTestInfo(dataSet []*conformanceData) {
 	fmt.Println(string(b))
 }
 
-func getConformanceDataFromStackTrace(fullstackstrace string) (*conformanceData, error) {
+func getConformanceDataFromStackTrace(fullstackstrace string) (*behaviors.ConformanceData, error) {
 	// The full stacktrace to parse from ginkgo is of the form:
 	// k8s.io/kubernetes/test/e2e/storage/utils.SIGDescribe(0x51f4c4f, 0xf, 0x53a0dd8, 0xc000ab6e01)\n\ttest/e2e/storage/utils/framework.go:23 +0x75\n ... ...
 	// So we need to split it into lines, remove whitespace, and then grab the files/lines.
@@ -240,7 +225,7 @@ func getConformanceDataFromStackTrace(fullstackstrace string) (*conformanceData,
 
 // scanFileForFrame will scan the target and look for a conformance comment attached to the function
 // described by the target frame. If the comment can't be found then nil, nil is returned.
-func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*conformanceData, error) {
+func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*behaviors.ConformanceData, error) {
 	fset := token.NewFileSet() // positions are relative to fset
 	f, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
 	if err != nil {
@@ -266,7 +251,7 @@ func validateTestName(s string) error {
 	return nil
 }
 
-func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *conformanceData {
+func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *behaviors.ConformanceData {
 	if !shouldProcessCommentGroup(fset, cg, f) {
 		return nil
 	}
@@ -290,10 +275,10 @@ func shouldProcessCommentGroup(fset *token.FileSet, cg *ast.CommentGroup, f fram
 	return lineDiff > 0 && lineDiff <= conformanceCommentsLineWindow
 }
 
-func commentToConformanceData(comment string) *conformanceData {
+func commentToConformanceData(comment string) *behaviors.ConformanceData {
 	lines := strings.Split(comment, "\n")
 	descLines := []string{}
-	cd := &conformanceData{}
+	cd := &behaviors.ConformanceData{}
 	var curLine string
 	for _, line := range lines {
 		line = strings.TrimSpace(line)

test/conformance/walk_test.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"reflect"
 	"testing"
+
+	"k8s.io/kubernetes/test/conformance/behaviors"
 )
 
 func TestConformance(t *testing.T) {
@@ -28,7 +30,7 @@ func TestConformance(t *testing.T) {
 		filename    string
 		code        string
 		targetFrame frame
-		output      *conformanceData
+		output      *behaviors.ConformanceData
 	}{
 		{
 			desc:     "Grabs comment above test",
@@ -45,7 +47,7 @@ func TestConformance(t *testing.T) {
 	*/
 	 framework.ConformanceIt("validates describe with ConformanceIt", func() {})
 	})`,
-			output: &conformanceData{
+			output: &behaviors.ConformanceData{
 				URL:         "https://github.com/kubernetes/kubernetes/tree/master/test/list/main_test.go#L11",
 				TestName:    "Kubelet-OutputToLogs",
 				Description: `By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.`,
@@ -65,7 +67,7 @@ func TestConformance(t *testing.T) {
 				   framework.ConformanceIt("should work", func() {})
 		   })
 	})`,
-			output: &conformanceData{
+			output: &behaviors.ConformanceData{
 				URL:         "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L8",
 				TestName:    "Test with spaces",
 				Description: `Should pick up testname even if it is not within 3 spaces even when executed from memory.`,
@@ -89,7 +91,7 @@ func TestConformance(t *testing.T) {
 				   framework.ConformanceIt("should work", func() {})
 		   })
 	})`,
-			output: &conformanceData{
+			output: &behaviors.ConformanceData{
 				URL:         "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L13",
 				TestName:    "Second test",
 				Description: `Should target the correct test/comment based on the line numbers`,
@@ -112,7 +114,7 @@ func TestConformance(t *testing.T) {
 				   framework.ConformanceIt("should work", func() {})
 		   })
 	})`,
-			output: &conformanceData{
+			output: &behaviors.ConformanceData{
 				URL:         "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L8",
 				TestName:    "First test",
 				Description: `Should target the correct test/comment based on the line numbers`,
@@ -139,7 +141,7 @@ func TestCommentToConformanceData(t *testing.T) {
 	tcs := []struct {
 		desc     string
 		input    string
-		expected *conformanceData
+		expected *behaviors.ConformanceData
 	}{
 		{
 			desc: "Empty comment leads to nil",
@@ -152,19 +154,19 @@ func TestCommentToConformanceData(t *testing.T) {
 		}, {
 			desc:     "Testname but no Release does not result in nil",
 			input:    "Testname: mytest\nDescription: foo",
-			expected: &conformanceData{TestName: "mytest", Description: "foo"},
+			expected: &behaviors.ConformanceData{TestName: "mytest", Description: "foo"},
 		}, {
 			desc:     "All fields parsed and newlines and whitespace removed from description",
 			input:    "Release: v1.1\n\t\tTestname: mytest\n\t\tDescription: foo\n\t\tbar\ndone",
-			expected: &conformanceData{TestName: "mytest", Release: "v1.1", Description: "foo bar done"},
+			expected: &behaviors.ConformanceData{TestName: "mytest", Release: "v1.1", Description: "foo bar done"},
 		}, {
 			desc:     "Behaviors are read",
 			input:    "Testname: behaviors\nBehaviors:\n- should behave\n- second behavior",
-			expected: &conformanceData{TestName: "behaviors", Behaviors: []string{"should behave", "second behavior"}},
+			expected: &behaviors.ConformanceData{TestName: "behaviors", Behaviors: []string{"should behave", "second behavior"}},
 		}, {
 			desc:     "Multiple behaviors are parsed",
 			input:    "Testname: behaviors2\nBehaviors:\n- first behavior\n- second behavior",
-			expected: &conformanceData{TestName: "behaviors2", Behaviors: []string{"first behavior", "second behavior"}},
+			expected: &behaviors.ConformanceData{TestName: "behaviors2", Behaviors: []string{"first behavior", "second behavior"}},
 		},
 	}