From 77edd42619b7d078ba269ab59f6fa3e19f54b802 Mon Sep 17 00:00:00 2001 From: Jefftree Date: Mon, 23 Mar 2020 17:49:53 -0700 Subject: [PATCH] Add CLI script for listing untested conformance behaviors --- test/conformance/BUILD | 3 + test/conformance/behaviors/types.go | 18 +++ test/conformance/kubetestlink/BUILD | 32 +++++ test/conformance/kubetestlink/kubetestlink.go | 119 ++++++++++++++++++ test/conformance/walk.go | 37 ++---- test/conformance/walk_test.go | 22 ++-- 6 files changed, 195 insertions(+), 36 deletions(-) create mode 100644 test/conformance/kubetestlink/BUILD create mode 100644 test/conformance/kubetestlink/kubetestlink.go diff --git a/test/conformance/BUILD b/test/conformance/BUILD index be4fd088544..16dec712176 100644 --- a/test/conformance/BUILD +++ b/test/conformance/BUILD @@ -9,6 +9,7 @@ go_library( importpath = "k8s.io/kubernetes/test/conformance", visibility = ["//visibility:private"], deps = [ + "//test/conformance/behaviors:go_default_library", "//vendor/github.com/onsi/ginkgo/types:go_default_library", "//vendor/gopkg.in/yaml.v2:go_default_library", ], @@ -33,6 +34,7 @@ filegroup( ":package-srcs", "//test/conformance/behaviors:all-srcs", "//test/conformance/kubetestgen:all-srcs", + "//test/conformance/kubetestlink:all-srcs", ], tags = ["automanaged"], visibility = ["//visibility:public"], @@ -77,6 +79,7 @@ go_test( srcs = ["walk_test.go"], data = glob(["testdata/**"]), embed = [":go_default_library"], + deps = ["//test/conformance/behaviors:go_default_library"], ) genrule( diff --git a/test/conformance/behaviors/types.go b/test/conformance/behaviors/types.go index 1632ab3d306..e394b87cafa 100644 --- a/test/conformance/behaviors/types.go +++ b/test/conformance/behaviors/types.go @@ -37,3 +37,21 @@ type Behavior struct { APIType string `json:"apiType,omitempty"` Description string `json:"description,omitempty"` } + +// ConformanceData describes the structure of the conformance.yaml file +type ConformanceData struct { + // A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number. + URL string `yaml:"-"` + // Extracted from the "Testname:" comment before the test + TestName string + // CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]` + CodeName string + // Extracted from the "Description:" comment before the test + Description string + // Version when this test is added or modified ex: v1.12, v1.13 + Release string + // File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes. + File string + // Behaviors is the list of conformance behaviors tested by a particular e2e test + Behaviors []string `yaml:"behaviors,omitempty"` +} diff --git a/test/conformance/kubetestlink/BUILD b/test/conformance/kubetestlink/BUILD new file mode 100644 index 00000000000..e7a2dd28ae1 --- /dev/null +++ b/test/conformance/kubetestlink/BUILD @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["kubetestlink.go"], + importpath = "k8s.io/kubernetes/test/conformance/kubetestlink", + visibility = ["//visibility:private"], + deps = [ + "//test/conformance/behaviors:go_default_library", + "//vendor/gopkg.in/yaml.v2:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_binary( + name = "kubetestlink", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/test/conformance/kubetestlink/kubetestlink.go b/test/conformance/kubetestlink/kubetestlink.go new file mode 100644 index 00000000000..91be0314c27 --- /dev/null +++ b/test/conformance/kubetestlink/kubetestlink.go @@ -0,0 +1,119 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + + "gopkg.in/yaml.v2" + + "k8s.io/kubernetes/test/conformance/behaviors" +) + +type options struct { + behaviorsDir string + testdata string + listMissing bool +} + +func parseFlags() *options { + o := &options{} + flag.StringVar(&o.behaviorsDir, "behaviors", "../behaviors/", "Path to the behaviors directory") + flag.StringVar(&o.testdata, "testdata", "../testdata/conformance.yaml", "YAML file containing test linkage data") + flag.BoolVar(&o.listMissing, "missing", true, "Only list behaviors missing tests") + flag.Parse() + return o +} + +func main() { + o := parseFlags() + + var behaviorFiles []string + behaviorsMapping := make(map[string][]string) + var conformanceDataList []behaviors.ConformanceData + + err := filepath.Walk(o.behaviorsDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + fmt.Printf("%v", err) + } + r, _ := regexp.Compile(".+.yaml$") + if r.MatchString(path) { + behaviorFiles = append(behaviorFiles, path) + } + return nil + }) + if err != nil { + fmt.Printf("%v", err) + return + } + + for _, behaviorFile := range behaviorFiles { + var suite behaviors.Suite + + yamlFile, err := ioutil.ReadFile(behaviorFile) + if err != nil { + fmt.Printf("%v", err) + return + } + err = yaml.UnmarshalStrict(yamlFile, &suite) + if err != nil { + fmt.Printf("%v", err) + return + } + + for _, behavior := range suite.Behaviors { + behaviorsMapping[behavior.ID] = nil + } + } + + conformanceYaml, err := ioutil.ReadFile(o.testdata) + + err = yaml.Unmarshal(conformanceYaml, &conformanceDataList) + if err != nil { + fmt.Printf("%v", err) + return + } + + for _, data := range conformanceDataList { + for _, behaviorID := range data.Behaviors { + if _, ok := behaviorsMapping[behaviorID]; !ok { + fmt.Printf("Error, cannot find behavior \"%s\"", behaviorID) + return + } + behaviorsMapping[behaviorID] = append(behaviorsMapping[behaviorID], data.CodeName) + } + } + printBehaviorsMapping(behaviorsMapping, o) +} + +func printBehaviorsMapping(behaviorsMapping map[string][]string, o *options) { + for behaviorID, tests := range behaviorsMapping { + if o.listMissing { + if tests == nil { + fmt.Println(behaviorID) + } else { + fmt.Println(behaviorID) + } + } + } +} diff --git a/test/conformance/walk.go b/test/conformance/walk.go index 34d232eacf0..99edc6cb06f 100644 --- a/test/conformance/walk.go +++ b/test/conformance/walk.go @@ -34,6 +34,8 @@ import ( "text/template" "github.com/onsi/ginkgo/types" + + "k8s.io/kubernetes/test/conformance/behaviors" ) var ( @@ -63,23 +65,6 @@ type frame struct { Line int } -type conformanceData struct { - // A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number. - URL string `yaml:"-"` - // Extracted from the "Testname:" comment before the test - TestName string - // CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]` - CodeName string - // Extracted from the "Description:" comment before the test - Description string - // Version when this test is added or modified ex: v1.12, v1.13 - Release string - // File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes. - File string - // Behaviors is the list of conformance behaviors tested by a particular e2e test - Behaviors []string `yaml:"behaviors,omitempty"` -} - func main() { flag.Parse() @@ -95,7 +80,7 @@ func main() { seenLines = map[string]struct{}{} dec := json.NewDecoder(f) - testInfos := []*conformanceData{} + testInfos := []*behaviors.ConformanceData{} for { var spec *types.SpecSummary if err := dec.Decode(&spec); err == io.EOF { @@ -124,8 +109,8 @@ func isConformance(spec *types.SpecSummary) bool { return strings.Contains(getTestName(spec), "[Conformance]") } -func getTestInfo(spec *types.SpecSummary) *conformanceData { - var c *conformanceData +func getTestInfo(spec *types.SpecSummary) *behaviors.ConformanceData { + var c *behaviors.ConformanceData var err error // The key to this working is that we don't need to parse every file or walk // every componentCodeLocation. The last componentCodeLocation is going to typically start @@ -155,7 +140,7 @@ func getTestName(spec *types.SpecSummary) string { return strings.Join(spec.ComponentTexts[1:], " ") } -func saveAllTestInfo(dataSet []*conformanceData) { +func saveAllTestInfo(dataSet []*behaviors.ConformanceData) { if *confDoc { // Note: this assumes that you're running from the root of the kube src repo templ, err := template.ParseFiles("./test/conformance/cf_header.md") @@ -186,7 +171,7 @@ func saveAllTestInfo(dataSet []*conformanceData) { fmt.Println(string(b)) } -func getConformanceDataFromStackTrace(fullstackstrace string) (*conformanceData, error) { +func getConformanceDataFromStackTrace(fullstackstrace string) (*behaviors.ConformanceData, error) { // The full stacktrace to parse from ginkgo is of the form: // k8s.io/kubernetes/test/e2e/storage/utils.SIGDescribe(0x51f4c4f, 0xf, 0x53a0dd8, 0xc000ab6e01)\n\ttest/e2e/storage/utils/framework.go:23 +0x75\n ... ... // So we need to split it into lines, remove whitespace, and then grab the files/lines. @@ -240,7 +225,7 @@ func getConformanceDataFromStackTrace(fullstackstrace string) (*conformanceData, // scanFileForFrame will scan the target and look for a conformance comment attached to the function // described by the target frame. If the comment can't be found then nil, nil is returned. -func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*conformanceData, error) { +func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*behaviors.ConformanceData, error) { fset := token.NewFileSet() // positions are relative to fset f, err := parser.ParseFile(fset, filename, src, parser.ParseComments) if err != nil { @@ -266,7 +251,7 @@ func validateTestName(s string) error { return nil } -func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *conformanceData { +func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *behaviors.ConformanceData { if !shouldProcessCommentGroup(fset, cg, f) { return nil } @@ -290,10 +275,10 @@ func shouldProcessCommentGroup(fset *token.FileSet, cg *ast.CommentGroup, f fram return lineDiff > 0 && lineDiff <= conformanceCommentsLineWindow } -func commentToConformanceData(comment string) *conformanceData { +func commentToConformanceData(comment string) *behaviors.ConformanceData { lines := strings.Split(comment, "\n") descLines := []string{} - cd := &conformanceData{} + cd := &behaviors.ConformanceData{} var curLine string for _, line := range lines { line = strings.TrimSpace(line) diff --git a/test/conformance/walk_test.go b/test/conformance/walk_test.go index 153f01183cb..f1b4d3ec430 100644 --- a/test/conformance/walk_test.go +++ b/test/conformance/walk_test.go @@ -20,6 +20,8 @@ import ( "fmt" "reflect" "testing" + + "k8s.io/kubernetes/test/conformance/behaviors" ) func TestConformance(t *testing.T) { @@ -28,7 +30,7 @@ func TestConformance(t *testing.T) { filename string code string targetFrame frame - output *conformanceData + output *behaviors.ConformanceData }{ { desc: "Grabs comment above test", @@ -45,7 +47,7 @@ func TestConformance(t *testing.T) { */ framework.ConformanceIt("validates describe with ConformanceIt", func() {}) })`, - output: &conformanceData{ + output: &behaviors.ConformanceData{ URL: "https://github.com/kubernetes/kubernetes/tree/master/test/list/main_test.go#L11", TestName: "Kubelet-OutputToLogs", Description: `By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.`, @@ -65,7 +67,7 @@ func TestConformance(t *testing.T) { framework.ConformanceIt("should work", func() {}) }) })`, - output: &conformanceData{ + output: &behaviors.ConformanceData{ URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L8", TestName: "Test with spaces", Description: `Should pick up testname even if it is not within 3 spaces even when executed from memory.`, @@ -89,7 +91,7 @@ func TestConformance(t *testing.T) { framework.ConformanceIt("should work", func() {}) }) })`, - output: &conformanceData{ + output: &behaviors.ConformanceData{ URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L13", TestName: "Second test", Description: `Should target the correct test/comment based on the line numbers`, @@ -112,7 +114,7 @@ func TestConformance(t *testing.T) { framework.ConformanceIt("should work", func() {}) }) })`, - output: &conformanceData{ + output: &behaviors.ConformanceData{ URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L8", TestName: "First test", Description: `Should target the correct test/comment based on the line numbers`, @@ -139,7 +141,7 @@ func TestCommentToConformanceData(t *testing.T) { tcs := []struct { desc string input string - expected *conformanceData + expected *behaviors.ConformanceData }{ { desc: "Empty comment leads to nil", @@ -152,19 +154,19 @@ func TestCommentToConformanceData(t *testing.T) { }, { desc: "Testname but no Release does not result in nil", input: "Testname: mytest\nDescription: foo", - expected: &conformanceData{TestName: "mytest", Description: "foo"}, + expected: &behaviors.ConformanceData{TestName: "mytest", Description: "foo"}, }, { desc: "All fields parsed and newlines and whitespace removed from description", input: "Release: v1.1\n\t\tTestname: mytest\n\t\tDescription: foo\n\t\tbar\ndone", - expected: &conformanceData{TestName: "mytest", Release: "v1.1", Description: "foo bar done"}, + expected: &behaviors.ConformanceData{TestName: "mytest", Release: "v1.1", Description: "foo bar done"}, }, { desc: "Behaviors are read", input: "Testname: behaviors\nBehaviors:\n- should behave\n- second behavior", - expected: &conformanceData{TestName: "behaviors", Behaviors: []string{"should behave", "second behavior"}}, + expected: &behaviors.ConformanceData{TestName: "behaviors", Behaviors: []string{"should behave", "second behavior"}}, }, { desc: "Multiple behaviors are parsed", input: "Testname: behaviors2\nBehaviors:\n- first behavior\n- second behavior", - expected: &conformanceData{TestName: "behaviors2", Behaviors: []string{"first behavior", "second behavior"}}, + expected: &behaviors.ConformanceData{TestName: "behaviors2", Behaviors: []string{"first behavior", "second behavior"}}, }, }