github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 19:56:01 +00:00
Code Issues Projects Releases Wiki Activity

Add CLI script for listing untested conformance behaviors

Browse Source
This commit is contained in:
Jefftree 2020-03-23 17:49:53 -07:00
parent c4fd09d80a
commit 77edd42619
6 changed files with 195 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
test/conformance/BUILD
View File

@ -9,6 +9,7 @@ go_library(
importpath = "k8s.io/kubernetes/test/conformance",
visibility = ["//visibility:private"],
deps = [
"//test/conformance/behaviors:go_default_library",
"//vendor/github.com/onsi/ginkgo/types:go_default_library",
"//vendor/gopkg.in/yaml.v2:go_default_library",
],
@ -33,6 +34,7 @@ filegroup(
":package-srcs",
"//test/conformance/behaviors:all-srcs",
"//test/conformance/kubetestgen:all-srcs",
"//test/conformance/kubetestlink:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
@ -77,6 +79,7 @@ go_test(
srcs = ["walk_test.go"],
data = glob(["testdata/**"]),
embed = [":go_default_library"],
deps = ["//test/conformance/behaviors:go_default_library"],
)
genrule(

18
test/conformance/behaviors/types.go
View File

@ -37,3 +37,21 @@ type Behavior struct {
APIType string `json:"apiType,omitempty"`
Description string `json:"description,omitempty"`
}
// ConformanceData describes the structure of the conformance.yaml file
type ConformanceData struct {
// A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number.
URL string `yaml:"-"`
// Extracted from the "Testname:" comment before the test
TestName string
// CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]`
CodeName string
// Extracted from the "Description:" comment before the test
Description string
// Version when this test is added or modified ex: v1.12, v1.13
Release string
// File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes.
File string
// Behaviors is the list of conformance behaviors tested by a particular e2e test
Behaviors []string `yaml:"behaviors,omitempty"`
}

32
test/conformance/kubetestlink/BUILD Normal file
View File

@ -0,0 +1,32 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
go_library(
name = "go_default_library",
srcs = ["kubetestlink.go"],
importpath = "k8s.io/kubernetes/test/conformance/kubetestlink",
visibility = ["//visibility:private"],
deps = [
"//test/conformance/behaviors:go_default_library",
"//vendor/gopkg.in/yaml.v2:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_binary(
name = "kubetestlink",
embed = [":go_default_library"],
visibility = ["//visibility:public"],
)

119
test/conformance/kubetestlink/kubetestlink.go Normal file
View File

@ -0,0 +1,119 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"flag"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"gopkg.in/yaml.v2"
"k8s.io/kubernetes/test/conformance/behaviors"
)
type options struct {
behaviorsDir string
testdata string
listMissing bool
}
func parseFlags() *options {
o := &options{}
flag.StringVar(&o.behaviorsDir, "behaviors", "../behaviors/", "Path to the behaviors directory")
flag.StringVar(&o.testdata, "testdata", "../testdata/conformance.yaml", "YAML file containing test linkage data")
flag.BoolVar(&o.listMissing, "missing", true, "Only list behaviors missing tests")
flag.Parse()
return o
}
func main() {
o := parseFlags()
var behaviorFiles []string
behaviorsMapping := make(map[string][]string)
var conformanceDataList []behaviors.ConformanceData
err := filepath.Walk(o.behaviorsDir,
func(path string, info os.FileInfo, err error) error {
if err != nil {
fmt.Printf("%v", err)
}
r, _ := regexp.Compile(".+.yaml$")
if r.MatchString(path) {
behaviorFiles = append(behaviorFiles, path)
}
return nil
})
if err != nil {
fmt.Printf("%v", err)
return
}
for _, behaviorFile := range behaviorFiles {
var suite behaviors.Suite
yamlFile, err := ioutil.ReadFile(behaviorFile)
if err != nil {
fmt.Printf("%v", err)
return
}
err = yaml.UnmarshalStrict(yamlFile, &suite)
if err != nil {
fmt.Printf("%v", err)
return
}
for _, behavior := range suite.Behaviors {
behaviorsMapping[behavior.ID] = nil
}
}
conformanceYaml, err := ioutil.ReadFile(o.testdata)
err = yaml.Unmarshal(conformanceYaml, &conformanceDataList)
if err != nil {
fmt.Printf("%v", err)
return
}
for _, data := range conformanceDataList {
for _, behaviorID := range data.Behaviors {
if _, ok := behaviorsMapping[behaviorID]; !ok {
fmt.Printf("Error, cannot find behavior \"%s\"", behaviorID)
return
}
behaviorsMapping[behaviorID] = append(behaviorsMapping[behaviorID], data.CodeName)
}
}
printBehaviorsMapping(behaviorsMapping, o)
}
func printBehaviorsMapping(behaviorsMapping map[string][]string, o *options) {
for behaviorID, tests := range behaviorsMapping {
if o.listMissing {
if tests == nil {
fmt.Println(behaviorID)
} else {
fmt.Println(behaviorID)
}
}
}
}

37
test/conformance/walk.go
View File

@ -34,6 +34,8 @@ import (
"text/template"
"github.com/onsi/ginkgo/types"
"k8s.io/kubernetes/test/conformance/behaviors"
)
var (
@ -63,23 +65,6 @@ type frame struct {
Line int
}
type conformanceData struct {
// A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number.
URL string `yaml:"-"`
// Extracted from the "Testname:" comment before the test
TestName string
// CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]`
CodeName string
// Extracted from the "Description:" comment before the test
Description string
// Version when this test is added or modified ex: v1.12, v1.13
Release string
// File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes.
File string
// Behaviors is the list of conformance behaviors tested by a particular e2e test
Behaviors []string `yaml:"behaviors,omitempty"`
}
func main() {
flag.Parse()
@ -95,7 +80,7 @@ func main() {
seenLines = map[string]struct{}{}
dec := json.NewDecoder(f)
testInfos := []*conformanceData{}
testInfos := []*behaviors.ConformanceData{}
for {
var spec *types.SpecSummary
if err := dec.Decode(&spec); err == io.EOF {
@ -124,8 +109,8 @@ func isConformance(spec *types.SpecSummary) bool {
return strings.Contains(getTestName(spec), "[Conformance]")
}
func getTestInfo(spec *types.SpecSummary) *conformanceData {
var c *conformanceData
func getTestInfo(spec *types.SpecSummary) *behaviors.ConformanceData {
var c *behaviors.ConformanceData
var err error
// The key to this working is that we don't need to parse every file or walk
// every componentCodeLocation. The last componentCodeLocation is going to typically start
@ -155,7 +140,7 @@ func getTestName(spec *types.SpecSummary) string {
return strings.Join(spec.ComponentTexts[1:], " ")
}
func saveAllTestInfo(dataSet []*conformanceData) {
func saveAllTestInfo(dataSet []*behaviors.ConformanceData) {
if *confDoc {
// Note: this assumes that you're running from the root of the kube src repo
templ, err := template.ParseFiles("./test/conformance/cf_header.md")
@ -186,7 +171,7 @@ func saveAllTestInfo(dataSet []*conformanceData) {
fmt.Println(string(b))
}
func getConformanceDataFromStackTrace(fullstackstrace string) (*conformanceData, error) {
func getConformanceDataFromStackTrace(fullstackstrace string) (*behaviors.ConformanceData, error) {
// The full stacktrace to parse from ginkgo is of the form:
// k8s.io/kubernetes/test/e2e/storage/utils.SIGDescribe(0x51f4c4f, 0xf, 0x53a0dd8, 0xc000ab6e01)\n\ttest/e2e/storage/utils/framework.go:23 +0x75\n ... ...
// So we need to split it into lines, remove whitespace, and then grab the files/lines.
@ -240,7 +225,7 @@ func getConformanceDataFromStackTrace(fullstackstrace string) (*conformanceData,
// scanFileForFrame will scan the target and look for a conformance comment attached to the function
// described by the target frame. If the comment can't be found then nil, nil is returned.
func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*conformanceData, error) {
func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*behaviors.ConformanceData, error) {
fset := token.NewFileSet() // positions are relative to fset
f, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
if err != nil {
@ -266,7 +251,7 @@ func validateTestName(s string) error {
return nil
}
func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *conformanceData {
func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *behaviors.ConformanceData {
if !shouldProcessCommentGroup(fset, cg, f) {
return nil
}
@ -290,10 +275,10 @@ func shouldProcessCommentGroup(fset *token.FileSet, cg *ast.CommentGroup, f fram
return lineDiff > 0 && lineDiff <= conformanceCommentsLineWindow
}
func commentToConformanceData(comment string) *conformanceData {
func commentToConformanceData(comment string) *behaviors.ConformanceData {
lines := strings.Split(comment, "\n")
descLines := []string{}
cd := &conformanceData{}
cd := &behaviors.ConformanceData{}
var curLine string
for _, line := range lines {
line = strings.TrimSpace(line)

22
test/conformance/walk_test.go
View File

@ -20,6 +20,8 @@ import (
"fmt"
"reflect"
"testing"
"k8s.io/kubernetes/test/conformance/behaviors"
)
func TestConformance(t *testing.T) {
@ -28,7 +30,7 @@ func TestConformance(t *testing.T) {
filename string
code string
targetFrame frame
output *conformanceData
output *behaviors.ConformanceData
}{
{
desc: "Grabs comment above test",
@ -45,7 +47,7 @@ func TestConformance(t *testing.T) {
*/
framework.ConformanceIt("validates describe with ConformanceIt", func() {})
})`,
output: &conformanceData{
output: &behaviors.ConformanceData{
URL: "https://github.com/kubernetes/kubernetes/tree/master/test/list/main_test.go#L11",
TestName: "Kubelet-OutputToLogs",
Description: `By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.`,
@ -65,7 +67,7 @@ func TestConformance(t *testing.T) {
framework.ConformanceIt("should work", func() {})
})
})`,
output: &conformanceData{
output: &behaviors.ConformanceData{
URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L8",
TestName: "Test with spaces",
Description: `Should pick up testname even if it is not within 3 spaces even when executed from memory.`,
@ -89,7 +91,7 @@ func TestConformance(t *testing.T) {
framework.ConformanceIt("should work", func() {})
})
})`,
output: &conformanceData{
output: &behaviors.ConformanceData{
URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L13",
TestName: "Second test",
Description: `Should target the correct test/comment based on the line numbers`,
@ -112,7 +114,7 @@ func TestConformance(t *testing.T) {
framework.ConformanceIt("should work", func() {})
})
})`,
output: &conformanceData{
output: &behaviors.ConformanceData{
URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L8",
TestName: "First test",
Description: `Should target the correct test/comment based on the line numbers`,
@ -139,7 +141,7 @@ func TestCommentToConformanceData(t *testing.T) {
tcs := []struct {
desc string
input string
expected *conformanceData
expected *behaviors.ConformanceData
}{
{
desc: "Empty comment leads to nil",
@ -152,19 +154,19 @@ func TestCommentToConformanceData(t *testing.T) {
}, {
desc: "Testname but no Release does not result in nil",
input: "Testname: mytest\nDescription: foo",
expected: &conformanceData{TestName: "mytest", Description: "foo"},
expected: &behaviors.ConformanceData{TestName: "mytest", Description: "foo"},
}, {
desc: "All fields parsed and newlines and whitespace removed from description",
input: "Release: v1.1\n\t\tTestname: mytest\n\t\tDescription: foo\n\t\tbar\ndone",
expected: &conformanceData{TestName: "mytest", Release: "v1.1", Description: "foo bar done"},
expected: &behaviors.ConformanceData{TestName: "mytest", Release: "v1.1", Description: "foo bar done"},
}, {
desc: "Behaviors are read",
input: "Testname: behaviors\nBehaviors:\n- should behave\n- second behavior",
expected: &conformanceData{TestName: "behaviors", Behaviors: []string{"should behave", "second behavior"}},
expected: &behaviors.ConformanceData{TestName: "behaviors", Behaviors: []string{"should behave", "second behavior"}},
}, {
desc: "Multiple behaviors are parsed",
input: "Testname: behaviors2\nBehaviors:\n- first behavior\n- second behavior",
expected: &conformanceData{TestName: "behaviors2", Behaviors: []string{"first behavior", "second behavior"}},
expected: &behaviors.ConformanceData{TestName: "behaviors2", Behaviors: []string{"first behavior", "second behavior"}},
},
}