From fd7eaffd9907b73544dc1c50869640f52565fa73 Mon Sep 17 00:00:00 2001 From: Jefftree Date: Tue, 15 Oct 2019 10:52:18 -0700 Subject: [PATCH] kubetestgen implementation --- go.mod | 1 + test/conformance/BUILD | 5 +- test/conformance/behaviors/OWNERS | 17 ++ test/conformance/kubetestgen/.gitignore | 1 + test/conformance/kubetestgen/BUILD | 38 ++++ test/conformance/kubetestgen/README.md | 19 ++ test/conformance/kubetestgen/kubetestgen.go | 234 ++++++++++++++++++++ test/conformance/kubetestgen/types.go | 39 ++++ 8 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 test/conformance/behaviors/OWNERS create mode 100644 test/conformance/kubetestgen/.gitignore create mode 100644 test/conformance/kubetestgen/BUILD create mode 100644 test/conformance/kubetestgen/README.md create mode 100644 test/conformance/kubetestgen/kubetestgen.go create mode 100644 test/conformance/kubetestgen/types.go diff --git a/go.mod b/go.mod index a066abc5d60..60168468aa2 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/evanphx/json-patch v4.2.0+incompatible github.com/fsnotify/fsnotify v1.4.7 github.com/go-bindata/go-bindata v3.1.1+incompatible + github.com/go-openapi/analysis v0.19.2 github.com/go-openapi/loads v0.19.2 github.com/go-openapi/spec v0.19.2 github.com/go-openapi/strfmt v0.19.0 diff --git a/test/conformance/BUILD b/test/conformance/BUILD index 249647e4226..31fafb4174f 100644 --- a/test/conformance/BUILD +++ b/test/conformance/BUILD @@ -22,7 +22,10 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//test/conformance/kubetestgen:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/test/conformance/behaviors/OWNERS b/test/conformance/behaviors/OWNERS new file mode 100644 index 00000000000..6f3dea12a0d --- /dev/null +++ b/test/conformance/behaviors/OWNERS @@ -0,0 +1,17 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +# To be owned by sig-architecture. +options: + no_parent_owners: true +reviewers: + - bgrant0607 + - smarterclayton + - spiffxp + - timothysc + - dims + - johnbelamaric +approvers: + - conformance-behavior-approvers +labels: + - area/conformance + - sig/architecture diff --git a/test/conformance/kubetestgen/.gitignore b/test/conformance/kubetestgen/.gitignore new file mode 100644 index 00000000000..0af3e9fa8fa --- /dev/null +++ b/test/conformance/kubetestgen/.gitignore @@ -0,0 +1 @@ +kubetestgen diff --git a/test/conformance/kubetestgen/BUILD b/test/conformance/kubetestgen/BUILD new file mode 100644 index 00000000000..d3529c14b0b --- /dev/null +++ b/test/conformance/kubetestgen/BUILD @@ -0,0 +1,38 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "kubetestgen.go", + "types.go", + ], + importpath = "k8s.io/kubernetes/test/conformance/kubetestgen", + visibility = ["//visibility:private"], + deps = [ + "//vendor/github.com/go-openapi/analysis:go_default_library", + "//vendor/github.com/go-openapi/loads:go_default_library", + "//vendor/github.com/go-openapi/spec:go_default_library", + "//vendor/gopkg.in/yaml.v2:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_binary( + name = "kubetestgen", + data = ["//api/openapi-spec"], + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/test/conformance/kubetestgen/README.md b/test/conformance/kubetestgen/README.md new file mode 100644 index 00000000000..62fb80b8051 --- /dev/null +++ b/test/conformance/kubetestgen/README.md @@ -0,0 +1,19 @@ +# Kubetestgen + +kubetestgen generates a list of behaviors for a resource based on the OpenAPI schema. The purpose is to bootstrap a list of behaviors, and not to produce the final list of behaviors. We expect that the resulting files will be curated to identify a meaningful set of behaviors for the conformance requirements of the targeted resource. This may include addition, modification, and removal of behaviors from the generated list. + +**Example usage for PodSpec:** + +``` +bazel build //test/conformance/kubetestgen:kubetestgen +/bazel-out/k8-fastbuild/bin/test/conformance/kubetestgen/linux_amd64_stripped/kubetestgen --resource io.k8s.api.core.v1.PodSpec --area pod --schema api/openapi-spec/swagger.json --dir test/conformance/behaviors/ +``` + +**Flags:** + +- `schema` - a URL or local file name pointing to the JSON OpenAPI schema +- `resource` - the specific OpenAPI definition for which to generate behaviors +- `area` - the name to use for the area +- `dir` - the path to the behaviors directory (default current directory) + +**Note**: The tool automatically generates suites based on the object type for a field. All primitive data types are grouped into a default suite, while object data types are grouped into their own suite, one per object. diff --git a/test/conformance/kubetestgen/kubetestgen.go b/test/conformance/kubetestgen/kubetestgen.go new file mode 100644 index 00000000000..87d9bbdb25f --- /dev/null +++ b/test/conformance/kubetestgen/kubetestgen.go @@ -0,0 +1,234 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + "sort" + "strings" + + "github.com/go-openapi/analysis" + "github.com/go-openapi/loads" + "github.com/go-openapi/spec" + "gopkg.in/yaml.v2" +) + +type options struct { + schemaPath string + resource string + area string + behaviorsDir string +} + +func parseFlags() *options { + o := &options{} + flag.StringVar(&o.schemaPath, "schema", "", "Path to the OpenAPI schema") + flag.StringVar(&o.resource, "resource", ".*", "Resource name") + flag.StringVar(&o.area, "area", "default", "Area name to use") + flag.StringVar(&o.behaviorsDir, "dir", "../behaviors/", "Path to the behaviors directory") + flag.Parse() + return o +} + +var defMap map[string]analysis.SchemaRef + +func main() { + defMap = make(map[string]analysis.SchemaRef) + o := parseFlags() + + d, err := loads.JSONSpec(o.schemaPath) + if err != nil { + fmt.Printf("ERROR: %s\n", err.Error()) + os.Exit(1) + } + defs := d.Analyzer.AllDefinitions() + sort.Slice(defs, func(i, j int) bool { return defs[i].Name < defs[j].Name }) + + for _, d := range defs { + if !d.TopLevel { + continue + } + defMap[d.Ref.String()] = d + } + + var suites []Suite + var suiteMapping = make(map[string]*Suite) + + for _, v := range defs { + if !v.TopLevel || o.resource != v.Name { + continue + } + name := trimObjectName(v.Name) + + defaultsuite := Suite{ + Suite: o.area + "/spec", + Description: "Base suite for " + o.area, + Behaviors: []Behavior{}, + } + + _ = defaultsuite + + for p, propSchema := range v.Schema.Properties { + id := o.area + p + "/" + + if propSchema.Ref.String() != "" || propSchema.Type[0] == "array" { + if _, ok := suiteMapping[id]; !ok { + newsuite := Suite{ + Suite: o.area + "/" + p, + Description: "Suite for " + o.area + "/" + p, + Behaviors: []Behavior{}, + } + suiteMapping[id] = &newsuite + } + behaviors := suiteMapping[id].Behaviors + behaviors = append(behaviors, schemaBehavior(o.area, name, p, propSchema)...) + suiteMapping[id].Behaviors = behaviors + } else { + if _, ok := suiteMapping["default"]; !ok { + newsuite := Suite{ + Suite: o.area + "/spec", + Description: "Base suite for " + o.area, + Behaviors: []Behavior{}, + } + suiteMapping["default"] = &newsuite + } + + behaviors := suiteMapping["default"].Behaviors + behaviors = append(behaviors, schemaBehavior(o.area, name, p, propSchema)...) + suiteMapping["default"].Behaviors = behaviors + + } + } + for _, v := range suiteMapping { + suites = append(suites, *v) + } + + break + } + + var area Area = Area{o.area, suites} + countFields(suites) + printYAML(o.behaviorsDir+o.area, area) +} + +func printYAML(fileName string, areaO Area) { + f, err := os.Create(fileName + ".yaml") + if err != nil { + fmt.Printf("ERROR: %s\n", err.Error()) + } + defer f.Close() + y, err := yaml.Marshal(areaO) + if err != nil { + fmt.Printf("ERROR: %s\n", err.Error()) + os.Exit(1) + } + + f.WriteString(string(y)) +} + +func countFields(suites []Suite) { + var fieldsMapping map[string]int + fieldsMapping = make(map[string]int) + for _, suite := range suites { + for _, behavior := range suite.Behaviors { + if _, exists := fieldsMapping[behavior.APIType]; exists { + fieldsMapping[behavior.APIType]++ + } else { + fieldsMapping[behavior.APIType] = 1 + } + } + } + for k, v := range fieldsMapping { + fmt.Printf("Type %v, Count %v\n", k, v) + } +} + +func trimObjectName(name string) string { + if strings.Index(name, "#/definitions/") == 0 { + name = name[len("#/definitions/"):] + } + if strings.Index(name, "io.k8s.api.") == 0 { + return name[len("io.k8s.api."):] + } + return name +} + +func objectBehaviors(id string, s *spec.Schema) []Behavior { + if strings.Contains(id, "openAPIV3Schema") || strings.Contains(id, "JSONSchema") || strings.Contains(s.Ref.String(), "JSONSchema") { + return []Behavior{} + } + + ref, ok := defMap[s.Ref.String()] + if !ok { + return []Behavior{} + } + + return schemaBehaviors(id, trimObjectName(ref.Name), ref.Schema) +} + +func schemaBehaviors(base, apiObject string, s *spec.Schema) []Behavior { + var behaviors []Behavior + for p, propSchema := range s.Properties { + b := schemaBehavior(base, apiObject, p, propSchema) + behaviors = append(behaviors, b...) + } + return behaviors +} + +func schemaBehavior(base, apiObject, p string, propSchema spec.Schema) []Behavior { + + id := strings.Join([]string{base, p}, "/") + if propSchema.Ref.String() != "" { + if apiObject == trimObjectName(propSchema.Ref.String()) { + return []Behavior{} + } + return objectBehaviors(id, &propSchema) + } + var b []Behavior + switch propSchema.Type[0] { + case "array": + b = objectBehaviors(id, propSchema.Items.Schema) + case "boolean": + b = []Behavior{ + { + ID: id, + APIObject: apiObject, + APIField: p, + APIType: propSchema.Type[0], + Description: "Boolean set to true. " + propSchema.Description, + }, + { + ID: id, + APIObject: apiObject, + APIField: p, + APIType: propSchema.Type[0], + Description: "Boolean set to false. " + propSchema.Description, + }, + } + default: + b = []Behavior{{ + ID: id, + APIObject: apiObject, + APIField: p, + APIType: propSchema.Type[0], + Description: propSchema.Description, + }} + } + return b +} diff --git a/test/conformance/kubetestgen/types.go b/test/conformance/kubetestgen/types.go new file mode 100644 index 00000000000..0cbf1b7d806 --- /dev/null +++ b/test/conformance/kubetestgen/types.go @@ -0,0 +1,39 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// Area is a conformance area composed of a list of test suites +type Area struct { + Area string `json:"area,omitempty"` + Suites []Suite `json:"suites,omitempty"` +} + +// Suite is a conformance test suite composed of a list of behaviors +type Suite struct { + Suite string `json:"suite,omitempty"` + Description string `json:"description,omitempty"` + Behaviors []Behavior `json:"behaviors,omitempty"` +} + +// Behavior describes the set of properties for a conformance behavior +type Behavior struct { + ID string `json:"id,omitempty"` + APIObject string `json:"apiObject,omitempty"` + APIField string `json:"apiField,omitempty"` + APIType string `json:"apiType,omitempty"` + Description string `json:"description,omitempty"` +}