Improvements on OpenAPI spec generation:

- Generating models using go2idl library (no reflection anymore)
- Remove dependencies on go-restful/swagger
- Generate one swagger.json file for each web-service
- Bugfix: fixed a bug in trie implementation

cmd/libs/go2idl/openapi-gen/generators/openapi.go

@@ -0,0 +1,519 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package generators
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
+
+	"k8s.io/kubernetes/cmd/libs/go2idl/args"
+	"k8s.io/kubernetes/cmd/libs/go2idl/generator"
+	"k8s.io/kubernetes/cmd/libs/go2idl/namer"
+	"k8s.io/kubernetes/cmd/libs/go2idl/openapi-gen/generators/common"
+	"k8s.io/kubernetes/cmd/libs/go2idl/types"
+	"k8s.io/kubernetes/pkg/util/sets"
+
+	"github.com/golang/glog"
+)
+
+// This is the comment tag that carries parameters for open API generation.
+const tagName = "k8s:openapi-gen"
+
+// Known values for the tag.
+const (
+	tagValueTrue  = "true"
+	tagValueFalse = "false"
+	// Should only be used only for test
+	tagTargetType = "target"
+)
+
+func hasOpenAPITagValue(comments []string, value string) bool {
+	tagValues := types.ExtractCommentTags("+", comments)[tagName]
+	if tagValues == nil {
+		return false
+	}
+	for _, val := range tagValues {
+		if val == value {
+			return true
+		}
+	}
+	return false
+}
+
+// NameSystems returns the name system used by the generators in this package.
+func NameSystems() namer.NameSystems {
+	return namer.NameSystems{
+		"raw": namer.NewRawNamer("", nil),
+	}
+}
+
+// DefaultNameSystem returns the default name system for ordering the types to be
+// processed by the generators in this package.
+func DefaultNameSystem() string {
+	return "raw"
+}
+
+func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages {
+	boilerplate, err := arguments.LoadGoBoilerplate()
+	if err != nil {
+		glog.Fatalf("Failed loading boilerplate: %v", err)
+	}
+	inputs := sets.NewString(context.Inputs...)
+	header := append([]byte(fmt.Sprintf("// +build !%s\n\n", arguments.GeneratedBuildTag)), boilerplate...)
+	header = append(header, []byte(
+		`
+// This file was autogenerated by openapi-gen. Do not edit it manually!
+
+`)...)
+
+	targets := []*types.Type{}
+	for i := range inputs {
+		glog.V(5).Infof("considering pkg %q", i)
+		pkg, ok := context.Universe[i]
+		if !ok {
+			// If the input had no Go files, for example.
+			continue
+		}
+		for _, t := range pkg.Types {
+			if hasOpenAPITagValue(t.CommentLines, tagTargetType) {
+				glog.V(5).Infof("target type : %q", t)
+				targets = append(targets, t)
+			}
+		}
+	}
+	switch len(targets) {
+	case 0:
+		// If no target package found, that means the generated file in target package is up to date
+		// and build excluded the target package.
+		return generator.Packages{}
+	case 1:
+		pkg := context.Universe[targets[0].Name.Package]
+		return generator.Packages{&generator.DefaultPackage{
+			PackageName: strings.Split(filepath.Base(pkg.Path), ".")[0],
+			PackagePath: pkg.Path,
+			HeaderText:  header,
+			GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
+				return []generator.Generator{NewOpenAPIGen(arguments.OutputFileBaseName, targets[0], context)}
+			},
+			FilterFunc: func(c *generator.Context, t *types.Type) bool {
+				// There is a conflict between this codegen and codecgen, we should avoid types generated for codecgen
+				if strings.HasPrefix(t.Name.Name, "codecSelfer") {
+					return false
+				}
+				pkg := context.Universe.Package(t.Name.Package)
+				if hasOpenAPITagValue(pkg.Comments, tagValueTrue) {
+					return !hasOpenAPITagValue(t.CommentLines, tagValueFalse)
+				}
+				if hasOpenAPITagValue(t.CommentLines, tagValueTrue) {
+					return true
+				}
+				return false
+			},
+		},
+		}
+	default:
+		glog.Fatalf("Duplicate target type found: %v", targets)
+	}
+	return generator.Packages{}
+}
+
+const (
+	specPackagePath          = "github.com/go-openapi/spec"
+	openAPICommonPackagePath = "k8s.io/kubernetes/cmd/libs/go2idl/openapi-gen/generators/common"
+)
+
+// openApiGen produces a file with auto-generated OpenAPI functions.
+type openAPIGen struct {
+	generator.DefaultGen
+	// TargetType is the type that will get OpenAPIDefinitions method returning all definitions.
+	targetType *types.Type
+	imports    namer.ImportTracker
+	context    *generator.Context
+}
+
+func NewOpenAPIGen(sanitizedName string, targetType *types.Type, context *generator.Context) generator.Generator {
+	return &openAPIGen{
+		DefaultGen: generator.DefaultGen{
+			OptionalName: sanitizedName,
+		},
+		imports:    generator.NewImportTracker(),
+		targetType: targetType,
+		context:    context,
+	}
+}
+
+func (g *openAPIGen) Namers(c *generator.Context) namer.NameSystems {
+	// Have the raw namer for this file track what it imports.
+	return namer.NameSystems{
+		"raw": namer.NewRawNamer(g.targetType.Name.Package, g.imports),
+	}
+}
+
+func (g *openAPIGen) Filter(c *generator.Context, t *types.Type) bool {
+	// There is a conflict between this codegen and codecgen, we should avoid types generated for codecgen
+	if strings.HasPrefix(t.Name.Name, "codecSelfer") {
+		return false
+	}
+	return true
+}
+
+func (g *openAPIGen) isOtherPackage(pkg string) bool {
+	if pkg == g.targetType.Name.Package {
+		return false
+	}
+	if strings.HasSuffix(pkg, "\""+g.targetType.Name.Package+"\"") {
+		return false
+	}
+	return true
+}
+
+func (g *openAPIGen) Imports(c *generator.Context) []string {
+	importLines := []string{}
+	for _, singleImport := range g.imports.ImportLines() {
+		importLines = append(importLines, singleImport)
+	}
+	return importLines
+}
+
+func argsFromType(t *types.Type) generator.Args {
+	return generator.Args{
+		"type":               t,
+		"OpenAPIDefinitions": types.Ref(openAPICommonPackagePath, "OpenAPIDefinitions"),
+		"OpenAPIDefinition":  types.Ref(openAPICommonPackagePath, "OpenAPIDefinition"),
+		"SpecSchemaType":     types.Ref(specPackagePath, "Schema"),
+	}
+}
+
+func (g *openAPIGen) Init(c *generator.Context, w io.Writer) error {
+	sw := generator.NewSnippetWriter(w, c, "$", "$")
+	sw.Do("func (_ $.type|raw$) OpenAPIDefinitions() *$.OpenAPIDefinitions|raw$ {\n", argsFromType(g.targetType))
+	sw.Do("return &$.OpenAPIDefinitions|raw${\n", argsFromType(nil))
+	return sw.Error()
+}
+
+func (g *openAPIGen) Finalize(c *generator.Context, w io.Writer) error {
+	sw := generator.NewSnippetWriter(w, c, "$", "$")
+	sw.Do("}\n}\n", nil)
+	return sw.Error()
+}
+
+func (g *openAPIGen) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error {
+	glog.V(5).Infof("generating for type %v", t)
+	sw := generator.NewSnippetWriter(w, c, "$", "$")
+	err := newOpenAPITypeWriter(sw).generate(t)
+	if err != nil {
+		return err
+	}
+	return sw.Error()
+}
+
+func getJsonTags(m *types.Member) []string {
+	jsonTag := reflect.StructTag(m.Tags).Get("json")
+	if jsonTag == "" {
+		return []string{}
+	}
+	return strings.Split(jsonTag, ",")
+}
+
+func getReferableName(m *types.Member) string {
+	jsonTags := getJsonTags(m)
+	if len(jsonTags) > 0 {
+		if jsonTags[0] == "-" {
+			return ""
+		} else {
+			return jsonTags[0]
+		}
+	} else {
+		return m.Name
+	}
+}
+
+func optionIndex(s, optionName string) int {
+	ret := 0
+	for s != "" {
+		var next string
+		i := strings.Index(s, ",")
+		if i >= 0 {
+			s, next = s[:i], s[i+1:]
+		}
+		if s == optionName {
+			return ret
+		}
+		s = next
+		ret++
+	}
+	return -1
+}
+
+func isPropertyRequired(m *types.Member) bool {
+	// A property is required if it does not have omitempty value in its json tag (documentation and implementation
+	// of json package requires omitempty to be at location 1 or higher.
+	// TODO: Move optional field definition from tags to comments.
+	return optionIndex(reflect.StructTag(m.Tags).Get("json"), "omitempty") < 1
+}
+
+type openAPITypeWriter struct {
+	*generator.SnippetWriter
+	refTypes               map[string]*types.Type
+	GetDefinitionInterface *types.Type
+}
+
+func newOpenAPITypeWriter(sw *generator.SnippetWriter) openAPITypeWriter {
+	return openAPITypeWriter{
+		SnippetWriter: sw,
+		refTypes:      map[string]*types.Type{},
+	}
+}
+
+func hasOpenAPIDefinitionMethod(t *types.Type) bool {
+	for mn, mt := range t.Methods {
+		if mn != "OpenAPIDefinition" {
+			continue
+		}
+		if len(mt.Signature.Parameters) != 0 || len(mt.Signature.Results) != 1 {
+			return false
+		}
+		r := mt.Signature.Results[0]
+		if r.Name.Name != "OpenAPIDefinition" || r.Name.Package != openAPICommonPackagePath {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+// typeShortName returns short package name (e.g. the name x appears in package x definition) dot type name.
+func typeShortName(t *types.Type) string {
+	return filepath.Base(t.Name.Package) + "." + t.Name.Name
+}
+
+func (g openAPITypeWriter) generate(t *types.Type) error {
+	// Only generate for struct type and ignore the rest
+	switch t.Kind {
+	case types.Struct:
+		args := argsFromType(t)
+		g.Do("\"$.$\": ", typeShortName(t))
+		if hasOpenAPIDefinitionMethod(t) {
+			g.Do("$.type|raw${}.OpenAPIDefinition(),", args)
+			return nil
+		}
+		g.Do("{\nSchema: spec.Schema{\nSchemaProps: spec.SchemaProps{\n", nil)
+		g.generateDescription(t.CommentLines)
+		g.Do("Properties: map[string]$.SpecSchemaType|raw${\n", args)
+		required := []string{}
+		for _, m := range t.Members {
+			if hasOpenAPITagValue(m.CommentLines, tagValueFalse) {
+				continue
+			}
+			name := getReferableName(&m)
+			if name == "" {
+				continue
+			}
+			if isPropertyRequired(&m) {
+				required = append(required, name)
+			}
+			if err := g.generateProperty(&m); err != nil {
+				return err
+			}
+		}
+		g.Do("},\n", nil)
+		if len(required) > 0 {
+			g.Do("Required: []string{\"$.$\"},\n", strings.Join(required, "\",\""))
+		}
+		g.Do("},\n},\n", nil)
+		g.Do("Dependencies: []string{\n", args)
+		// Map order is undefined, sort them or we may get a different file generated each time.
+		keys := []string{}
+		for k := range g.refTypes {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, k := range keys {
+			v := g.refTypes[k]
+			if t, _ := common.GetOpenAPITypeFormat(v.String()); t != "" {
+				// This is a known type, we do not need a reference to it
+				// Will eliminate special case of time.Time
+				continue
+			}
+			g.Do("\"$.$\",", k)
+		}
+		g.Do("},\n},\n", nil)
+	}
+	return nil
+}
+
+func (g openAPITypeWriter) generateDescription(CommentLines []string) {
+	var buffer bytes.Buffer
+	delPrevChar := func() {
+		if buffer.Len() > 0 {
+			buffer.Truncate(buffer.Len() - 1) // Delete the last " " or "\n"
+		}
+	}
+
+	for _, line := range CommentLines {
+		// Ignore all lines after ---
+		if line == "---" {
+			break
+		}
+		line = strings.TrimRight(line, " ")
+		leading := strings.TrimLeft(line, " ")
+		switch {
+		case len(line) == 0: // Keep paragraphs
+			delPrevChar()
+			buffer.WriteString("\n\n")
+		case strings.HasPrefix(leading, "TODO"): // Ignore one line TODOs
+		case strings.HasPrefix(leading, "+"): // Ignore instructions to go2idl
+		default:
+			if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") {
+				delPrevChar()
+				line = "\n" + line + "\n" // Replace it with newline. This is useful when we have a line with: "Example:\n\tJSON-someting..."
+			} else {
+				line += " "
+			}
+			buffer.WriteString(line)
+		}
+	}
+
+	postDoc := strings.TrimRight(buffer.String(), "\n")
+	postDoc = strings.Replace(postDoc, "\\\"", "\"", -1) // replace user's \" to "
+	postDoc = strings.Replace(postDoc, "\"", "\\\"", -1) // Escape "
+	postDoc = strings.Replace(postDoc, "\n", "\\n", -1)
+	postDoc = strings.Replace(postDoc, "\t", "\\t", -1)
+	postDoc = strings.Trim(postDoc, " ")
+	if postDoc != "" {
+		g.Do("Description: \"$.$\",\n", postDoc)
+	}
+}
+
+func (g openAPITypeWriter) generateProperty(m *types.Member) error {
+	name := getReferableName(m)
+	if name == "" {
+		return nil
+	}
+	g.Do("\"$.$\": {\n", name)
+	g.Do("SchemaProps: spec.SchemaProps{\n", nil)
+	g.generateDescription(m.CommentLines)
+	jsonTags := getJsonTags(m)
+	if len(jsonTags) > 1 && jsonTags[1] == "string" {
+		g.generateSimpleProperty("string", "")
+		g.Do("},\n},\n", nil)
+		return nil
+	}
+	t := resolveAliasAndPtrType(m.Type)
+	// If we can get a openAPI type and format for this type, we consider it to be simple property
+	typeString, format := common.GetOpenAPITypeFormat(t.String())
+	if typeString != "" {
+		g.generateSimpleProperty(typeString, format)
+		g.Do("},\n},\n", nil)
+		return nil
+	}
+	switch t.Kind {
+	case types.Builtin:
+		return fmt.Errorf("please add type %v to getOpenAPITypeFormat function.", t)
+	case types.Map:
+		if err := g.generateMapProperty(t); err != nil {
+			return err
+		}
+	case types.Slice, types.Array:
+		if err := g.generateSliceProperty(t); err != nil {
+			return err
+		}
+	case types.Struct, types.Interface:
+		g.generateReferenceProperty(t)
+	default:
+		return fmt.Errorf("cannot generate spec for type %v.", t)
+	}
+	g.Do("},\n},\n", nil)
+	return g.Error()
+}
+
+func (g openAPITypeWriter) generateSimpleProperty(typeString, format string) {
+	g.Do("Type: []string{\"$.$\"},\n", typeString)
+	g.Do("Format: \"$.$\",\n", format)
+}
+
+func (g openAPITypeWriter) generateReferenceProperty(t *types.Type) {
+	var name string
+	if t.Name.Package == "" {
+		name = t.Name.Name
+	} else {
+		name = filepath.Base(t.Name.Package) + "." + t.Name.Name
+	}
+	g.refTypes[name] = t
+	g.Do("Ref: spec.MustCreateRef(\"#/definitions/$.$\"),\n", name)
+}
+
+func resolveAliasAndPtrType(t *types.Type) *types.Type {
+	var prev *types.Type
+	for prev != t {
+		prev = t
+		if t.Kind == types.Alias {
+			t = t.Underlying
+		}
+		if t.Kind == types.Pointer {
+			t = t.Elem
+		}
+	}
+	return t
+}
+
+func (g openAPITypeWriter) generateMapProperty(t *types.Type) error {
+	keyType := resolveAliasAndPtrType(t.Key)
+	elemType := resolveAliasAndPtrType(t.Elem)
+
+	// According to OpenAPI examples, only map from string is supported
+	if keyType.Name.Name != "string" {
+		return fmt.Errorf("map with non-string keys are not supported by OpenAPI in %v", t)
+	}
+	g.Do("Type: []string{\"object\"},\n", nil)
+	g.Do("AdditionalProperties: &spec.SchemaOrBool{\nSchema: &spec.Schema{\nSchemaProps: spec.SchemaProps{\n", nil)
+	switch elemType.Kind {
+	case types.Builtin:
+		typeString, format := common.GetOpenAPITypeFormat(elemType.String())
+		g.generateSimpleProperty(typeString, format)
+	case types.Struct:
+		g.generateReferenceProperty(t.Elem)
+	case types.Slice, types.Array:
+		g.generateSliceProperty(elemType)
+	default:
+		return fmt.Errorf("map Element kind %v is not supported in %v", elemType.Kind, t.Name)
+	}
+	g.Do("},\n},\n},\n", nil)
+	return nil
+}
+
+func (g openAPITypeWriter) generateSliceProperty(t *types.Type) error {
+	elemType := resolveAliasAndPtrType(t.Elem)
+	g.Do("Type: []string{\"array\"},\n", nil)
+	g.Do("Items: &spec.SchemaOrArray{\nSchema: &spec.Schema{\nSchemaProps: spec.SchemaProps{\n", nil)
+	switch elemType.Kind {
+	case types.Builtin:
+		typeString, format := common.GetOpenAPITypeFormat(elemType.String())
+		g.generateSimpleProperty(typeString, format)
+	case types.Struct:
+		g.generateReferenceProperty(t.Elem)
+	default:
+		return fmt.Errorf("slice Element kind %v is not supported in %v", elemType.Kind, t)
+	}
+	g.Do("},\n},\n},\n", nil)
+	return nil
+}