Merge pull request #23272 from wojtek-t/conversion_generator_with_framework

Auto commit by PR queue bot

cmd/libs/go2idl/conversion-gen/generators/conversion.go

@@ -0,0 +1,514 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package generators
+
+import (
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/kubernetes/cmd/libs/go2idl/args"
+	"k8s.io/kubernetes/cmd/libs/go2idl/generator"
+	"k8s.io/kubernetes/cmd/libs/go2idl/namer"
+	"k8s.io/kubernetes/cmd/libs/go2idl/types"
+	"k8s.io/kubernetes/pkg/util/sets"
+
+	"github.com/golang/glog"
+)
+
+// TODO: This is created only to reduce number of changes in a single PR.
+// Remove it and use PublicNamer instead.
+func conversionNamer() *namer.NameStrategy {
+	return &namer.NameStrategy{
+		Join: func(pre string, in []string, post string) string {
+			return strings.Join(in, "_")
+		},
+		PrependPackageNames: 1,
+	}
+}
+
+// NameSystems returns the name system used by the generators in this package.
+func NameSystems() namer.NameSystems {
+	return namer.NameSystems{
+		"public": conversionNamer(),
+		"raw":    namer.NewRawNamer("", nil),
+	}
+}
+
+// DefaultNameSystem returns the default name system for ordering the types to be
+// processed by the generators in this package.
+func DefaultNameSystem() string {
+	return "public"
+}
+
+func getInternalTypeFor(context *generator.Context, t *types.Type) (*types.Type, bool) {
+	internalPackage := filepath.Dir(t.Name.Package)
+	if !context.Universe.Package(internalPackage).Has(t.Name.Name) {
+		return nil, false
+	}
+	return context.Universe.Package(internalPackage).Type(t.Name.Name), true
+}
+
+func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages {
+	boilerplate, err := arguments.LoadGoBoilerplate()
+	if err != nil {
+		glog.Fatalf("Failed loading boilerplate: %v", err)
+	}
+
+	inputs := sets.NewString(arguments.InputDirs...)
+	packages := generator.Packages{}
+	header := append([]byte(
+		`
+// +build !ignore_autogenerated
+
+`), boilerplate...)
+	header = append(header, []byte(
+		`
+// This file was autogenerated by conversion-gen. Do not edit it manually!
+
+`)...)
+
+	// We are generating conversions only for packages that are explicitly
+	// passed as InputDir, and only for those that have a corresponding type
+	// (in the directory one above) and can be automatically converted to.
+	for _, p := range context.Universe {
+		path := p.Path
+		if !inputs.Has(path) {
+			continue
+		}
+
+		convertibleType := false
+		for _, t := range p.Types {
+			// Check whether this type can be auto-converted to the internal
+			// version.
+			internalType, exists := getInternalTypeFor(context, t)
+			if !exists {
+				// There is no corresponding type in the internal package.
+				continue
+			}
+			if isConvertible(t, internalType) && isConvertible(internalType, t) {
+				convertibleType = true
+			}
+		}
+
+		if convertibleType {
+			packages = append(packages,
+				&generator.DefaultPackage{
+					PackageName: filepath.Base(path),
+					PackagePath: path,
+					HeaderText:  header,
+					GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
+						generators = []generator.Generator{}
+						generators = append(
+							generators, NewGenConversion("conversion_generated", path))
+						return generators
+					},
+					FilterFunc: func(c *generator.Context, t *types.Type) bool {
+						return t.Name.Package == path
+					},
+				})
+		}
+	}
+	return packages
+}
+
+func findMember(t *types.Type, name string) (types.Member, bool) {
+	if t.Kind != types.Struct {
+		return types.Member{}, false
+	}
+	for _, member := range t.Members {
+		if member.Name == name {
+			return member, true
+		}
+	}
+	return types.Member{}, false
+}
+
+func isConvertible(in, out *types.Type) bool {
+	// FIXME: Check manually-written conversion functions.
+
+	// If one of the types is Alias, resolve it.
+	if in.Kind == types.Alias {
+		return isConvertible(in.Underlying, out)
+	}
+	if out.Kind == types.Alias {
+		return isConvertible(in, out.Underlying)
+	}
+
+	if in.Kind != out.Kind {
+		return false
+	}
+	switch in.Kind {
+	case types.Builtin, types.Struct, types.Map, types.Slice, types.Pointer:
+	default:
+		// We don't support conversion of other types yet.
+		return false
+	}
+	switch out.Kind {
+	case types.Builtin, types.Struct, types.Map, types.Slice, types.Pointer:
+	default:
+		// We don't support conversion of other types yet.
+		return false
+	}
+
+	switch in.Kind {
+	case types.Builtin:
+		// FIXME: Enough to be convertible - see AWSElastic
+		return in.Name == out.Name
+	case types.Struct:
+		convertible := true
+		for _, inMember := range in.Members {
+			// Check if there is an out member with that name.
+			outMember, found := findMember(out, inMember.Name)
+			if !found {
+				return false
+			}
+			convertible = convertible && isConvertible(inMember.Type, outMember.Type)
+		}
+		return convertible
+	case types.Map:
+		return isConvertible(in.Key, out.Key) && isConvertible(in.Elem, out.Elem)
+	case types.Slice:
+		return isConvertible(in.Elem, out.Elem)
+	case types.Pointer:
+		return isConvertible(in.Elem, out.Elem)
+	}
+	glog.Fatalf("All other types should be filtered before")
+	return false
+}
+
+const (
+	apiPackagePath        = "k8s.io/kubernetes/pkg/api"
+	conversionPackagePath = "k8s.io/kubernetes/pkg/conversion"
+)
+
+// genConversion produces a file with a autogenerated conversions.
+type genConversion struct {
+	generator.DefaultGen
+	targetPackage string
+	imports       namer.ImportTracker
+	typesForInit  []*types.Type
+}
+
+func NewGenConversion(sanitizedName, targetPackage string) generator.Generator {
+	return &genConversion{
+		DefaultGen: generator.DefaultGen{
+			OptionalName: sanitizedName,
+		},
+		targetPackage: targetPackage,
+		imports:       generator.NewImportTracker(),
+		typesForInit:  make([]*types.Type, 0),
+	}
+}
+
+func (g *genConversion) Namers(c *generator.Context) namer.NameSystems {
+	// Have the raw namer for this file track what it imports.
+	return namer.NameSystems{"raw": namer.NewRawNamer(g.targetPackage, g.imports)}
+}
+
+func (g *genConversion) convertibleOnlyWithinPackage(inType, outType *types.Type) bool {
+	var t *types.Type
+	if inType.Name.Package == g.targetPackage {
+		t = inType
+	} else {
+		t = outType
+	}
+
+	if t.Name.Package != g.targetPackage {
+		return false
+	}
+	if types.ExtractCommentTags("+", t.CommentLines)["genConversion"] == "false" {
+		return false
+	}
+	// TODO: Consider generating functions for other kinds too.
+	if t.Kind != types.Struct {
+		return false
+	}
+	// Also, filter out private types.
+	if namer.IsPrivateGoName(t.Name.Name) {
+		return false
+	}
+	return true
+}
+
+func (g *genConversion) Filter(c *generator.Context, t *types.Type) bool {
+	internalType, exists := getInternalTypeFor(c, t)
+	if !g.convertibleOnlyWithinPackage(t, internalType) {
+		return false
+	}
+	if exists && isConvertible(t, internalType) && isConvertible(internalType, t) {
+		g.typesForInit = append(g.typesForInit, t)
+		return true
+	}
+	return false
+}
+
+func (g *genConversion) isOtherPackage(pkg string) bool {
+	if pkg == g.targetPackage {
+		return false
+	}
+	if strings.HasSuffix(pkg, `"`+g.targetPackage+`"`) {
+		return false
+	}
+	return true
+}
+
+func (g *genConversion) Imports(c *generator.Context) (imports []string) {
+	importLines := []string{"reflect \"reflect\""}
+	if g.isOtherPackage(apiPackagePath) {
+		importLines = append(importLines, "api \""+apiPackagePath+"\"")
+	}
+	if g.isOtherPackage(conversionPackagePath) {
+		importLines = append(importLines, "conversion \""+conversionPackagePath+"\"")
+	}
+	for _, singleImport := range g.imports.ImportLines() {
+		if g.isOtherPackage(singleImport) {
+			importLines = append(importLines, singleImport)
+		}
+	}
+	return importLines
+}
+
+func argsFromType(inType, outType *types.Type) interface{} {
+	return map[string]interface{}{
+		"inType":  inType,
+		"outType": outType,
+	}
+}
+
+func (g *genConversion) funcNameTmpl(inType, outType *types.Type) string {
+	tmpl := "Convert_$.inType|public$_To_$.outType|public$"
+	g.imports.AddType(inType)
+	g.imports.AddType(outType)
+	return tmpl
+}
+
+func (g *genConversion) Init(c *generator.Context, w io.Writer) error {
+	sw := generator.NewSnippetWriter(w, c, "$", "$")
+	sw.Do("func init() {\n", nil)
+	if g.targetPackage == apiPackagePath {
+		sw.Do("if err := Scheme.AddGeneratedConversionFuncs(\n", nil)
+	} else {
+		sw.Do("if err := api.Scheme.AddGeneratedConversionFuncs(\n", nil)
+	}
+	for _, t := range g.typesForInit {
+		internalType, _ := getInternalTypeFor(c, t)
+		sw.Do(fmt.Sprintf("%s,\n", g.funcNameTmpl(t, internalType)), argsFromType(t, internalType))
+		sw.Do(fmt.Sprintf("%s,\n", g.funcNameTmpl(internalType, t)), argsFromType(internalType, t))
+	}
+	sw.Do("); err != nil {\n", nil)
+	sw.Do("// if one of the conversion functions is malformed, detect it immediately.\n", nil)
+	sw.Do("panic(err)\n", nil)
+	sw.Do("}\n", nil)
+	sw.Do("}\n\n", nil)
+	return sw.Error()
+}
+
+func (g *genConversion) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error {
+	sw := generator.NewSnippetWriter(w, c, "$", "$")
+	internalType, _ := getInternalTypeFor(c, t)
+	g.generateConversion(t, internalType, sw)
+	g.generateConversion(internalType, t, sw)
+
+	return sw.Error()
+}
+
+func (g *genConversion) generateConversion(inType, outType *types.Type, sw *generator.SnippetWriter) {
+	funcName := g.funcNameTmpl(inType, outType)
+	if g.targetPackage == conversionPackagePath {
+		sw.Do(fmt.Sprintf("func %s(in $.inType|raw$, out *$.outType|raw$, s *Scope) error {\n", funcName), argsFromType(inType, outType))
+	} else {
+		sw.Do(fmt.Sprintf("func %s(in $.inType|raw$, out *$.outType|raw$, s *conversion.Scope) error {\n", funcName), argsFromType(inType, outType))
+	}
+	// FIXME: Generate defaulting.
+	g.generateFor(inType, outType, sw)
+	sw.Do("return nil\n", nil)
+	sw.Do("}\n\n", nil)
+}
+
+// we use the system of shadowing 'in' and 'out' so that the same code is valid
+// at any nesting level. This makes the autogenerator easy to understand, and
+// the compiler shouldn't care.
+func (g *genConversion) generateFor(inType, outType *types.Type, sw *generator.SnippetWriter) {
+	var f func(*types.Type, *types.Type, *generator.SnippetWriter)
+	switch inType.Kind {
+	case types.Builtin:
+		f = g.doBuiltin
+	case types.Map:
+		f = g.doMap
+	case types.Slice:
+		f = g.doSlice
+	case types.Struct:
+		f = g.doStruct
+	case types.Pointer:
+		f = g.doPointer
+	case types.Alias:
+		f = g.doAlias
+	default:
+		f = g.doUnknown
+	}
+	f(inType, outType, sw)
+}
+
+func (g *genConversion) doBuiltin(inType, outType *types.Type, sw *generator.SnippetWriter) {
+	sw.Do("*out = in\n", nil)
+}
+
+func (g *genConversion) doMap(inType, outType *types.Type, sw *generator.SnippetWriter) {
+	sw.Do("*out = make($.|raw$)\n", outType)
+	if outType.Key.IsAssignable() {
+		sw.Do("for key, val := range in {\n", nil)
+		if outType.Elem.IsAssignable() {
+			if inType.Elem == outType.Elem {
+				sw.Do("(*out)[key] = val\n", nil)
+			} else {
+				sw.Do("(*out)[key] = $.|raw$(val)\n", outType.Elem)
+			}
+		} else {
+			sw.Do("newVal := new($.|raw$)\n", outType.Elem)
+			if g.convertibleOnlyWithinPackage(inType.Elem, outType.Elem) {
+				funcName := g.funcNameTmpl(inType.Elem, outType.Elem)
+				sw.Do(fmt.Sprintf("if err := %s(val, newVal, s); err != nil {\n", funcName), argsFromType(inType.Elem, outType.Elem))
+			} else {
+				sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil)
+				sw.Do("if err := s.Convert(val, newVal, 0); err != nil {\n", nil)
+			}
+			sw.Do("return err\n", nil)
+			sw.Do("}\n", nil)
+			sw.Do("(*out)[key] = *newVal\n", nil)
+		}
+	} else {
+		// TODO: Implement it when necessary.
+		sw.Do("for range in {\n", nil)
+		sw.Do("// FIXME: Converting unassignable keys unsupported $.|raw$\n", inType.Key)
+	}
+	sw.Do("}\n", nil)
+}
+
+func (g *genConversion) doSlice(inType, outType *types.Type, sw *generator.SnippetWriter) {
+	sw.Do("*out = make($.|raw$, len(in))\n", outType)
+	if inType.Elem == outType.Elem && inType.Elem.Kind == types.Builtin {
+		sw.Do("copy(*out, in)\n", nil)
+	} else {
+		sw.Do("for i := range in {\n", nil)
+		if outType.Elem.IsAssignable() {
+			if inType.Elem == outType.Elem {
+				sw.Do("(*out)[i] = in[i]\n", nil)
+			} else {
+				sw.Do("(*out)[i] = $.|raw$(in[i])\n", outType.Elem)
+			}
+		} else {
+			if g.convertibleOnlyWithinPackage(inType.Elem, outType.Elem) {
+				funcName := g.funcNameTmpl(inType.Elem, outType.Elem)
+				sw.Do(fmt.Sprintf("if err := %s(in[i], &(*out)[i], c); err != nil {\n", funcName), argsFromType(inType.Elem, outType.Elem))
+			} else {
+				sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil)
+				sw.Do("if err := s.Convert(in[i], &out[i], 0); err != nil {\n", nil)
+			}
+			sw.Do("return err\n", nil)
+			sw.Do("}\n", nil)
+		}
+		sw.Do("}\n", nil)
+	}
+}
+
+func (g *genConversion) doStruct(inType, outType *types.Type, sw *generator.SnippetWriter) {
+	for _, m := range inType.Members {
+		outMember, _ := findMember(outType, m.Name)
+		args := map[string]interface{}{
+			"inType":  m.Type,
+			"outType": outMember.Type,
+			"name":    m.Name,
+		}
+		switch m.Type.Kind {
+		case types.Builtin:
+			sw.Do("out.$.name$ = in.$.name$\n", args)
+		case types.Map, types.Slice, types.Pointer:
+			sw.Do("if in.$.name$ != nil {\n", args)
+			sw.Do("in, out := in.$.name$, &out.$.name$\n", args)
+			g.generateFor(m.Type, outMember.Type, sw)
+			sw.Do("} else {\n", nil)
+			sw.Do("out.$.name$ = nil\n", args)
+			sw.Do("}\n", nil)
+		case types.Struct:
+			if g.convertibleOnlyWithinPackage(m.Type, outMember.Type) {
+				funcName := g.funcNameTmpl(m.Type, outMember.Type)
+				sw.Do(fmt.Sprintf("if err := %s(in.$.name$, &out.$.name$, c); err != nil {\n", funcName), args)
+			} else {
+				sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil)
+				sw.Do("if err := s.Convert(in.$.name$, &out.$.name$, 0); err != nil {\n", args)
+			}
+			sw.Do("return err\n", nil)
+			sw.Do("}\n", nil)
+		case types.Alias:
+			if outMember.Type.IsAssignable() {
+				sw.Do("out.$.name$ = $.outType|raw$(in.$.name$)\n", args)
+			} else {
+				if g.convertibleOnlyWithinPackage(m.Type, outMember.Type) {
+					funcName := g.funcNameTmpl(m.Type, outMember.Type)
+					sw.Do(fmt.Sprintf("if err := %s(in.$.name$, &out.$.name$, c); err != nil {\n", funcName), args)
+				} else {
+					sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil)
+					sw.Do("if err := s.Convert(in.$.name$, &out.$.name$, 0); err != nil {\n", args)
+				}
+				sw.Do("return err\n", nil)
+				sw.Do("}\n", nil)
+			}
+		default:
+			if g.convertibleOnlyWithinPackage(m.Type, outMember.Type) {
+				funcName := g.funcNameTmpl(m.Type, outMember.Type)
+				sw.Do(fmt.Sprintf("if err := %s(in.$.name$, &out.$.name$, c); err != nil {\n", funcName), args)
+			} else {
+				sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil)
+				sw.Do("if err := s.Convert(in.$.name$, &out.$.name$, 0); err != nil {\n", args)
+			}
+			sw.Do("return err\n", nil)
+			sw.Do("}\n", nil)
+		}
+	}
+}
+
+func (g *genConversion) doPointer(inType, outType *types.Type, sw *generator.SnippetWriter) {
+	sw.Do("*out = new($.Elem|raw$)\n", outType)
+	if outType.Elem.IsAssignable() {
+		if inType.Elem == outType.Elem {
+			sw.Do("**out = *in\n", nil)
+		} else {
+			sw.Do("**out = $.|raw$(*in)\n", outType.Elem)
+		}
+	} else {
+		if g.convertibleOnlyWithinPackage(inType.Elem, outType.Elem) {
+			funcName := g.funcNameTmpl(inType.Elem, outType.Elem)
+			sw.Do(fmt.Sprintf("if err := %s(*in, out, c); err != nil {\n", funcName), argsFromType(inType.Elem, outType.Elem))
+		} else {
+			sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil)
+			sw.Do("if err := s.Convert(*in, out, 0); err != nil {\n", nil)
+		}
+		sw.Do("return err\n", nil)
+		sw.Do("}\n", nil)
+	}
+}
+
+func (g *genConversion) doAlias(inType, outType *types.Type, sw *generator.SnippetWriter) {
+	// TODO: Add support for aliases.
+	g.doUnknown(inType, outType, sw)
+}
+
+func (g *genConversion) doUnknown(inType, outType *types.Type, sw *generator.SnippetWriter) {
+	sw.Do("// FIXME: Type $.|raw$ is unsupported.\n", inType)
+}

cmd/libs/go2idl/conversion-gen/main.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// conversion-gen is a tool for auto-generating Conversion functions.
+//
+// Structs in the input directories with the below line in their comments
+// will be ignored during generation.
+// // +genconversion=false
+package main
+
+import (
+	"k8s.io/kubernetes/cmd/libs/go2idl/args"
+	"k8s.io/kubernetes/cmd/libs/go2idl/conversion-gen/generators"
+
+	"github.com/golang/glog"
+)
+
+func main() {
+	arguments := args.Default()
+
+	// Override defaults. These are Kubernetes specific input locations.
+	arguments.InputDirs = []string{
+		"k8s.io/kubernetes/pkg/api/v1",
+	}
+
+	if err := arguments.Execute(
+		generators.NameSystems(),
+		generators.DefaultNameSystem(),
+		generators.Packages,
+	); err != nil {
+		glog.Fatalf("Error: %v", err)
+	}
+	glog.Info("Completed successfully.")
+}

cmd/libs/go2idl/deepcopy-gen/generators/deepcopy.go

@@ -95,10 +95,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat
 						return generators
 					},
 					FilterFunc: func(c *generator.Context, t *types.Type) bool {
-						if t.Name.Package != path {
+						return t.Name.Package == path
-							return false
-						}
-						return copyableWithinPackage(t)
 					},
 				})
 		}
@@ -111,7 +108,7 @@ const (
 	conversionPackagePath = "k8s.io/kubernetes/pkg/conversion"
 )
 
-// genDeepCopy produces a file with a set for a single type.
+// genDeepCopy produces a file with autogenerated deep-copy functions.
 type genDeepCopy struct {
 	generator.DefaultGen
 	targetPackage    string
@@ -137,7 +134,6 @@ func (g *genDeepCopy) Namers(c *generator.Context) namer.NameSystems {
 	return namer.NameSystems{"raw": namer.NewRawNamer(g.targetPackage, g.imports)}
 }
 
-// Filter ignores all but one type because we're making a single file per type.
 func (g *genDeepCopy) Filter(c *generator.Context, t *types.Type) bool {
 	// Filter out all types not copyable within the package.
 	copyable := copyableWithinPackage(t)
@@ -231,7 +227,6 @@ func (g *genDeepCopy) Init(c *generator.Context, w io.Writer) error {
 	return sw.Error()
 }
 
-// GenerateType makes the body of a file implementing a set for type t.
 func (g *genDeepCopy) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error {
 	sw := generator.NewSnippetWriter(w, c, "$", "$")
 	funcName := g.funcNameTmpl(t)