From b1e01875a10e6e28c3e971a8f660093fd95c43d3 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Thu, 26 Nov 2015 18:13:55 -0500 Subject: [PATCH] go-to-protobuf: generate protobuf IDL and marshalers for Go structs --- cmd/libs/go2idl/go-to-protobuf/.gitignore | 1 + cmd/libs/go2idl/go-to-protobuf/main.go | 36 + .../go2idl/go-to-protobuf/protobuf/cmd.go | 246 ++++++ .../go-to-protobuf/protobuf/generator.go | 705 ++++++++++++++++++ .../go-to-protobuf/protobuf/import_tracker.go | 117 +++ .../go2idl/go-to-protobuf/protobuf/namer.go | 188 +++++ .../go2idl/go-to-protobuf/protobuf/package.go | 169 +++++ .../go2idl/go-to-protobuf/protobuf/parser.go | 99 +++ .../go-to-protobuf/protoc-gen-gogo/main.go | 32 + hack/after-build/update-generated-protobuf.sh | 44 ++ hack/after-build/verify-generated-protobuf.sh | 63 ++ hack/update-generated-protobuf.sh | 30 + hack/verify-all.sh | 3 +- hack/verify-generated-protobuf.sh | 30 + 14 files changed, 1762 insertions(+), 1 deletion(-) create mode 100644 cmd/libs/go2idl/go-to-protobuf/.gitignore create mode 100644 cmd/libs/go2idl/go-to-protobuf/main.go create mode 100644 cmd/libs/go2idl/go-to-protobuf/protobuf/cmd.go create mode 100644 cmd/libs/go2idl/go-to-protobuf/protobuf/generator.go create mode 100644 cmd/libs/go2idl/go-to-protobuf/protobuf/import_tracker.go create mode 100644 cmd/libs/go2idl/go-to-protobuf/protobuf/namer.go create mode 100644 cmd/libs/go2idl/go-to-protobuf/protobuf/package.go create mode 100644 cmd/libs/go2idl/go-to-protobuf/protobuf/parser.go create mode 100644 cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo/main.go create mode 100755 hack/after-build/update-generated-protobuf.sh create mode 100755 hack/after-build/verify-generated-protobuf.sh create mode 100755 hack/update-generated-protobuf.sh create mode 100755 hack/verify-generated-protobuf.sh diff --git a/cmd/libs/go2idl/go-to-protobuf/.gitignore b/cmd/libs/go2idl/go-to-protobuf/.gitignore new file mode 100644 index 00000000000..0e9aa466bba --- /dev/null +++ b/cmd/libs/go2idl/go-to-protobuf/.gitignore @@ -0,0 +1 @@ +go-to-protobuf diff --git a/cmd/libs/go2idl/go-to-protobuf/main.go b/cmd/libs/go2idl/go-to-protobuf/main.go new file mode 100644 index 00000000000..b6fac17ad47 --- /dev/null +++ b/cmd/libs/go2idl/go-to-protobuf/main.go @@ -0,0 +1,36 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any +// existing IDL tags on the Go struct. +package main + +import ( + "k8s.io/kubernetes/cmd/libs/go2idl/go-to-protobuf/protobuf" + + flag "github.com/spf13/pflag" +) + +var g = protobuf.New() + +func init() { + g.BindFlags(flag.CommandLine) +} + +func main() { + flag.Parse() + protobuf.Run(g) +} diff --git a/cmd/libs/go2idl/go-to-protobuf/protobuf/cmd.go b/cmd/libs/go2idl/go-to-protobuf/protobuf/cmd.go new file mode 100644 index 00000000000..cfc19d09353 --- /dev/null +++ b/cmd/libs/go2idl/go-to-protobuf/protobuf/cmd.go @@ -0,0 +1,246 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any +// existing IDL tags on the Go struct. +package protobuf + +import ( + "bytes" + "fmt" + "log" + "os/exec" + "path/filepath" + "strings" + + "k8s.io/kubernetes/cmd/libs/go2idl/args" + "k8s.io/kubernetes/cmd/libs/go2idl/generator" + "k8s.io/kubernetes/cmd/libs/go2idl/namer" + "k8s.io/kubernetes/cmd/libs/go2idl/parser" + "k8s.io/kubernetes/cmd/libs/go2idl/types" + + flag "github.com/spf13/pflag" +) + +type Generator struct { + Common args.GeneratorArgs + Packages string + OutputBase string + ProtoImport []string + Conditional string + Clean bool + OnlyIDL bool + SkipGeneratedRewrite bool + DropEmbeddedFields string +} + +func New() *Generator { + sourceTree := args.DefaultSourceTree() + common := args.GeneratorArgs{ + OutputBase: sourceTree, + GoHeaderFilePath: filepath.Join(sourceTree, "k8s.io/kubernetes/hack/boilerplate/boilerplate.go.txt"), + } + defaultProtoImport := filepath.Join(sourceTree, "k8s.io", "kubernetes", "Godeps", "_workspace", "src", "github.com", "gogo", "protobuf", "protobuf") + return &Generator{ + Common: common, + OutputBase: sourceTree, + ProtoImport: []string{defaultProtoImport}, + Packages: `+k8s.io/kubernetes/pkg/util/intstr,` + + `+k8s.io/kubernetes/pkg/api/resource,` + + `+k8s.io/kubernetes/pkg/runtime,` + + `k8s.io/kubernetes/pkg/api/unversioned,` + + `k8s.io/kubernetes/pkg/api/v1,` + + `k8s.io/kubernetes/pkg/apis/extensions/v1beta1`, + DropEmbeddedFields: "k8s.io/kubernetes/pkg/api/unversioned.TypeMeta", + } +} + +func (g *Generator) BindFlags(flag *flag.FlagSet) { + flag.StringVarP(&g.Common.GoHeaderFilePath, "go-header-file", "h", g.Common.GoHeaderFilePath, "File containing boilerplate header text. The string YEAR will be replaced with the current 4-digit year.") + flag.BoolVar(&g.Common.VerifyOnly, "verify-only", g.Common.VerifyOnly, "If true, only verify existing output, do not write anything.") + flag.StringVarP(&g.Packages, "packages", "p", g.Packages, "comma-separated list of directories to get input types from. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.") + flag.StringVarP(&g.OutputBase, "output-base", "o", g.OutputBase, "Output base; defaults to $GOPATH/src/") + flag.StringSliceVar(&g.ProtoImport, "proto-import", g.ProtoImport, "The search path for the core protobuf .protos, required, defaults to GODEPS on path.") + flag.StringVar(&g.Conditional, "conditional", g.Conditional, "An optional Golang build tag condition to add to the generated Go code") + flag.BoolVar(&g.Clean, "clean", g.Clean, "If true, remove all generated files for the specified Packages.") + flag.BoolVar(&g.OnlyIDL, "only-idl", g.OnlyIDL, "If true, only generate the IDL for each package.") + flag.BoolVar(&g.SkipGeneratedRewrite, "skip-generated-rewrite", g.SkipGeneratedRewrite, "If true, skip fixing up the generated.pb.go file (debugging only).") + flag.StringVar(&g.DropEmbeddedFields, "drop-embedded-fields", g.DropEmbeddedFields, "Comma-delimited list of embedded Go types to omit from generated protobufs") +} + +const ( + typesKindProtobuf = "Protobuf" +) + +func Run(g *Generator) { + if g.Common.VerifyOnly { + g.OnlyIDL = true + g.Clean = false + } + + b := parser.New() + b.AddBuildTags("proto") + + omitTypes := map[types.Name]struct{}{} + for _, t := range strings.Split(g.DropEmbeddedFields, ",") { + name := types.Name{} + if i := strings.LastIndex(t, "."); i != -1 { + name.Package, name.Name = t[:i], t[i+1:] + } else { + name.Name = t + } + if len(name.Name) == 0 { + log.Fatalf("--drop-embedded-types requires names in the form of [GOPACKAGE.]TYPENAME: %v", t) + } + omitTypes[name] = struct{}{} + } + + boilerplate, err := g.Common.LoadGoBoilerplate() + if err != nil { + log.Fatalf("Failed loading boilerplate: %v", err) + } + + protobufNames := NewProtobufNamer() + outputPackages := generator.Packages{} + for _, d := range strings.Split(g.Packages, ",") { + generateAllTypes, outputPackage := true, true + switch { + case strings.HasPrefix(d, "+"): + d = d[1:] + generateAllTypes = false + case strings.HasPrefix(d, "-"): + d = d[1:] + outputPackage = false + } + name := protoSafePackage(d) + parts := strings.SplitN(d, "=", 2) + if len(parts) > 1 { + d = parts[0] + name = parts[1] + } + p := newProtobufPackage(d, name, generateAllTypes, omitTypes) + header := append([]byte{}, boilerplate...) + header = append(header, p.HeaderText...) + p.HeaderText = header + protobufNames.Add(p) + if outputPackage { + outputPackages = append(outputPackages, p) + } + } + + if !g.Common.VerifyOnly { + for _, p := range outputPackages { + if err := p.(*protobufPackage).Clean(g.OutputBase); err != nil { + log.Fatalf("Unable to clean package %s: %v", p.Name(), err) + } + } + } + + if g.Clean { + return + } + + for _, p := range protobufNames.List() { + if err := b.AddDir(p.Path()); err != nil { + log.Fatalf("Unable to add directory %q: %v", p.Path(), err) + } + } + + c, err := generator.NewContext( + b, + namer.NameSystems{ + "public": namer.NewPublicNamer(3), + "proto": protobufNames, + }, + "public", + ) + c.Verify = g.Common.VerifyOnly + c.FileTypes["protoidl"] = protoIDLFileType{} + + if err != nil { + log.Fatalf("Failed making a context: %v", err) + } + + if err := protobufNames.AssignTypesToPackages(c); err != nil { + log.Fatalf("Failed to identify Common types: %v", err) + } + + if err := c.ExecutePackages(g.OutputBase, outputPackages); err != nil { + log.Fatalf("Failed executing generator: %v", err) + } + + if g.OnlyIDL { + return + } + + if _, err := exec.LookPath("protoc"); err != nil { + log.Fatalf("Unable to find 'protoc': %v", err) + } + + searchArgs := []string{"-I", ".", "-I", g.OutputBase} + if len(g.ProtoImport) != 0 { + for _, s := range g.ProtoImport { + searchArgs = append(searchArgs, "-I", s) + } + } + args := append(searchArgs, fmt.Sprintf("--gogo_out=%s", g.OutputBase)) + + buf := &bytes.Buffer{} + if len(g.Conditional) > 0 { + fmt.Fprintf(buf, "// +build %s\n\n", g.Conditional) + } + buf.Write(boilerplate) + + for _, outputPackage := range outputPackages { + p := outputPackage.(*protobufPackage) + path := filepath.Join(g.OutputBase, p.ImportPath()) + outputPath := filepath.Join(g.OutputBase, p.OutputPath()) + cmd := exec.Command("protoc", append(args, path)...) + out, err := cmd.CombinedOutput() + if len(out) > 0 { + log.Printf(string(out)) + } + if err != nil { + log.Println(strings.Join(cmd.Args, " ")) + log.Fatalf("Unable to generate protoc on %s: %v", p.PackageName, err) + } + if !g.SkipGeneratedRewrite { + if err := RewriteGeneratedGogoProtobufFile(outputPath, p.GoPackageName(), p.HasGoType, buf.Bytes()); err != nil { + log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err) + } + + cmd := exec.Command("goimports", "-w", outputPath) + out, err := cmd.CombinedOutput() + if len(out) > 0 { + log.Printf(string(out)) + } + if err != nil { + log.Println(strings.Join(cmd.Args, " ")) + log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err) + } + + cmd = exec.Command("gofmt", "-s", "-w", outputPath) + out, err = cmd.CombinedOutput() + if len(out) > 0 { + log.Printf(string(out)) + } + if err != nil { + log.Println(strings.Join(cmd.Args, " ")) + log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err) + } + } + } +} diff --git a/cmd/libs/go2idl/go-to-protobuf/protobuf/generator.go b/cmd/libs/go2idl/go-to-protobuf/protobuf/generator.go new file mode 100644 index 00000000000..f63307e444d --- /dev/null +++ b/cmd/libs/go2idl/go-to-protobuf/protobuf/generator.go @@ -0,0 +1,705 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protobuf + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + + "k8s.io/kubernetes/cmd/libs/go2idl/generator" + "k8s.io/kubernetes/cmd/libs/go2idl/namer" + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +// genProtoIDL produces a .proto IDL. +type genProtoIDL struct { + generator.DefaultGen + localPackage types.Name + localGoPackage types.Name + imports *ImportTracker + + generateAll bool + omitFieldTypes map[types.Name]struct{} +} + +func (g *genProtoIDL) PackageVars(c *generator.Context) []string { + return []string{ + "option (gogoproto.marshaler_all) = true;", + "option (gogoproto.sizer_all) = true;", + "option (gogoproto.unmarshaler_all) = true;", + "option (gogoproto.goproto_unrecognized_all) = false;", + "option (gogoproto.goproto_enum_prefix_all) = false;", + "option (gogoproto.goproto_getters_all) = false;", + fmt.Sprintf("option go_package = %q;", g.localGoPackage.Name), + } +} +func (g *genProtoIDL) Filename() string { return g.OptionalName + ".proto" } +func (g *genProtoIDL) FileType() string { return "protoidl" } +func (g *genProtoIDL) Namers(c *generator.Context) namer.NameSystems { + return namer.NameSystems{ + // The local namer returns the correct protobuf name for a proto type + // in the context of a package + "local": localNamer{g.localPackage}, + } +} + +// Filter ignores types that are identified as not exportable. +func (g *genProtoIDL) Filter(c *generator.Context, t *types.Type) bool { + flags := types.ExtractCommentTags("+", t.CommentLines) + switch { + case flags["protobuf"] == "false": + return false + case flags["protobuf"] == "true": + return true + case !g.generateAll: + return false + } + seen := map[*types.Type]bool{} + ok := isProtoable(seen, t) + return ok +} + +func isProtoable(seen map[*types.Type]bool, t *types.Type) bool { + if seen[t] { + // be optimistic in the case of type cycles. + return true + } + seen[t] = true + switch t.Kind { + case types.Builtin: + return true + case types.Alias: + return isProtoable(seen, t.Underlying) + case types.Slice, types.Pointer: + return isProtoable(seen, t.Elem) + case types.Map: + return isProtoable(seen, t.Key) && isProtoable(seen, t.Elem) + case types.Struct: + for _, m := range t.Members { + if isProtoable(seen, m.Type) { + return true + } + } + return false + case types.Func, types.Chan: + return false + case types.DeclarationOf, types.Unknown, types.Unsupported: + return false + case types.Interface: + return false + default: + log.Printf("WARNING: type %q is not protable: %s", t.Kind, t.Name) + return false + } +} + +func (g *genProtoIDL) Imports(c *generator.Context) (imports []string) { + return g.imports.ImportLines() +} + +// GenerateType makes the body of a file implementing a set for type t. +func (g *genProtoIDL) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { + sw := generator.NewSnippetWriter(w, c, "$", "$") + b := bodyGen{ + locator: &protobufLocator{ + namer: c.Namers["proto"].(ProtobufFromGoNamer), + tracker: g.imports, + + localGoPackage: g.localGoPackage.Package, + }, + localPackage: g.localPackage, + omitFieldTypes: g.omitFieldTypes, + + t: t, + } + switch t.Kind { + case types.Struct: + return b.doStruct(sw) + default: + return b.unknown(sw) + } + return sw.Error() +} + +// ProtobufFromGoNamer finds the protobuf name of a type (and its package, and +// the package path) from its Go name. +type ProtobufFromGoNamer interface { + GoNameToProtoName(name types.Name) types.Name +} + +type ProtobufLocator interface { + ProtoTypeFor(t *types.Type) (*types.Type, error) + CastTypeName(name types.Name) string +} + +type protobufLocator struct { + namer ProtobufFromGoNamer + tracker namer.ImportTracker + + localGoPackage string +} + +// CastTypeName returns the cast type name of a Go type +// TODO: delegate to a new localgo namer? +func (p protobufLocator) CastTypeName(name types.Name) string { + if name.Package == p.localGoPackage { + return name.Name + } + return name.String() +} + +// ProtoTypeFor locates a Protobuf type for the provided Go type (if possible). +func (p protobufLocator) ProtoTypeFor(t *types.Type) (*types.Type, error) { + switch { + // we've already converted the type, or it's a map + case t.Kind == typesKindProtobuf || t.Kind == types.Map: + p.tracker.AddType(t) + return t, nil + } + // it's a fundamental type + if t, ok := isFundamentalProtoType(t); ok { + p.tracker.AddType(t) + return t, nil + } + // it's a message + if t.Kind == types.Struct { + t := &types.Type{ + Name: p.namer.GoNameToProtoName(t.Name), + Kind: typesKindProtobuf, + + CommentLines: t.CommentLines, + } + p.tracker.AddType(t) + return t, nil + } + return nil, errUnrecognizedType +} + +type bodyGen struct { + locator ProtobufLocator + localPackage types.Name + omitFieldTypes map[types.Name]struct{} + + t *types.Type +} + +func (b bodyGen) unknown(sw *generator.SnippetWriter) error { + return fmt.Errorf("not sure how to generate: %#v", b.t) +} + +func (b bodyGen) doStruct(sw *generator.SnippetWriter) error { + if len(b.t.Name.Name) == 0 { + return nil + } + if isPrivateGoName(b.t.Name.Name) { + return nil + } + + var fields []protoField + options := []string{} + allOptions := types.ExtractCommentTags("+", b.t.CommentLines) + for k, v := range allOptions { + switch { + case strings.HasPrefix(k, "protobuf.options."): + key := strings.TrimPrefix(k, "protobuf.options.") + switch key { + case "marshal": + if v == "false" { + options = append(options, + "(gogoproto.marshaler) = false", + "(gogoproto.unmarshaler) = false", + "(gogoproto.sizer) = false", + ) + } + default: + options = append(options, fmt.Sprintf("%s = %s", key, v)) + } + case k == "protobuf.embed": + fields = []protoField{ + { + Tag: 1, + Name: v, + Type: &types.Type{ + Name: types.Name{ + Name: v, + Package: b.localPackage.Package, + Path: b.localPackage.Path, + }, + }, + }, + } + } + } + + if fields == nil { + memberFields, err := membersToFields(b.locator, b.t, b.localPackage, b.omitFieldTypes) + if err != nil { + return fmt.Errorf("type %v cannot be converted to protobuf: %v", b.t, err) + } + fields = memberFields + } + + out := sw.Out() + genComment(out, b.t.CommentLines, "") + sw.Do(`message $.Name.Name$ { +`, b.t) + + if len(options) > 0 { + sort.Sort(sort.StringSlice(options)) + for _, s := range options { + fmt.Fprintf(out, " option %s;\n", s) + } + fmt.Fprintln(out) + } + + for i, field := range fields { + genComment(out, field.CommentLines, " ") + fmt.Fprintf(out, " ") + switch { + case field.Map: + case field.Repeated: + fmt.Fprintf(out, "repeated ") + case field.Required: + fmt.Fprintf(out, "required ") + default: + fmt.Fprintf(out, "optional ") + } + sw.Do(`$.Type|local$ $.Name$ = $.Tag$`, field) + if len(field.Extras) > 0 { + fmt.Fprintf(out, " [") + extras := []string{} + for k, v := range field.Extras { + extras = append(extras, fmt.Sprintf("%s = %s", k, v)) + } + sort.Sort(sort.StringSlice(extras)) + fmt.Fprint(out, strings.Join(extras, ", ")) + fmt.Fprintf(out, "]") + } + fmt.Fprintf(out, ";\n") + if i != len(fields)-1 { + fmt.Fprintf(out, "\n") + } + } + fmt.Fprintf(out, "}\n\n") + return nil +} + +type protoField struct { + LocalPackage types.Name + + Tag int + Name string + Type *types.Type + Map bool + Repeated bool + Optional bool + Required bool + Nullable bool + Extras map[string]string + + CommentLines string + + OptionalSet bool +} + +var ( + errUnrecognizedType = fmt.Errorf("did not recognize the provided type") +) + +func isFundamentalProtoType(t *types.Type) (*types.Type, bool) { + // TODO: when we enable proto3, also include other fundamental types in the google.protobuf package + // switch { + // case t.Kind == types.Struct && t.Name == types.Name{Package: "time", Name: "Time"}: + // return &types.Type{ + // Kind: typesKindProtobuf, + // Name: types.Name{Path: "google/protobuf/timestamp.proto", Package: "google.protobuf", Name: "Timestamp"}, + // }, true + // } + switch t.Kind { + case types.Slice: + if t.Elem.Name.Name == "byte" && len(t.Elem.Name.Package) == 0 { + return &types.Type{Name: types.Name{Name: "bytes"}, Kind: typesKindProtobuf}, true + } + case types.Builtin: + switch t.Name.Name { + case "string", "uint32", "int32", "uint64", "int64", "bool": + return &types.Type{Name: types.Name{Name: t.Name.Name}, Kind: typesKindProtobuf}, true + case "int": + return &types.Type{Name: types.Name{Name: "int64"}, Kind: typesKindProtobuf}, true + case "uint": + return &types.Type{Name: types.Name{Name: "uint64"}, Kind: typesKindProtobuf}, true + case "float64", "float": + return &types.Type{Name: types.Name{Name: "double"}, Kind: typesKindProtobuf}, true + case "float32": + return &types.Type{Name: types.Name{Name: "float"}, Kind: typesKindProtobuf}, true + case "uintptr": + return &types.Type{Name: types.Name{Name: "uint64"}, Kind: typesKindProtobuf}, true + } + // TODO: complex? + } + return t, false +} + +func memberTypeToProtobufField(locator ProtobufLocator, field *protoField, t *types.Type) error { + var err error + switch t.Kind { + case typesKindProtobuf: + field.Type, err = locator.ProtoTypeFor(t) + case types.Builtin: + field.Type, err = locator.ProtoTypeFor(t) + case types.Map: + valueField := &protoField{} + if err := memberTypeToProtobufField(locator, valueField, t.Elem); err != nil { + return err + } + keyField := &protoField{} + if err := memberTypeToProtobufField(locator, keyField, t.Key); err != nil { + return err + } + field.Type = &types.Type{ + Kind: types.Map, + Key: keyField.Type, + Elem: valueField.Type, + } + if !strings.HasPrefix(t.Name.Name, "map[") { + field.Extras["(gogoproto.casttype)"] = strconv.Quote(locator.CastTypeName(t.Name)) + } + if k, ok := keyField.Extras["(gogoproto.casttype)"]; ok { + field.Extras["(gogoproto.castkey)"] = k + } + if v, ok := valueField.Extras["(gogoproto.casttype)"]; ok { + field.Extras["(gogoproto.castvalue)"] = v + } + field.Map = true + case types.Pointer: + if err := memberTypeToProtobufField(locator, field, t.Elem); err != nil { + return err + } + field.Nullable = true + case types.Alias: + if err := memberTypeToProtobufField(locator, field, t.Underlying); err != nil { + log.Printf("failed to alias: %s %s: err", t.Name, t.Underlying.Name, err) + return err + } + if field.Extras == nil { + field.Extras = make(map[string]string) + } + field.Extras["(gogoproto.casttype)"] = strconv.Quote(locator.CastTypeName(t.Name)) + case types.Slice: + if t.Elem.Name.Name == "byte" && len(t.Elem.Name.Package) == 0 { + field.Type = &types.Type{Name: types.Name{Name: "bytes"}, Kind: typesKindProtobuf} + return nil + } + if err := memberTypeToProtobufField(locator, field, t.Elem); err != nil { + return err + } + field.Repeated = true + case types.Struct: + if len(t.Name.Name) == 0 { + return errUnrecognizedType + } + field.Type, err = locator.ProtoTypeFor(t) + field.Nullable = false + default: + return errUnrecognizedType + } + return err +} + +// protobufTagToField extracts information from an existing protobuf tag +// TODO: take a current package +func protobufTagToField(tag string, field *protoField, m types.Member, t *types.Type, localPackage types.Name) error { + if len(tag) == 0 { + return nil + } + + // protobuf:"bytes,3,opt,name=Id,customtype=github.com/gogo/protobuf/test.Uuid" + parts := strings.Split(tag, ",") + if len(parts) < 3 { + return fmt.Errorf("member %q of %q malformed 'protobuf' tag, not enough segments\n", m.Name, t.Name) + } + protoTag, err := strconv.Atoi(parts[1]) + if err != nil { + return fmt.Errorf("member %q of %q malformed 'protobuf' tag, field ID is %q which is not an integer: %v\n", m.Name, t.Name, parts[1], err) + } + field.Tag = protoTag + // TODO: we are converting a Protobuf type back into an internal type, which is questionable + if last := strings.LastIndex(parts[0], "."); last != -1 { + prefix := parts[0][:last] + field.Type = &types.Type{ + Name: types.Name{ + Name: parts[0][last+1:], + Package: prefix, + // TODO: this probably needs to be a lookup into a namer + Path: strings.Replace(prefix, ".", "/", -1), + }, + Kind: typesKindProtobuf, + } + } else { + field.Type = &types.Type{ + Name: types.Name{ + Name: parts[0], + Package: localPackage.Package, + Path: localPackage.Path, + }, + Kind: typesKindProtobuf, + } + } + switch parts[2] { + case "rep": + field.Repeated = true + case "opt": + field.Optional = true + case "req": + default: + return fmt.Errorf("member %q of %q malformed 'protobuf' tag, field mode is %q not recognized\n", m.Name, t.Name, parts[2]) + } + field.OptionalSet = true + + protoExtra := make(map[string]string) + for i, extra := range parts[3:] { + parts := strings.SplitN(extra, "=", 2) + if len(parts) != 2 { + return fmt.Errorf("member %q of %q malformed 'protobuf' tag, tag %d should be key=value, got %q\n", m.Name, t.Name, i+4, extra) + } + protoExtra[parts[0]] = parts[1] + } + + field.Extras = protoExtra + if name, ok := protoExtra["name"]; ok { + field.Name = name + delete(protoExtra, "name") + } + + return nil +} + +func membersToFields(locator ProtobufLocator, t *types.Type, localPackage types.Name, omitFieldTypes map[types.Name]struct{}) ([]protoField, error) { + fields := []protoField{} + + for _, m := range t.Members { + if isPrivateGoName(m.Name) { + // skip private fields + continue + } + if _, ok := omitFieldTypes[types.Name{Name: m.Type.Name.Name, Package: m.Type.Name.Package}]; ok { + continue + } + tags := reflect.StructTag(m.Tags) + field := protoField{ + LocalPackage: localPackage, + + Tag: -1, + Extras: make(map[string]string), + } + + if err := protobufTagToField(tags.Get("protobuf"), &field, m, t, localPackage); err != nil { + return nil, err + } + + // extract information from JSON field tag + if tag := tags.Get("json"); len(tag) > 0 { + parts := strings.Split(tag, ",") + if len(field.Name) == 0 && len(parts[0]) != 0 { + field.Name = parts[0] + } + if field.Name == "-" { + continue + } + } + + if field.Type == nil { + if err := memberTypeToProtobufField(locator, &field, m.Type); err != nil { + return nil, fmt.Errorf("unable to embed type %q as field %q in %q: %v", m.Type, field.Name, t.Name, err) + } + } + if len(field.Name) == 0 { + field.Name = strings.ToLower(m.Name[:1]) + m.Name[1:] + } + + if field.Map && field.Repeated { + // maps cannot be repeated + field.Repeated = false + field.Nullable = true + } + + if !field.Nullable { + field.Extras["(gogoproto.nullable)"] = "false" + } + if (field.Type.Name.Name == "bytes" && field.Type.Name.Package == "") || (field.Repeated && field.Type.Name.Package == "" && isPrivateGoName(field.Type.Name.Name)) { + delete(field.Extras, "(gogoproto.nullable)") + } + if field.Name != m.Name { + field.Extras["(gogoproto.customname)"] = strconv.Quote(m.Name) + } + field.CommentLines = m.CommentLines + fields = append(fields, field) + } + + // assign tags + highest := 0 + byTag := make(map[int]*protoField) + // fields are in Go struct order, which we preserve + for i := range fields { + field := &fields[i] + tag := field.Tag + if tag != -1 { + if existing, ok := byTag[tag]; ok { + return nil, fmt.Errorf("field %q and %q in %q both have tag %d", field.Name, existing.Name, tag) + } + byTag[tag] = field + } + if tag > highest { + highest = tag + } + } + // starting from the highest observed tag, assign new field tags + for i := range fields { + field := &fields[i] + if field.Tag != -1 { + continue + } + highest++ + field.Tag = highest + byTag[field.Tag] = field + } + return fields, nil +} + +func genComment(out io.Writer, comment, indent string) { + lines := strings.Split(comment, "\n") + for { + l := len(lines) + if l == 0 || len(lines[l-1]) != 0 { + break + } + lines = lines[:l-1] + } + for _, c := range lines { + fmt.Fprintf(out, "%s// %s\n", indent, c) + } +} + +type protoIDLFileType struct{} + +func (ft protoIDLFileType) AssembleFile(f *generator.File, pathname string) error { + log.Printf("Assembling IDL file %q", pathname) + destFile, err := os.Create(pathname) + if err != nil { + return err + } + defer destFile.Close() + + b := &bytes.Buffer{} + et := generator.NewErrorTracker(b) + ft.assemble(et, f) + if et.Error() != nil { + return et.Error() + } + + // TODO: is there an IDL formatter? + _, err = destFile.Write(b.Bytes()) + return err +} + +func (ft protoIDLFileType) VerifyFile(f *generator.File, pathname string) error { + log.Printf("Verifying IDL file %q", pathname) + friendlyName := filepath.Join(f.PackageName, f.Name) + b := &bytes.Buffer{} + et := generator.NewErrorTracker(b) + ft.assemble(et, f) + if et.Error() != nil { + return et.Error() + } + formatted := b.Bytes() + existing, err := ioutil.ReadFile(pathname) + if err != nil { + return fmt.Errorf("unable to read file %q for comparison: %v", friendlyName, err) + } + if bytes.Compare(formatted, existing) == 0 { + return nil + } + // Be nice and find the first place where they differ + i := 0 + for i < len(formatted) && i < len(existing) && formatted[i] == existing[i] { + i++ + } + eDiff, fDiff := existing[i:], formatted[i:] + if len(eDiff) > 100 { + eDiff = eDiff[:100] + } + if len(fDiff) > 100 { + fDiff = fDiff[:100] + } + return fmt.Errorf("output for %q differs; first existing/expected diff: \n %q\n %q", friendlyName, string(eDiff), string(fDiff)) +} + +func (ft protoIDLFileType) assemble(w io.Writer, f *generator.File) { + w.Write(f.Header) + + fmt.Fprint(w, "syntax = 'proto2';\n\n") + + if len(f.PackageName) > 0 { + fmt.Fprintf(w, "package %v;\n\n", f.PackageName) + } + + if len(f.Imports) > 0 { + imports := []string{} + for i := range f.Imports { + imports = append(imports, i) + } + sort.Strings(imports) + for _, s := range imports { + fmt.Fprintf(w, "import %q;\n", s) + } + fmt.Fprint(w, "\n") + } + + if f.Vars.Len() > 0 { + fmt.Fprintf(w, "%s\n", f.Vars.String()) + } + + w.Write(f.Body.Bytes()) +} + +func isPackable(t *types.Type) bool { + if t.Kind != typesKindProtobuf { + return false + } + switch t.Name.Name { + case "int32", "int64", "varint": + return true + default: + return false + } +} + +func isPrivateGoName(name string) bool { + if len(name) == 0 { + return true + } + return strings.ToLower(name[:1]) == name[:1] +} diff --git a/cmd/libs/go2idl/go-to-protobuf/protobuf/import_tracker.go b/cmd/libs/go2idl/go-to-protobuf/protobuf/import_tracker.go new file mode 100644 index 00000000000..6603fcb4c4e --- /dev/null +++ b/cmd/libs/go2idl/go-to-protobuf/protobuf/import_tracker.go @@ -0,0 +1,117 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protobuf + +import ( + "fmt" + "sort" + + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +// ImportTracker handles Protobuf package imports +// +// TODO: pay attention to the package name (instead of renaming every package). +// TODO: Figure out the best way to make names for packages that collide. +type ImportTracker struct { + pathToName map[string]string + // forbidden names are in here. (e.g. "go" is a directory in which + // there is code, but "go" is not a legal name for a package, so we put + // it here to prevent us from naming any package "go") + nameToPath map[string]string + local types.Name +} + +func NewImportTracker(local types.Name, types ...*types.Type) *ImportTracker { + tracker := &ImportTracker{ + local: local, + pathToName: map[string]string{}, + nameToPath: map[string]string{ + // Add other forbidden keywords that also happen to be + // package names here. + }, + } + tracker.AddTypes(types...) + return tracker +} + +// AddNullable ensures that support for the nullable Gogo-protobuf extension is added. +func (tracker *ImportTracker) AddNullable() { + tracker.AddType(&types.Type{ + Kind: typesKindProtobuf, + Name: types.Name{ + Name: "nullable", + Package: "gogoproto", + Path: "github.com/gogo/protobuf/gogoproto/gogo.proto", + }, + }) +} + +func (tracker *ImportTracker) AddTypes(types ...*types.Type) { + for _, t := range types { + tracker.AddType(t) + } +} +func (tracker *ImportTracker) AddType(t *types.Type) { + if tracker.local.Package == t.Name.Package { + return + } + // Golang type + if t.Kind != typesKindProtobuf { + // ignore built in package + if t.Kind == types.Builtin { + return + } + if _, ok := tracker.nameToPath[t.Name.Package]; !ok { + tracker.nameToPath[t.Name.Package] = "" + } + return + } + // ignore default proto package + if len(t.Name.Package) == 0 { + return + } + path := t.Name.Path + if len(path) == 0 { + panic(fmt.Sprintf("imported proto package %s must also have a path", t.Name.Package)) + } + if _, ok := tracker.pathToName[path]; ok { + return + } + tracker.nameToPath[t.Name.Package] = path + tracker.pathToName[path] = t.Name.Package +} + +func (tracker *ImportTracker) ImportLines() []string { + for k, v := range tracker.nameToPath { + if len(v) == 0 { + panic(fmt.Sprintf("tracking import Go package %s from %s, but no matching proto path set", k, tracker.local.Package)) + } + } + out := []string{} + for path := range tracker.pathToName { + out = append(out, path) + } + sort.Sort(sort.StringSlice(out)) + return out +} + +// LocalNameOf returns the name you would use to refer to the package at the +// specified path within the body of a file. +func (tracker *ImportTracker) LocalNameOf(path string) string { + return tracker.pathToName[path] +} diff --git a/cmd/libs/go2idl/go-to-protobuf/protobuf/namer.go b/cmd/libs/go2idl/go-to-protobuf/protobuf/namer.go new file mode 100644 index 00000000000..f7e83d2e215 --- /dev/null +++ b/cmd/libs/go2idl/go-to-protobuf/protobuf/namer.go @@ -0,0 +1,188 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protobuf + +import ( + "fmt" + "reflect" + "strings" + + "k8s.io/kubernetes/cmd/libs/go2idl/generator" + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +type localNamer struct { + localPackage types.Name +} + +func (n localNamer) Name(t *types.Type) string { + if t.Kind == types.Map { + return fmt.Sprintf("map<%s, %s>", n.Name(t.Key), n.Name(t.Elem)) + } + if len(n.localPackage.Package) != 0 && n.localPackage.Package == t.Name.Package { + return t.Name.Name + } + return t.Name.String() +} + +type protobufNamer struct { + packages []*protobufPackage + packagesByPath map[string]*protobufPackage +} + +func NewProtobufNamer() *protobufNamer { + return &protobufNamer{ + packagesByPath: make(map[string]*protobufPackage), + } +} + +func (n *protobufNamer) Name(t *types.Type) string { + if t.Kind == types.Map { + return fmt.Sprintf("map<%s, %s>", n.Name(t.Key), n.Name(t.Elem)) + } + return t.Name.String() +} + +func (n *protobufNamer) List() []generator.Package { + packages := make([]generator.Package, 0, len(n.packages)) + for i := range n.packages { + packages = append(packages, n.packages[i]) + } + return packages +} + +func (n *protobufNamer) Add(p *protobufPackage) { + n.packagesByPath[p.PackagePath] = p + n.packages = append(n.packages, p) +} + +func (n *protobufNamer) GoNameToProtoName(name types.Name) types.Name { + if p, ok := n.packagesByPath[name.Package]; ok { + return types.Name{ + Name: name.Name, + Package: p.PackageName, + Path: p.ImportPath(), + } + } + for _, p := range n.packages { + if _, ok := p.FilterTypes[name]; ok { + return types.Name{ + Name: name.Name, + Package: p.PackageName, + Path: p.ImportPath(), + } + } + } + return types.Name{Name: name.Name} +} + +func (n *protobufNamer) protoNameForTypeName(name types.Name) string { + packageName := name.Package + if len(name.Package) != 0 { + if p, ok := n.packagesByPath[packageName]; ok { + packageName = p.Name() + } else { + packageName = protoSafePackage(packageName) + } + } + if len(name.Name) == 0 { + return packageName + } + if len(packageName) > 0 { + return packageName + "." + name.Name + } + return name.Name +} + +func protoSafePackage(name string) string { + return strings.Replace(name, "/", ".", -1) +} + +type typeNameSet map[types.Name]*protobufPackage + +// assignGoTypeToProtoPackage looks for Go and Protobuf types that are referenced by a type in +// a package. It will not recurse into protobuf types. +func assignGoTypeToProtoPackage(p *protobufPackage, t *types.Type, local, global typeNameSet) { + newT, isProto := isFundamentalProtoType(t) + if isProto { + t = newT + } + if otherP, ok := global[t.Name]; ok { + if _, ok := local[t.Name]; !ok { + p.Imports.AddType(&types.Type{ + Kind: typesKindProtobuf, + Name: otherP.ProtoTypeName(), + }) + } + return + } + global[t.Name] = p + if _, ok := local[t.Name]; ok { + return + } + // don't recurse into existing proto types + if isProto { + p.Imports.AddType(t) + return + } + + local[t.Name] = p + for _, m := range t.Members { + if isPrivateGoName(m.Name) { + continue + } + field := &protoField{} + if err := protobufTagToField(reflect.StructTag(m.Tags).Get("protobuf"), field, m, t, p.ProtoTypeName()); err == nil && field.Type != nil { + assignGoTypeToProtoPackage(p, field.Type, local, global) + continue + } + assignGoTypeToProtoPackage(p, m.Type, local, global) + } + // TODO: should methods be walked? + if t.Elem != nil { + assignGoTypeToProtoPackage(p, t.Elem, local, global) + } + if t.Key != nil { + assignGoTypeToProtoPackage(p, t.Key, local, global) + } + if t.Underlying != nil { + assignGoTypeToProtoPackage(p, t.Underlying, local, global) + } +} + +func (n *protobufNamer) AssignTypesToPackages(c *generator.Context) error { + global := make(typeNameSet) + for _, p := range n.packages { + local := make(typeNameSet) + p.Imports = NewImportTracker(p.ProtoTypeName()) + for _, t := range c.Order { + if t.Name.Package != p.PackagePath { + continue + } + assignGoTypeToProtoPackage(p, t, local, global) + } + p.FilterTypes = make(map[types.Name]struct{}) + p.LocalNames = make(map[string]struct{}) + for k, v := range local { + if v == p { + p.FilterTypes[k] = struct{}{} + p.LocalNames[k.Name] = struct{}{} + } + } + } + return nil +} diff --git a/cmd/libs/go2idl/go-to-protobuf/protobuf/package.go b/cmd/libs/go2idl/go-to-protobuf/protobuf/package.go new file mode 100644 index 00000000000..6d9111c5a6d --- /dev/null +++ b/cmd/libs/go2idl/go-to-protobuf/protobuf/package.go @@ -0,0 +1,169 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protobuf + +import ( + "fmt" + "os" + "path/filepath" + + "k8s.io/kubernetes/cmd/libs/go2idl/generator" + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +func newProtobufPackage(packagePath, packageName string, generateAll bool, omitFieldTypes map[types.Name]struct{}) *protobufPackage { + pkg := &protobufPackage{ + // The protobuf package name (foo.bar.baz) + PackageName: packageName, + // A path segment relative to the GOPATH root (foo/bar/baz) + PackagePath: packagePath, + GenerateAll: generateAll, + OmitFieldTypes: omitFieldTypes, + HeaderText: []byte( + ` +// This file was autogenerated by the command: +// $ ` + os.Args[0] + ` +// Do not edit it manually! + +`), + PackageDocumentation: []byte(fmt.Sprintf( + `// Package %s is an autogenerated protobuf IDL. +`, packageName)), + } + return pkg +} + +// protobufPackage contains the protobuf implentation of Package. +type protobufPackage struct { + // Short name of package, used in the "package xxxx" line. + PackageName string + // Import path of the package, and the location on disk of the package. + PackagePath string + // If true, generate protobuf serializations for all public types. + // If false, only generate protobuf serializations for structs that + // request serialization. + GenerateAll bool + + // Emitted at the top of every file. + HeaderText []byte + + // Emitted only for a "doc.go" file; appended to the HeaderText for + // that file. + PackageDocumentation []byte + + // A list of types to filter to; if not specified all types will be included. + FilterTypes map[types.Name]struct{} + + // A list of field types that will be excluded from the output struct + OmitFieldTypes map[types.Name]struct{} + + // A list of names that this package exports + LocalNames map[string]struct{} + + // An import tracker for this package + Imports *ImportTracker +} + +func (p *protobufPackage) Clean(outputBase string) error { + for _, s := range []string{p.ImportPath(), p.OutputPath()} { + if err := os.Remove(filepath.Join(outputBase, s)); err != nil && !os.IsNotExist(err) { + return err + } + } + return nil +} + +func (p *protobufPackage) ProtoTypeName() types.Name { + return types.Name{ + Name: p.Path(), // the go path "foo/bar/baz" + Package: p.Name(), // the protobuf package "foo.bar.baz" + Path: p.ImportPath(), // the path of the import to get the proto + } +} + +func (p *protobufPackage) Name() string { return p.PackageName } +func (p *protobufPackage) Path() string { return p.PackagePath } + +func (p *protobufPackage) Filter(c *generator.Context, t *types.Type) bool { + switch t.Kind { + case types.Func, types.Chan: + return false + case types.Struct: + if t.Name.Name == "struct{}" { + return false + } + case types.Builtin: + return false + case types.Alias: + return false + case types.Slice, types.Array, types.Map: + return false + case types.Pointer: + return false + } + if _, ok := isFundamentalProtoType(t); ok { + return false + } + _, ok := p.FilterTypes[t.Name] + return ok +} + +func (p *protobufPackage) HasGoType(name string) bool { + _, ok := p.LocalNames[name] + return ok +} + +func (p *protobufPackage) Generators(c *generator.Context) []generator.Generator { + generators := []generator.Generator{} + + p.Imports.AddNullable() + + generators = append(generators, &genProtoIDL{ + DefaultGen: generator.DefaultGen{ + OptionalName: "generated", + }, + localPackage: types.Name{Package: p.PackageName, Path: p.PackagePath}, + localGoPackage: types.Name{Package: p.PackagePath, Name: p.GoPackageName()}, + imports: p.Imports, + generateAll: p.GenerateAll, + omitFieldTypes: p.OmitFieldTypes, + }) + return generators +} + +func (p *protobufPackage) Header(filename string) []byte { + if filename == "doc.go" { + return append(p.HeaderText, p.PackageDocumentation...) + } + return p.HeaderText +} + +func (p *protobufPackage) GoPackageName() string { + return filepath.Base(p.PackagePath) +} + +func (p *protobufPackage) ImportPath() string { + return filepath.Join(p.PackagePath, "generated.proto") +} + +func (p *protobufPackage) OutputPath() string { + return filepath.Join(p.PackagePath, "generated.pb.go") +} + +var ( + _ = generator.Package(&protobufPackage{}) +) diff --git a/cmd/libs/go2idl/go-to-protobuf/protobuf/parser.go b/cmd/libs/go2idl/go-to-protobuf/protobuf/parser.go new file mode 100644 index 00000000000..9a1cbfbddcd --- /dev/null +++ b/cmd/libs/go2idl/go-to-protobuf/protobuf/parser.go @@ -0,0 +1,99 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protobuf + +import ( + "bytes" + "go/format" + "io/ioutil" + "os" + + "k8s.io/kubernetes/third_party/golang/go/ast" + "k8s.io/kubernetes/third_party/golang/go/parser" + "k8s.io/kubernetes/third_party/golang/go/printer" + "k8s.io/kubernetes/third_party/golang/go/token" +) + +func RewriteGeneratedGogoProtobufFile(name string, packageName string, typeExistsFn func(string) bool, header []byte) error { + fset := token.NewFileSet() + src, err := ioutil.ReadFile(name) + if err != nil { + return err + } + file, err := parser.ParseFile(fset, name, src, parser.DeclarationErrors|parser.ParseComments) + if err != nil { + return err + } + cmap := ast.NewCommentMap(fset, file, file.Comments) + + // remove types that are already declared + decls := []ast.Decl{} + for _, d := range file.Decls { + if !dropExistingTypeDeclarations(d, typeExistsFn) { + decls = append(decls, d) + } + } + file.Decls = decls + + // remove unmapped comments + file.Comments = cmap.Filter(file).Comments() + + b := &bytes.Buffer{} + b.Write(header) + if err := printer.Fprint(b, fset, file); err != nil { + return err + } + + body, err := format.Source(b.Bytes()) + if err != nil { + return err + } + + f, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer f.Close() + if _, err := f.Write(body); err != nil { + return err + } + return f.Close() +} + +func dropExistingTypeDeclarations(decl ast.Decl, existsFn func(string) bool) bool { + switch t := decl.(type) { + case *ast.GenDecl: + if t.Tok != token.TYPE { + return false + } + specs := []ast.Spec{} + for _, s := range t.Specs { + switch spec := s.(type) { + case *ast.TypeSpec: + if existsFn(spec.Name.Name) { + continue + } + specs = append(specs, spec) + } + } + if len(specs) == 0 { + return true + } + t.Specs = specs + } + return false +} diff --git a/cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo/main.go b/cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo/main.go new file mode 100644 index 00000000000..df56c8a68c1 --- /dev/null +++ b/cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo/main.go @@ -0,0 +1,32 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package main defines the protoc-gen-gogo binary we use to generate our proto go files, +// as well as takes dependencies on the correct gogo/protobuf packages for godeps. +package main + +import ( + "github.com/gogo/protobuf/vanity/command" + + // dependencies that are required for our packages + _ "github.com/gogo/protobuf/gogoproto" + _ "github.com/gogo/protobuf/proto" + _ "github.com/gogo/protobuf/sortkeys" +) + +func main() { + command.Write(command.Generate(command.Read())) +} diff --git a/hack/after-build/update-generated-protobuf.sh b/hack/after-build/update-generated-protobuf.sh new file mode 100755 index 00000000000..0ff8a883af8 --- /dev/null +++ b/hack/after-build/update-generated-protobuf.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +# TODO: preinstall protobuf +if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then + echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and" + echo "install the platform appropriate Protobuf package for your OS: " + echo + echo " https://github.com/google/protobuf/releases" + echo + echo "WARNING: Protobuf changes are not being validated" + # TODO: make error when protobuf is installed + exit 0 +fi + +gotoprotobuf=$(kube::util::find-binary "go-to-protobuf") + +# requires the 'proto' tag to build (will remove when ready) +# searches for the protoc-gen-gogo extension in the output directory +# satisfies import of github.com/gogo/protobuf/gogoproto/gogo.proto and the core Google protobuf types +PATH="${KUBE_ROOT}/_output/local/go/bin:${PATH}" "${gotoprotobuf}" \ + --conditional="proto" \ + --proto-import="${KUBE_ROOT}/Godeps/_workspace/src" \ + --proto-import="${KUBE_ROOT}/third_party/protobuf" diff --git a/hack/after-build/verify-generated-protobuf.sh b/hack/after-build/verify-generated-protobuf.sh new file mode 100755 index 00000000000..0239d480b14 --- /dev/null +++ b/hack/after-build/verify-generated-protobuf.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +# TODO: preinstall protobuf +if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then + echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and" + echo "install the platform appropriate Protobuf package for your OS: " + echo + echo " https://github.com/google/protobuf/releases" + echo + echo "WARNING: Protobuf changes are not being validated" + # TODO: make error when protobuf is installed + exit 0 +fi + +APIROOTS=${APIROOTS:-pkg/api pkg/apis/extensions pkg/apis/metrics pkg/runtime pkg/util/intstr} +_tmp="${KUBE_ROOT}/_tmp" + +cleanup() { + rm -rf "${_tmp}" +} + +trap "cleanup" EXIT SIGINT + +for APIROOT in ${APIROOTS}; do + mkdir -p "${_tmp}/${APIROOT%/*}" + cp -a "${KUBE_ROOT}/${APIROOT}" "${_tmp}/${APIROOT}" +done + +"${KUBE_ROOT}/hack/after-build/update-generated-protobuf.sh" +for APIROOT in ${APIROOTS}; do + TMP_APIROOT="${_tmp}/${APIROOT}" + echo "diffing ${APIROOT} against freshly generated protobuf" + ret=0 + diff -Naupr -I 'Auto generated by' "${KUBE_ROOT}/${APIROOT}" "${TMP_APIROOT}" || ret=$? + cp -a "${TMP_APIROOT}" "${KUBE_ROOT}/${APIROOT%/*}" + if [[ $ret -eq 0 ]]; then + echo "${APIROOT} up to date." + else + echo "${APIROOT} is out of date. Please run hack/update-generated-protobuf.sh" + exit 1 + fi +done diff --git a/hack/update-generated-protobuf.sh b/hack/update-generated-protobuf.sh new file mode 100755 index 00000000000..df3df7bb1b3 --- /dev/null +++ b/hack/update-generated-protobuf.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +kube::golang::setup_env + +"${KUBE_ROOT}/hack/build-go.sh" cmd/libs/go2idl/go-to-protobuf cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo + +"${KUBE_ROOT}/hack/after-build/update-generated-protobuf.sh" "$@" + +# ex: ts=2 sw=2 et filetype=sh diff --git a/hack/verify-all.sh b/hack/verify-all.sh index 17fcfb41818..f30eecdcc01 100755 --- a/hack/verify-all.sh +++ b/hack/verify-all.sh @@ -59,7 +59,8 @@ if $SILENT ; then echo "Running in the silent mode, run with -v if you want to see script logs." fi -EXCLUDE="verify-godeps.sh" +# remove protobuf until it is part of direct generation +EXCLUDE="verify-godeps.sh verify-generated-protobuf.sh" ret=0 for t in `ls $KUBE_ROOT/hack/verify-*.sh` diff --git a/hack/verify-generated-protobuf.sh b/hack/verify-generated-protobuf.sh new file mode 100755 index 00000000000..043aa42bd8e --- /dev/null +++ b/hack/verify-generated-protobuf.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +kube::golang::setup_env + +"${KUBE_ROOT}/hack/build-go.sh" cmd/libs/go2idl/go-to-protobuf cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo + +"${KUBE_ROOT}/hack/after-build/verify-generated-protobuf.sh" "$@" + +# ex: ts=2 sw=2 et filetype=sh