From 72df8bbfbe4cfa3c7799cc1ac8f1522e24534bc2 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Thu, 26 Nov 2015 18:16:26 -0500 Subject: [PATCH] Allow go2idl to generate non-Go file types Remove some indentation from file generation for specific languages, allow SnippetWriter's io.Writer to be accessed, and add Path to types.Name for languages where Package and Path can be disjoint. --- .../generators/generator-for-group.go | 12 +-- .../generators/generator-for-type.go | 10 +-- .../go2idl/generator/default_generator.go | 5 ++ cmd/libs/go2idl/generator/execute.go | 80 ++++++++++--------- cmd/libs/go2idl/generator/generator.go | 27 +++++++ cmd/libs/go2idl/generator/snippet_writer.go | 4 + cmd/libs/go2idl/namer/namer_test.go | 10 +-- cmd/libs/go2idl/parser/parse.go | 5 ++ cmd/libs/go2idl/parser/parse_test.go | 6 +- cmd/libs/go2idl/types/flatten_test.go | 6 +- cmd/libs/go2idl/types/types.go | 53 ++++++++---- cmd/libs/go2idl/types/types_test.go | 4 +- 12 files changed, 146 insertions(+), 76 deletions(-) diff --git a/cmd/libs/go2idl/client-gen/generators/generator-for-group.go b/cmd/libs/go2idl/client-gen/generators/generator-for-group.go index 354ff5dc8cf..20aea8303d2 100644 --- a/cmd/libs/go2idl/client-gen/generators/generator-for-group.go +++ b/cmd/libs/go2idl/client-gen/generators/generator-for-group.go @@ -57,12 +57,12 @@ func (g *genGroup) GenerateType(c *generator.Context, t *types.Type, w io.Writer "group": g.group, "Group": namer.IC(g.group), "types": g.types, - "Config": c.Universe.Get(types.Name{pkgUnversioned, "Config"}), - "DefaultKubernetesUserAgent": c.Universe.Get(types.Name{pkgUnversioned, "DefaultKubernetesUserAgent"}), - "RESTClient": c.Universe.Get(types.Name{pkgUnversioned, "RESTClient"}), - "RESTClientFor": c.Universe.Get(types.Name{pkgUnversioned, "RESTClientFor"}), - "latestGroup": c.Universe.Get(types.Name{pkgLatest, "Group"}), - "GroupOrDie": c.Universe.Get(types.Name{pkgLatest, "GroupOrDie"}), + "Config": c.Universe.Get(types.Name{Package: pkgUnversioned, Name: "Config"}), + "DefaultKubernetesUserAgent": c.Universe.Get(types.Name{Package: pkgUnversioned, Name: "DefaultKubernetesUserAgent"}), + "RESTClient": c.Universe.Get(types.Name{Package: pkgUnversioned, Name: "RESTClient"}), + "RESTClientFor": c.Universe.Get(types.Name{Package: pkgUnversioned, Name: "RESTClientFor"}), + "latestGroup": c.Universe.Get(types.Name{Package: pkgLatest, Name: "Group"}), + "GroupOrDie": c.Universe.Get(types.Name{Package: pkgLatest, Name: "GroupOrDie"}), } sw.Do(groupInterfaceTemplate, m) sw.Do(groupClientTemplate, m) diff --git a/cmd/libs/go2idl/client-gen/generators/generator-for-type.go b/cmd/libs/go2idl/client-gen/generators/generator-for-type.go index 5e51a84515a..22cea110f65 100644 --- a/cmd/libs/go2idl/client-gen/generators/generator-for-type.go +++ b/cmd/libs/go2idl/client-gen/generators/generator-for-type.go @@ -54,11 +54,11 @@ func (g *genClientForType) GenerateType(c *generator.Context, t *types.Type, w i "type": t, "package": pkg, "Package": namer.IC(pkg), - "fieldSelector": c.Universe.Get(types.Name{"k8s.io/kubernetes/pkg/fields", "Selector"}), - "labelSelector": c.Universe.Get(types.Name{"k8s.io/kubernetes/pkg/labels", "Selector"}), - "watchInterface": c.Universe.Get(types.Name{"k8s.io/kubernetes/pkg/watch", "Interface"}), - "apiDeleteOptions": c.Universe.Get(types.Name{"k8s.io/kubernetes/pkg/api", "DeleteOptions"}), - "apiListOptions": c.Universe.Get(types.Name{"k8s.io/kubernetes/pkg/api", "ListOptions"}), + "fieldSelector": c.Universe.Get(types.Name{Package: "k8s.io/kubernetes/pkg/fields", Name: "Selector"}), + "labelSelector": c.Universe.Get(types.Name{Package: "k8s.io/kubernetes/pkg/labels", Name: "Selector"}), + "watchInterface": c.Universe.Get(types.Name{Package: "k8s.io/kubernetes/pkg/watch", Name: "Interface"}), + "apiDeleteOptions": c.Universe.Get(types.Name{Package: "k8s.io/kubernetes/pkg/api", Name: "DeleteOptions"}), + "apiListOptions": c.Universe.Get(types.Name{Package: "k8s.io/kubernetes/pkg/api", Name: "ListOptions"}), } sw.Do(namespacerTemplate, m) sw.Do(interfaceTemplate, m) diff --git a/cmd/libs/go2idl/generator/default_generator.go b/cmd/libs/go2idl/generator/default_generator.go index 734cf2d732f..0a4845cce0c 100644 --- a/cmd/libs/go2idl/generator/default_generator.go +++ b/cmd/libs/go2idl/generator/default_generator.go @@ -23,6 +23,10 @@ import ( "k8s.io/kubernetes/cmd/libs/go2idl/types" ) +const ( + GolangFileType = "golang" +) + // DefaultGen implements a do-nothing Generator. // // It can be used to implement static content files. @@ -45,6 +49,7 @@ func (d DefaultGen) PackageVars(*Context) []string { retur func (d DefaultGen) PackageConsts(*Context) []string { return []string{} } func (d DefaultGen) GenerateType(*Context, *types.Type, io.Writer) error { return nil } func (d DefaultGen) Filename() string { return d.OptionalName + ".go" } +func (d DefaultGen) FileType() string { return GolangFileType } func (d DefaultGen) Init(c *Context, w io.Writer) error { _, err := w.Write(d.OptionalBody) diff --git a/cmd/libs/go2idl/generator/execute.go b/cmd/libs/go2idl/generator/execute.go index 0495a3debbc..7aa333c1917 100644 --- a/cmd/libs/go2idl/generator/execute.go +++ b/cmd/libs/go2idl/generator/execute.go @@ -44,17 +44,9 @@ func (c *Context) ExecutePackages(outDir string, packages Packages) error { return nil } -type file struct { - name string - packageName string - header []byte - imports map[string]struct{} - vars bytes.Buffer - consts bytes.Buffer - body bytes.Buffer -} +type golangFileType struct{} -func (f *file) assembleToFile(pathname string) error { +func (ft golangFileType) AssembleFile(f *File, pathname string) error { log.Printf("Assembling file %q", pathname) destFile, err := os.Create(pathname) if err != nil { @@ -64,7 +56,7 @@ func (f *file) assembleToFile(pathname string) error { b := &bytes.Buffer{} et := NewErrorTracker(b) - f.assemble(et) + ft.assemble(et, f) if et.Error() != nil { return et.Error() } @@ -78,14 +70,14 @@ func (f *file) assembleToFile(pathname string) error { } } -func (f *file) assemble(w io.Writer) { - w.Write(f.header) - fmt.Fprintf(w, "package %v\n\n", f.packageName) +func (ft golangFileType) assemble(w io.Writer, f *File) { + w.Write(f.Header) + fmt.Fprintf(w, "package %v\n\n", f.PackageName) - if len(f.imports) > 0 { + if len(f.Imports) > 0 { fmt.Fprint(w, "import (\n") // TODO: sort imports like goimports does. - for i := range f.imports { + for i := range f.Imports { if strings.Contains(i, "\"") { // they included quotes, or are using the // `name "path/to/pkg"` format. @@ -97,27 +89,27 @@ func (f *file) assemble(w io.Writer) { fmt.Fprint(w, ")\n\n") } - if f.vars.Len() > 0 { + if f.Vars.Len() > 0 { fmt.Fprint(w, "var (\n") - w.Write(f.vars.Bytes()) + w.Write(f.Vars.Bytes()) fmt.Fprint(w, ")\n\n") } - if f.consts.Len() > 0 { + if f.Consts.Len() > 0 { fmt.Fprint(w, "const (\n") - w.Write(f.consts.Bytes()) + w.Write(f.Consts.Bytes()) fmt.Fprint(w, ")\n\n") } - w.Write(f.body.Bytes()) + w.Write(f.Body.Bytes()) } // format should be one line only, and not end with \n. func addIndentHeaderComment(b *bytes.Buffer, format string, args ...interface{}) { if b.Len() > 0 { - fmt.Fprintf(b, "\n\t// "+format+"\n", args...) + fmt.Fprintf(b, "\n// "+format+"\n", args...) } else { - fmt.Fprintf(b, "\t// "+format+"\n", args...) + fmt.Fprintf(b, "// "+format+"\n", args...) } } @@ -161,52 +153,66 @@ func (c *Context) ExecutePackage(outDir string, p Package) error { // Filter out any types the *package* doesn't care about. packageContext := c.filteredBy(p.Filter) os.MkdirAll(path, 0755) - files := map[string]*file{} + files := map[string]*File{} for _, g := range p.Generators(packageContext) { // Filter out types the *generator* doesn't care about. genContext := packageContext.filteredBy(g.Filter) // Now add any extra name systems defined by this generator genContext = genContext.addNameSystems(g.Namers(genContext)) + fileType := g.FileType() + if len(fileType) == 0 { + return fmt.Errorf("generator %q must specify a file type", g.Name()) + } f := files[g.Filename()] if f == nil { // This is the first generator to reference this file, so start it. - f = &file{ - name: g.Filename(), - packageName: p.Name(), - header: p.Header(g.Filename()), - imports: map[string]struct{}{}, + f = &File{ + Name: g.Filename(), + FileType: fileType, + PackageName: p.Name(), + Header: p.Header(g.Filename()), + Imports: map[string]struct{}{}, + } + files[f.Name] = f + } else { + if f.FileType != g.FileType() { + return fmt.Errorf("file %q already has type %q, but generator %q wants to use type %q", f.Name, f.FileType, g.Name(), g.FileType()) } - files[f.name] = f } + if vars := g.PackageVars(genContext); len(vars) > 0 { - addIndentHeaderComment(&f.vars, "Package-wide variables from generator %q.", g.Name()) + addIndentHeaderComment(&f.Vars, "Package-wide variables from generator %q.", g.Name()) for _, v := range vars { - if _, err := fmt.Fprintf(&f.vars, "\t%s\n", v); err != nil { + if _, err := fmt.Fprintf(&f.Vars, "%s\n", v); err != nil { return err } } } if consts := g.PackageVars(genContext); len(consts) > 0 { - addIndentHeaderComment(&f.consts, "Package-wide consts from generator %q.", g.Name()) + addIndentHeaderComment(&f.Consts, "Package-wide consts from generator %q.", g.Name()) for _, v := range consts { - if _, err := fmt.Fprintf(&f.consts, "\t%s\n", v); err != nil { + if _, err := fmt.Fprintf(&f.Consts, "%s\n", v); err != nil { return err } } } - if err := genContext.executeBody(&f.body, g); err != nil { + if err := genContext.executeBody(&f.Body, g); err != nil { return err } if imports := g.Imports(genContext); len(imports) > 0 { for _, i := range imports { - f.imports[i] = struct{}{} + f.Imports[i] = struct{}{} } } } for _, f := range files { - if err := f.assembleToFile(filepath.Join(path, f.name)); err != nil { + assembler, ok := c.FileTypes[f.FileType] + if !ok { + return fmt.Errorf("the file type %q registered for file %q does not exist in the context", f.FileType, f.Name) + } + if err := assembler.AssembleFile(f, filepath.Join(path, f.Name)); err != nil { return err } } diff --git a/cmd/libs/go2idl/generator/generator.go b/cmd/libs/go2idl/generator/generator.go index d77f6d7d5cb..6edd5ee781b 100644 --- a/cmd/libs/go2idl/generator/generator.go +++ b/cmd/libs/go2idl/generator/generator.go @@ -17,6 +17,7 @@ limitations under the License. package generator import ( + "bytes" "io" "k8s.io/kubernetes/cmd/libs/go2idl/namer" @@ -48,6 +49,21 @@ type Package interface { Generators(*Context) []Generator } +type File struct { + Name string + FileType string + PackageName string + Header []byte + Imports map[string]struct{} + Vars bytes.Buffer + Consts bytes.Buffer + Body bytes.Buffer +} + +type FileType interface { + AssembleFile(f *File, path string) error +} + // Packages is a list of packages to generate. type Packages []Package @@ -120,6 +136,10 @@ type Generator interface { // TODO: provide per-file import tracking, removing the requirement // that generators coordinate.. Filename() string + + // A registered file type in the context to generate this file with. If + // the FileType is not found in the context, execution will stop. + FileType() string } // Context is global context for individual generators to consume. @@ -134,6 +154,10 @@ type Context struct { // The canonical ordering of the types (will be filtered by both the // Package's and Generator's Filter methods). Order []*types.Type + + // A set of types this context can process. If this is empty or nil, + // the default "golang" filetype will be provided. + FileTypes map[string]FileType } // NewContext generates a context from the given builder, naming systems, and @@ -147,6 +171,9 @@ func NewContext(b *parser.Builder, nameSystems namer.NameSystems, canonicalOrder c := &Context{ Namers: namer.NameSystems{}, Universe: u, + FileTypes: map[string]FileType{ + GolangFileType: golangFileType{}, + }, } for name, systemNamer := range nameSystems { diff --git a/cmd/libs/go2idl/generator/snippet_writer.go b/cmd/libs/go2idl/generator/snippet_writer.go index f1dfd1e55c6..344b960fed4 100644 --- a/cmd/libs/go2idl/generator/snippet_writer.go +++ b/cmd/libs/go2idl/generator/snippet_writer.go @@ -116,6 +116,10 @@ func (s *SnippetWriter) Do(format string, args interface{}) *SnippetWriter { return s } +func (s *SnippetWriter) Out() io.Writer { + return s.w +} + // Error returns any encountered error. func (s *SnippetWriter) Error() error { return s.err diff --git a/cmd/libs/go2idl/namer/namer_test.go b/cmd/libs/go2idl/namer/namer_test.go index 322b5c9ff33..5e7ce3bf667 100644 --- a/cmd/libs/go2idl/namer/namer_test.go +++ b/cmd/libs/go2idl/namer/namer_test.go @@ -27,26 +27,26 @@ func TestNameStrategy(t *testing.T) { u := types.Universe{} // Add some types. - base := u.Get(types.Name{"foo/bar", "Baz"}) + base := u.Get(types.Name{Package: "foo/bar", Name: "Baz"}) base.Kind = types.Struct - tmp := u.Get(types.Name{"", "[]bar.Baz"}) + tmp := u.Get(types.Name{Package: "", Name: "[]bar.Baz"}) tmp.Kind = types.Slice tmp.Elem = base - tmp = u.Get(types.Name{"", "map[string]bar.Baz"}) + tmp = u.Get(types.Name{Package: "", Name: "map[string]bar.Baz"}) tmp.Kind = types.Map tmp.Key = types.String tmp.Elem = base - tmp = u.Get(types.Name{"foo/other", "Baz"}) + tmp = u.Get(types.Name{Package: "foo/other", Name: "Baz"}) tmp.Kind = types.Struct tmp.Members = []types.Member{{ Embedded: true, Type: base, }} - u.Get(types.Name{"", "string"}) + u.Get(types.Name{Package: "", Name: "string"}) o := Orderer{NewPublicNamer(0)} order := o.Order(u) diff --git a/cmd/libs/go2idl/parser/parse.go b/cmd/libs/go2idl/parser/parse.go index 93db9e1c7a9..dfac95b2b59 100644 --- a/cmd/libs/go2idl/parser/parse.go +++ b/cmd/libs/go2idl/parser/parse.go @@ -83,6 +83,11 @@ func New() *Builder { } } +// AddBuildTags adds the specified build tags to the parse context. +func (b *Builder) AddBuildTags(tags ...string) { + b.context.BuildTags = append(b.context.BuildTags, tags...) +} + // Get package information from the go/build package. Automatically excludes // e.g. test files and files for other platforms-- there is quite a bit of // logic of that nature in the build package. diff --git a/cmd/libs/go2idl/parser/parse_test.go b/cmd/libs/go2idl/parser/parse_test.go index 2e1d1eeeb0b..4858edf1c7a 100644 --- a/cmd/libs/go2idl/parser/parse_test.go +++ b/cmd/libs/go2idl/parser/parse_test.go @@ -175,7 +175,7 @@ type Blah struct { _, u, o := construct(t, structTest, namer.NewPublicNamer(0)) t.Logf("%#v", o) - blahT := u.Get(types.Name{"base/foo/proto", "Blah"}) + blahT := u.Get(types.Name{Package: "base/foo/proto", Name: "Blah"}) if blahT == nil { t.Fatal("type not found") } @@ -344,11 +344,11 @@ type Interface interface{Method(a, b string) (c, d string)} } // Also do some one-off checks - gtest := u.Get(types.Name{"g", "Test"}) + gtest := u.Get(types.Name{Package: "g", Name: "Test"}) if e, a := 1, len(gtest.Methods); e != a { t.Errorf("expected %v but found %v methods: %#v", e, a, gtest) } - iface := u.Get(types.Name{"g", "Interface"}) + iface := u.Get(types.Name{Package: "g", Name: "Interface"}) if e, a := 1, len(iface.Methods); e != a { t.Errorf("expected %v but found %v methods: %#v", e, a, iface) } diff --git a/cmd/libs/go2idl/types/flatten_test.go b/cmd/libs/go2idl/types/flatten_test.go index 280dc55bf0f..fcfa77059e8 100644 --- a/cmd/libs/go2idl/types/flatten_test.go +++ b/cmd/libs/go2idl/types/flatten_test.go @@ -23,7 +23,7 @@ import ( func TestFlatten(t *testing.T) { mapType := &Type{ - Name: Name{"", "map[string]string"}, + Name: Name{Package: "", Name: "map[string]string"}, Kind: Map, Key: String, Elem: String, @@ -33,7 +33,7 @@ func TestFlatten(t *testing.T) { Name: "Baz", Embedded: true, Type: &Type{ - Name: Name{"pkg", "Baz"}, + Name: Name{Package: "pkg", Name: "Baz"}, Kind: Struct, Members: []Member{ {Name: "Foo", Type: String}, @@ -41,7 +41,7 @@ func TestFlatten(t *testing.T) { Name: "Qux", Embedded: true, Type: &Type{ - Name: Name{"pkg", "Qux"}, + Name: Name{Package: "pkg", Name: "Qux"}, Kind: Struct, Members: []Member{{Name: "Zot", Type: String}}, }, diff --git a/cmd/libs/go2idl/types/types.go b/cmd/libs/go2idl/types/types.go index 6849bf87277..08a318b9691 100644 --- a/cmd/libs/go2idl/types/types.go +++ b/cmd/libs/go2idl/types/types.go @@ -18,10 +18,13 @@ package types // A type name may have a package qualifier. type Name struct { - // Empty if embedded or builtin. This is the package path. + // Empty if embedded or builtin. This is the package path unless Path is specified. Package string // The type name. Name string + // An optional location of the type definition for languages that can have disjoint + // packages and paths. + Path string } // String returns the name formatted as a string. @@ -104,7 +107,7 @@ func (p *Package) Get(typeName string) *Type { return t } } - t := &Type{Name: Name{p.Path, typeName}} + t := &Type{Name: Name{Package: p.Path, Name: typeName}} p.Types[typeName] = t return t } @@ -280,6 +283,22 @@ var ( Name: Name{Name: "uint"}, Kind: Builtin, } + Uintptr = &Type{ + Name: Name{Name: "uintptr"}, + Kind: Builtin, + } + Float64 = &Type{ + Name: Name{Name: "float64"}, + Kind: Builtin, + } + Float32 = &Type{ + Name: Name{Name: "float32"}, + Kind: Builtin, + } + Float = &Type{ + Name: Name{Name: "float"}, + Kind: Builtin, + } Bool = &Type{ Name: Name{Name: "bool"}, Kind: Builtin, @@ -291,19 +310,23 @@ var ( builtins = &Package{ Types: map[string]*Type{ - "bool": Bool, - "string": String, - "int": Int, - "int64": Int64, - "int32": Int32, - "int16": Int16, - "int8": Byte, - "uint": Uint, - "uint64": Uint64, - "uint32": Uint32, - "uint16": Uint16, - "uint8": Byte, - "byte": Byte, + "bool": Bool, + "string": String, + "int": Int, + "int64": Int64, + "int32": Int32, + "int16": Int16, + "int8": Byte, + "uint": Uint, + "uint64": Uint64, + "uint32": Uint32, + "uint16": Uint16, + "uint8": Byte, + "uintptr": Uintptr, + "byte": Byte, + "float": Float, + "float64": Float64, + "float32": Float32, }, Imports: map[string]*Package{}, Path: "", diff --git a/cmd/libs/go2idl/types/types_test.go b/cmd/libs/go2idl/types/types_test.go index 33893525536..4a7e08b499e 100644 --- a/cmd/libs/go2idl/types/types_test.go +++ b/cmd/libs/go2idl/types/types_test.go @@ -25,7 +25,7 @@ func TestGetBuiltin(t *testing.T) { if builtinPkg := u.Package(""); builtinPkg.Has("string") { t.Errorf("Expected builtin package to not have builtins until they're asked for explicitly. %#v", builtinPkg) } - s := u.Get(Name{"", "string"}) + s := u.Get(Name{Package: "", Name: "string"}) if s != String { t.Errorf("Expected canonical string type.") } @@ -39,7 +39,7 @@ func TestGetBuiltin(t *testing.T) { func TestGetMarker(t *testing.T) { u := Universe{} - n := Name{"path/to/package", "Foo"} + n := Name{Package: "path/to/package", Name: "Foo"} f := u.Get(n) if f == nil || f.Name != n { t.Errorf("Expected marker type.")