diff --git a/cmd/libs/go2idl/generator/default_generator.go b/cmd/libs/go2idl/generator/default_generator.go new file mode 100644 index 00000000000..734cf2d732f --- /dev/null +++ b/cmd/libs/go2idl/generator/default_generator.go @@ -0,0 +1,56 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "io" + + "k8s.io/kubernetes/cmd/libs/go2idl/namer" + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +// DefaultGen implements a do-nothing Generator. +// +// It can be used to implement static content files. +type DefaultGen struct { + // OptionalName, if present, will be used for the generator's name, and + // the filename (with ".go" appended). + OptionalName string + + // OptionalBody, if present, will be used as the return from the "Init" + // method. This causes it to be static content for the entire file if + // no other generator touches the file. + OptionalBody []byte +} + +func (d DefaultGen) Name() string { return d.OptionalName } +func (d DefaultGen) Filter(*Context, *types.Type) bool { return true } +func (d DefaultGen) Namers(*Context) namer.NameSystems { return nil } +func (d DefaultGen) Imports(*Context) []string { return []string{} } +func (d DefaultGen) PackageVars(*Context) []string { return []string{} } +func (d DefaultGen) PackageConsts(*Context) []string { return []string{} } +func (d DefaultGen) GenerateType(*Context, *types.Type, io.Writer) error { return nil } +func (d DefaultGen) Filename() string { return d.OptionalName + ".go" } + +func (d DefaultGen) Init(c *Context, w io.Writer) error { + _, err := w.Write(d.OptionalBody) + return err +} + +var ( + _ = Generator(DefaultGen{}) +) diff --git a/cmd/libs/go2idl/generator/default_package.go b/cmd/libs/go2idl/generator/default_package.go new file mode 100644 index 00000000000..77f20bbb291 --- /dev/null +++ b/cmd/libs/go2idl/generator/default_package.go @@ -0,0 +1,72 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +// DefaultPackage contains a default implentation of Package. +type DefaultPackage struct { + // Short name of package, used in the "package xxxx" line. + PackageName string + // Import path of the package, and the location on disk of the package. + PackagePath string + + // Emitted at the top of every file. + HeaderText []byte + + // Emitted only for a "doc.go" file; appended to the HeaderText for + // that file. + PackageDocumentation []byte + + // If non-nil, will be called on "Generators"; otherwise, the static + // list will be used. So you should set only one of these two fields. + GeneratorFunc func(*Context) []Generator + GeneratorList []Generator + + // Optional; filters the types exposed to the generators. + FilterFunc func(*Context, *types.Type) bool +} + +func (d *DefaultPackage) Name() string { return d.PackageName } +func (d *DefaultPackage) Path() string { return d.PackagePath } + +func (d *DefaultPackage) Filter(c *Context, t *types.Type) bool { + if d.FilterFunc != nil { + return d.FilterFunc(c, t) + } + return true +} + +func (d *DefaultPackage) Generators(c *Context) []Generator { + if d.GeneratorFunc != nil { + return d.GeneratorFunc(c) + } + return d.GeneratorList +} + +func (d *DefaultPackage) Header(filename string) []byte { + if filename == "doc.go" { + return append(d.HeaderText, d.PackageDocumentation...) + } + return d.HeaderText +} + +var ( + _ = Package(&DefaultPackage{}) +) diff --git a/cmd/libs/go2idl/generator/doc.go b/cmd/libs/go2idl/generator/doc.go new file mode 100644 index 00000000000..8aba4817ff3 --- /dev/null +++ b/cmd/libs/go2idl/generator/doc.go @@ -0,0 +1,31 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package generator defines an interface for code generators to implement. +// +// To use this package, you'll implement the "Package" and "Generator" +// interfaces; you'll call NewContext to load up the types you want to work +// with, and then you'll call one or more of the Execute methods. See the +// interface definitions for explanations. All output will have gofmt called on +// it automatically, so you do not need to worry about generating correct +// indentation. +// +// This package also exposes SnippetWriter. SnippetWriter reduces to a minimum +// the boilerplate involved in setting up a template from go's text/template +// package. Additionally, all naming systems in the Context will be added as +// functions to the parsed template, so that they can be called directly from +// your templates! +package generator diff --git a/cmd/libs/go2idl/generator/error_tracker.go b/cmd/libs/go2idl/generator/error_tracker.go new file mode 100644 index 00000000000..2ee907420be --- /dev/null +++ b/cmd/libs/go2idl/generator/error_tracker.go @@ -0,0 +1,50 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "io" +) + +// ErrorTracker tracks errors to the underlying writer, so that you can ignore +// them until you're ready to return. +type ErrorTracker struct { + io.Writer + err error +} + +// NewErrorTracker makes a new error tracker; note that it implements io.Writer. +func NewErrorTracker(w io.Writer) *ErrorTracker { + return &ErrorTracker{Writer: w} +} + +// Write intercepts calls to Write. +func (et *ErrorTracker) Write(p []byte) (n int, err error) { + if et.err != nil { + return 0, et.err + } + n, err = et.Writer.Write(p) + if err != nil { + et.err = err + } + return n, err +} + +// Error returns nil if no error has occurred, otherwise it returns the error. +func (et *ErrorTracker) Error() error { + return et.err +} diff --git a/cmd/libs/go2idl/generator/execute.go b/cmd/libs/go2idl/generator/execute.go new file mode 100644 index 00000000000..ddf405cd332 --- /dev/null +++ b/cmd/libs/go2idl/generator/execute.go @@ -0,0 +1,226 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "bytes" + "fmt" + "go/format" + "io" + "log" + "os" + "path/filepath" + "strings" + + "k8s.io/kubernetes/cmd/libs/go2idl/namer" + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +// ExecutePackages runs the generators for every package in 'packages'. 'outDir' +// is the base directory in which to place all the generated packages; it +// should be a physical path on disk, not an import path. e.g.: +// /path/to/home/path/to/gopath/src/ +// Each package has its import path already, this will be appended to 'outDir'. +func (c *Context) ExecutePackages(outDir string, packages Packages) error { + for _, p := range packages { + if err := c.ExecutePackage(outDir, p); err != nil { + return err + } + } + return nil +} + +type file struct { + name string + packageName string + header []byte + imports map[string]struct{} + vars bytes.Buffer + consts bytes.Buffer + body bytes.Buffer +} + +func (f *file) assembleToFile(pathname string) error { + log.Printf("Assembling file %q", pathname) + destFile, err := os.Create(pathname) + if err != nil { + return err + } + defer destFile.Close() + + b := &bytes.Buffer{} + et := NewErrorTracker(b) + f.assemble(et) + if et.Error() != nil { + return et.Error() + } + if formatted, err := format.Source(b.Bytes()); err != nil { + log.Printf("Warning: unable to run gofmt on %q (%v).", pathname, err) + _, err = destFile.Write(b.Bytes()) + return err + } else { + _, err = destFile.Write(formatted) + return err + } +} + +func (f *file) assemble(w io.Writer) { + w.Write(f.header) + fmt.Fprintf(w, "package %v\n\n", f.packageName) + + if len(f.imports) > 0 { + fmt.Fprint(w, "import (\n") + for i := range f.imports { + if strings.Contains(i, "\"") { + // they included quotes, or are using the + // `name "path/to/pkg"` format. + fmt.Fprintf(w, "\t%s\n", i) + } else { + fmt.Fprintf(w, "\t%q\n", i) + } + } + fmt.Fprint(w, ")\n\n") + } + + if f.vars.Len() > 0 { + fmt.Fprint(w, "var (\n") + w.Write(f.vars.Bytes()) + fmt.Fprint(w, ")\n\n") + } + + if f.consts.Len() > 0 { + fmt.Fprint(w, "const (\n") + w.Write(f.consts.Bytes()) + fmt.Fprint(w, ")\n\n") + } + + w.Write(f.body.Bytes()) +} + +// format should be one line only, and not end with \n. +func addIndentHeaderComment(b *bytes.Buffer, format string, args ...interface{}) { + if b.Len() > 0 { + fmt.Fprintf(b, "\n\t// "+format+"\n", args...) + } else { + fmt.Fprintf(b, "\t// "+format+"\n", args...) + } +} + +func (c *Context) filteredBy(f func(*Context, *types.Type) bool) *Context { + c2 := *c + c2.Order = []*types.Type{} + for _, t := range c.Order { + if f(c, t) { + c2.Order = append(c2.Order, t) + } + } + return &c2 +} + +// make a new context; inheret c.Namers, but add on 'namers'. In case of a name +// collision, the namer in 'namers' wins. +func (c *Context) addNameSystems(namers namer.NameSystems) *Context { + if namers == nil { + return c + } + c2 := *c + // Copy the existing name systems so we don't corrupt a parent context + c2.Namers = namer.NameSystems{} + for k, v := range c.Namers { + c2.Namers[k] = v + } + + for name, namer := range namers { + c2.Namers[name] = namer + } + return &c2 +} + +// ExecutePackage executes a single package. 'outDir' is the base directory in +// which to place the package; it should be a physical path on disk, not an +// import path. e.g.: '/path/to/home/path/to/gopath/src/' The package knows its +// import path already, this will be appended to 'outDir'. +func (c *Context) ExecutePackage(outDir string, p Package) error { + path := filepath.Join(outDir, p.Path()) + log.Printf("Executing package %v into %v", p.Name(), path) + // Filter out any types the *package* doesn't care about. + packageContext := c.filteredBy(p.Filter) + os.MkdirAll(path, 0755) + files := map[string]*file{} + for _, g := range p.Generators(packageContext) { + // Filter out types the *generator* doesn't care about. + genContext := packageContext.filteredBy(g.Filter) + // Now add any extra name systems defined by this generator + genContext = genContext.addNameSystems(g.Namers(genContext)) + + f := files[g.Filename()] + if f == nil { + // This is the first generator to reference this file, so start it. + f = &file{ + name: g.Filename(), + packageName: p.Name(), + header: p.Header(g.Filename()), + imports: map[string]struct{}{}, + } + files[f.name] = f + } + if vars := g.PackageVars(genContext); len(vars) > 0 { + addIndentHeaderComment(&f.vars, "Package-wide variables from generator %q.", g.Name()) + for _, v := range vars { + if _, err := fmt.Fprintf(&f.vars, "\t%s\n", v); err != nil { + return err + } + } + } + if consts := g.PackageVars(genContext); len(consts) > 0 { + addIndentHeaderComment(&f.consts, "Package-wide consts from generator %q.", g.Name()) + for _, v := range consts { + if _, err := fmt.Fprintf(&f.consts, "\t%s\n", v); err != nil { + return err + } + } + } + if err := genContext.executeBody(&f.body, g); err != nil { + return err + } + if imports := g.Imports(genContext); len(imports) > 0 { + for _, i := range imports { + f.imports[i] = struct{}{} + } + } + } + + for _, f := range files { + if err := f.assembleToFile(filepath.Join(path, f.name)); err != nil { + return err + } + } + return nil +} + +func (c *Context) executeBody(w io.Writer, generator Generator) error { + et := NewErrorTracker(w) + if err := generator.Init(c, et); err != nil { + return err + } + for _, t := range c.Order { + if err := generator.GenerateType(c, t, et); err != nil { + return err + } + } + return et.Error() +} diff --git a/cmd/libs/go2idl/generator/generator.go b/cmd/libs/go2idl/generator/generator.go new file mode 100644 index 00000000000..d77f6d7d5cb --- /dev/null +++ b/cmd/libs/go2idl/generator/generator.go @@ -0,0 +1,160 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "io" + + "k8s.io/kubernetes/cmd/libs/go2idl/namer" + "k8s.io/kubernetes/cmd/libs/go2idl/parser" + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +// Package contains the contract for generating a package. +type Package interface { + // Name returns the package short name. + Name() string + // Path returns the package import path. + Path() string + + // Filter should return true if this package cares about this type. + // Otherwise, this type will be ommitted from the type ordering for + // this package. + Filter(*Context, *types.Type) bool + + // Header should return a header for the file, including comment markers. + // Useful for copyright notices and doc strings. Include an + // autogeneration notice! Do not include the "package x" line. + Header(filename string) []byte + + // Generators returns the list of generators for this package. It is + // allowed for more than one generator to write to the same file. + // A Context is passed in case the list of generators depends on the + // input types. + Generators(*Context) []Generator +} + +// Packages is a list of packages to generate. +type Packages []Package + +// Generator is the contract for anything that wants to do auto-generation. +// It's expected that the io.Writers passed to the below functions will be +// ErrorTrackers; this allows implementations to not check for io errors, +// making more readable code. +// +// The call order for the functions that take a Context is: +// 1. Filter() // Subsequent calls see only types that pass this. +// 2. Namers() // Subsequent calls see the namers provided by this. +// 3. PackageVars() +// 4. PackageConsts() +// 5. Init() +// 6. GenerateType() // Called N times, once per type in the context's Order. +// 7. Imports() +// +// You may have multiple generators for the same file. +type Generator interface { + // The name of this generator. Will be included in generated comments. + Name() string + + // Filter should return true if this generator cares about this type. + // (otherwise, GenerateType will not be called.) + // + // Filter is called before any of the generator's other functions; + // subsequent calls will get a context with only the types that passed + // this filter. + Filter(*Context, *types.Type) bool + + // If this generator needs special namers, return them here. These will + // override the original namers in the context if there is a collision. + // You may return nil if you don't need special names. These names will + // be available in the context passed to the rest of the generator's + // functions. + // + // A use case for this is to return a namer that tracks imports. + Namers(*Context) namer.NameSystems + + // Init should write an init function, and any other content that's not + // generated per-type. (It's not intended for generator specific + // initialization! Do that when your Package constructs the + // Generators.) + Init(*Context, io.Writer) error + + // PackageVars should emit an array of variable lines. They will be + // placed in a var ( ... ) block. There's no need to include a leading + // \t or trailing \n. + PackageVars(*Context) []string + + // PackageConsts should emit an array of constant lines. They will be + // placed in a const ( ... ) block. There's no need to include a leading + // \t or trailing \n. + PackageConsts(*Context) []string + + // GenerateType should emit the code for a particular type. + GenerateType(*Context, *types.Type, io.Writer) error + + // Imports should return a list of necessary imports. They will be + // formatted correctly. You do not need to include quotation marks, + // return only the package name; alternatively, you can also return + // imports in the format `name "path/to/pkg"`. Imports will be called + // after Init, PackageVars, PackageConsts, and GenerateType, to allow + // you to keep track of what imports you actually need. + Imports(*Context) []string + + // Preferred file name of this generator, not including a path. It is + // allowed for multiple generators to use the same filename, but it's + // up to you to make sure they don't have colliding import names. + // TODO: provide per-file import tracking, removing the requirement + // that generators coordinate.. + Filename() string +} + +// Context is global context for individual generators to consume. +type Context struct { + // A map from the naming system to the names for that system. E.g., you + // might have public names and several private naming systems. + Namers namer.NameSystems + + // All the types, in case you want to look up something. + Universe types.Universe + + // The canonical ordering of the types (will be filtered by both the + // Package's and Generator's Filter methods). + Order []*types.Type +} + +// NewContext generates a context from the given builder, naming systems, and +// the naming system you wish to construct the canonical ordering from. +func NewContext(b *parser.Builder, nameSystems namer.NameSystems, canonicalOrderName string) (*Context, error) { + u, err := b.FindTypes() + if err != nil { + return nil, err + } + + c := &Context{ + Namers: namer.NameSystems{}, + Universe: u, + } + + for name, systemNamer := range nameSystems { + c.Namers[name] = systemNamer + if name == canonicalOrderName { + orderer := namer.Orderer{systemNamer} + c.Order = orderer.Order(u) + } + } + return c, nil +} diff --git a/cmd/libs/go2idl/generator/import_tracker.go b/cmd/libs/go2idl/generator/import_tracker.go new file mode 100644 index 00000000000..1e429096cd8 --- /dev/null +++ b/cmd/libs/go2idl/generator/import_tracker.go @@ -0,0 +1,97 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "path/filepath" + "strings" + + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +// ImportTracker may be passed to a namer.RawNamer, to track the imports needed +// for the types it names. +// +// TODO: pay attention to the package name (instead of renaming every package). +// TODO: Figure out the best way to make names for packages that collide. +type ImportTracker struct { + pathToName map[string]string + // forbidden names are in here. (e.g. "go" is a directory in which + // there is code, but "go" is not a legal name for a package, so we put + // it here to prevent us from naming any package "go") + nameToPath map[string]string +} + +func NewImportTracker(types ...*types.Type) *ImportTracker { + tracker := &ImportTracker{ + pathToName: map[string]string{}, + nameToPath: map[string]string{ + "go": "", + // Add other forbidden keywords that also happen to be + // package names here. + }, + } + tracker.AddTypes(types...) + return tracker +} + +func (tracker *ImportTracker) AddTypes(types ...*types.Type) { + for _, t := range types { + tracker.AddType(t) + } +} +func (tracker *ImportTracker) AddType(t *types.Type) { + path := t.Name.Package + if path == "" { + return + } + if _, ok := tracker.pathToName[path]; ok { + return + } + dirs := strings.Split(path, string(filepath.Separator)) + for n := len(dirs) - 1; n >= 0; n-- { + // TODO: bikeshed about whether it's more readable to have an + // _, something else, or nothing between directory names. + name := strings.Join(dirs[n:], "_") + // These characters commonly appear in import paths for go + // packages, but aren't legal go names. So we'll sanitize. + name = strings.Replace(name, ".", "_", -1) + name = strings.Replace(name, "-", "_", -1) + if _, found := tracker.nameToPath[name]; found { + // This name collides with some other package + continue + } + tracker.nameToPath[name] = path + tracker.pathToName[path] = name + return + } + panic("can't find import for " + path) +} + +func (tracker *ImportTracker) ImportLines() []string { + out := []string{} + for path, name := range tracker.pathToName { + out = append(out, name+" \""+path+"\"") + } + return out +} + +// LocalNameOf returns the name you would use to refer to the package at the +// specified path within the body of a file. +func (tracker *ImportTracker) LocalNameOf(path string) string { + return tracker.pathToName[path] +} diff --git a/cmd/libs/go2idl/generator/snippet_writer.go b/cmd/libs/go2idl/generator/snippet_writer.go new file mode 100644 index 00000000000..f1dfd1e55c6 --- /dev/null +++ b/cmd/libs/go2idl/generator/snippet_writer.go @@ -0,0 +1,122 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "fmt" + "io" + "runtime" + "text/template" +) + +// SnippetWriter is an attempt to make the template library usable. +// Methods are chainable, and you don't have to check Error() until you're all +// done. +type SnippetWriter struct { + w io.Writer + context *Context + // Left & right delimiters. text/template defaults to "{{" and "}}" + // which is totally unusable for go code based templates. + left, right string + funcMap template.FuncMap + err error +} + +// w is the destination; left and right are the delimiters; @ and $ are both +// reasonable choices. +// +// c is used to make a function for every naming system, to which you can pass +// a type and get the corresponding name. +func NewSnippetWriter(w io.Writer, c *Context, left, right string) *SnippetWriter { + sw := &SnippetWriter{ + w: w, + context: c, + left: left, + right: right, + funcMap: template.FuncMap{}, + } + for name, namer := range c.Namers { + sw.funcMap[name] = namer.Name + } + return sw +} + +// Do parses format and runs args through it. You can have arbitrary logic in +// the format (see the text/template documentation), but consider running many +// short templaces, with ordinary go logic in between--this may be more +// readable. Do is chainable. Any error causes every other call to do to be +// ignored, and the error will be returned by Error(). So you can check it just +// once, at the end of your function. +// +// 'args' can be quite literally anything; read the text/template documentation +// for details. Maps and structs work particularly nicely. Conveniently, the +// types package is designed to have structs that are easily referencable from +// the template language. +// +// Example: +// +// sw := generator.NewSnippetWriter(outBuffer, context, "$", "$") +// sw.Do(`The public type name is: $.type|public$`, map[string]interface{}{"type": t}) +// return sw.Error() +// +// Where: +// * "$" starts a template directive +// * "." references the entire thing passed as args +// * "type" therefore sees a map and looks up the key "type" +// * "|" means "pass the thing on the left to the thing on the right" +// * "public" is the name of a naming system, so the SnippetWriter has given +// the template a function called "public" that takes a *types.Type and +// returns the naming system's name. E.g., if the type is "string" this might +// return "String". +// * the second "$" ends the template directive. +// +// The map is actually not necessary. The below does the same thing: +// +// sw.Do(`The public type name is: $.|public$`, t) +// +// You may or may not find it more readable to use the map with a descriptive +// key, but if you want to pass more than one arg, the map or a custom struct +// becomes a requirement. You can do arbitrary logic inside these templates, +// but you should consider doing the logic in go and stitching them together +// for the sake of your readers. +func (s *SnippetWriter) Do(format string, args interface{}) *SnippetWriter { + if s.err != nil { + return s + } + // Name the template by source file:line so it can be found when + // there's an error. + _, file, line, _ := runtime.Caller(1) + tmpl, err := template. + New(fmt.Sprintf("%s:%d", file, line)). + Delims(s.left, s.right). + Funcs(s.funcMap). + Parse(format) + if err != nil { + s.err = err + return s + } + err = tmpl.Execute(s.w, args) + if err != nil { + s.err = err + } + return s +} + +// Error returns any encountered error. +func (s *SnippetWriter) Error() error { + return s.err +} diff --git a/cmd/libs/go2idl/generator/snippet_writer_test.go b/cmd/libs/go2idl/generator/snippet_writer_test.go new file mode 100644 index 00000000000..19ef83ddd52 --- /dev/null +++ b/cmd/libs/go2idl/generator/snippet_writer_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator_test + +import ( + "bytes" + "strings" + "testing" + + "k8s.io/kubernetes/cmd/libs/go2idl/generator" + "k8s.io/kubernetes/cmd/libs/go2idl/namer" + "k8s.io/kubernetes/cmd/libs/go2idl/parser" +) + +func construct(t *testing.T, files map[string]string) *generator.Context { + b := parser.New() + for name, src := range files { + if err := b.AddFile(name, []byte(src)); err != nil { + t.Fatal(err) + } + } + c, err := generator.NewContext(b, namer.NameSystems{ + "public": namer.NewPublicNamer(0), + "private": namer.NewPrivateNamer(0), + }, "public") + if err != nil { + t.Fatal(err) + } + return c +} + +func TestSnippetWriter(t *testing.T) { + var structTest = map[string]string{ + "base/foo/proto/foo.go": ` +package foo + +// Blah is a test. +// A test, I tell you. +type Blah struct { + // A is the first field. + A int64 ` + "`" + `json:"a"` + "`" + ` + + // B is the second field. + // Multiline comments work. + B string ` + "`" + `json:"b"` + "`" + ` +} +`, + } + + c := construct(t, structTest) + b := &bytes.Buffer{} + err := generator.NewSnippetWriter(b, c, "$", "$"). + Do("$.|public$$.|private$", c.Order[0]). + Error() + if err != nil { + t.Errorf("Unexpected error %v", err) + } + if e, a := "Blahblah", b.String(); e != a { + t.Errorf("Expected %q, got %q", e, a) + } + + err = generator.NewSnippetWriter(b, c, "$", "$"). + Do("$.|public", c.Order[0]). + Error() + if err == nil { + t.Errorf("expected error on invalid template") + } else { + // Dear reader, I apologize for making the worst change + // detection test in the history of ever. + if e, a := "snippet_writer_test.go:78", err.Error(); !strings.Contains(a, e) { + t.Errorf("Expected %q but didn't find it in %q", e, a) + } + } +} diff --git a/cmd/libs/go2idl/namer/doc.go b/cmd/libs/go2idl/namer/doc.go new file mode 100644 index 00000000000..2282c65a0db --- /dev/null +++ b/cmd/libs/go2idl/namer/doc.go @@ -0,0 +1,31 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package namer has support for making different type naming systems. +// +// This is because sometimes you want to refer to the literal type, sometimes +// you want to make a name for the thing you're generating, and you want to +// make the name based on the type. For example, if you have `type foo string`, +// you want to be able to generate something like `func FooPrinter(f *foo) { +// Print(string(*f)) }`; that is, you want to refer to a public name, a literal +// name, and the underlying literal name. +// +// This package supports the idea of a "Namer" and a set of "NameSystems" to +// support these use cases. +// +// Additionally, a "RawNamer" can optionally keep track of what needs to be +// imported. +package namer diff --git a/cmd/libs/go2idl/namer/namer.go b/cmd/libs/go2idl/namer/namer.go new file mode 100644 index 00000000000..760c3652851 --- /dev/null +++ b/cmd/libs/go2idl/namer/namer.go @@ -0,0 +1,347 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package namer + +import ( + "path/filepath" + "strings" + + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +// NewPublicNamer is a helper function that returns a namer that makes +// CamelCase names. See the NameStrategy struct for an explanation of the +// arguments to this constructor. +func NewPublicNamer(prependPackageNames int, ignoreWords ...string) *NameStrategy { + n := &NameStrategy{ + Join: Joiner(IC, IC), + IgnoreWords: map[string]bool{}, + PrependPackageNames: prependPackageNames, + } + for _, w := range ignoreWords { + n.IgnoreWords[w] = true + } + return n +} + +// NewPrivateNamer is a helper function that returns a namer that makes +// camelCase names. See the NameStrategy struct for an explanation of the +// arguments to this constructor. +func NewPrivateNamer(prependPackageNames int, ignoreWords ...string) *NameStrategy { + n := &NameStrategy{ + Join: Joiner(IL, IC), + IgnoreWords: map[string]bool{}, + PrependPackageNames: prependPackageNames, + } + for _, w := range ignoreWords { + n.IgnoreWords[w] = true + } + return n +} + +// NewRawNamer will return a Namer that makes a name by which you would +// directly refer to a type, optionally keeping track of the import paths +// necessary to reference the names it provides. Tracker may be nil. +// +// For example, if the type is map[string]int, a raw namer will literally +// return "map[string]int". +// +// Or if the type, in package foo, is "type Bar struct { ... }", then the raw +// namer will return "foo.Bar" as the name of the type, and if 'tracker' was +// not nil, will record that package foo needs to be imported. +func NewRawNamer(tracker ImportTracker) *rawNamer { + return &rawNamer{tracker: tracker} +} + +// Names is a map from Type to name, as defined by some Namer. +type Names map[*types.Type]string + +// Namer takes a type, and assigns a name. +// +// The purpose of this complexity is so that you can assign coherent +// side-by-side systems of names for the types. For example, you might want a +// public interface, a private implementation struct, and also to reference +// literally the type name. +// +// Note that it is safe to call your own Name() function recursively to find +// the names of keys, elements, etc. This is because anonymous types can't have +// cycles in their names, and named types don't require the sort of recursion +// that would be problematic. +type Namer interface { + Name(*types.Type) string +} + +// NameSystems is a map of a system name to a namer for that system. +type NameSystems map[string]Namer + +// NameStrategy is a general Namer. The easiest way to use it is to copy the +// Public/PrivateNamer variables, and modify the members you wish to change. +// +// The Name method produces a name for the given type, of the forms: +// Anonymous types: +// Named types: +// +// In all cases, every part of the name is run through the capitalization +// functions. +// +// The IgnoreWords map can be set if you have directory names that are +// semantically meaningless for naming purposes, e.g. "proto". +// +// Prefix and Suffix can be used to disambiguate parallel systems of type +// names. For example, if you want to generate an interface and an +// implementation, you might want to suffix one with "Interface" and the other +// with "Implementation". Another common use-- if you want to generate private +// types, and one of your source types could be "string", you can't use the +// default lowercase private namer. You'll have to add a suffix or prefix. +type NameStrategy struct { + Prefix, Suffix string + Join func(pre string, parts []string, post string) string + + // Add non-meaningful package directory names here (e.g. "proto") and + // they will be ignored. + IgnoreWords map[string]bool + + // If > 0, prepend exactly that many package directory names (or as + // many as there are). Package names listed in "IgnoreWords" will be + // ignored. + // + // For example, if Ignore words lists "proto" and type Foo is in + // pkg/server/frobbing/proto, then a value of 1 will give a type name + // of FrobbingFoo, 2 gives ServerFrobbingFoo, etc. + PrependPackageNames int + + // A cache of names thus far assigned by this namer. + Names +} + +// IC ensures the first character is uppercase. +func IC(in string) string { + if in == "" { + return in + } + return strings.ToUpper(in[:1]) + in[1:] +} + +// IL ensures the first character is lowercase. +func IL(in string) string { + if in == "" { + return in + } + return strings.ToLower(in[:1]) + in[1:] +} + +// Joiner lets you specify functions that preprocess the various components of +// a name before joining them. You can construct e.g. camelCase or CamelCase or +// any other way of joining words. (See the IC and IL convenience functions.) +func Joiner(first, others func(string) string) func(pre string, in []string, post string) string { + return func(pre string, in []string, post string) string { + tmp := []string{others(pre)} + for i := range in { + tmp = append(tmp, others(in[i])) + } + tmp = append(tmp, others(post)) + return first(strings.Join(tmp, "")) + } +} + +func (ns *NameStrategy) removePrefixAndSuffix(s string) string { + // The join function may have changed capitalization. + lowerIn := strings.ToLower(s) + lowerP := strings.ToLower(ns.Prefix) + lowerS := strings.ToLower(ns.Suffix) + b, e := 0, len(s) + if strings.HasPrefix(lowerIn, lowerP) { + b = len(ns.Prefix) + } + if strings.HasSuffix(lowerIn, lowerS) { + e -= len(ns.Suffix) + } + return s[b:e] +} + +var ( + importPathNameSanitizer = strings.NewReplacer("-", "_", ".", "") +) + +// filters out unwanted directory names and sanitizes remaining names. +func (ns *NameStrategy) filterDirs(path string) []string { + allDirs := strings.Split(path, string(filepath.Separator)) + dirs := make([]string, 0, len(allDirs)) + for _, p := range allDirs { + if ns.IgnoreWords == nil || !ns.IgnoreWords[p] { + dirs = append(dirs, importPathNameSanitizer.Replace(p)) + } + } + return dirs +} + +// See the comment on NameStrategy. +func (ns *NameStrategy) Name(t *types.Type) string { + if ns.Names == nil { + ns.Names = Names{} + } + if s, ok := ns.Names[t]; ok { + return s + } + + if t.Name.Package != "" { + dirs := append(ns.filterDirs(t.Name.Package), t.Name.Name) + i := ns.PrependPackageNames + 1 + dn := len(dirs) + if i > dn { + i = dn + } + name := ns.Join(ns.Prefix, dirs[dn-i:], ns.Suffix) + ns.Names[t] = name + return name + } + + // Only anonymous types remain. + var name string + switch t.Kind { + case types.Builtin: + name = ns.Join(ns.Prefix, []string{t.Name.Name}, ns.Suffix) + case types.Map: + name = ns.Join(ns.Prefix, []string{ + "Map", + ns.removePrefixAndSuffix(ns.Name(t.Key)), + "To", + ns.removePrefixAndSuffix(ns.Name(t.Elem)), + }, ns.Suffix) + case types.Slice: + name = ns.Join(ns.Prefix, []string{ + "Slice", + ns.removePrefixAndSuffix(ns.Name(t.Elem)), + }, ns.Suffix) + case types.Pointer: + name = ns.Join(ns.Prefix, []string{ + "Pointer", + ns.removePrefixAndSuffix(ns.Name(t.Elem)), + }, ns.Suffix) + case types.Struct: + names := []string{"Struct"} + for _, m := range t.Members { + names = append(names, ns.removePrefixAndSuffix(ns.Name(m.Type))) + } + name = ns.Join(ns.Prefix, names, ns.Suffix) + // TODO: add types.Chan + case types.Interface: + // TODO: add to name test + names := []string{"Interface"} + for _, m := range t.Methods { + // TODO: include function signature + names = append(names, m.Name.Name) + } + name = ns.Join(ns.Prefix, names, ns.Suffix) + case types.Func: + // TODO: add to name test + parts := []string{"Func"} + for _, pt := range t.Signature.Parameters { + parts = append(parts, ns.removePrefixAndSuffix(ns.Name(pt))) + } + parts = append(parts, "Returns") + for _, rt := range t.Signature.Results { + parts = append(parts, ns.removePrefixAndSuffix(ns.Name(rt))) + } + name = ns.Join(ns.Prefix, parts, ns.Suffix) + default: + name = "unnameable_" + string(t.Kind) + } + ns.Names[t] = name + return name +} + +// ImportTracker allows a raw namer to keep track of the packages needed for +// import. You can implement yourself or use the one in the generation package. +type ImportTracker interface { + AddType(*types.Type) + LocalNameOf(packagePath string) string +} + +type rawNamer struct { + tracker ImportTracker + Names +} + +// Name makes a name the way you'd write it to literally refer to type t, +// making ordinary assumptions about how you've imported t's package (or using +// r.tracker to specifically track the package imports). +func (r *rawNamer) Name(t *types.Type) string { + if r.Names == nil { + r.Names = Names{} + } + if name, ok := r.Names[t]; ok { + return name + } + if t.Name.Package != "" { + var name string + if r.tracker != nil { + r.tracker.AddType(t) + name = r.tracker.LocalNameOf(t.Name.Package) + "." + t.Name.Name + } else { + name = filepath.Base(t.Name.Package) + "." + t.Name.Name + } + r.Names[t] = name + return name + } + var name string + switch t.Kind { + case types.Builtin: + name = t.Name.Name + case types.Map: + name = "map[" + r.Name(t.Key) + "]" + r.Name(t.Elem) + case types.Slice: + name = "[]" + r.Name(t.Elem) + case types.Pointer: + name = "*" + r.Name(t.Elem) + case types.Struct: + elems := []string{} + for _, m := range t.Members { + elems = append(elems, m.Name+" "+r.Name(m.Type)) + } + name = "struct{" + strings.Join(elems, "; ") + "}" + // TODO: add types.Chan + case types.Interface: + // TODO: add to name test + elems := []string{} + for _, m := range t.Methods { + // TODO: include function signature + elems = append(elems, m.Name.Name) + } + name = "interface{" + strings.Join(elems, "; ") + "}" + case types.Func: + // TODO: add to name test + params := []string{} + for _, pt := range t.Signature.Parameters { + params = append(params, r.Name(pt)) + } + results := []string{} + for _, rt := range t.Signature.Results { + results = append(results, r.Name(rt)) + } + name = "func(" + strings.Join(params, ",") + ")" + if len(results) == 1 { + name += " " + results[0] + } else if len(results) > 1 { + name += " (" + strings.Join(results, ",") + ")" + } + default: + name = "unnameable_" + string(t.Kind) + } + r.Names[t] = name + return name +} diff --git a/cmd/libs/go2idl/namer/namer_test.go b/cmd/libs/go2idl/namer/namer_test.go new file mode 100644 index 00000000000..537c0ee6187 --- /dev/null +++ b/cmd/libs/go2idl/namer/namer_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package namer + +import ( + "reflect" + "testing" + + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +func TestNameStrategy(t *testing.T) { + u := types.Universe{} + + // Add some types. + base := u.Get(types.Name{"foo/bar", "Baz"}) + base.Kind = types.Struct + + tmp := u.Get(types.Name{"", "[]bar.Baz"}) + tmp.Kind = types.Slice + tmp.Elem = base + + tmp = u.Get(types.Name{"", "map[string]bar.Baz"}) + tmp.Kind = types.Map + tmp.Key = types.String + tmp.Elem = base + + tmp = u.Get(types.Name{"foo/other", "Baz"}) + tmp.Kind = types.Struct + tmp.Members = []types.Member{{ + Embedded: true, + Type: base, + }} + + u.Get(types.Name{"", "string"}) + + o := Orderer{NewPublicNamer(0)} + order := o.Order(u) + orderedNames := make([]string, len(order)) + for i, t := range order { + orderedNames[i] = o.Name(t) + } + expect := []string{"Baz", "Baz", "MapStringToBaz", "SliceBaz", "String"} + if e, a := expect, orderedNames; !reflect.DeepEqual(e, a) { + t.Errorf("Wanted %#v, got %#v", e, a) + } + + o = Orderer{NewRawNamer(nil)} + order = o.Order(u) + orderedNames = make([]string, len(order)) + for i, t := range order { + orderedNames[i] = o.Name(t) + } + + expect = []string{"[]bar.Baz", "bar.Baz", "map[string]bar.Baz", "other.Baz", "string"} + if e, a := expect, orderedNames; !reflect.DeepEqual(e, a) { + t.Errorf("Wanted %#v, got %#v", e, a) + } + + o = Orderer{NewPublicNamer(1)} + order = o.Order(u) + orderedNames = make([]string, len(order)) + for i, t := range order { + orderedNames[i] = o.Name(t) + } + expect = []string{"BarBaz", "MapStringToBarBaz", "OtherBaz", "SliceBarBaz", "String"} + if e, a := expect, orderedNames; !reflect.DeepEqual(e, a) { + t.Errorf("Wanted %#v, got %#v", e, a) + } +} diff --git a/cmd/libs/go2idl/namer/order.go b/cmd/libs/go2idl/namer/order.go new file mode 100644 index 00000000000..24aa6c2494c --- /dev/null +++ b/cmd/libs/go2idl/namer/order.go @@ -0,0 +1,52 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package namer + +import ( + "sort" + + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +// Orderer produces an ordering of types given a Namer. +type Orderer struct { + Namer +} + +// Order assigns a name to every type, and returns a list sorted by those +// names. +func (o *Orderer) Order(u types.Universe) []*types.Type { + list := tList{ + namer: o.Namer, + } + for _, p := range u { + for _, t := range p.Types { + list.types = append(list.types, t) + } + } + sort.Sort(list) + return list.types +} + +type tList struct { + namer Namer + types []*types.Type +} + +func (t tList) Len() int { return len(t.types) } +func (t tList) Less(i, j int) bool { return t.namer.Name(t.types[i]) < t.namer.Name(t.types[j]) } +func (t tList) Swap(i, j int) { t.types[i], t.types[j] = t.types[j], t.types[i] } diff --git a/cmd/libs/go2idl/parser/doc.go b/cmd/libs/go2idl/parser/doc.go new file mode 100644 index 00000000000..a82398b80ea --- /dev/null +++ b/cmd/libs/go2idl/parser/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package parser provides code to parse go files, type-check them, extract the +// types. +package parser diff --git a/cmd/libs/go2idl/parser/parse.go b/cmd/libs/go2idl/parser/parse.go new file mode 100644 index 00000000000..d0a0de5e6b4 --- /dev/null +++ b/cmd/libs/go2idl/parser/parse.go @@ -0,0 +1,497 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + tc "go/types" + "io/ioutil" + "os/exec" + "path/filepath" + "strings" + + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +// Builder lets you add all the go files in all the packages that you care +// about, then constructs the type source data. +type Builder struct { + context *build.Context + buildInfo map[string]*build.Package + + fset *token.FileSet + // map of package id to list of parsed files + parsed map[string][]*ast.File + + // Set by makePackages, used by importer() and friends. + pkgs map[string]*tc.Package + + // Map of package path to whether the user requested it or it was from + // an import. + userRequested map[string]bool + + // All comments from everywhere in every parsed file. + endLineToCommentGroup map[fileLine]*ast.CommentGroup + + // map of package to list of packages it imports. + importGraph map[string]map[string]struct{} +} + +// key type for finding comments. +type fileLine struct { + file string + line int +} + +// New constructs a new builder. +func New() *Builder { + c := build.Default + if c.GOROOT == "" { + if p, err := exec.Command("which", "go").CombinedOutput(); err == nil { + // The returned string will have some/path/bin/go, so remove the last two elements. + c.GOROOT = filepath.Dir(filepath.Dir(strings.Trim(string(p), "\n"))) + } else { + fmt.Printf("Warning: $GOROOT not set, and unable to run `which go` to find it: %v\n", err) + } + } + return &Builder{ + context: &c, + buildInfo: map[string]*build.Package{}, + fset: token.NewFileSet(), + parsed: map[string][]*ast.File{}, + userRequested: map[string]bool{}, + endLineToCommentGroup: map[fileLine]*ast.CommentGroup{}, + importGraph: map[string]map[string]struct{}{}, + } +} + +// Get package information from the go/build package. Automatically excludes +// e.g. test files and files for other platforms-- there is quite a bit of +// logic of that nature in the build package. +func (b *Builder) buildPackage(pkgPath string) (*build.Package, error) { + // First, find it, so we know what path to use. + pkg, err := b.context.Import(pkgPath, ".", build.FindOnly) + if err != nil { + return nil, fmt.Errorf("unable to *find* %q: %v", pkgPath, err) + } + + pkgPath = pkg.ImportPath + + if pkg, ok := b.buildInfo[pkgPath]; ok { + return pkg, nil + } + pkg, err = b.context.Import(pkgPath, ".", build.ImportComment) + if err != nil { + return nil, fmt.Errorf("unable to import %q: %v", pkgPath, err) + } + b.buildInfo[pkgPath] = pkg + + if b.importGraph[pkgPath] == nil { + b.importGraph[pkgPath] = map[string]struct{}{} + } + for _, p := range pkg.Imports { + b.importGraph[pkgPath][p] = struct{}{} + } + return pkg, nil +} + +// AddFile adds a file to the set. The name must be of the form canonical/pkg/path/file.go. +func (b *Builder) AddFile(name string, src []byte) error { + return b.addFile(name, src, true) +} + +// addFile adds a file to the set. The name must be of the form +// canonical/pkg/path/file.go. A flag indicates whether this file was +// user-requested or just from following the import graph. +func (b *Builder) addFile(name string, src []byte, userRequested bool) error { + p, err := parser.ParseFile(b.fset, name, src, parser.DeclarationErrors|parser.ParseComments) + if err != nil { + return err + } + pkg := filepath.Dir(name) + b.parsed[pkg] = append(b.parsed[pkg], p) + b.userRequested[pkg] = userRequested + for _, c := range p.Comments { + position := b.fset.Position(c.End()) + b.endLineToCommentGroup[fileLine{position.Filename, position.Line}] = c + } + + // We have to get the packages from this specific file, in case the + // user added individual files instead of entire directories. + if b.importGraph[pkg] == nil { + b.importGraph[pkg] = map[string]struct{}{} + } + for _, im := range p.Imports { + importedPath := strings.Trim(im.Path.Value, `"`) + b.importGraph[pkg][importedPath] = struct{}{} + } + return nil +} + +// AddDir adds an entire directory, scanning it for go files. 'dir' should have +// a single go package in it. GOPATH, GOROOT, and the location of your go +// binary (`which go`) will all be searched if dir doesn't literally resolve. +func (b *Builder) AddDir(dir string) error { + return b.addDir(dir, true) +} + +// The implementation of AddDir. A flag indicates whether this directory was +// user-requested or just from following the import graph. +func (b *Builder) addDir(dir string, userRequested bool) error { + pkg, err := b.buildPackage(dir) + if err != nil { + return err + } + dir = pkg.Dir + // Check in case this package was added (maybe dir was not canonical) + if _, alreadyAdded := b.parsed[dir]; alreadyAdded { + return nil + } + + for _, n := range pkg.GoFiles { + if !strings.HasSuffix(n, ".go") { + continue + } + absPath := filepath.Join(pkg.Dir, n) + pkgPath := filepath.Join(pkg.ImportPath, n) + data, err := ioutil.ReadFile(absPath) + if err != nil { + return fmt.Errorf("while loading %q: %v", absPath, err) + } + err = b.addFile(pkgPath, data, userRequested) + if err != nil { + return fmt.Errorf("while parsing %q: %v", pkgPath, err) + } + } + return nil +} + +// importer is a function that will be called by the type check package when it +// needs to import a go package. 'path' is the import path. go1.5 changes the +// interface, and importAdapter below implements the new interface in terms of +// the old one. +func (b *Builder) importer(imports map[string]*tc.Package, path string) (*tc.Package, error) { + if pkg, ok := imports[path]; ok { + return pkg, nil + } + ignoreError := false + if _, ours := b.parsed[path]; !ours { + // Ignore errors in paths that we're importing solely because + // they're referenced by other packages. + ignoreError = true + // fmt.Printf("trying to import %q\n", path) + if err := b.addDir(path, false); err != nil { + return nil, err + } + } + pkg, err := b.typeCheckPackage(path) + if err != nil { + if ignoreError && pkg != nil { + fmt.Printf("type checking encountered some errors in %q, but ignoring.\n", path) + } else { + return nil, err + } + } + imports[path] = pkg + return pkg, nil +} + +type importAdapter struct { + b *Builder +} + +func (a importAdapter) Import(path string) (*tc.Package, error) { + return a.b.importer(a.b.pkgs, path) +} + +// typeCheckPackage will attempt to return the package even if there are some +// errors, so you may check whether the package is nil or not even if you get +// an error. +func (b *Builder) typeCheckPackage(id string) (*tc.Package, error) { + if pkg, ok := b.pkgs[id]; ok { + if pkg != nil { + return pkg, nil + } + // We store a nil right before starting work on a package. So + // if we get here and it's present and nil, that means there's + // another invocation of this function on the call stack + // already processing this package. + return nil, fmt.Errorf("circular dependency for %q", id) + } + files, ok := b.parsed[id] + if !ok { + return nil, fmt.Errorf("No files for pkg %q: %#v", id, b.parsed) + } + b.pkgs[id] = nil + c := tc.Config{ + IgnoreFuncBodies: true, + // Note that importAdater can call b.import which calls this + // method. So there can't be cycles in the import graph. + Importer: importAdapter{b}, + Error: func(err error) { + fmt.Printf("type checker error: %v\n", err) + }, + } + pkg, err := c.Check(id, b.fset, files, nil) + b.pkgs[id] = pkg // record the result whether or not there was an error + return pkg, err +} + +func (b *Builder) makePackages() error { + b.pkgs = map[string]*tc.Package{} + for id := range b.parsed { + // We have to check here even though we made a new one above, + // because typeCheckPackage follows the import graph, which may + // cause a package to be filled before we get to it in this + // loop. + if _, done := b.pkgs[id]; done { + continue + } + if _, err := b.typeCheckPackage(id); err != nil { + return err + } + } + return nil +} + +// FindTypes finalizes the package imports, and searches through all the +// packages for types. +func (b *Builder) FindTypes() (types.Universe, error) { + if err := b.makePackages(); err != nil { + return nil, err + } + + u := types.Universe{} + + for pkgName, pkg := range b.pkgs { + if !b.userRequested[pkgName] { + // Since walkType is recursive, all types that the + // packages they asked for depend on will be included. + // But we don't need to include all types in all + // *packages* they depend on. + continue + } + s := pkg.Scope() + for _, n := range s.Names() { + obj := s.Lookup(n) + tn, ok := obj.(*tc.TypeName) + if !ok { + continue + } + t := b.walkType(u, nil, tn.Type()) + t.CommentLines = b.priorCommentLines(obj.Pos()) + } + for p := range b.importGraph[pkgName] { + u.AddImports(pkgName, p) + } + } + return u, nil +} + +// if there's a comment on the line before pos, return its text, otherwise "". +func (b *Builder) priorCommentLines(pos token.Pos) string { + position := b.fset.Position(pos) + key := fileLine{position.Filename, position.Line - 1} + if c, ok := b.endLineToCommentGroup[key]; ok { + return c.Text() + } + return "" +} + +func tcNameToName(in string) types.Name { + // Detect anonymous type names. (These may have '.' characters because + // embedded types may have packages, so we detect them specially.) + if strings.HasPrefix(in, "struct{") || + strings.HasPrefix(in, "*") || + strings.HasPrefix(in, "map[") || + strings.HasPrefix(in, "[") { + return types.Name{Name: in} + } + + // Otherwise, if there are '.' characters present, the name has a + // package path in front. + nameParts := strings.Split(in, ".") + name := types.Name{Name: in} + if n := len(nameParts); n >= 2 { + // The final "." is the name of the type--previous ones must + // have been in the package path. + name.Package, name.Name = strings.Join(nameParts[:n-1], "."), nameParts[n-1] + } + return name +} + +func (b *Builder) convertSignature(u types.Universe, t *tc.Signature) *types.Signature { + signature := &types.Signature{} + for i := 0; i < t.Params().Len(); i++ { + signature.Parameters = append(signature.Parameters, b.walkType(u, nil, t.Params().At(i).Type())) + } + for i := 0; i < t.Results().Len(); i++ { + signature.Results = append(signature.Results, b.walkType(u, nil, t.Results().At(i).Type())) + } + if r := t.Recv(); r != nil { + signature.Receiver = b.walkType(u, nil, r.Type()) + } + signature.Variadic = t.Variadic() + return signature +} + +// walkType adds the type, and any necessary child types. +func (b *Builder) walkType(u types.Universe, useName *types.Name, in tc.Type) *types.Type { + // Most of the cases are underlying types of the named type. + name := tcNameToName(in.String()) + if useName != nil { + name = *useName + } + + switch t := in.(type) { + case *tc.Struct: + out := u.Get(name) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Struct + for i := 0; i < t.NumFields(); i++ { + f := t.Field(i) + m := types.Member{ + Name: f.Name(), + Embedded: f.Anonymous(), + Tags: t.Tag(i), + Type: b.walkType(u, nil, f.Type()), + CommentLines: b.priorCommentLines(f.Pos()), + } + out.Members = append(out.Members, m) + } + return out + case *tc.Map: + out := u.Get(name) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Map + out.Elem = b.walkType(u, nil, t.Elem()) + out.Key = b.walkType(u, nil, t.Key()) + return out + case *tc.Pointer: + out := u.Get(name) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Pointer + out.Elem = b.walkType(u, nil, t.Elem()) + return out + case *tc.Slice: + out := u.Get(name) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Slice + out.Elem = b.walkType(u, nil, t.Elem()) + return out + case *tc.Array: + out := u.Get(name) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Array + out.Elem = b.walkType(u, nil, t.Elem()) + // TODO: need to store array length, otherwise raw type name + // cannot be properly written. + return out + case *tc.Chan: + out := u.Get(name) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Chan + out.Elem = b.walkType(u, nil, t.Elem()) + // TODO: need to store direction, otherwise raw type name + // cannot be properly written. + return out + case *tc.Basic: + out := u.Get(types.Name{ + Package: "", + Name: t.Name(), + }) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Unsupported + return out + case *tc.Signature: + out := u.Get(name) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Func + out.Signature = b.convertSignature(u, t) + return out + case *tc.Interface: + out := u.Get(name) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Interface + t.Complete() + for i := 0; i < t.NumMethods(); i++ { + out.Methods = append(out.Methods, b.walkType(u, nil, t.Method(i).Type())) + } + return out + case *tc.Named: + switch t.Underlying().(type) { + case *tc.Named, *tc.Basic: + name := tcNameToName(t.String()) + out := u.Get(name) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Alias + out.Underlying = b.walkType(u, nil, t.Underlying()) + return out + default: + // tc package makes everything "named" with an + // underlying anonymous type--we remove that annoying + // "feature" for users. This flattens those types + // together. + name := tcNameToName(t.String()) + if out := u.Get(name); out.Kind != types.Unknown { + return out // short circuit if we've already made this. + } + out := b.walkType(u, &name, t.Underlying()) + if len(out.Methods) == 0 { + // If the underlying type didn't already add + // methods, add them. (Interface types will + // have already added methods.) + for i := 0; i < t.NumMethods(); i++ { + out.Methods = append(out.Methods, b.walkType(u, nil, t.Method(i).Type())) + } + } + return out + } + default: + out := u.Get(name) + if out.Kind != types.Unknown { + return out + } + out.Kind = types.Unsupported + fmt.Printf("Making unsupported type entry %q for: %#v\n", out, t) + return out + } +} diff --git a/cmd/libs/go2idl/parser/parse_test.go b/cmd/libs/go2idl/parser/parse_test.go new file mode 100644 index 00000000000..ab5201c2475 --- /dev/null +++ b/cmd/libs/go2idl/parser/parse_test.go @@ -0,0 +1,343 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser_test + +import ( + "bytes" + "reflect" + "testing" + "text/template" + + "k8s.io/kubernetes/cmd/libs/go2idl/namer" + "k8s.io/kubernetes/cmd/libs/go2idl/parser" + "k8s.io/kubernetes/cmd/libs/go2idl/types" +) + +func construct(t *testing.T, files map[string]string, testNamer namer.Namer) (*parser.Builder, types.Universe, []*types.Type) { + b := parser.New() + for name, src := range files { + if err := b.AddFile(name, []byte(src)); err != nil { + t.Fatal(err) + } + } + u, err := b.FindTypes() + if err != nil { + t.Fatal(err) + } + orderer := namer.Orderer{testNamer} + o := orderer.Order(u) + return b, u, o +} + +func TestBuilder(t *testing.T) { + var testFiles = map[string]string{ + "base/foo/proto/foo.go": ` +package foo + +import ( + "base/common/proto" +) + +type Blah struct { + common.Object + Count int64 + Frobbers map[string]*Frobber + Baz []Object + Nickname *string + NumberIsAFavorite map[int]bool +} + +type Frobber struct { + Name string + Amount int64 +} + +type Object struct { + common.Object +} + +`, + "base/common/proto/common.go": ` +package common + +type Object struct { + ID int64 +} +`, + } + + var tmplText = ` +package o +{{define "Struct"}}type {{Name .}} interface { {{range $m := .Members}}{{$n := Name $m.Type}} + {{if $m.Embedded}}{{$n}}{{else}}{{$m.Name}}() {{$n}}{{if $m.Type.Elem}}{{else}} + Set{{$m.Name}}({{$n}}){{end}}{{end}}{{end}} +} + +{{end}} +{{range $t := .}}{{if eq $t.Kind "Struct"}}{{template "Struct" $t}}{{end}}{{end}}` + + var expect = ` +package o + +type CommonObject interface { + ID() Int64 + SetID(Int64) +} + +type FooBlah interface { + CommonObject + Count() Int64 + SetCount(Int64) + Frobbers() MapStringToPointerFooFrobber + Baz() SliceFooObject + Nickname() PointerString + NumberIsAFavorite() MapIntToBool +} + +type FooFrobber interface { + Name() String + SetName(String) + Amount() Int64 + SetAmount(Int64) +} + +type FooObject interface { + CommonObject +} + +` + testNamer := namer.NewPublicNamer(1, "proto") + _, u, o := construct(t, testFiles, testNamer) + t.Logf("\n%v\n\n", o) + tmpl := template.Must( + template.New(""). + Funcs( + map[string]interface{}{ + "Name": testNamer.Name, + }). + Parse(tmplText), + ) + buf := &bytes.Buffer{} + tmpl.Execute(buf, o) + if e, a := expect, buf.String(); e != a { + t.Errorf("Wanted, got:\n%v\n-----\n%v\n", e, a) + } + + if p := u.Package("base/foo/proto"); !p.HasImport("base/common/proto") { + t.Errorf("Unexpected lack of import line: %#s", p.Imports) + } +} + +func TestStructParse(t *testing.T) { + var structTest = map[string]string{ + "base/foo/proto/foo.go": ` +package foo + +// Blah is a test. +// A test, I tell you. +type Blah struct { + // A is the first field. + A int64 ` + "`" + `json:"a"` + "`" + ` + + // B is the second field. + // Multiline comments work. + B string ` + "`" + `json:"b"` + "`" + ` +} +`, + } + + _, u, o := construct(t, structTest, namer.NewPublicNamer(0)) + t.Logf("%#v", o) + blahT := u.Get(types.Name{"base/foo/proto", "Blah"}) + if blahT == nil { + t.Fatal("type not found") + } + if e, a := types.Struct, blahT.Kind; e != a { + t.Errorf("struct kind wrong, wanted %v, got %v", e, a) + } + if e, a := "Blah is a test.\nA test, I tell you.\n", blahT.CommentLines; e != a { + t.Errorf("struct comment wrong, wanted %v, got %v", e, a) + } + m := types.Member{ + Name: "B", + Embedded: false, + CommentLines: "B is the second field.\nMultiline comments work.\n", + Tags: `json:"b"`, + Type: types.String, + } + if e, a := m, blahT.Members[1]; !reflect.DeepEqual(e, a) { + t.Errorf("wanted, got:\n%#v\n%#v", e, a) + } +} + +func TestTypeKindParse(t *testing.T) { + var testFiles = map[string]string{ + "a/foo.go": "package a\ntype Test string\n", + "b/foo.go": "package b\ntype Test map[int]string\n", + "c/foo.go": "package c\ntype Test []string\n", + "d/foo.go": "package d\ntype Test struct{a int; b struct{a int}; c map[int]string; d *string}\n", + "e/foo.go": "package e\ntype Test *string\n", + "f/foo.go": ` +package f +import ( + "a" + "b" +) +type Test []a.Test +type Test2 *a.Test +type Test3 map[a.Test]b.Test +type Test4 struct { + a struct {a a.Test; b b.Test} + b map[a.Test]b.Test + c *a.Test + d []a.Test + e []string +} +`, + "g/foo.go": ` +package g +type Test func(a, b string) (c, d string) +func (t Test) Method(a, b string) (c, d string) { return t(a, b) } +type Interface interface{Method(a, b string) (c, d string)} +`, + } + + // Check that the right types are found, and the namers give the expected names. + + assertions := []struct { + Package, Name string + k types.Kind + names []string + }{ + { + Package: "a", Name: "Test", k: types.Alias, + names: []string{"Test", "ATest", "test", "aTest", "a.Test"}, + }, + { + Package: "b", Name: "Test", k: types.Map, + names: []string{"Test", "BTest", "test", "bTest", "b.Test"}, + }, + { + Package: "c", Name: "Test", k: types.Slice, + names: []string{"Test", "CTest", "test", "cTest", "c.Test"}, + }, + { + Package: "d", Name: "Test", k: types.Struct, + names: []string{"Test", "DTest", "test", "dTest", "d.Test"}, + }, + { + Package: "e", Name: "Test", k: types.Pointer, + names: []string{"Test", "ETest", "test", "eTest", "e.Test"}, + }, + { + Package: "f", Name: "Test", k: types.Slice, + names: []string{"Test", "FTest", "test", "fTest", "f.Test"}, + }, + { + Package: "g", Name: "Test", k: types.Func, + names: []string{"Test", "GTest", "test", "gTest", "g.Test"}, + }, + { + Package: "g", Name: "Interface", k: types.Interface, + names: []string{"Interface", "GInterface", "interface", "gInterface", "g.Interface"}, + }, + { + Package: "", Name: "string", k: types.Builtin, + names: []string{"String", "String", "string", "string", "string"}, + }, + { + Package: "", Name: "int", k: types.Builtin, + names: []string{"Int", "Int", "int", "int", "int"}, + }, + { + Package: "", Name: "struct{a int}", k: types.Struct, + names: []string{"StructInt", "StructInt", "structInt", "structInt", "struct{a int}"}, + }, + { + Package: "", Name: "struct{a a.Test; b b.Test}", k: types.Struct, + names: []string{"StructTestTest", "StructATestBTest", "structTestTest", "structATestBTest", "struct{a a.Test; b b.Test}"}, + }, + { + Package: "", Name: "map[int]string", k: types.Map, + names: []string{"MapIntToString", "MapIntToString", "mapIntToString", "mapIntToString", "map[int]string"}, + }, + { + Package: "", Name: "map[a.Test]b.Test", k: types.Map, + names: []string{"MapTestToTest", "MapATestToBTest", "mapTestToTest", "mapATestToBTest", "map[a.Test]b.Test"}, + }, + { + Package: "", Name: "[]string", k: types.Slice, + names: []string{"SliceString", "SliceString", "sliceString", "sliceString", "[]string"}, + }, + { + Package: "", Name: "[]a.Test", k: types.Slice, + names: []string{"SliceTest", "SliceATest", "sliceTest", "sliceATest", "[]a.Test"}, + }, + { + Package: "", Name: "*string", k: types.Pointer, + names: []string{"PointerString", "PointerString", "pointerString", "pointerString", "*string"}, + }, + { + Package: "", Name: "*a.Test", k: types.Pointer, + names: []string{"PointerTest", "PointerATest", "pointerTest", "pointerATest", "*a.Test"}, + }, + } + + namers := []namer.Namer{ + namer.NewPublicNamer(0), + namer.NewPublicNamer(1), + namer.NewPrivateNamer(0), + namer.NewPrivateNamer(1), + namer.NewRawNamer(nil), + } + + for nameIndex, namer := range namers { + _, u, _ := construct(t, testFiles, namer) + t.Logf("Found types:\n") + for pkgName, pkg := range u { + for typeName, cur := range pkg.Types { + t.Logf("%q-%q: %s %s", pkgName, typeName, cur.Name, cur.Kind) + } + } + t.Logf("\n\n") + + for _, item := range assertions { + n := types.Name{Package: item.Package, Name: item.Name} + thisType := u.Get(n) + if thisType == nil { + t.Errorf("type %s not found", n) + continue + } + if e, a := item.k, thisType.Kind; e != a { + t.Errorf("%v-%s: type kind wrong, wanted %v, got %v (%#v)", nameIndex, n, e, a, thisType) + } + if e, a := item.names[nameIndex], namer.Name(thisType); e != a { + t.Errorf("%v-%s: Expected %q, got %q", nameIndex, n, e, a) + } + } + + // Also do some one-off checks + gtest := u.Get(types.Name{"g", "Test"}) + if e, a := 1, len(gtest.Methods); e != a { + t.Errorf("expected %v but found %v methods: %#v", e, a, gtest) + } + iface := u.Get(types.Name{"g", "Interface"}) + if e, a := 1, len(iface.Methods); e != a { + t.Errorf("expected %v but found %v methods: %#v", e, a, iface) + } + } +} diff --git a/cmd/libs/go2idl/types/comments.go b/cmd/libs/go2idl/types/comments.go new file mode 100644 index 00000000000..71d3063a09e --- /dev/null +++ b/cmd/libs/go2idl/types/comments.go @@ -0,0 +1,64 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package types contains go type information, packaged in a way that makes +// auto-generation convenient, whether by template or straight go functions. +package types + +import ( + "strings" +) + +// ExtractCommentTags parses comments for lines of the form: +// +// 'marker'+"key1=value1,key2=value2". +// +// Values are optional; 'true' is the default. If a key is set multiple times, +// the last one wins. +// +// Example: if you pass "+" for 'marker', and the following two lines are in +// the comments: +// +foo=value1,bar +// +foo=value2,baz="frobber" +// Then this function will return: +// map[string]string{"foo":"value2", "bar": "true", "baz": "frobber"} +// +// TODO: Basically we need to define a standard way of giving instructions to +// autogenerators in the comments of a type. This is a first iteration of that. +// TODO: allow multiple values per key? +func ExtractCommentTags(marker, allLines string) map[string]string { + lines := strings.Split(allLines, "\n") + out := map[string]string{} + for _, line := range lines { + line = strings.Trim(line, " ") + if len(line) == 0 { + continue + } + if !strings.HasPrefix(line, marker) { + continue + } + pairs := strings.Split(line[len(marker):], ",") + for _, p := range pairs { + kv := strings.Split(p, "=") + if len(kv) == 2 { + out[kv[0]] = kv[1] + } else if len(kv) == 1 { + out[kv[0]] = "true" + } + } + } + return out +} diff --git a/cmd/libs/go2idl/types/comments_test.go b/cmd/libs/go2idl/types/comments_test.go new file mode 100644 index 00000000000..622d23bd996 --- /dev/null +++ b/cmd/libs/go2idl/types/comments_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "reflect" + "testing" +) + +func TestExtractCommentTags(t *testing.T) { + commentLines := ` +Human comment that is ignored. ++foo=value1,bar ++foo=value2,baz=frobber +` + a := ExtractCommentTags("+", commentLines) + e := map[string]string{"foo": "value2", "bar": "true", "baz": "frobber"} + if !reflect.DeepEqual(e, a) { + t.Errorf("Wanted %#v, got %#v", e, a) + } +} diff --git a/cmd/libs/go2idl/types/doc.go b/cmd/libs/go2idl/types/doc.go new file mode 100644 index 00000000000..9dd9c5efcc3 --- /dev/null +++ b/cmd/libs/go2idl/types/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package types contains go type information, packaged in a way that makes +// auto-generation convenient, whether by template or straight go functions. +package types diff --git a/cmd/libs/go2idl/types/flatten.go b/cmd/libs/go2idl/types/flatten.go new file mode 100644 index 00000000000..a4f72bfabdc --- /dev/null +++ b/cmd/libs/go2idl/types/flatten.go @@ -0,0 +1,57 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +// FlattenMembers recursively takes any embedded members and puts them in the +// top level, correctly hiding them if the top level hides them. There must not +// be a cycle-- that implies infinite members. +// +// This is useful for e.g. computing all the valid keys in a json struct, +// properly considering any configuration of embedded structs. +func FlattenMembers(m []Member) []Member { + embedded := []Member{} + normal := []Member{} + type nameInfo struct { + top bool + i int + } + names := map[string]nameInfo{} + for i := range m { + if m[i].Embedded && m[i].Type.Kind == Struct { + embedded = append(embedded, m[i]) + } else { + normal = append(normal, m[i]) + names[m[i].Name] = nameInfo{true, len(normal) - 1} + } + } + for i := range embedded { + for _, e := range FlattenMembers(embedded[i].Type.Members) { + if info, found := names[e.Name]; found { + if info.top { + continue + } + if n := normal[info.i]; n.Name == e.Name && n.Type == e.Type { + continue + } + panic("conflicting members") + } + normal = append(normal, e) + names[e.Name] = nameInfo{false, len(normal) - 1} + } + } + return normal +} diff --git a/cmd/libs/go2idl/types/flatten_test.go b/cmd/libs/go2idl/types/flatten_test.go new file mode 100644 index 00000000000..280dc55bf0f --- /dev/null +++ b/cmd/libs/go2idl/types/flatten_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "reflect" + "testing" +) + +func TestFlatten(t *testing.T) { + mapType := &Type{ + Name: Name{"", "map[string]string"}, + Kind: Map, + Key: String, + Elem: String, + } + m := []Member{ + { + Name: "Baz", + Embedded: true, + Type: &Type{ + Name: Name{"pkg", "Baz"}, + Kind: Struct, + Members: []Member{ + {Name: "Foo", Type: String}, + { + Name: "Qux", + Embedded: true, + Type: &Type{ + Name: Name{"pkg", "Qux"}, + Kind: Struct, + Members: []Member{{Name: "Zot", Type: String}}, + }, + }, + }, + }, + }, + {Name: "Bar", Type: String}, + { + Name: "NotSureIfLegal", + Embedded: true, + Type: mapType, + }, + } + e := []Member{ + {Name: "Bar", Type: String}, + {Name: "NotSureIfLegal", Type: mapType, Embedded: true}, + {Name: "Foo", Type: String}, + {Name: "Zot", Type: String}, + } + if a := FlattenMembers(m); !reflect.DeepEqual(e, a) { + t.Errorf("Expected \n%#v\n, got \n%#v\n", e, a) + } +} diff --git a/cmd/libs/go2idl/types/types.go b/cmd/libs/go2idl/types/types.go new file mode 100644 index 00000000000..c48796c1992 --- /dev/null +++ b/cmd/libs/go2idl/types/types.go @@ -0,0 +1,307 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +// A type name may have a package qualifier. +type Name struct { + // Empty if embedded or builtin. This is the package path. + Package string + // The type name. + Name string +} + +// String returns the name formatted as a string. +func (n Name) String() string { + if n.Package == "" { + return n.Name + } + return n.Package + "." + n.Name +} + +// The possible classes of types. +type Kind string + +const ( + // Builtin is a primitive, like bool, string, int. + Builtin Kind = "Builtin" + Struct Kind = "Struct" + Map Kind = "Map" + Slice Kind = "Slice" + Pointer Kind = "Pointer" + + // Alias is an alias of another type, e.g. in: + // type Foo string + // type Bar Foo + // Bar is an alias of Foo. + // + // In the real go type system, Foo is a "Named" string; but to simplify + // generation, this type system will just say that Foo *is* a builtin. + // We then need "Alias" as a way for us to say that Bar *is* a Foo. + Alias Kind = "Alias" + + // Interface is any type that could have differing types at run time. + Interface Kind = "Interface" + + // The remaining types are included for completeness, but are not well + // supported. + Array Kind = "Array" // Array is just like slice, but has a fixed length. + Chan Kind = "Chan" + Func Kind = "Func" + Unknown Kind = "" + Unsupported Kind = "Unsupported" +) + +// Package holds package-level information. +// Fields are public, as everything in this package, to enable consumption by +// templates (for example). But it is strongly encouraged for code to build by +// using the provided functions. +type Package struct { + // Canonical name of this package-- its path. + Path string + + // Short name of this package; the name that appears in the + // 'package x' line. + Name string + + // Types within this package, indexed by their name (*not* including + // package name). + Types map[string]*Type + + // Packages imported by this package, indexed by (canonicalized) + // package path. + Imports map[string]*Package +} + +// Has returns true if the given name references a type known to this package. +func (p *Package) Has(name string) bool { + _, has := p.Types[name] + return has +} + +// Get (or add) the given type +func (p *Package) Get(typeName string) *Type { + if t, ok := p.Types[typeName]; ok { + return t + } + if p.Path == "" { + // Import the standard builtin types! + if t, ok := builtins.Types[typeName]; ok { + p.Types[typeName] = t + return t + } + } + t := &Type{Name: Name{p.Path, typeName}} + p.Types[typeName] = t + return t +} + +// HasImport returns true if p imports packageName. Package names include the +// package directory. +func (p *Package) HasImport(packageName string) bool { + _, has := p.Imports[packageName] + return has +} + +// Universe is a map of all packages. The key is the package name, but you +// should use Get() or Package() instead of direct access. +type Universe map[string]*Package + +// Get returns the canonical type for the given fully-qualified name. Builtin +// types will always be found, even if they haven't been explicitly added to +// the map. If a non-existing type is requested, u will create (a marker for) +// it. +func (u Universe) Get(n Name) *Type { + return u.Package(n.Package).Get(n.Name) +} + +// AddImports registers import lines for packageName. May be called multiple times. +// You are responsible for canonicalizing all package paths. +func (u Universe) AddImports(packagePath string, importPaths ...string) { + p := u.Package(packagePath) + for _, i := range importPaths { + p.Imports[i] = u.Package(i) + } +} + +// Get (create if needed) the package. +func (u Universe) Package(packagePath string) *Package { + if p, ok := u[packagePath]; ok { + return p + } + p := &Package{ + Path: packagePath, + Types: map[string]*Type{}, + Imports: map[string]*Package{}, + } + u[packagePath] = p + return p +} + +// Type represents a subset of possible go types. +type Type struct { + // There are two general categories of types, those explicitly named + // and those anonymous. Named ones will have a non-empty package in the + // name field. + Name Name + + // The general kind of this type. + Kind Kind + + // If there are comment lines immediately before the type definition, + // they will be recorded here. + CommentLines string + + // If Kind == Struct + Members []Member + + // If Kind == Map, Slice, Pointer, or Chan + Elem *Type + + // If Kind == Map, this is the map's key type. + Key *Type + + // If Kind == Alias, this is the underlying type. + Underlying *Type + + // If Kind == Interface, this is the list of all required functions. + // Otherwise, if this is a named type, this is the list of methods that + // type has. (All elements will have Kind=="Func") + Methods []*Type + + // If Kind == func, this is the signature of the function. + Signature *Signature + + // TODO: Add: + // * channel direction + // * array length +} + +// String returns the name of the type. +func (t *Type) String() string { + return t.Name.String() +} + +// A single struct member +type Member struct { + // The name of the member. + Name string + + // If the member is embedded (anonymous) this will be true, and the + // Name will be the type name. + Embedded bool + + // If there are comment lines immediately before the member in the type + // definition, they will be recorded here. + CommentLines string + + // If there are tags along with this member, they will be saved here. + Tags string + + // The type of this member. + Type *Type +} + +// String returns the name and type of the member. +func (m Member) String() string { + return m.Name + " " + m.Type.String() +} + +// Signature is a function's signature. +type Signature struct { + // TODO: store the parameter names, not just types. + + // If a method of some type, this is the type it's a member of. + Receiver *Type + Parameters []*Type + Results []*Type + + // True if the last in parameter is of the form ...T. + Variadic bool + + // If there are comment lines immediately before this + // signature/method/function declaration, they will be recorded here. + CommentLines string +} + +// Built in types. +var ( + String = &Type{ + Name: Name{Name: "string"}, + Kind: Builtin, + } + Int64 = &Type{ + Name: Name{Name: "int64"}, + Kind: Builtin, + } + Int32 = &Type{ + Name: Name{Name: "int32"}, + Kind: Builtin, + } + Int16 = &Type{ + Name: Name{Name: "int16"}, + Kind: Builtin, + } + Int = &Type{ + Name: Name{Name: "int"}, + Kind: Builtin, + } + Uint64 = &Type{ + Name: Name{Name: "uint64"}, + Kind: Builtin, + } + Uint32 = &Type{ + Name: Name{Name: "uint32"}, + Kind: Builtin, + } + Uint16 = &Type{ + Name: Name{Name: "uint16"}, + Kind: Builtin, + } + Uint = &Type{ + Name: Name{Name: "uint"}, + Kind: Builtin, + } + Bool = &Type{ + Name: Name{Name: "bool"}, + Kind: Builtin, + } + Byte = &Type{ + Name: Name{Name: "byte"}, + Kind: Builtin, + } + + builtins = &Package{ + Types: map[string]*Type{ + "bool": Bool, + "string": String, + "int": Int, + "int64": Int64, + "int32": Int32, + "int16": Int16, + "int8": Byte, + "uint": Uint, + "uint64": Uint64, + "uint32": Uint32, + "uint16": Uint16, + "uint8": Byte, + "byte": Byte, + }, + Imports: map[string]*Package{}, + Path: "", + Name: "", + } +) diff --git a/cmd/libs/go2idl/types/types_test.go b/cmd/libs/go2idl/types/types_test.go new file mode 100644 index 00000000000..33893525536 --- /dev/null +++ b/cmd/libs/go2idl/types/types_test.go @@ -0,0 +1,47 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "testing" +) + +func TestGetBuiltin(t *testing.T) { + u := Universe{} + if builtinPkg := u.Package(""); builtinPkg.Has("string") { + t.Errorf("Expected builtin package to not have builtins until they're asked for explicitly. %#v", builtinPkg) + } + s := u.Get(Name{"", "string"}) + if s != String { + t.Errorf("Expected canonical string type.") + } + if builtinPkg := u.Package(""); !builtinPkg.Has("string") { + t.Errorf("Expected builtin package to exist and have builtins by default. %#v", builtinPkg) + } + if builtinPkg := u.Package(""); len(builtinPkg.Types) != 1 { + t.Errorf("Expected builtin package to not have builtins until they're asked for explicitly. %#v", builtinPkg) + } +} + +func TestGetMarker(t *testing.T) { + u := Universe{} + n := Name{"path/to/package", "Foo"} + f := u.Get(n) + if f == nil || f.Name != n { + t.Errorf("Expected marker type.") + } +}