From b2280db7241b70936064e4d9a748a604b4171cd4 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Wed, 13 May 2015 15:52:53 +0200 Subject: [PATCH] Deep-copy functions autogeneration. --- .../conversion.go | 0 cmd/gendeepcopy/deep_copy.go | 82 +++ hack/lib/golang.sh | 2 + hack/update-generated-conversions.sh | 2 +- hack/update-generated-deep-copies.sh | 51 ++ hack/verify-generated-conversions.sh | 6 +- hack/verify-generated-deep-copies.sh | 60 +++ pkg/api/deep_copy_test.go | 7 +- pkg/conversion/cloner.go | 226 ++++++++ pkg/conversion/scheme.go | 33 ++ pkg/runtime/conversion_generation_test.go | 7 +- pkg/runtime/deep_copy_generation_test.go | 87 ++++ pkg/runtime/deep_copy_generator.go | 481 ++++++++++++++++++ pkg/runtime/scheme.go | 18 + 14 files changed, 1049 insertions(+), 13 deletions(-) rename cmd/{kube-conversion => genconversion}/conversion.go (100%) create mode 100644 cmd/gendeepcopy/deep_copy.go create mode 100755 hack/update-generated-deep-copies.sh create mode 100755 hack/verify-generated-deep-copies.sh create mode 100644 pkg/conversion/cloner.go create mode 100644 pkg/runtime/deep_copy_generation_test.go create mode 100644 pkg/runtime/deep_copy_generator.go diff --git a/cmd/kube-conversion/conversion.go b/cmd/genconversion/conversion.go similarity index 100% rename from cmd/kube-conversion/conversion.go rename to cmd/genconversion/conversion.go diff --git a/cmd/gendeepcopy/deep_copy.go b/cmd/gendeepcopy/deep_copy.go new file mode 100644 index 00000000000..864c22743c0 --- /dev/null +++ b/cmd/gendeepcopy/deep_copy.go @@ -0,0 +1,82 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "io" + "os" + "runtime" + "strings" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + _ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1" + _ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta1" + _ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta2" + _ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta3" + pkg_runtime "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" + + "github.com/golang/glog" + flag "github.com/spf13/pflag" +) + +var ( + functionDest = flag.StringP("func-dest", "f", "-", "Output for deep copy functions; '-' means stdout") + version = flag.StringP("version", "v", "v1beta3", "Version for deep copies.") + overwrites = flag.StringP("overwrites", "o", "", "Comma-separated overwrites for package names") +) + +func main() { + runtime.GOMAXPROCS(runtime.NumCPU()) + flag.Parse() + + var funcOut io.Writer + if *functionDest == "-" { + funcOut = os.Stdout + } else { + file, err := os.Create(*functionDest) + if err != nil { + glog.Fatalf("Couldn't open %v: %v", *functionDest, err) + } + defer file.Close() + funcOut = file + } + + knownVersion := *version + if knownVersion == "api" { + knownVersion = api.Scheme.Raw().InternalVersion + } + generator := pkg_runtime.NewDeepCopyGenerator(api.Scheme.Raw()) + + for _, overwrite := range strings.Split(*overwrites, ",") { + vals := strings.Split(overwrite, "=") + generator.OverwritePackage(vals[0], vals[1]) + } + for _, knownType := range api.Scheme.KnownTypes(knownVersion) { + if err := generator.AddType(knownType); err != nil { + glog.Errorf("error while generating deep copy functions for %v: %v", knownType, err) + } + } + if err := generator.WriteImports(funcOut, *version); err != nil { + glog.Fatalf("error while writing imports: %v", err) + } + if err := generator.WriteDeepCopyFunctions(funcOut); err != nil { + glog.Fatalf("error while writing deep copy functions: %v", err) + } + if err := generator.RegisterDeepCopyFunctions(funcOut, *version); err != nil { + glog.Fatalf("error while registering deep copy functions: %v", err) + } +} diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index 96caa0ae52c..aaa328b068a 100644 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -48,6 +48,8 @@ readonly KUBE_TEST_TARGETS=( cmd/gendocs cmd/genman cmd/genbashcomp + cmd/genconversion + cmd/gendeepcopy examples/k8petstore/web-server github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test diff --git a/hack/update-generated-conversions.sh b/hack/update-generated-conversions.sh index 773de0c3d6f..1236079ee7c 100755 --- a/hack/update-generated-conversions.sh +++ b/hack/update-generated-conversions.sh @@ -39,7 +39,7 @@ import ( // AUTO-GENERATED FUNCTIONS START HERE EOF - GOPATH=$(godep path):$GOPATH go run cmd/kube-conversion/conversion.go -v ${version} -f - >> $TMPFILE + GOPATH=$(godep path):$GOPATH go run cmd/genconversion/conversion.go -v ${version} -f - >> $TMPFILE cat >> $TMPFILE < $TMPFILE + cat >> $TMPFILE <&2 exit 1 fi diff --git a/hack/verify-generated-deep-copies.sh b/hack/verify-generated-deep-copies.sh new file mode 100755 index 00000000000..20a25089327 --- /dev/null +++ b/hack/verify-generated-deep-copies.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +kube::golang::setup_env +"${KUBE_ROOT}/hack/build-go.sh" cmd/gendeepcopy + +genconversion=$(kube::util::find-binary "gendeepcopy") + +if [[ ! -x "$genconversion" ]]; then + { + echo "It looks as if you don't have a compiled conversion binary" + echo + echo "If you are running from a clone of the git repo, please run" + echo "'./hack/build-go.sh cmd/gendeepcopy'." + } >&2 + exit 1 +fi + +APIROOT="${KUBE_ROOT}/pkg/api" +TMP_APIROOT="${KUBE_ROOT}/_tmp/api" +_tmp="${KUBE_ROOT}/_tmp" + +mkdir -p "${_tmp}" +cp -a "${APIROOT}" "${TMP_APIROOT}" + +"${KUBE_ROOT}/hack/update-generated-deep-copies.sh" +echo "diffing ${APIROOT} against freshly generated deep copies" +ret=0 +diff -Naupr -I 'Auto generated by' "${APIROOT}" "${TMP_APIROOT}" || ret=$? +cp -a ${TMP_APIROOT} "${KUBE_ROOT}/pkg" +rm -rf "${_tmp}" +if [[ $ret -eq 0 ]] +then + echo "${APIROOT} up to date." +else + echo "${APIROOT} is out of date. Please run hack/update-generated-deep-copies.sh" + exit 1 +fi + +# ex: ts=2 sw=2 et filetype=sh diff --git a/pkg/api/deep_copy_test.go b/pkg/api/deep_copy_test.go index ca2b62eff55..232ad7d67dd 100644 --- a/pkg/api/deep_copy_test.go +++ b/pkg/api/deep_copy_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" - "github.com/GoogleCloudPlatform/kubernetes/pkg/conversion" ) func BenchmarkPodCopy(b *testing.B) { @@ -36,7 +35,7 @@ func BenchmarkPodCopy(b *testing.B) { var result *api.Pod for i := 0; i < b.N; i++ { - obj, err := conversion.DeepCopy(&pod) + obj, err := api.Scheme.DeepCopy(&pod) if err != nil { b.Fatalf("Unexpected error copying pod: %v", err) } @@ -59,7 +58,7 @@ func BenchmarkNodeCopy(b *testing.B) { var result *api.Node for i := 0; i < b.N; i++ { - obj, err := conversion.DeepCopy(&node) + obj, err := api.Scheme.DeepCopy(&node) if err != nil { b.Fatalf("Unexpected error copying node: %v", err) } @@ -82,7 +81,7 @@ func BenchmarkReplicationControllerCopy(b *testing.B) { var result *api.ReplicationController for i := 0; i < b.N; i++ { - obj, err := conversion.DeepCopy(&replicationController) + obj, err := api.Scheme.DeepCopy(&replicationController) if err != nil { b.Fatalf("Unexpected error copying replication controller: %v", err) } diff --git a/pkg/conversion/cloner.go b/pkg/conversion/cloner.go new file mode 100644 index 00000000000..d42922159dc --- /dev/null +++ b/pkg/conversion/cloner.go @@ -0,0 +1,226 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "fmt" + "reflect" +) + +// Cloner knows how to copy one type to another. +type Cloner struct { + // Map from the type to a function which can do the deep copy. + deepCopyFuncs map[reflect.Type]reflect.Value + generatedDeepCopyFuncs map[reflect.Type]reflect.Value +} + +// NewCloner creates a new Cloner object. +func NewCloner() *Cloner { + c := &Cloner{ + deepCopyFuncs: map[reflect.Type]reflect.Value{}, + generatedDeepCopyFuncs: map[reflect.Type]reflect.Value{}, + } + if err := c.RegisterDeepCopyFunc(byteSliceDeepCopy); err != nil { + // If one of the deep-copy functions is malformed, detect it immediately. + panic(err) + } + return c +} + +// Prevent recursing into every byte... +func byteSliceDeepCopy(in []byte, out *[]byte, c *Cloner) error { + *out = make([]byte, len(in)) + copy(*out, in) + return nil +} + +// Verifies whether a deep-copy function has a correct signature. +func verifyDeepCopyFunctionSignature(ft reflect.Type) error { + if ft.Kind() != reflect.Func { + return fmt.Errorf("expected func, got: %v", ft) + } + if ft.NumIn() != 3 { + return fmt.Errorf("expected three 'in' params, got $v", ft) + } + if ft.NumOut() != 1 { + return fmt.Errorf("expected one 'out' param, got %v", ft) + } + if ft.In(1).Kind() != reflect.Ptr { + return fmt.Errorf("expected pointer arg for 'in' param 1, got: %v", ft) + } + if ft.In(1).Elem() != ft.In(0) { + return fmt.Errorf("expected 'in' param 0 the same as param 1, got: %v", ft) + } + var forClonerType Cloner + if expected := reflect.TypeOf(&forClonerType); ft.In(2) != expected { + return fmt.Errorf("expected '%v' arg for 'in' param 2, got: '%v'", expected, ft.In(2)) + } + var forErrorType error + // This convolution is necessary, otherwise TypeOf picks up on the fact + // that forErrorType is nil + errorType := reflect.TypeOf(&forErrorType).Elem() + if ft.Out(0) != errorType { + return fmt.Errorf("expected error return, got: %v", ft) + } + return nil +} + +// RegisterGeneratedDeepCopyFunc registers a copying func with the Cloner. +// deepCopyFunc must take two parameters: a type and a pointer to Cloner +// and return an element of the same type and an error. +// +// Example: +// c.RegisterGeneratedDeepCopyFunc( +// func(in Pod, c *Cloner) (Pod, error) { +// // deep copy logic... +// return copy, nil +// }) +func (c *Cloner) RegisterDeepCopyFunc(deepCopyFunc interface{}) error { + fv := reflect.ValueOf(deepCopyFunc) + ft := fv.Type() + if err := verifyDeepCopyFunctionSignature(ft); err != nil { + return err + } + c.deepCopyFuncs[ft.In(0)] = fv + return nil +} + +// Similar to RegisterDeepCopyFunc, but registers deep copy function that were +// automatically generated. +func (c *Cloner) RegisterGeneratedDeepCopyFunc(deepCopyFunc interface{}) error { + fv := reflect.ValueOf(deepCopyFunc) + ft := fv.Type() + if err := verifyDeepCopyFunctionSignature(ft); err != nil { + return err + } + c.generatedDeepCopyFuncs[ft.In(0)] = fv + return nil +} + +// DeepCopy will perform a deep copy of a given object. +func (c *Cloner) DeepCopy(in interface{}) (interface{}, error) { + inValue := reflect.ValueOf(in) + outValue, err := c.deepCopy(inValue) + if err != nil { + return nil, err + } + return outValue.Interface(), nil +} + +func (c *Cloner) deepCopy(src reflect.Value) (reflect.Value, error) { + inType := src.Type() + + if fv, ok := c.deepCopyFuncs[inType]; ok { + return c.customDeepCopy(src, fv) + } + if fv, ok := c.generatedDeepCopyFuncs[inType]; ok { + return c.customDeepCopy(src, fv) + } + return c.defaultDeepCopy(src) +} + +func (c *Cloner) customDeepCopy(src, fv reflect.Value) (reflect.Value, error) { + outValue := reflect.New(src.Type()) + args := []reflect.Value{src, outValue, reflect.ValueOf(c)} + result := fv.Call(args)[0].Interface() + // This convolution is necessary because nil interfaces won't convert + // to error. + if result == nil { + return outValue.Elem(), nil + } + return outValue.Elem(), result.(error) +} + +func (c *Cloner) defaultDeepCopy(src reflect.Value) (reflect.Value, error) { + switch src.Kind() { + case reflect.Chan, reflect.Func, reflect.UnsafePointer, reflect.Uintptr: + return src, fmt.Errorf("cannot deep copy kind: %s", src.Kind()) + case reflect.Array: + dst := reflect.New(src.Type()) + for i := 0; i < src.Len(); i++ { + copyVal, err := c.deepCopy(src.Index(i)) + if err != nil { + return src, err + } + dst.Elem().Index(i).Set(copyVal) + } + return dst.Elem(), nil + case reflect.Interface: + if src.IsNil() { + return src, nil + } + return c.deepCopy(src.Elem()) + case reflect.Map: + if src.IsNil() { + return src, nil + } + dst := reflect.MakeMap(src.Type()) + for _, k := range src.MapKeys() { + copyVal, err := c.deepCopy(src.MapIndex(k)) + if err != nil { + return src, err + } + dst.SetMapIndex(k, copyVal) + } + return dst, nil + case reflect.Ptr: + if src.IsNil() { + return src, nil + } + dst := reflect.New(src.Type().Elem()) + copyVal, err := c.deepCopy(src.Elem()) + if err != nil { + return src, err + } + dst.Elem().Set(copyVal) + return dst, nil + case reflect.Slice: + if src.IsNil() { + return src, nil + } + dst := reflect.MakeSlice(src.Type(), 0, src.Len()) + for i := 0; i < src.Len(); i++ { + copyVal, err := c.deepCopy(src.Index(i)) + if err != nil { + return src, err + } + dst = reflect.Append(dst, copyVal) + } + return dst, nil + case reflect.Struct: + dst := reflect.New(src.Type()) + for i := 0; i < src.NumField(); i++ { + if !dst.Elem().Field(i).CanSet() { + // Can't set private fields. At this point, the + // best we can do is a shallow copy. For + // example, time.Time is a value type with + // private members that can be shallow copied. + return src, nil + } + copyVal, err := c.deepCopy(src.Field(i)) + if err != nil { + return src, err + } + dst.Elem().Field(i).Set(copyVal) + } + return dst.Elem(), nil + + default: + // Value types like numbers, booleans, and strings. + return src, nil + } +} diff --git a/pkg/conversion/scheme.go b/pkg/conversion/scheme.go index b2724eb496d..e534345a1ae 100644 --- a/pkg/conversion/scheme.go +++ b/pkg/conversion/scheme.go @@ -40,6 +40,10 @@ type Scheme struct { // default coverting behavior. converter *Converter + // cloner stores all registered copy functions. It also has default + // deep copy behavior. + cloner *Cloner + // Indent will cause the JSON output from Encode to be indented, iff it is true. Indent bool @@ -60,6 +64,7 @@ func NewScheme() *Scheme { typeToVersion: map[reflect.Type]string{}, typeToKind: map[reflect.Type][]string{}, converter: NewConverter(), + cloner: NewCloner(), InternalVersion: "", MetaFactory: DefaultMetaFactory, } @@ -212,6 +217,29 @@ func (s *Scheme) AddGeneratedConversionFuncs(conversionFuncs ...interface{}) err return nil } +// AddDeepCopyFuncs adds functions to the list of deep copy functions. +// Note that to copy sub-objects, you can use the conversion.Cloner object that +// will be passed to your deep-copy function. +func (s *Scheme) AddDeepCopyFuncs(deepCopyFuncs ...interface{}) error { + for _, f := range deepCopyFuncs { + if err := s.cloner.RegisterDeepCopyFunc(f); err != nil { + return err + } + } + return nil +} + +// Similar to AddDeepCopyFuncs, but registers deep copy functions that were +// automatically generated. +func (s *Scheme) AddGeneratedDeepCopyFuncs(deepCopyFuncs ...interface{}) error { + for _, f := range deepCopyFuncs { + if err := s.cloner.RegisterGeneratedDeepCopyFunc(f); err != nil { + return err + } + } + return nil +} + // AddStructFieldConversion allows you to specify a mechanical copy for a moved // or renamed struct field without writing an entire conversion function. See // the comment in Converter.SetStructFieldCopy for parameter details. @@ -262,6 +290,11 @@ func (s *Scheme) RegisterInputDefaults(in interface{}, fn FieldMappingFunc, defa return s.converter.RegisterInputDefaults(in, fn, defaultFlags) } +// Performs a deep copy of the given object. +func (s *Scheme) DeepCopy(in interface{}) (interface{}, error) { + return s.cloner.DeepCopy(in) +} + // Convert will attempt to convert in into out. Both must be pointers. For easy // testing of conversion functions. Returns an error if the conversion isn't // possible. You can call this with types that haven't been registered (for example, diff --git a/pkg/runtime/conversion_generation_test.go b/pkg/runtime/conversion_generation_test.go index 9a69515cad2..006494d8614 100644 --- a/pkg/runtime/conversion_generation_test.go +++ b/pkg/runtime/conversion_generation_test.go @@ -47,9 +47,6 @@ func generateConversions(t *testing.T, version string) bytes.Buffer { if err := g.WriteConversionFunctions(functionsWriter); err != nil { t.Fatalf("couldn't generate conversion functions: %v", err) } - if err := functionsWriter.Flush(); err != nil { - t.Fatalf("error while flushing writer") - } if err := g.RegisterConversionFunctions(functionsWriter); err != nil { t.Fatalf("couldn't generate conversion function names: %v", err) } @@ -81,7 +78,7 @@ func readLinesUntil(t *testing.T, reader *bufio.Reader, stop string, buffer *byt return nil } -func bufferExistingConversions(t *testing.T, fileName string) bytes.Buffer { +func bufferExistingGeneratedCode(t *testing.T, fileName string) bytes.Buffer { file, err := os.Open(fileName) if err != nil { t.Fatalf("couldn't open file %s", fileName) @@ -139,7 +136,7 @@ func TestNoManualChangesToGenerateConversions(t *testing.T) { for _, version := range versions { fileName := fmt.Sprintf("../../pkg/api/%s/conversion_generated.go", version) - existingFunctions := bufferExistingConversions(t, fileName) + existingFunctions := bufferExistingGeneratedCode(t, fileName) generatedFunctions := generateConversions(t, version) functionsTxt := fmt.Sprintf("%s.functions.txt", version) diff --git a/pkg/runtime/deep_copy_generation_test.go b/pkg/runtime/deep_copy_generation_test.go new file mode 100644 index 00000000000..47c12208f2e --- /dev/null +++ b/pkg/runtime/deep_copy_generation_test.go @@ -0,0 +1,87 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtime_test + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + _ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1" + _ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta3" + "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" + + "github.com/golang/glog" +) + +func generateDeepCopies(t *testing.T, version string) bytes.Buffer { + g := runtime.NewDeepCopyGenerator(api.Scheme.Raw()) + g.OverwritePackage(version, "") + testedVersion := version + if version == "api" { + testedVersion = api.Scheme.Raw().InternalVersion + } + for _, knownType := range api.Scheme.KnownTypes(testedVersion) { + if err := g.AddType(knownType); err != nil { + glog.Errorf("error while generating deep-copy functions for %v: %v", knownType, err) + } + } + + var functions bytes.Buffer + functionsWriter := bufio.NewWriter(&functions) + if err := g.WriteImports(functionsWriter, version); err != nil { + t.Fatalf("couldn't generate deep-copy function imports: %v", err) + } + if err := g.WriteDeepCopyFunctions(functionsWriter); err != nil { + t.Fatalf("couldn't generate deep-copy functions: %v", err) + } + if err := g.RegisterDeepCopyFunctions(functionsWriter, version); err != nil { + t.Fatalf("couldn't generate deep-copy function names: %v", err) + } + if err := functionsWriter.Flush(); err != nil { + t.Fatalf("error while flushing writer") + } + + return functions +} + +func TestNoManualChangesToGenerateDeepCopies(t *testing.T) { + versions := []string{"api", "v1beta3", "v1"} + + for _, version := range versions { + fileName := "" + if version == "api" { + fileName = "../../pkg/api/deep_copy_generated.go" + } else { + fileName = fmt.Sprintf("../../pkg/api/%s/deep_copy_generated.go", version) + } + + existingFunctions := bufferExistingGeneratedCode(t, fileName) + generatedFunctions := generateDeepCopies(t, version) + + functionsTxt := fmt.Sprintf("%s.deep_copy.txt", version) + ioutil.WriteFile(functionsTxt, generatedFunctions.Bytes(), os.FileMode(0644)) + + if ok := compareBuffers(t, functionsTxt, existingFunctions, generatedFunctions); ok { + os.Remove(functionsTxt) + } + } +} diff --git a/pkg/runtime/deep_copy_generator.go b/pkg/runtime/deep_copy_generator.go new file mode 100644 index 00000000000..1c35633c2e8 --- /dev/null +++ b/pkg/runtime/deep_copy_generator.go @@ -0,0 +1,481 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtime + +import ( + "fmt" + "io" + "reflect" + "sort" + "strings" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/conversion" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" +) + +type DeepCopyGenerator interface { + // Adds a type to a generator. + // If the type is non-struct, it will return an error, otherwise deep-copy + // functions for this type and all nested types will be generated. + AddType(inType reflect.Type) error + + // Writes all imports that are necessary for deep-copy function and + // their registration. + WriteImports(w io.Writer, pkg string) error + + // Writes deel-copy functions for all types added via AddType() method + // and their nested types. + WriteDeepCopyFunctions(w io.Writer) error + + // Writes an init() function that registers all the generated deep-copy + // functions. + RegisterDeepCopyFunctions(w io.Writer, pkg string) error + + // When generating code, all references to "pkg" package name will be + // replaced with "overwrite". It is used mainly to replace references + // to name of the package in which the code will be created with empty + // string. + OverwritePackage(pkg, overwrite string) +} + +func NewDeepCopyGenerator(scheme *conversion.Scheme) DeepCopyGenerator { + return &deepCopyGenerator{ + scheme: scheme, + copyables: make(map[reflect.Type]bool), + imports: util.StringSet{}, + pkgOverwrites: make(map[string]string), + } +} + +type deepCopyGenerator struct { + scheme *conversion.Scheme + copyables map[reflect.Type]bool + imports util.StringSet + pkgOverwrites map[string]string +} + +func (g *deepCopyGenerator) addAllRecursiveTypes(inType reflect.Type) error { + if _, found := g.copyables[inType]; found { + return nil + } + switch inType.Kind() { + case reflect.Map: + if err := g.addAllRecursiveTypes(inType.Key()); err != nil { + return err + } + if err := g.addAllRecursiveTypes(inType.Elem()); err != nil { + return err + } + case reflect.Slice, reflect.Ptr: + if err := g.addAllRecursiveTypes(inType.Elem()); err != nil { + return err + } + case reflect.Interface: + g.imports.Insert(inType.PkgPath()) + return nil + case reflect.Struct: + g.imports.Insert(inType.PkgPath()) + if !strings.HasPrefix(inType.PkgPath(), "github.com/GoogleCloudPlatform/kubernetes") { + return nil + } + for i := 0; i < inType.NumField(); i++ { + inField := inType.Field(i) + if err := g.addAllRecursiveTypes(inField.Type); err != nil { + return err + } + } + g.copyables[inType] = true + default: + // Simple types should be copied automatically. + } + return nil +} + +func (g *deepCopyGenerator) AddType(inType reflect.Type) error { + if inType.Kind() != reflect.Struct { + return fmt.Errorf("non-struct copies are not supported") + } + return g.addAllRecursiveTypes(inType) +} + +func (g *deepCopyGenerator) WriteImports(w io.Writer, pkg string) error { + var packages []string + packages = append(packages, "github.com/GoogleCloudPlatform/kubernetes/pkg/api") + packages = append(packages, "github.com/GoogleCloudPlatform/kubernetes/pkg/conversion") + for key := range g.imports { + packages = append(packages, key) + } + sort.Strings(packages) + + buffer := newBuffer() + indent := 0 + buffer.addLine("import (\n", indent) + for _, importPkg := range packages { + if strings.HasSuffix(importPkg, pkg) { + continue + } + buffer.addLine(fmt.Sprintf("\"%s\"\n", importPkg), indent+1) + } + buffer.addLine(")\n", indent) + buffer.addLine("\n", indent) + if err := buffer.flushLines(w); err != nil { + return err + } + return nil +} + +type byPkgAndName []reflect.Type + +func (s byPkgAndName) Len() int { + return len(s) +} + +func (s byPkgAndName) Less(i, j int) bool { + fullNameI := s[i].PkgPath() + "/" + s[i].Name() + fullNameJ := s[j].PkgPath() + "/" + s[j].Name() + return fullNameI < fullNameJ +} + +func (s byPkgAndName) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (g *deepCopyGenerator) typeName(inType reflect.Type) string { + switch inType.Kind() { + case reflect.Map: + return fmt.Sprintf("map[%s]%s", g.typeName(inType.Key()), g.typeName(inType.Elem())) + case reflect.Slice: + return fmt.Sprintf("[]%s", g.typeName(inType.Elem())) + case reflect.Ptr: + return fmt.Sprintf("*%s", g.typeName(inType.Elem())) + default: + typeWithPkg := fmt.Sprintf("%s", inType) + slices := strings.Split(typeWithPkg, ".") + if len(slices) == 1 { + // Default package. + return slices[0] + } + if len(slices) == 2 { + pkg := slices[0] + if val, found := g.pkgOverwrites[pkg]; found { + pkg = val + } + if pkg != "" { + pkg = pkg + "." + } + return pkg + slices[1] + } + panic("Incorrect type name: " + typeWithPkg) + } +} + +func (g *deepCopyGenerator) deepCopyFunctionName(inType reflect.Type) string { + funcNameFormat := "deepCopy_%s_%s" + inPkg := packageForName(inType) + funcName := fmt.Sprintf(funcNameFormat, inPkg, inType.Name()) + return funcName +} + +func (g *deepCopyGenerator) writeHeader(b *buffer, inType reflect.Type, indent int) { + format := "func %s(in %s, out *%s, c *conversion.Cloner) error {\n" + stmt := fmt.Sprintf(format, g.deepCopyFunctionName(inType), g.typeName(inType), g.typeName(inType)) + b.addLine(stmt, indent) +} + +func (g *deepCopyGenerator) writeFooter(b *buffer, indent int) { + b.addLine("return nil\n", indent+1) + b.addLine("}\n", indent) +} + +func (g *deepCopyGenerator) WriteDeepCopyFunctions(w io.Writer) error { + var keys []reflect.Type + for key := range g.copyables { + keys = append(keys, key) + } + sort.Sort(byPkgAndName(keys)) + + buffer := newBuffer() + indent := 0 + for _, inType := range keys { + if err := g.writeDeepCopyForType(buffer, inType, indent); err != nil { + return err + } + buffer.addLine("\n", 0) + } + if err := buffer.flushLines(w); err != nil { + return err + } + return nil +} + +func (g *deepCopyGenerator) writeDeepCopyForMap(b *buffer, inField reflect.StructField, indent int) error { + ifFormat := "if in.%s != nil {\n" + ifStmt := fmt.Sprintf(ifFormat, inField.Name) + b.addLine(ifStmt, indent) + newFormat := "out.%s = make(%s)\n" + newStmt := fmt.Sprintf(newFormat, inField.Name, g.typeName(inField.Type)) + b.addLine(newStmt, indent+1) + forFormat := "for key, val := range in.%s {\n" + forStmt := fmt.Sprintf(forFormat, inField.Name) + b.addLine(forStmt, indent+1) + + switch inField.Type.Key().Kind() { + case reflect.Map, reflect.Ptr, reflect.Slice, reflect.Interface, reflect.Struct: + return fmt.Errorf("not supported") + default: + switch inField.Type.Elem().Kind() { + case reflect.Map, reflect.Ptr, reflect.Slice, reflect.Interface, reflect.Struct: + if _, found := g.copyables[inField.Type.Elem()]; found { + newFormat := "newVal := new(%s)\n" + newStmt := fmt.Sprintf(newFormat, g.typeName(inField.Type.Elem())) + b.addLine(newStmt, indent+2) + assignFormat := "if err := %s(val, newVal, c); err != nil {\n" + funcName := g.deepCopyFunctionName(inField.Type.Elem()) + assignStmt := fmt.Sprintf(assignFormat, funcName) + b.addLine(assignStmt, indent+2) + b.addLine("return err\n", indent+3) + b.addLine("}\n", indent+2) + setFormat := "out.%s[key] = *newVal\n" + setStmt := fmt.Sprintf(setFormat, inField.Name) + b.addLine(setStmt, indent+2) + } else { + ifStmt := "if newVal, err := c.DeepCopy(val); err != nil {\n" + b.addLine(ifStmt, indent+2) + b.addLine("return err\n", indent+3) + b.addLine("} else {\n", indent+2) + assignFormat := "out.%s[key] = newVal.(%s)\n" + assignStmt := fmt.Sprintf(assignFormat, inField.Name, g.typeName(inField.Type.Elem())) + b.addLine(assignStmt, indent+3) + b.addLine("}\n", indent+2) + } + default: + assignFormat := "out.%s[key] = val\n" + assignStmt := fmt.Sprintf(assignFormat, inField.Name) + b.addLine(assignStmt, indent+2) + } + } + b.addLine("}\n", indent+1) + b.addLine("} else {\n", indent) + elseFormat := "out.%s = nil\n" + elseStmt := fmt.Sprintf(elseFormat, inField.Name) + b.addLine(elseStmt, indent+1) + b.addLine("}\n", indent) + return nil +} + +func (g *deepCopyGenerator) writeDeepCopyForPtr(b *buffer, inField reflect.StructField, indent int) error { + ifFormat := "if in.%s != nil {\n" + ifStmt := fmt.Sprintf(ifFormat, inField.Name) + b.addLine(ifStmt, indent) + + switch inField.Type.Elem().Kind() { + case reflect.Map, reflect.Ptr, reflect.Slice, reflect.Interface, reflect.Struct: + if _, found := g.copyables[inField.Type.Elem()]; found { + newFormat := "out.%s = new(%s)\n" + newStmt := fmt.Sprintf(newFormat, inField.Name, g.typeName(inField.Type.Elem())) + b.addLine(newStmt, indent+1) + assignFormat := "if err := %s(*in.%s, out.%s, c); err != nil {\n" + funcName := g.deepCopyFunctionName(inField.Type.Elem()) + assignStmt := fmt.Sprintf(assignFormat, funcName, inField.Name, inField.Name) + b.addLine(assignStmt, indent+1) + b.addLine("return err\n", indent+2) + b.addLine("}\n", indent+1) + } else { + ifFormat := "if newVal, err := c.DeepCopy(in.%s); err != nil {\n" + ifStmt := fmt.Sprintf(ifFormat, inField.Name) + b.addLine(ifStmt, indent+1) + b.addLine("return err\n", indent+2) + b.addLine("} else {\n", indent+1) + assignFormat := "out.%s = newVal.(%s)\n" + assignStmt := fmt.Sprintf(assignFormat, inField.Name, g.typeName(inField.Type)) + b.addLine(assignStmt, indent+2) + b.addLine("}\n", indent+1) + } + default: + newFormat := "out.%s = new(%s)\n" + newStmt := fmt.Sprintf(newFormat, inField.Name, g.typeName(inField.Type.Elem())) + b.addLine(newStmt, indent+1) + assignFormat := "*out.%s = *in.%s\n" + assignStmt := fmt.Sprintf(assignFormat, inField.Name, inField.Name) + b.addLine(assignStmt, indent+1) + } + b.addLine("} else {\n", indent) + elseFormat := "out.%s = nil\n" + elseStmt := fmt.Sprintf(elseFormat, inField.Name) + b.addLine(elseStmt, indent+1) + b.addLine("}\n", indent) + return nil +} + +func (g *deepCopyGenerator) writeDeepCopyForSlice(b *buffer, inField reflect.StructField, indent int) error { + ifFormat := "if in.%s != nil {\n" + ifStmt := fmt.Sprintf(ifFormat, inField.Name) + b.addLine(ifStmt, indent) + newFormat := "out.%s = make(%s, len(in.%s))\n" + newStmt := fmt.Sprintf(newFormat, inField.Name, g.typeName(inField.Type), inField.Name) + b.addLine(newStmt, indent+1) + forFormat := "for i := range in.%s {\n" + forStmt := fmt.Sprintf(forFormat, inField.Name) + b.addLine(forStmt, indent+1) + + switch inField.Type.Elem().Kind() { + case reflect.Map, reflect.Ptr, reflect.Slice, reflect.Interface, reflect.Struct: + if _, found := g.copyables[inField.Type.Elem()]; found { + assignFormat := "if err := %s(in.%s[i], &out.%s[i], c); err != nil {\n" + funcName := g.deepCopyFunctionName(inField.Type.Elem()) + assignStmt := fmt.Sprintf(assignFormat, funcName, inField.Name, inField.Name) + b.addLine(assignStmt, indent+2) + b.addLine("return err\n", indent+3) + b.addLine("}\n", indent+2) + } else { + ifFormat := "if newVal, err := c.DeepCopy(in.%s[i]); err != nil {\n" + ifStmt := fmt.Sprintf(ifFormat, inField.Name) + b.addLine(ifStmt, indent+2) + b.addLine("return err\n", indent+3) + b.addLine("} else {\n", indent+2) + assignFormat := "out.%s[i] = newVal.(%s)\n" + assignStmt := fmt.Sprintf(assignFormat, inField.Name, g.typeName(inField.Type.Elem())) + b.addLine(assignStmt, indent+3) + b.addLine("}\n", indent+2) + } + default: + assignFormat := "out.%s[i] = in.%s[i]\n" + assignStmt := fmt.Sprintf(assignFormat, inField.Name, inField.Name) + b.addLine(assignStmt, indent+2) + } + b.addLine("}\n", indent+1) + b.addLine("} else {\n", indent) + elseFormat := "out.%s = nil\n" + elseStmt := fmt.Sprintf(elseFormat, inField.Name) + b.addLine(elseStmt, indent+1) + b.addLine("}\n", indent) + return nil +} + +func (g *deepCopyGenerator) writeDeepCopyForStruct(b *buffer, inType reflect.Type, indent int) error { + for i := 0; i < inType.NumField(); i++ { + inField := inType.Field(i) + switch inField.Type.Kind() { + case reflect.Map: + if err := g.writeDeepCopyForMap(b, inField, indent); err != nil { + return err + } + case reflect.Ptr: + if err := g.writeDeepCopyForPtr(b, inField, indent); err != nil { + return err + } + case reflect.Slice: + if err := g.writeDeepCopyForSlice(b, inField, indent); err != nil { + return err + } + case reflect.Interface: + ifFormat := "if newVal, err := c.DeepCopy(in.%s); err != nil {\n" + ifStmt := fmt.Sprintf(ifFormat, inField.Name) + b.addLine(ifStmt, indent) + b.addLine("return err\n", indent+1) + b.addLine("} else {\n", indent) + copyFormat := "out.%s = newVal.(%s)\n" + copyStmt := fmt.Sprintf(copyFormat, inField.Name, g.typeName(inField.Type)) + b.addLine(copyStmt, indent+1) + b.addLine("}\n", indent) + case reflect.Struct: + if _, found := g.copyables[inField.Type]; found { + ifFormat := "if err := %s(in.%s, &out.%s, c); err != nil {\n" + funcName := g.deepCopyFunctionName(inField.Type) + ifStmt := fmt.Sprintf(ifFormat, funcName, inField.Name, inField.Name) + b.addLine(ifStmt, indent) + b.addLine("return err\n", indent+1) + b.addLine("}\n", indent) + } else { + ifFormat := "if newVal, err := c.DeepCopy(in.%s); err != nil {\n" + ifStmt := fmt.Sprintf(ifFormat, inField.Name) + b.addLine(ifStmt, indent) + b.addLine("return err\n", indent+1) + b.addLine("} else {\n", indent) + assignFormat := "out.%s = newVal.(%s)\n" + assignStmt := fmt.Sprintf(assignFormat, inField.Name, g.typeName(inField.Type)) + b.addLine(assignStmt, indent+1) + b.addLine("}\n", indent) + } + default: + // This should handle all simple types. + assignFormat := "out.%s = in.%s\n" + assignStmt := fmt.Sprintf(assignFormat, inField.Name, inField.Name) + b.addLine(assignStmt, indent) + } + } + return nil +} + +func (g *deepCopyGenerator) writeDeepCopyForType(b *buffer, inType reflect.Type, indent int) error { + g.writeHeader(b, inType, indent) + switch inType.Kind() { + case reflect.Struct: + if err := g.writeDeepCopyForStruct(b, inType, indent+1); err != nil { + return err + } + default: + return fmt.Errorf("type not supported: %v", inType) + } + g.writeFooter(b, indent) + return nil +} + +func (g *deepCopyGenerator) writeRegisterHeader(b *buffer, pkg string, indent int) { + b.addLine("func init() {\n", indent) + registerFormat := "err := %sScheme.AddGeneratedDeepCopyFuncs(\n" + if pkg == "api" { + b.addLine(fmt.Sprintf(registerFormat, ""), indent+1) + } else { + b.addLine(fmt.Sprintf(registerFormat, "api."), indent+1) + } +} + +func (g *deepCopyGenerator) writeRegisterFooter(b *buffer, indent int) { + b.addLine(")\n", indent+1) + b.addLine("if err != nil {\n", indent+1) + b.addLine("// if one of the deep copy functions is malformed, detect it immediately.\n", indent+2) + b.addLine("panic(err)\n", indent+2) + b.addLine("}\n", indent+1) + b.addLine("}\n", indent) + b.addLine("\n", indent) +} + +func (g *deepCopyGenerator) RegisterDeepCopyFunctions(w io.Writer, pkg string) error { + var keys []reflect.Type + for key := range g.copyables { + keys = append(keys, key) + } + sort.Sort(byPkgAndName(keys)) + + buffer := newBuffer() + indent := 0 + g.writeRegisterHeader(buffer, pkg, indent) + for _, inType := range keys { + funcStmt := fmt.Sprintf("%s,\n", g.deepCopyFunctionName(inType)) + buffer.addLine(funcStmt, indent+2) + } + g.writeRegisterFooter(buffer, indent) + if err := buffer.flushLines(w); err != nil { + return err + } + return nil +} + +func (g *deepCopyGenerator) OverwritePackage(pkg, overwrite string) { + g.pkgOverwrites[pkg] = overwrite +} diff --git a/pkg/runtime/scheme.go b/pkg/runtime/scheme.go index a1445f83078..a5c5b50b304 100644 --- a/pkg/runtime/scheme.go +++ b/pkg/runtime/scheme.go @@ -321,6 +321,19 @@ func (s *Scheme) AddGeneratedConversionFuncs(conversionFuncs ...interface{}) err return s.raw.AddGeneratedConversionFuncs(conversionFuncs...) } +// AddDeepCopyFuncs adds a function to the list of deep-copy functions. +// For the expected format of deep-copy function, see the comment for +// Copier.RegisterDeepCopyFunction. +func (s *Scheme) AddDeepCopyFuncs(deepCopyFuncs ...interface{}) error { + return s.raw.AddDeepCopyFuncs(deepCopyFuncs...) +} + +// Similar to AddDeepCopyFuncs, but registers deep-copy functions that were +// automatically generated. +func (s *Scheme) AddGeneratedDeepCopyFuncs(deepCopyFuncs ...interface{}) error { + return s.raw.AddGeneratedDeepCopyFuncs(deepCopyFuncs...) +} + // AddFieldLabelConversionFunc adds a conversion function to convert field selectors // of the given kind from the given version to internal version representation. func (s *Scheme) AddFieldLabelConversionFunc(version, kind string, conversionFunc FieldLabelConversionFunc) error { @@ -347,6 +360,11 @@ func (s *Scheme) AddDefaultingFuncs(defaultingFuncs ...interface{}) error { return s.raw.AddDefaultingFuncs(defaultingFuncs...) } +// Performs a deep copy of the given object. +func (s *Scheme) DeepCopy(src interface{}) (interface{}, error) { + return s.raw.DeepCopy(src) +} + // Convert will attempt to convert in into out. Both must be pointers. // For easy testing of conversion functions. Returns an error if the conversion isn't // possible.