Scheme should support Unstructured conversion natively

To reduce the impact of transitioning away from internal, make
Unstructured conversion a natural part of the scheme. Convert and
ConvertToVersion now automatically manage converting unstructured
objects into versioned types as necessary.

Alter the signature of Unstructured to make set possible.
This commit is contained in:
Clayton Coleman 2017-11-13 22:24:50 -05:00
parent 557f9ddfe6
commit e7e313d6d2
No known key found for this signature in database
GPG Key ID: 3D16906B4F1C5CB3
6 changed files with 402 additions and 61 deletions

View File

@ -49,8 +49,6 @@ var _ runtime.Unstructured = &Unstructured{}
func (obj *Unstructured) GetObjectKind() schema.ObjectKind { return obj }
func (obj *Unstructured) IsUnstructuredObject() {}
func (obj *Unstructured) IsList() bool {
if obj.Object != nil {
_, ok := obj.Object["items"]
@ -90,6 +88,10 @@ func (obj *Unstructured) UnstructuredContent() map[string]interface{} {
return obj.Object
}
func (obj *Unstructured) SetUnstructuredContent(content map[string]interface{}) {
obj.Object = content
}
// MarshalJSON ensures that the unstructured object produces proper
// JSON when passed to Go's standard JSON library.
func (u *Unstructured) MarshalJSON() ([]byte, error) {

View File

@ -41,8 +41,6 @@ type UnstructuredList struct {
func (u *UnstructuredList) GetObjectKind() schema.ObjectKind { return u }
func (u *UnstructuredList) IsUnstructuredObject() {}
func (u *UnstructuredList) IsList() bool { return true }
func (u *UnstructuredList) EachListItem(fn func(runtime.Object) error) error {
@ -73,6 +71,33 @@ func (u *UnstructuredList) UnstructuredContent() map[string]interface{} {
return out
}
// SetUnstructuredContent obeys the conventions of List and keeps Items and the items
// array in sync. If items is not an array of objects in the incoming map, then any
// mismatched item will be removed.
func (obj *UnstructuredList) SetUnstructuredContent(content map[string]interface{}) {
obj.Object = content
if content == nil {
obj.Items = nil
return
}
items, ok := obj.Object["items"].([]interface{})
if !ok || items == nil {
items = []interface{}{}
}
unstructuredItems := make([]Unstructured, 0, len(items))
newItems := make([]interface{}, 0, len(items))
for _, item := range items {
o, ok := item.(map[string]interface{})
if !ok {
continue
}
unstructuredItems = append(unstructuredItems, Unstructured{Object: o})
newItems = append(newItems, o)
}
obj.Items = unstructuredItems
obj.Object["items"] = newItems
}
func (u *UnstructuredList) DeepCopy() *UnstructuredList {
if u == nil {
return nil

View File

@ -233,13 +233,13 @@ type Object interface {
// Unstructured objects store values as map[string]interface{}, with only values that can be serialized
// to JSON allowed.
type Unstructured interface {
// IsUnstructuredObject is a marker interface to allow objects that can be serialized but not introspected
// to bypass conversion.
IsUnstructuredObject()
Object
// UnstructuredContent returns a non-nil, mutable map of the contents of this object. Values may be
// []interface{}, map[string]interface{}, or any primitive type. Contents are typically serialized to
// and from JSON.
UnstructuredContent() map[string]interface{}
// SetUnstructuredContent updates the object content to match the provided map.
SetUnstructuredContent(map[string]interface{})
// IsList returns true if this type is a list or matches the list convention - has an array called "items".
IsList() bool
// EachListItem should pass a single item out of the list as an Object to the provided function. Any

View File

@ -338,7 +338,7 @@ func (s *Scheme) AddConversionFuncs(conversionFuncs ...interface{}) error {
return nil
}
// Similar to AddConversionFuncs, but registers conversion functions that were
// AddGeneratedConversionFuncs registers conversion functions that were
// automatically generated.
func (s *Scheme) AddGeneratedConversionFuncs(conversionFuncs ...interface{}) error {
for _, f := range conversionFuncs {
@ -396,10 +396,67 @@ func (s *Scheme) Default(src Object) {
// testing of conversion functions. Returns an error if the conversion isn't
// possible. You can call this with types that haven't been registered (for example,
// a to test conversion of types that are nested within registered types). The
// context interface is passed to the convertor.
// TODO: identify whether context should be hidden, or behind a formal context/scope
// interface
// context interface is passed to the convertor. Convert also supports Unstructured
// types and will convert them intelligently.
func (s *Scheme) Convert(in, out interface{}, context interface{}) error {
unstructuredIn, okIn := in.(Unstructured)
unstructuredOut, okOut := out.(Unstructured)
switch {
case okIn && okOut:
// converting unstructured input to an unstructured output is a straight copy - unstructured
// is a "smart holder" and the contents are passed by reference between the two objects
unstructuredOut.SetUnstructuredContent(unstructuredIn.UnstructuredContent())
return nil
case okOut:
// if the output is an unstructured object, use the standard Go type to unstructured
// conversion. The object must not be internal.
obj, ok := in.(Object)
if !ok {
return fmt.Errorf("unable to convert object type %T to Unstructured, must be a runtime.Object", in)
}
gvk, unversioned, err := s.ObjectKind(obj)
if err != nil {
return err
}
// if no conversion is necessary, convert immediately
if unversioned || gvk.Version != APIVersionInternal {
content, err := DefaultUnstructuredConverter.ToUnstructured(in)
if err != nil {
return err
}
unstructuredOut.SetUnstructuredContent(content)
return nil
}
// attempt to convert the object to an external version first.
target, ok := context.(GroupVersioner)
if !ok {
return fmt.Errorf("unable to convert the internal object type %T to Unstructured without providing a preferred version to convert to", in)
}
// Convert is implicitly unsafe, so we don't need to perform a safe conversion
versioned, err := s.UnsafeConvertToVersion(obj, target)
if err != nil {
return err
}
content, err := DefaultUnstructuredConverter.ToUnstructured(versioned)
if err != nil {
return err
}
unstructuredOut.SetUnstructuredContent(content)
return nil
case okIn:
// converting an unstructured object to any type is modeled by first converting
// the input to a versioned type, then running standard conversions
typed, err := s.unstructuredToTyped(unstructuredIn)
if err != nil {
return err
}
in = typed
}
flags, meta := s.generateConvertMeta(in)
meta.Context = context
if flags == 0 {
@ -408,8 +465,8 @@ func (s *Scheme) Convert(in, out interface{}, context interface{}) error {
return s.converter.Convert(in, out, flags, meta)
}
// Converts the given field label and value for an kind field selector from
// versioned representation to an unversioned one.
// ConvertFieldLabel alters the given field label and value for an kind field selector from
// versioned representation to an unversioned one or returns an error.
func (s *Scheme) ConvertFieldLabel(version, kind, label, value string) (string, string, error) {
if s.fieldLabelConversionFuncs[version] == nil {
return DefaultMetaV1FieldSelectorConversion(label, value)
@ -439,15 +496,30 @@ func (s *Scheme) UnsafeConvertToVersion(in Object, target GroupVersioner) (Objec
// convertToVersion handles conversion with an optional copy.
func (s *Scheme) convertToVersion(copy bool, in Object, target GroupVersioner) (Object, error) {
// determine the incoming kinds with as few allocations as possible.
t := reflect.TypeOf(in)
if t.Kind() != reflect.Ptr {
return nil, fmt.Errorf("only pointer types may be converted: %v", t)
}
t = t.Elem()
if t.Kind() != reflect.Struct {
return nil, fmt.Errorf("only pointers to struct types may be converted: %v", t)
var t reflect.Type
if u, ok := in.(Unstructured); ok {
typed, err := s.unstructuredToTyped(u)
if err != nil {
return nil, err
}
in = typed
// unstructuredToTyped returns an Object, which must be a pointer to a struct.
t = reflect.TypeOf(in).Elem()
} else {
// determine the incoming kinds with as few allocations as possible.
t = reflect.TypeOf(in)
if t.Kind() != reflect.Ptr {
return nil, fmt.Errorf("only pointer types may be converted: %v", t)
}
t = t.Elem()
if t.Kind() != reflect.Struct {
return nil, fmt.Errorf("only pointers to struct types may be converted: %v", t)
}
}
kinds, ok := s.typeToGVK[t]
if !ok || len(kinds) == 0 {
return nil, NewNotRegisteredErrForType(t)
@ -463,7 +535,6 @@ func (s *Scheme) convertToVersion(copy bool, in Object, target GroupVersioner) (
}
return copyAndSetTargetKind(copy, in, unversionedKind)
}
return nil, NewNotRegisteredErrForTarget(t, target)
}
@ -501,6 +572,25 @@ func (s *Scheme) convertToVersion(copy bool, in Object, target GroupVersioner) (
return out, nil
}
// unstructuredToTyped attempts to transform an unstructured object to a typed
// object if possible. It will return an error if conversion is not possible, or the versioned
// Go form of the object. Note that this conversion will lose fields.
func (s *Scheme) unstructuredToTyped(in Unstructured) (Object, error) {
// the type must be something we recognize
gvks, _, err := s.ObjectKinds(in)
if err != nil {
return nil, err
}
typed, err := s.New(gvks[0])
if err != nil {
return nil, err
}
if err := DefaultUnstructuredConverter.FromUnstructured(in.UnstructuredContent(), typed); err != nil {
return nil, fmt.Errorf("unable to convert unstructured object to %v: %v", gvks[0], err)
}
return typed, nil
}
// generateConvertMeta constructs the meta value we pass to Convert.
func (s *Scheme) generateConvertMeta(in interface{}) (conversion.FieldMatchingFlags, *conversion.Meta) {
return s.converter.DefaultMeta(reflect.TypeOf(in))

View File

@ -17,6 +17,7 @@ limitations under the License.
package runtime_test
import (
"fmt"
"reflect"
"strings"
"testing"
@ -126,8 +127,61 @@ func TestScheme(t *testing.T) {
t.Errorf("Expected %v, got %v", e, a)
}
// Test convert internal to unstructured
unstructuredObj := &runtimetesting.Unstructured{}
err = scheme.Convert(simple, unstructuredObj, nil)
if err == nil || !strings.Contains(err.Error(), "to Unstructured without providing a preferred version to convert to") {
t.Fatalf("Unexpected non-error: %v", err)
}
err = scheme.Convert(simple, unstructuredObj, schema.GroupVersion{Group: "test.group", Version: "testExternal"})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if e, a := simple.TestString, unstructuredObj.Object["testString"].(string); e != a {
t.Errorf("Expected %v, got %v", e, a)
}
if e := unstructuredObj.GetObjectKind().GroupVersionKind(); !reflect.DeepEqual(e, schema.GroupVersionKind{Group: "test.group", Version: "testExternal", Kind: "Simple"}) {
t.Errorf("Unexpected object kind: %#v", e)
}
// Test convert external to unstructured
unstructuredObj = &runtimetesting.Unstructured{}
err = scheme.Convert(external, unstructuredObj, nil)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if e, a := simple.TestString, unstructuredObj.Object["testString"].(string); e != a {
t.Errorf("Expected %v, got %v", e, a)
}
if e := unstructuredObj.GetObjectKind().GroupVersionKind(); !reflect.DeepEqual(e, schema.GroupVersionKind{Group: "test.group", Version: "testExternal", Kind: "Simple"}) {
t.Errorf("Unexpected object kind: %#v", e)
}
// Test convert unstructured to unstructured
uIn := &runtimetesting.Unstructured{Object: map[string]interface{}{
"test": []interface{}{"other", "test"},
}}
uOut := &runtimetesting.Unstructured{}
err = scheme.Convert(uIn, uOut, nil)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if !reflect.DeepEqual(uIn.Object, uOut.Object) {
t.Errorf("Unexpected object contents: %#v", uOut.Object)
}
// Test convert unstructured to structured
externalOut := &runtimetesting.ExternalSimple{}
err = scheme.Convert(unstructuredObj, externalOut, nil)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if !reflect.DeepEqual(external, externalOut) {
t.Errorf("Unexpected object contents: %#v", externalOut)
}
// Encode and Convert should each have caused an increment.
if e, a := 2, internalToExternalCalls; e != a {
if e, a := 3, internalToExternalCalls; e != a {
t.Errorf("Expected %v, got %v", e, a)
}
// DecodeInto and Decode should each have caused an increment because of a conversion
@ -367,6 +421,7 @@ func GetTestScheme() *runtime.Scheme {
internalGV := schema.GroupVersion{Version: "__internal"}
externalGV := schema.GroupVersion{Version: "v1"}
alternateExternalGV := schema.GroupVersion{Group: "custom", Version: "v1"}
alternateInternalGV := schema.GroupVersion{Group: "custom", Version: "__internal"}
differentExternalGV := schema.GroupVersion{Group: "other", Version: "v2"}
s := runtime.NewScheme()
@ -380,10 +435,15 @@ func GetTestScheme() *runtime.Scheme {
s.AddKnownTypeWithName(internalGV.WithKind("TestType3"), &runtimetesting.TestType1{})
s.AddKnownTypeWithName(externalGV.WithKind("TestType3"), &runtimetesting.ExternalTestType1{})
s.AddKnownTypeWithName(externalGV.WithKind("TestType4"), &runtimetesting.ExternalTestType1{})
s.AddKnownTypeWithName(alternateInternalGV.WithKind("TestType3"), &runtimetesting.TestType1{})
s.AddKnownTypeWithName(alternateExternalGV.WithKind("TestType3"), &runtimetesting.ExternalTestType1{})
s.AddKnownTypeWithName(alternateExternalGV.WithKind("TestType5"), &runtimetesting.ExternalTestType1{})
s.AddKnownTypeWithName(differentExternalGV.WithKind("TestType1"), &runtimetesting.ExternalTestType1{})
s.AddUnversionedTypes(externalGV, &runtimetesting.UnversionedType{})
s.AddConversionFuncs(func(in *runtimetesting.TestType1, out *runtimetesting.ExternalTestType1, s conversion.Scope) {
out.A = in.A
})
return s
}
@ -540,6 +600,28 @@ func TestConvertToVersion(t *testing.T) {
gv: schema.GroupVersion{Version: "__internal"},
out: &runtimetesting.TestType1{A: "test"},
},
// converts from unstructured to internal
{
scheme: GetTestScheme(),
in: &runtimetesting.Unstructured{Object: map[string]interface{}{
"apiVersion": "custom/v1",
"kind": "TestType3",
"A": "test",
}},
gv: schema.GroupVersion{Version: "__internal"},
out: &runtimetesting.TestType1{A: "test"},
},
// converts from unstructured to external
{
scheme: GetTestScheme(),
in: &runtimetesting.Unstructured{Object: map[string]interface{}{
"apiVersion": "custom/v1",
"kind": "TestType3",
"A": "test",
}},
gv: schema.GroupVersion{Group: "custom", Version: "v1"},
out: &runtimetesting.ExternalTestType1{MyWeirdCustomEmbeddedVersionKindField: runtimetesting.MyWeirdCustomEmbeddedVersionKindField{APIVersion: "custom/v1", ObjectKind: "TestType3"}, A: "test"},
},
// prefers the best match
{
scheme: GetTestScheme(),
@ -711,51 +793,88 @@ func TestConvertToVersion(t *testing.T) {
},
}
for i, test := range testCases {
original := test.in.DeepCopyObject()
out, err := test.scheme.ConvertToVersion(test.in, test.gv)
switch {
case test.errFn != nil:
if !test.errFn(err) {
t.Errorf("%d: unexpected error: %v", i, err)
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
original := test.in.DeepCopyObject()
out, err := test.scheme.ConvertToVersion(test.in, test.gv)
switch {
case test.errFn != nil:
if !test.errFn(err) {
t.Fatalf("unexpected error: %v", err)
}
return
case err != nil:
t.Fatalf("unexpected error: %v", err)
}
if out == test.in {
t.Fatalf("ConvertToVersion should always copy out: %#v", out)
}
continue
case err != nil:
t.Errorf("%d: unexpected error: %v", i, err)
continue
}
if out == test.in {
t.Errorf("%d: ConvertToVersion should always copy out: %#v", i, out)
continue
}
if test.same {
if !reflect.DeepEqual(original, test.in) {
t.Errorf("%d: unexpected mutation of input: %s", i, diff.ObjectReflectDiff(original, test.in))
continue
if test.same {
if !reflect.DeepEqual(original, test.in) {
t.Fatalf("unexpected mutation of input: %s", diff.ObjectReflectDiff(original, test.in))
}
if !reflect.DeepEqual(out, test.out) {
t.Fatalf("unexpected out: %s", diff.ObjectReflectDiff(out, test.out))
}
unsafe, err := test.scheme.UnsafeConvertToVersion(test.in, test.gv)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !reflect.DeepEqual(unsafe, test.out) {
t.Fatalf("unexpected unsafe: %s", diff.ObjectReflectDiff(unsafe, test.out))
}
if unsafe != test.in {
t.Fatalf("UnsafeConvertToVersion should return same object: %#v", unsafe)
}
return
}
if !reflect.DeepEqual(out, test.out) {
t.Errorf("%d: unexpected out: %s", i, diff.ObjectReflectDiff(out, test.out))
continue
t.Fatalf("unexpected out: %s", diff.ObjectReflectDiff(out, test.out))
}
unsafe, err := test.scheme.UnsafeConvertToVersion(test.in, test.gv)
if err != nil {
t.Errorf("%d: unexpected error: %v", i, err)
continue
})
}
}
func TestConvert(t *testing.T) {
testCases := []struct {
scheme *runtime.Scheme
in runtime.Object
into runtime.Object
gv runtime.GroupVersioner
out runtime.Object
errFn func(error) bool
}{
// converts from internal to unstructured, given a target version
{
scheme: GetTestScheme(),
in: &runtimetesting.TestType1{A: "test"},
into: &runtimetesting.Unstructured{},
out: &runtimetesting.Unstructured{Object: map[string]interface{}{
"myVersionKey": "custom/v1",
"myKindKey": "TestType3",
"A": "test",
}},
gv: schema.GroupVersion{Group: "custom", Version: "v1"},
},
}
for i, test := range testCases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
err := test.scheme.Convert(test.in, test.into, test.gv)
switch {
case test.errFn != nil:
if !test.errFn(err) {
t.Fatalf("unexpected error: %v", err)
}
return
case err != nil:
t.Fatalf("unexpected error: %v", err)
return
}
if !reflect.DeepEqual(unsafe, test.out) {
t.Errorf("%d: unexpected unsafe: %s", i, diff.ObjectReflectDiff(unsafe, test.out))
continue
if !reflect.DeepEqual(test.into, test.out) {
t.Fatalf("unexpected out: %s", diff.ObjectReflectDiff(test.into, test.out))
}
if unsafe != test.in {
t.Errorf("%d: UnsafeConvertToVersion should return same object: %#v", i, unsafe)
continue
}
continue
}
if !reflect.DeepEqual(out, test.out) {
t.Errorf("%d: unexpected out: %s", i, diff.ObjectReflectDiff(out, test.out))
continue
}
})
}
}

View File

@ -17,8 +17,11 @@ limitations under the License.
package testing
import (
"fmt"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/json"
)
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@ -213,3 +216,105 @@ func (obj *MyWeirdCustomEmbeddedVersionKindField) GroupVersionKind() schema.Grou
func (obj *TestType2) GetObjectKind() schema.ObjectKind { return schema.EmptyObjectKind }
func (obj *ExternalTestType2) GetObjectKind() schema.ObjectKind { return schema.EmptyObjectKind }
// +k8s:deepcopy-gen=false
type Unstructured struct {
// Object is a JSON compatible map with string, float, int, bool, []interface{}, or
// map[string]interface{}
// children.
Object map[string]interface{}
}
var _ runtime.Unstructured = &Unstructured{}
func (obj *Unstructured) GetObjectKind() schema.ObjectKind { return obj }
func (obj *Unstructured) IsList() bool {
if obj.Object != nil {
_, ok := obj.Object["items"]
return ok
}
return false
}
func (obj *Unstructured) EachListItem(fn func(runtime.Object) error) error {
if obj.Object == nil {
return fmt.Errorf("content is not a list")
}
field, ok := obj.Object["items"]
if !ok {
return fmt.Errorf("content is not a list")
}
items, ok := field.([]interface{})
if !ok {
return nil
}
for _, item := range items {
child, ok := item.(map[string]interface{})
if !ok {
return fmt.Errorf("items member is not an object")
}
if err := fn(&Unstructured{Object: child}); err != nil {
return err
}
}
return nil
}
func (obj *Unstructured) UnstructuredContent() map[string]interface{} {
if obj.Object == nil {
obj.Object = make(map[string]interface{})
}
return obj.Object
}
func (obj *Unstructured) SetUnstructuredContent(content map[string]interface{}) {
obj.Object = content
}
// MarshalJSON ensures that the unstructured object produces proper
// JSON when passed to Go's standard JSON library.
func (u *Unstructured) MarshalJSON() ([]byte, error) {
return json.Marshal(u.Object)
}
// UnmarshalJSON ensures that the unstructured object properly decodes
// JSON when passed to Go's standard JSON library.
func (u *Unstructured) UnmarshalJSON(b []byte) error {
return json.Unmarshal(b, &u.Object)
}
func (in *Unstructured) DeepCopyObject() runtime.Object {
return in.DeepCopy()
}
func (in *Unstructured) DeepCopy() *Unstructured {
if in == nil {
return nil
}
out := new(Unstructured)
*out = *in
out.Object = runtime.DeepCopyJSON(in.Object)
return out
}
func (u *Unstructured) GroupVersionKind() schema.GroupVersionKind {
apiVersion, ok := u.Object["apiVersion"].(string)
if !ok {
return schema.GroupVersionKind{}
}
gv, err := schema.ParseGroupVersion(apiVersion)
if err != nil {
return schema.GroupVersionKind{}
}
kind, ok := u.Object["kind"].(string)
if ok {
return gv.WithKind(kind)
}
return schema.GroupVersionKind{}
}
func (u *Unstructured) SetGroupVersionKind(gvk schema.GroupVersionKind) {
u.Object["apiVersion"] = gvk.GroupVersion().String()
u.Object["kind"] = gvk.Kind
}