Do not automatically decode runtime.RawExtension

Make clients opt in to decoding objects that are stored
in the generic api.List object by invoking runtime.DecodeList()
with a set of schemes. Makes it easier to handle unknown
schema objects because decoding is in the control of the code.

Add runtime.Unstructured, which is a simple in memory
representation of an external object.

pkg/runtime/codec.go

@@ -21,8 +21,8 @@ import (
 )
 
 // CodecFor returns a Codec that invokes Encode with the provided version.
-func CodecFor(scheme *Scheme, version string) Codec {
-	return &codecWrapper{scheme, version}
+func CodecFor(codec ObjectCodec, version string) Codec {
+	return &codecWrapper{codec, version}
 }
 
 // yamlCodec converts YAML passed to the Decoder methods to JSON.
@@ -69,11 +69,11 @@ func EncodeOrDie(codec Codec, obj Object) string {
 // codecWrapper implements encoding to an alternative
 // default version for a scheme.
 type codecWrapper struct {
-	*Scheme
+	ObjectCodec
 	version string
 }
 
 // Encode implements Codec
 func (c *codecWrapper) Encode(obj Object) ([]byte, error) {
-	return c.Scheme.EncodeToVersion(obj, c.version)
+	return c.EncodeToVersion(obj, c.version)
 }

pkg/runtime/embedded_test.go

@@ -63,9 +63,22 @@ func TestDecodeEmptyRawExtensionAsObject(t *testing.T) {
 	s.AddKnownTypes("", &ObjectTest{})
 	s.AddKnownTypeWithName("v1test", "ObjectTest", &ObjectTestExternal{})
 
-	_, err := s.Decode([]byte(`{"kind":"ObjectTest","apiVersion":"v1test","items":[{}]}`))
-	if err == nil {
-		t.Fatalf("unexpected non-error")
+	obj, err := s.Decode([]byte(`{"kind":"ObjectTest","apiVersion":"v1test","items":[{}]}`))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	test := obj.(*ObjectTest)
+	if unk, ok := test.Items[0].(*runtime.Unknown); !ok || unk.Kind != "" || unk.APIVersion != "" || string(unk.RawJSON) != "{}" {
+		t.Fatalf("unexpected object: %#v", test.Items[0])
+	}
+
+	obj, err = s.Decode([]byte(`{"kind":"ObjectTest","apiVersion":"v1test","items":[{"kind":"Other","apiVersion":"v1"}]}`))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	test = obj.(*ObjectTest)
+	if unk, ok := test.Items[0].(*runtime.Unknown); !ok || unk.Kind != "Other" || unk.APIVersion != "v1" || string(unk.RawJSON) != `{"kind":"Other","apiVersion":"v1"}` {
+		t.Fatalf("unexpected object: %#v", test.Items[0])
 	}
 }
 
@@ -99,17 +112,34 @@ func TestArrayOfRuntimeObject(t *testing.T) {
 	if err := json.Unmarshal(wire, obj); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	t.Logf("exact wire is: %#v", string(obj.Items[0].RawJSON))
+	t.Logf("exact wire is: %s", string(obj.Items[0].RawJSON))
 
 	decoded, err := s.Decode(wire)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
+	list, err := runtime.ExtractList(decoded)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if errs := runtime.DecodeList(list, s); len(errs) > 0 {
+		t.Fatalf("unexpected error: %v", errs)
+	}
+
+	list2, err := runtime.ExtractList(list[3])
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if errs := runtime.DecodeList(list2, s); len(errs) > 0 {
+		t.Fatalf("unexpected error: %v", errs)
+	}
+	if err := runtime.SetList(list[3], list2); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
 
 	internal.Items[2].(*runtime.Unknown).Kind = "OtherTest"
 	internal.Items[2].(*runtime.Unknown).APIVersion = "unknown"
-	if e, a := internal, decoded; !reflect.DeepEqual(e, a) {
-		t.Log(string(decoded.(*ObjectTest).Items[2].(*runtime.Unknown).RawJSON))
+	if e, a := internal.Items, list; !reflect.DeepEqual(e, a) {
 		t.Errorf("mismatched decoded: %s", util.ObjectDiff(e, a))
 	}
 }

pkg/runtime/helper.go

@@ -23,7 +23,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/conversion"
 )
 
-// TODO: move me to pkg/api/meta
+// IsListType returns true if the provided Object has a slice called Items
 func IsListType(obj Object) bool {
 	_, err := GetItemsPtr(obj)
 	return err == nil
@@ -33,7 +33,6 @@ func IsListType(obj Object) bool {
 // If 'list' doesn't have an Items member, it's not really a list type
 // and an error will be returned.
 // This function will either return a pointer to a slice, or an error, but not both.
-// TODO: move me to pkg/api/meta
 func GetItemsPtr(list Object) (interface{}, error) {
 	v, err := conversion.EnforcePtr(list)
 	if err != nil {
@@ -150,9 +149,36 @@ func FieldPtr(v reflect.Value, fieldName string, dest interface{}) error {
 	return fmt.Errorf("couldn't assign/convert %v to %v", field.Type(), v.Type())
 }
 
+// DecodeList alters the list in place, attempting to decode any objects found in
+// the list that have the runtime.Unknown type. Any errors that occur are returned
+// after the entire list is processed. Decoders are tried in order.
+func DecodeList(objects []Object, decoders ...ObjectDecoder) []error {
+	errs := []error(nil)
+	for i, obj := range objects {
+		switch t := obj.(type) {
+		case *Unknown:
+			for _, decoder := range decoders {
+				if !decoder.Recognizes(t.APIVersion, t.Kind) {
+					continue
+				}
+				obj, err := decoder.Decode(t.RawJSON)
+				if err != nil {
+					errs = append(errs, err)
+					break
+				}
+				objects[i] = obj
+				break
+			}
+		}
+	}
+	return errs
+}
+
 // MultiObjectTyper returns the types of objects across multiple schemes in order.
 type MultiObjectTyper []ObjectTyper
 
+var _ ObjectTyper = MultiObjectTyper{}
+
 func (m MultiObjectTyper) DataVersionAndKind(data []byte) (version, kind string, err error) {
 	for _, t := range m {
 		version, kind, err = t.DataVersionAndKind(data)
@@ -162,6 +188,7 @@ func (m MultiObjectTyper) DataVersionAndKind(data []byte) (version, kind string,
 	}
 	return
 }
+
 func (m MultiObjectTyper) ObjectVersionAndKind(obj Object) (version, kind string, err error) {
 	for _, t := range m {
 		version, kind, err = t.ObjectVersionAndKind(obj)
@@ -171,3 +198,12 @@ func (m MultiObjectTyper) ObjectVersionAndKind(obj Object) (version, kind string
 	}
 	return
 }
+
+func (m MultiObjectTyper) Recognizes(version, kind string) bool {
+	for _, t := range m {
+		if t.Recognizes(version, kind) {
+			return true
+		}
+	}
+	return false
+}

pkg/runtime/helper_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta3"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 
@@ -132,6 +133,22 @@ func TestExtractListOfValuePtrs(t *testing.T) {
 	}
 }
 
+func TestDecodeList(t *testing.T) {
+	pl := &api.List{
+		Items: []runtime.Object{
+			&api.Pod{ObjectMeta: api.ObjectMeta{Name: "1"}},
+			&runtime.Unknown{TypeMeta: runtime.TypeMeta{Kind: "Pod", APIVersion: "v1beta3"}, RawJSON: []byte(`{"kind":"Pod","apiVersion":"v1beta3","metadata":{"name":"test"}}`)},
+			&runtime.Unstructured{TypeMeta: runtime.TypeMeta{Kind: "Foo", APIVersion: "Bar"}, Object: map[string]interface{}{"test": "value"}},
+		},
+	}
+	if errs := runtime.DecodeList(pl.Items, api.Scheme); len(errs) != 0 {
+		t.Fatalf("unexpected error %v", errs)
+	}
+	if pod, ok := pl.Items[1].(*api.Pod); !ok || pod.Name != "test" {
+		t.Errorf("object not converted: %#v", pl.Items[1])
+	}
+}
+
 func TestSetList(t *testing.T) {
 	pl := &api.PodList{}
 	list := []runtime.Object{

pkg/runtime/interfaces.go

@@ -16,6 +16,24 @@ limitations under the License.
 
 package runtime
 
+// ObjectScheme represents common conversions between formal external API versions
+// and the internal Go structs. ObjectScheme is typically used with ObjectCodec to
+// transform internal Go structs into serialized versions. There may be many valid
+// ObjectCodecs for each ObjectScheme.
+type ObjectScheme interface {
+	ObjectConvertor
+	ObjectTyper
+	ObjectCreater
+	ObjectCopier
+}
+
+// ObjectCodec represents the common mechanisms for converting to and from a particular
+// binary representation of an object.
+type ObjectCodec interface {
+	ObjectEncoder
+	Decoder
+}
+
 // Decoder defines methods for deserializing API objects into a given type
 type Decoder interface {
 	Decode(data []byte) (Object, error)
@@ -33,6 +51,22 @@ type Codec interface {
 	Encoder
 }
 
+// ObjectCopier duplicates an object.
+type ObjectCopier interface {
+	// Copy returns an exact copy of the provided Object, or an error if the
+	// copy could not be completed.
+	Copy(Object) (Object, error)
+}
+
+// ObjectEncoder turns an object into a byte array. This interface is a
+// general form of the Encoder interface
+type ObjectEncoder interface {
+	// EncodeToVersion convert and serializes an object in the internal format
+	// to a specified output version. An error is returned if the object
+	// cannot be converted for any reason.
+	EncodeToVersion(obj Object, outVersion string) ([]byte, error)
+}
+
 // ObjectConvertor converts an object to a different version.
 type ObjectConvertor interface {
 	Convert(in, out interface{}) error
@@ -43,8 +77,17 @@ type ObjectConvertor interface {
 // ObjectTyper contains methods for extracting the APIVersion and Kind
 // of objects.
 type ObjectTyper interface {
+	// DataVersionAndKind returns the version and kind of the provided data, or an error
+	// if another problem is detected. In many cases this method can be as expensive to
+	// invoke as the Decode method.
 	DataVersionAndKind([]byte) (version, kind string, err error)
+	// ObjectVersionAndKind returns the version and kind of the provided object, or an
+	// error if the object is not recognized (IsNotRegisteredError will return true).
 	ObjectVersionAndKind(Object) (version, kind string, err error)
+	// Recognizes returns true if the scheme is able to handle the provided version and kind,
+	// or more precisely that the provided version is a possible conversion or decoding
+	// target.
+	Recognizes(version, kind string) bool
 }
 
 // ObjectCreater contains methods for instantiating an object by kind and version.
@@ -52,6 +95,20 @@ type ObjectCreater interface {
 	New(version, kind string) (out Object, err error)
 }
 
+// ObjectDecoder is a convenience interface for identifying serialized versions of objects
+// and transforming them into Objects. It intentionally overlaps with ObjectTyper and
+// Decoder for use in decode only paths.
+type ObjectDecoder interface {
+	Decoder
+	// DataVersionAndKind returns the version and kind of the provided data, or an error
+	// if another problem is detected. In many cases this method can be as expensive to
+	// invoke as the Decode method.
+	DataVersionAndKind([]byte) (version, kind string, err error)
+	// Recognizes returns true if the scheme is able to handle the provided version and kind
+	// of an object.
+	Recognizes(version, kind string) bool
+}
+
 // ResourceVersioner provides methods for setting and retrieving
 // the resource version from an API object.
 type ResourceVersioner interface {

pkg/runtime/scheme.go

@@ -17,6 +17,7 @@ limitations under the License.
 package runtime
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/url"
 	"reflect"
@@ -164,7 +165,15 @@ func (self *Scheme) runtimeObjectToRawExtensionArray(in *[]Object, out *[]RawExt
 	for i := range src {
 		switch t := src[i].(type) {
 		case *Unknown:
+			// TODO: this should be decoupled from the scheme (since it is JSON specific)
 			dest[i].RawJSON = t.RawJSON
+		case *Unstructured:
+			// TODO: this should be decoupled from the scheme (since it is JSON specific)
+			data, err := json.Marshal(t.Object)
+			if err != nil {
+				return err
+			}
+			dest[i].RawJSON = data
 		default:
 			version := outVersion
 			// if the object exists
@@ -192,24 +201,17 @@ func (self *Scheme) rawExtensionToRuntimeObjectArray(in *[]RawExtension, out *[]
 
 	for i := range src {
 		data := src[i].RawJSON
-		obj, err := scheme.Decode(data)
+		version, kind, err := scheme.raw.DataVersionAndKind(data)
 		if err != nil {
-			if !IsNotRegisteredError(err) {
-				return err
-			}
-			version, kind, err := scheme.raw.DataVersionAndKind(data)
-			if err != nil {
-				return err
-			}
-			obj = &Unknown{
-				TypeMeta: TypeMeta{
-					APIVersion: version,
-					Kind:       kind,
-				},
-				RawJSON: data,
-			}
+			return err
+		}
+		dest[i] = &Unknown{
+			TypeMeta: TypeMeta{
+				APIVersion: version,
+				Kind:       kind,
+			},
+			RawJSON: data,
 		}
-		dest[i] = obj
 	}
 	*out = dest
 	return nil
@@ -275,6 +277,12 @@ func (s *Scheme) ObjectVersionAndKind(obj Object) (version, kind string, err err
 	return s.raw.ObjectVersionAndKind(obj)
 }
 
+// Recognizes returns true if the scheme is able to handle the provided version and kind
+// of an object.
+func (s *Scheme) Recognizes(version, kind string) bool {
+	return s.raw.Recognizes(version, kind)
+}
+
 // New returns a new API object of the given version ("" for internal
 // representation) and name, or an error if it hasn't been registered.
 func (s *Scheme) New(versionName, typeName string) (Object, error) {

pkg/runtime/types.go

@@ -116,3 +116,17 @@ type Unknown struct {
 }
 
 func (*Unknown) IsAnAPIObject() {}
+
+// Unstructured allows objects that do not have Golang structs registered to be manipulated
+// generically. This can be used to deal with the API objects from a plug-in. Unstructured
+// objects still have functioning TypeMeta features-- kind, version, etc.
+// TODO: Make this object have easy access to field based accessors and settors for
+// metadata and field mutatation.
+type Unstructured struct {
+	TypeMeta `json:",inline"`
+	// Object is a JSON compatible map with string, float, int, []interface{}, or map[string]interface{}
+	// children.
+	Object map[string]interface{}
+}
+
+func (*Unstructured) IsAnAPIObject() {}

pkg/runtime/unstructured.go

@@ -0,0 +1,89 @@
+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package runtime
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/conversion"
+)
+
+// UnstructuredJSONScheme is capable of converting JSON data into the Unstructured
+// type, which can be used for generic access to objects without a predefined scheme.
+var UnstructuredJSONScheme ObjectDecoder = unstructuredJSONScheme{}
+
+type unstructuredJSONScheme struct{}
+
+// Recognizes returns true for any version or kind that is specified (internal
+// versions are specifically excluded).
+func (unstructuredJSONScheme) Recognizes(version, kind string) bool {
+	return len(version) > 0 && len(kind) > 0
+}
+
+func (s unstructuredJSONScheme) Decode(data []byte) (Object, error) {
+	unstruct := &Unstructured{}
+	if err := s.DecodeInto(data, unstruct); err != nil {
+		return nil, err
+	}
+	return unstruct, nil
+}
+
+func (unstructuredJSONScheme) DecodeInto(data []byte, obj Object) error {
+	unstruct, ok := obj.(*Unstructured)
+	if !ok {
+		return fmt.Errorf("the unstructured JSON scheme does not recognize %v", reflect.TypeOf(obj))
+	}
+
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(data, &m); err != nil {
+		return err
+	}
+	if v, ok := m["kind"]; ok {
+		if s, ok := v.(string); ok {
+			unstruct.Kind = s
+		}
+	}
+	if v, ok := m["apiVersion"]; ok {
+		if s, ok := v.(string); ok {
+			unstruct.APIVersion = s
+		}
+	}
+	if len(unstruct.APIVersion) == 0 {
+		return conversion.NewMissingVersionErr(string(data))
+	}
+	if len(unstruct.Kind) == 0 {
+		return conversion.NewMissingKindErr(string(data))
+	}
+	unstruct.Object = m
+	return nil
+}
+
+func (unstructuredJSONScheme) DataVersionAndKind(data []byte) (version, kind string, err error) {
+	obj := TypeMeta{}
+	if err := json.Unmarshal(data, &obj); err != nil {
+		return "", "", err
+	}
+	if len(obj.APIVersion) == 0 {
+		return "", "", conversion.NewMissingVersionErr(string(data))
+	}
+	if len(obj.Kind) == 0 {
+		return "", "", conversion.NewMissingKindErr(string(data))
+	}
+	return obj.APIVersion, obj.Kind, nil
+}

pkg/runtime/unstructured_test.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2014 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package runtime_test
+
+import (
+	"testing"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
+)
+
+func TestDecodeUnstructured(t *testing.T) {
+	pl := &api.List{
+		Items: []runtime.Object{
+			&api.Pod{ObjectMeta: api.ObjectMeta{Name: "1"}},
+			&runtime.Unknown{TypeMeta: runtime.TypeMeta{Kind: "Pod", APIVersion: "v1beta3"}, RawJSON: []byte(`{"kind":"Pod","apiVersion":"v1beta3","metadata":{"name":"test"}}`)},
+			&runtime.Unknown{TypeMeta: runtime.TypeMeta{Kind: "", APIVersion: "v1beta3"}, RawJSON: []byte(`{"kind":"Pod","apiVersion":"v1beta3","metadata":{"name":"test"}}`)},
+			&runtime.Unstructured{TypeMeta: runtime.TypeMeta{Kind: "Foo", APIVersion: "Bar"}, Object: map[string]interface{}{"test": "value"}},
+		},
+	}
+	if errs := runtime.DecodeList(pl.Items, runtime.UnstructuredJSONScheme); len(errs) == 1 {
+		t.Fatalf("unexpected error %v", errs)
+	}
+	if pod, ok := pl.Items[1].(*runtime.Unstructured); !ok || pod.Object["kind"] != "Pod" || pod.Object["metadata"].(map[string]interface{})["name"] != "test" {
+		t.Errorf("object not converted: %#v", pl.Items[1])
+	}
+	if _, ok := pl.Items[2].(*runtime.Unknown); !ok {
+		t.Errorf("object should not have been converted: %#v", pl.Items[2])
+	}
+}