From 6b2f70d5532728715179882b13d8fed321d3de01 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Mon, 21 Dec 2015 00:12:51 -0500 Subject: [PATCH] Provide a JSON and YAML serializer, and a versioning wrapper Add a recognizer that is capable of sniffing content type from data by asking each serializer to try to decode - this is for a "universal decoder/deserializer" which can be used by client logic. Add codec factory, which provides the core primitives for content type negotiation. Codec factories depend only on schemes, serializers, and groupversion pairs. --- pkg/runtime/serializer/codec_factory.go | 176 ++++++++ pkg/runtime/serializer/codec_test.go | 400 ++++++++++++++++++ pkg/runtime/serializer/json/json.go | 191 +++++++++ pkg/runtime/serializer/json/json_test.go | 269 ++++++++++++ pkg/runtime/serializer/json/meta.go | 61 +++ pkg/runtime/serializer/json/meta_test.go | 45 ++ pkg/runtime/serializer/protobuf/doc.go | 18 - .../serializer/recognizer/recognizer.go | 79 ++++ .../serializer/recognizer/recognizer_test.go | 57 +++ .../serializer/versioning/versioning.go | 243 +++++++++++ .../serializer/versioning/versioning_test.go | 300 +++++++++++++ pkg/runtime/serializer/yaml/yaml.go | 46 ++ pkg/util/yaml/decoder.go | 6 +- 13 files changed, 1870 insertions(+), 21 deletions(-) create mode 100644 pkg/runtime/serializer/codec_factory.go create mode 100644 pkg/runtime/serializer/codec_test.go create mode 100644 pkg/runtime/serializer/json/json.go create mode 100644 pkg/runtime/serializer/json/json_test.go create mode 100644 pkg/runtime/serializer/json/meta.go create mode 100644 pkg/runtime/serializer/json/meta_test.go delete mode 100644 pkg/runtime/serializer/protobuf/doc.go create mode 100644 pkg/runtime/serializer/recognizer/recognizer.go create mode 100644 pkg/runtime/serializer/recognizer/recognizer_test.go create mode 100644 pkg/runtime/serializer/versioning/versioning.go create mode 100644 pkg/runtime/serializer/versioning/versioning_test.go create mode 100644 pkg/runtime/serializer/yaml/yaml.go diff --git a/pkg/runtime/serializer/codec_factory.go b/pkg/runtime/serializer/codec_factory.go new file mode 100644 index 00000000000..6f6310d9c83 --- /dev/null +++ b/pkg/runtime/serializer/codec_factory.go @@ -0,0 +1,176 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serializer + +import ( + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/runtime/serializer/json" + "k8s.io/kubernetes/pkg/runtime/serializer/recognizer" + "k8s.io/kubernetes/pkg/runtime/serializer/versioning" +) + +type serializerType struct { + AcceptContentTypes []string + ContentType string + FileExtensions []string + Serializer runtime.Serializer + PrettySerializer runtime.Serializer +} + +// NewCodecFactory provides methods for retrieving serializers for the supported wire formats +// and conversion wrappers to define preferred internal and external versions. In the future, +// as the internal version is used less, callers may instead use a defaulting serializer and +// only convert objects which are shared internally (Status, common API machinery). +// TODO: allow other codecs to be compiled in? +// TODO: accept a scheme interface +func NewCodecFactory(scheme *runtime.Scheme) CodecFactory { + return newCodecFactory(scheme, json.DefaultMetaFactory) +} + +// newCodecFactory is a helper for testing that allows a different metafactory to be specified. +func newCodecFactory(scheme *runtime.Scheme, mf json.MetaFactory) CodecFactory { + jsonSerializer := json.NewSerializer(mf, scheme, runtime.ObjectTyperToTyper(scheme), false) + jsonPrettySerializer := json.NewSerializer(mf, scheme, runtime.ObjectTyperToTyper(scheme), true) + yamlSerializer := json.NewYAMLSerializer(mf, scheme, runtime.ObjectTyperToTyper(scheme)) + serializers := []serializerType{ + { + AcceptContentTypes: []string{"application/json"}, + ContentType: "application/json", + FileExtensions: []string{"json"}, + Serializer: jsonSerializer, + PrettySerializer: jsonPrettySerializer, + }, + { + AcceptContentTypes: []string{"application/yaml"}, + ContentType: "application/yaml", + FileExtensions: []string{"yaml"}, + Serializer: yamlSerializer, + }, + } + decoders := make([]runtime.Decoder, 0, len(serializers)) + accepts := []string{} + alreadyAccepted := make(map[string]struct{}) + for _, d := range serializers { + decoders = append(decoders, d.Serializer) + for _, mediaType := range d.AcceptContentTypes { + if _, ok := alreadyAccepted[mediaType]; ok { + continue + } + alreadyAccepted[mediaType] = struct{}{} + accepts = append(accepts, mediaType) + } + } + return CodecFactory{ + scheme: scheme, + serializers: serializers, + universal: recognizer.NewDecoder(decoders...), + accepts: accepts, + + legacySerializer: jsonSerializer, + } +} + +// CodecFactory provides methods for retrieving codecs and serializers for specific +// versions and content types. +type CodecFactory struct { + scheme *runtime.Scheme + serializers []serializerType + universal runtime.Decoder + accepts []string + + legacySerializer runtime.Serializer +} + +var _ runtime.NegotiatedSerializer = &CodecFactory{} + +// SupportedMediaTypes returns the RFC2046 media types that this factory has serializers for. +func (f CodecFactory) SupportedMediaTypes() []string { + return f.accepts +} + +// LegacyCodec encodes output to a given API version, and decodes output into the internal form from +// any recognized source. The returned codec will always encode output to JSON. +// +// This method is deprecated - clients and servers should negotiate a serializer by mime-type and +// invoke CodecForVersions. Callers that need only to read data should use UniversalDecoder(). +func (f CodecFactory) LegacyCodec(version ...unversioned.GroupVersion) runtime.Codec { + return f.CodecForVersions(runtime.NewCodec(f.legacySerializer, f.universal), version, nil) +} + +// UniversalDeserializer can convert any stored data recognized by this factory into a Go object that satisfies +// runtime.Object. It does not perform conversion. It does not perform defaulting. +func (f CodecFactory) UniversalDeserializer() runtime.Decoder { + return f.universal +} + +// UniversalDecoder returns a runtime.Decoder capable of decoding all known API objects in all known formats. Used +// by clients that do not need to encode objects but want to deserialize API objects stored on disk. Only decodes +// objects in groups registered with the scheme. The GroupVersions passed may be used to select alternate +// versions of objects to return - by default, runtime.APIVersionInternal is used. If any versions are specified, +// unrecognized groups will be returned in the version they are encoded as (no conversion). This decoder performs +// defaulting. +// +// TODO: the decoder will eventually be removed in favor of dealing with objects in their versioned form +func (f CodecFactory) UniversalDecoder(versions ...unversioned.GroupVersion) runtime.Decoder { + return f.CodecForVersions(runtime.NoopEncoder{f.universal}, nil, versions) +} + +// CodecFor creates a codec with the provided serializer. If an object is decoded and its group is not in the list, +// it will default to runtime.APIVersionInternal. If encode is not specified for an object's group, the object is not +// converted. If encode or decode are nil, no conversion is performed. +func (f CodecFactory) CodecForVersions(serializer runtime.Serializer, encode []unversioned.GroupVersion, decode []unversioned.GroupVersion) runtime.Codec { + return versioning.NewCodecForScheme(f.scheme, serializer, encode, decode) +} + +// DecoderToVersion returns a decoder that targets the provided group version. +func (f CodecFactory) DecoderToVersion(serializer runtime.Serializer, gv unversioned.GroupVersion) runtime.Decoder { + return f.CodecForVersions(serializer, nil, []unversioned.GroupVersion{gv}) +} + +// EncoderForVersion returns an encoder that targets the provided group version. +func (f CodecFactory) EncoderForVersion(serializer runtime.Serializer, gv unversioned.GroupVersion) runtime.Encoder { + return f.CodecForVersions(serializer, []unversioned.GroupVersion{gv}, nil) +} + +// SerializerForMediaType returns a serializer that matches the provided RFC2046 mediaType, or false if no such +// serializer exists +func (f CodecFactory) SerializerForMediaType(mediaType string, options map[string]string) (runtime.Serializer, bool) { + for _, s := range f.serializers { + for _, accepted := range s.AcceptContentTypes { + if accepted == mediaType { + if v, ok := options["pretty"]; ok && v == "1" && s.PrettySerializer != nil { + return s.PrettySerializer, true + } + return s.Serializer, true + } + } + } + return nil, false +} + +// SerializerForFileExtension returns a serializer for the provided extension, or false if no serializer matches. +func (f CodecFactory) SerializerForFileExtension(extension string) (runtime.Serializer, bool) { + for _, s := range f.serializers { + for _, ext := range s.FileExtensions { + if extension == ext { + return s.Serializer, true + } + } + } + return nil, false +} diff --git a/pkg/runtime/serializer/codec_test.go b/pkg/runtime/serializer/codec_test.go new file mode 100644 index 00000000000..6c1b2ff9547 --- /dev/null +++ b/pkg/runtime/serializer/codec_test.go @@ -0,0 +1,400 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serializer + +import ( + "encoding/json" + "fmt" + "log" + "os" + "reflect" + "strings" + "testing" + + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/conversion" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/util" + + "github.com/ghodss/yaml" + "github.com/google/gofuzz" + flag "github.com/spf13/pflag" +) + +var fuzzIters = flag.Int("fuzz-iters", 50, "How many fuzzing iterations to do.") + +type testMetaFactory struct{} + +func (testMetaFactory) Interpret(data []byte) (*unversioned.GroupVersionKind, error) { + findKind := struct { + APIVersion string `json:"myVersionKey,omitempty"` + ObjectKind string `json:"myKindKey,omitempty"` + }{} + // yaml is a superset of json, so we use it to decode here. That way, + // we understand both. + if err := yaml.Unmarshal(data, &findKind); err != nil { + return nil, fmt.Errorf("couldn't get version/kind: %v", err) + } + gv, err := unversioned.ParseGroupVersion(findKind.APIVersion) + if err != nil { + return nil, err + } + return &unversioned.GroupVersionKind{Group: gv.Group, Version: gv.Version, Kind: findKind.ObjectKind}, nil +} + +// Test a weird version/kind embedding format. +type MyWeirdCustomEmbeddedVersionKindField struct { + ID string `json:"ID,omitempty"` + APIVersion string `json:"myVersionKey,omitempty"` + ObjectKind string `json:"myKindKey,omitempty"` + Z string `json:"Z,omitempty"` + Y uint64 `json:"Y,omitempty"` +} + +type TestType1 struct { + MyWeirdCustomEmbeddedVersionKindField `json:",inline"` + A string `json:"A,omitempty"` + B int `json:"B,omitempty"` + C int8 `json:"C,omitempty"` + D int16 `json:"D,omitempty"` + E int32 `json:"E,omitempty"` + F int64 `json:"F,omitempty"` + G uint `json:"G,omitempty"` + H uint8 `json:"H,omitempty"` + I uint16 `json:"I,omitempty"` + J uint32 `json:"J,omitempty"` + K uint64 `json:"K,omitempty"` + L bool `json:"L,omitempty"` + M map[string]int `json:"M,omitempty"` + N map[string]TestType2 `json:"N,omitempty"` + O *TestType2 `json:"O,omitempty"` + P []TestType2 `json:"Q,omitempty"` +} + +type TestType2 struct { + A string `json:"A,omitempty"` + B int `json:"B,omitempty"` +} + +type ExternalTestType2 struct { + A string `json:"A,omitempty"` + B int `json:"B,omitempty"` +} +type ExternalTestType1 struct { + MyWeirdCustomEmbeddedVersionKindField `json:",inline"` + A string `json:"A,omitempty"` + B int `json:"B,omitempty"` + C int8 `json:"C,omitempty"` + D int16 `json:"D,omitempty"` + E int32 `json:"E,omitempty"` + F int64 `json:"F,omitempty"` + G uint `json:"G,omitempty"` + H uint8 `json:"H,omitempty"` + I uint16 `json:"I,omitempty"` + J uint32 `json:"J,omitempty"` + K uint64 `json:"K,omitempty"` + L bool `json:"L,omitempty"` + M map[string]int `json:"M,omitempty"` + N map[string]ExternalTestType2 `json:"N,omitempty"` + O *ExternalTestType2 `json:"O,omitempty"` + P []ExternalTestType2 `json:"Q,omitempty"` +} + +type ExternalInternalSame struct { + MyWeirdCustomEmbeddedVersionKindField `json:",inline"` + A TestType2 `json:"A,omitempty"` +} + +// TestObjectFuzzer can randomly populate all the above objects. +var TestObjectFuzzer = fuzz.New().NilChance(.5).NumElements(1, 100).Funcs( + func(j *MyWeirdCustomEmbeddedVersionKindField, c fuzz.Continue) { + c.FuzzNoCustom(j) + j.APIVersion = "" + j.ObjectKind = "" + }, +) + +func (obj *MyWeirdCustomEmbeddedVersionKindField) GetObjectKind() unversioned.ObjectKind { return obj } +func (obj *MyWeirdCustomEmbeddedVersionKindField) SetGroupVersionKind(gvk *unversioned.GroupVersionKind) { + obj.APIVersion, obj.ObjectKind = gvk.ToAPIVersionAndKind() +} +func (obj *MyWeirdCustomEmbeddedVersionKindField) GroupVersionKind() *unversioned.GroupVersionKind { + return unversioned.FromAPIVersionAndKind(obj.APIVersion, obj.ObjectKind) +} + +func (obj *ExternalInternalSame) GetObjectKind() unversioned.ObjectKind { + return &obj.MyWeirdCustomEmbeddedVersionKindField +} + +func (obj *TestType1) GetObjectKind() unversioned.ObjectKind { + return &obj.MyWeirdCustomEmbeddedVersionKindField +} + +func (obj *ExternalTestType1) GetObjectKind() unversioned.ObjectKind { + return &obj.MyWeirdCustomEmbeddedVersionKindField +} + +func (obj *TestType2) GetObjectKind() unversioned.ObjectKind { return unversioned.EmptyObjectKind } +func (obj *ExternalTestType2) GetObjectKind() unversioned.ObjectKind { + return unversioned.EmptyObjectKind +} + +// Returns a new Scheme set up with the test objects. +func GetTestScheme() (*runtime.Scheme, runtime.Codec) { + internalGV := unversioned.GroupVersion{Version: runtime.APIVersionInternal} + externalGV := unversioned.GroupVersion{Version: "v1"} + externalGV2 := unversioned.GroupVersion{Version: "v2"} + + s := runtime.NewScheme() + // Ordinarily, we wouldn't add TestType2, but because this is a test and + // both types are from the same package, we need to get it into the system + // so that converter will match it with ExternalType2. + s.AddKnownTypes(internalGV, &TestType1{}, &TestType2{}, &ExternalInternalSame{}) + s.AddKnownTypes(externalGV, &ExternalInternalSame{}) + s.AddKnownTypeWithName(externalGV.WithKind("TestType1"), &ExternalTestType1{}) + s.AddKnownTypeWithName(externalGV.WithKind("TestType2"), &ExternalTestType2{}) + s.AddKnownTypeWithName(internalGV.WithKind("TestType3"), &TestType1{}) + s.AddKnownTypeWithName(externalGV.WithKind("TestType3"), &ExternalTestType1{}) + s.AddKnownTypeWithName(externalGV2.WithKind("TestType1"), &ExternalTestType1{}) + + s.AddUnversionedTypes(externalGV, &unversioned.Status{}) + + cf := newCodecFactory(s, testMetaFactory{}) + codec := cf.LegacyCodec(unversioned.GroupVersion{Version: "v1"}) + return s, codec +} + +func objDiff(a, b interface{}) string { + ab, err := json.Marshal(a) + if err != nil { + panic("a") + } + bb, err := json.Marshal(b) + if err != nil { + panic("b") + } + return util.StringDiff(string(ab), string(bb)) + + // An alternate diff attempt, in case json isn't showing you + // the difference. (reflect.DeepEqual makes a distinction between + // nil and empty slices, for example.) + //return util.StringDiff( + // fmt.Sprintf("%#v", a), + // fmt.Sprintf("%#v", b), + //) +} + +var semantic = conversion.EqualitiesOrDie( + func(a, b MyWeirdCustomEmbeddedVersionKindField) bool { + a.APIVersion, a.ObjectKind = "", "" + b.APIVersion, b.ObjectKind = "", "" + return a == b + }, +) + +func runTest(t *testing.T, source interface{}) { + name := reflect.TypeOf(source).Elem().Name() + TestObjectFuzzer.Fuzz(source) + + _, codec := GetTestScheme() + data, err := runtime.Encode(codec, source.(runtime.Object)) + if err != nil { + t.Errorf("%v: %v (%#v)", name, err, source) + return + } + obj2, err := runtime.Decode(codec, data) + if err != nil { + t.Errorf("%v: %v (%v)", name, err, string(data)) + return + } + if !semantic.DeepEqual(source, obj2) { + t.Errorf("1: %v: diff: %v", name, util.ObjectGoPrintSideBySide(source, obj2)) + return + } + obj3 := reflect.New(reflect.TypeOf(source).Elem()).Interface() + if err := runtime.DecodeInto(codec, data, obj3.(runtime.Object)); err != nil { + t.Errorf("2: %v: %v", name, err) + return + } + if !semantic.DeepEqual(source, obj3) { + t.Errorf("3: %v: diff: %v", name, objDiff(source, obj3)) + return + } +} + +func TestTypes(t *testing.T) { + table := []interface{}{ + &TestType1{}, + &ExternalInternalSame{}, + } + for _, item := range table { + // Try a few times, since runTest uses random values. + for i := 0; i < *fuzzIters; i++ { + runTest(t, item) + } + } +} + +func TestVersionedEncoding(t *testing.T) { + s, codec := GetTestScheme() + out, err := runtime.Encode(codec, &TestType1{}, unversioned.GroupVersion{Version: "v2"}) + if err != nil { + t.Fatal(err) + } + if string(out) != `{"myVersionKey":"v2","myKindKey":"TestType1"}`+"\n" { + t.Fatal(string(out)) + } + _, err = runtime.Encode(codec, &TestType1{}, unversioned.GroupVersion{Version: "v3"}) + if err == nil { + t.Fatal(err) + } + + cf := newCodecFactory(s, testMetaFactory{}) + encoder, _ := cf.SerializerForFileExtension("json") + + // codec that is unversioned uses the target version + unversionedCodec := cf.CodecForVersions(encoder, nil, nil) + _, err = runtime.Encode(unversionedCodec, &TestType1{}, unversioned.GroupVersion{Version: "v3"}) + if err == nil || !runtime.IsNotRegisteredError(err) { + t.Fatal(err) + } + + // unversioned encode with no versions is written directly to wire + out, err = runtime.Encode(unversionedCodec, &TestType1{}) + if err != nil { + t.Fatal(err) + } + if string(out) != `{"myVersionKey":"__internal","myKindKey":"TestType1"}`+"\n" { + t.Fatal(string(out)) + } +} + +func TestMultipleNames(t *testing.T) { + _, codec := GetTestScheme() + + obj, _, err := codec.Decode([]byte(`{"myKindKey":"TestType3","myVersionKey":"v1","A":"value"}`), nil, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + internal := obj.(*TestType1) + if internal.A != "value" { + t.Fatalf("unexpected decoded object: %#v", internal) + } + + out, err := runtime.Encode(codec, internal) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(string(out), `"myKindKey":"TestType1"`) { + t.Errorf("unexpected encoded output: %s", string(out)) + } +} + +func TestConvertTypesWhenDefaultNamesMatch(t *testing.T) { + internalGV := unversioned.GroupVersion{Version: runtime.APIVersionInternal} + externalGV := unversioned.GroupVersion{Version: "v1"} + + s := runtime.NewScheme() + // create two names internally, with TestType1 being preferred + s.AddKnownTypeWithName(internalGV.WithKind("TestType1"), &TestType1{}) + s.AddKnownTypeWithName(internalGV.WithKind("OtherType1"), &TestType1{}) + // create two names externally, with TestType1 being preferred + s.AddKnownTypeWithName(externalGV.WithKind("TestType1"), &ExternalTestType1{}) + s.AddKnownTypeWithName(externalGV.WithKind("OtherType1"), &ExternalTestType1{}) + + ext := &ExternalTestType1{} + ext.APIVersion = "v1" + ext.ObjectKind = "OtherType1" + ext.A = "test" + data, err := json.Marshal(ext) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + expect := &TestType1{A: "test"} + + codec := newCodecFactory(s, testMetaFactory{}).LegacyCodec(unversioned.GroupVersion{Version: "v1"}) + + obj, err := runtime.Decode(codec, data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !semantic.DeepEqual(expect, obj) { + t.Errorf("unexpected object: %#v", obj) + } + + into := &TestType1{} + if err := runtime.DecodeInto(codec, data, into); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !semantic.DeepEqual(expect, into) { + t.Errorf("unexpected object: %#v", obj) + } +} + +func TestEncode_Ptr(t *testing.T) { + _, codec := GetTestScheme() + tt := &TestType1{A: "I am a pointer object"} + data, err := runtime.Encode(codec, tt) + obj2, err2 := runtime.Decode(codec, data) + if err != nil || err2 != nil { + t.Fatalf("Failure: '%v' '%v'\n%s", err, err2, data) + } + if _, ok := obj2.(*TestType1); !ok { + t.Fatalf("Got wrong type") + } + if !semantic.DeepEqual(obj2, tt) { + t.Errorf("Expected:\n %#v,\n Got:\n %#v", tt, obj2) + } +} + +func TestBadJSONRejection(t *testing.T) { + log.SetOutput(os.Stderr) + _, codec := GetTestScheme() + badJSONs := [][]byte{ + []byte(`{"myVersionKey":"v1"}`), // Missing kind + []byte(`{"myVersionKey":"v1","myKindKey":"bar"}`), // Unknown kind + []byte(`{"myVersionKey":"bar","myKindKey":"TestType1"}`), // Unknown version + []byte(`{"myKindKey":"TestType1"}`), // Missing version + } + for _, b := range badJSONs { + if _, err := runtime.Decode(codec, b); err == nil { + t.Errorf("Did not reject bad json: %s", string(b)) + } + } + badJSONKindMismatch := []byte(`{"myVersionKey":"v1","myKindKey":"ExternalInternalSame"}`) + if err := runtime.DecodeInto(codec, badJSONKindMismatch, &TestType1{}); err == nil { + t.Errorf("Kind is set but doesn't match the object type: %s", badJSONKindMismatch) + } + if err := runtime.DecodeInto(codec, []byte(``), &TestType1{}); err != nil { + t.Errorf("Should allow empty decode: %v", err) + } + if _, _, err := codec.Decode([]byte(``), &unversioned.GroupVersionKind{Kind: "ExternalInternalSame"}, nil); err == nil { + t.Errorf("Did not give error for empty data with only kind default") + } + if _, _, err := codec.Decode([]byte(`{"myVersionKey":"v1"}`), &unversioned.GroupVersionKind{Kind: "ExternalInternalSame"}, nil); err != nil { + t.Errorf("Gave error for version and kind default") + } + if _, _, err := codec.Decode([]byte(`{"myKindKey":"ExternalInternalSame"}`), &unversioned.GroupVersionKind{Version: "v1"}, nil); err != nil { + t.Errorf("Gave error for version and kind default") + } + if _, _, err := codec.Decode([]byte(``), &unversioned.GroupVersionKind{Kind: "ExternalInternalSame", Version: "v1"}, nil); err != nil { + t.Errorf("Gave error for version and kind defaulted: %v", err) + } + if _, err := runtime.Decode(codec, []byte(``)); err == nil { + t.Errorf("Did not give error for empty data") + } +} diff --git a/pkg/runtime/serializer/json/json.go b/pkg/runtime/serializer/json/json.go new file mode 100644 index 00000000000..f9fb4bbfb22 --- /dev/null +++ b/pkg/runtime/serializer/json/json.go @@ -0,0 +1,191 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json + +import ( + "encoding/json" + "io" + + "github.com/ghodss/yaml" + "github.com/ugorji/go/codec" + + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/runtime" + utilyaml "k8s.io/kubernetes/pkg/util/yaml" +) + +// NewSerializer creates a JSON serializer that handles encoding versioned objects into the proper JSON form. If typer +// is not nil, the object has the group, version, and kind fields set. +func NewSerializer(meta MetaFactory, creater runtime.ObjectCreater, typer runtime.Typer, pretty bool) runtime.Serializer { + return &Serializer{ + meta: meta, + creater: creater, + typer: typer, + yaml: false, + pretty: pretty, + } +} + +// NewYAMLSerializer creates a YAML serializer that handles encoding versioned objects into the proper YAML form. If typer +// is not nil, the object has the group, version, and kind fields set. This serializer supports only the subset of YAML that +// matches JSON, and will error if constructs are used that do not serialize to JSON. +func NewYAMLSerializer(meta MetaFactory, creater runtime.ObjectCreater, typer runtime.Typer) runtime.Serializer { + return &Serializer{ + meta: meta, + creater: creater, + typer: typer, + yaml: true, + } +} + +type Serializer struct { + meta MetaFactory + creater runtime.ObjectCreater + typer runtime.Typer + yaml bool + pretty bool +} + +// Decode attempts to convert the provided data into YAML or JSON, extract the stored schema kind, apply the provided default gvk, and then +// load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown, the raw data will be +// extracted and no decoding will be performed. If into is not registered with the typer, then the object will be straight decoded using +// normal JSON/YAML unmarshalling. If into is provided and the original data is not fully qualified with kind/version/group, the type of +// the into will be used to alter the returned gvk. On success or most errors, the method will return the calculated schema kind. +func (s *Serializer) Decode(originalData []byte, gvk *unversioned.GroupVersionKind, into runtime.Object) (runtime.Object, *unversioned.GroupVersionKind, error) { + if versioned, ok := into.(*runtime.VersionedObjects); ok { + into = versioned.Last() + obj, actual, err := s.Decode(originalData, gvk, into) + if err != nil { + return nil, actual, err + } + versioned.Objects = []runtime.Object{obj} + return versioned, actual, nil + } + + data := originalData + if s.yaml { + altered, err := yaml.YAMLToJSON(data) + if err != nil { + return nil, nil, err + } + data = altered + } + + actual, err := s.meta.Interpret(data) + if err != nil { + return nil, nil, err + } + + if gvk != nil { + // apply kind and version defaulting from provided default + if len(actual.Kind) == 0 { + actual.Kind = gvk.Kind + } + if len(actual.Version) == 0 && len(actual.Group) == 0 { + actual.Group = gvk.Group + actual.Version = gvk.Version + } + if len(actual.Version) == 0 && actual.Group == gvk.Group { + actual.Version = gvk.Version + } + } + + if unk, ok := into.(*runtime.Unknown); ok && unk != nil { + unk.RawJSON = originalData + // TODO: set content type here + unk.GetObjectKind().SetGroupVersionKind(actual) + return unk, actual, nil + } + + if into != nil { + typed, _, err := s.typer.ObjectKind(into) + switch { + case runtime.IsNotRegisteredError(err): + if err := codec.NewDecoderBytes(data, new(codec.JsonHandle)).Decode(into); err != nil { + return nil, actual, err + } + return into, actual, nil + case err != nil: + return nil, actual, err + default: + if len(actual.Kind) == 0 { + actual.Kind = typed.Kind + } + if len(actual.Version) == 0 && len(actual.Group) == 0 { + actual.Group = typed.Group + actual.Version = typed.Version + } + if len(actual.Version) == 0 && actual.Group == typed.Group { + actual.Version = typed.Version + } + } + } + + if len(actual.Kind) == 0 { + return nil, actual, runtime.NewMissingKindErr(string(originalData)) + } + if len(actual.Version) == 0 { + return nil, actual, runtime.NewMissingVersionErr(string(originalData)) + } + + // use the target if necessary + obj, err := runtime.UseOrCreateObject(s.typer, s.creater, *actual, into) + if err != nil { + return nil, actual, err + } + + if err := codec.NewDecoderBytes(data, new(codec.JsonHandle)).Decode(obj); err != nil { + return nil, actual, err + } + return obj, actual, nil +} + +// EncodeToStream serializes the provided object to the given writer. Overrides is ignored. +func (s *Serializer) EncodeToStream(obj runtime.Object, w io.Writer, overrides ...unversioned.GroupVersion) error { + if s.yaml { + json, err := json.Marshal(obj) + if err != nil { + return err + } + data, err := yaml.JSONToYAML(json) + if err != nil { + return err + } + _, err = w.Write(data) + return err + } + + if s.pretty { + data, err := json.MarshalIndent(obj, "", " ") + if err != nil { + return err + } + _, err = w.Write(data) + return err + } + encoder := json.NewEncoder(w) + return encoder.Encode(obj) +} + +// RecognizesData implements the RecognizingDecoder interface. +func (s *Serializer) RecognizesData(peek io.Reader) (bool, error) { + _, ok := utilyaml.GuessJSONStream(peek, 2048) + if s.yaml { + return !ok, nil + } + return ok, nil +} diff --git a/pkg/runtime/serializer/json/json_test.go b/pkg/runtime/serializer/json/json_test.go new file mode 100644 index 00000000000..f9a744f8718 --- /dev/null +++ b/pkg/runtime/serializer/json/json_test.go @@ -0,0 +1,269 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json_test + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/conversion" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/runtime/serializer/json" + "k8s.io/kubernetes/pkg/util" +) + +type testDecodable struct { + Other string + Value int `json:"value"` + gvk *unversioned.GroupVersionKind +} + +func (d *testDecodable) GetObjectKind() unversioned.ObjectKind { return d } +func (d *testDecodable) SetGroupVersionKind(gvk *unversioned.GroupVersionKind) { d.gvk = gvk } +func (d *testDecodable) GroupVersionKind() *unversioned.GroupVersionKind { return d.gvk } + +func TestDecode(t *testing.T) { + testCases := []struct { + creater runtime.ObjectCreater + typer runtime.Typer + yaml bool + pretty bool + + data []byte + defaultGVK *unversioned.GroupVersionKind + into runtime.Object + + errFn func(error) bool + expectedObject runtime.Object + expectedGVK *unversioned.GroupVersionKind + }{ + { + data: []byte("{}"), + + expectedGVK: &unversioned.GroupVersionKind{}, + errFn: func(err error) bool { return strings.Contains(err.Error(), "Object 'Kind' is missing in") }, + }, + { + data: []byte("{}"), + defaultGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + creater: &mockCreater{err: fmt.Errorf("fake error")}, + + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + errFn: func(err error) bool { return err.Error() == "fake error" }, + }, + { + data: []byte("{}"), + defaultGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + creater: &mockCreater{err: fmt.Errorf("fake error")}, + + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + errFn: func(err error) bool { return err.Error() == "fake error" }, + }, + { + data: []byte("{}"), + defaultGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + creater: &mockCreater{obj: &testDecodable{}}, + expectedObject: &testDecodable{ + gvk: nil, // json serializer does NOT set GVK + }, + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + }, + + // version without group is not defaulted + { + data: []byte(`{"apiVersion":"blah"}`), + defaultGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + creater: &mockCreater{obj: &testDecodable{}}, + expectedObject: &testDecodable{ + gvk: nil, // json serializer does NOT set GVK + }, + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "", Version: "blah"}, + }, + // group without version is defaulted + { + data: []byte(`{"apiVersion":"other/"}`), + defaultGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + creater: &mockCreater{obj: &testDecodable{}}, + expectedObject: &testDecodable{ + gvk: nil, // json serializer does NOT set GVK + }, + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + }, + + // accept runtime.Unknown as into and bypass creator + { + data: []byte(`{}`), + into: &runtime.Unknown{}, + + expectedGVK: &unversioned.GroupVersionKind{}, + expectedObject: &runtime.Unknown{ + RawJSON: []byte(`{}`), + }, + }, + { + data: []byte(`{"test":"object"}`), + into: &runtime.Unknown{}, + + expectedGVK: &unversioned.GroupVersionKind{}, + expectedObject: &runtime.Unknown{ + RawJSON: []byte(`{"test":"object"}`), + }, + }, + { + data: []byte(`{"test":"object"}`), + into: &runtime.Unknown{}, + defaultGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + expectedObject: &runtime.Unknown{ + TypeMeta: runtime.TypeMeta{APIVersion: "other/blah", Kind: "Test"}, + RawJSON: []byte(`{"test":"object"}`), + }, + }, + + // unregistered objects can be decoded into directly + { + data: []byte(`{"kind":"Test","apiVersion":"other/blah","value":1,"Other":"test"}`), + into: &testDecodable{}, + typer: &mockTyper{err: conversion.NewNotRegisteredErr(unversioned.GroupVersionKind{}, nil)}, + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + expectedObject: &testDecodable{ + Other: "test", + Value: 1, + }, + }, + // registered types get defaulted by the into object kind + { + data: []byte(`{"value":1,"Other":"test"}`), + into: &testDecodable{}, + typer: &mockTyper{gvk: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}}, + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + expectedObject: &testDecodable{ + Other: "test", + Value: 1, + }, + }, + // registered types get defaulted by the into object kind even without version, but return an error + { + data: []byte(`{"value":1,"Other":"test"}`), + into: &testDecodable{}, + typer: &mockTyper{gvk: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: ""}}, + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: ""}, + errFn: func(err error) bool { return strings.Contains(err.Error(), "Object 'apiVersion' is missing in") }, + expectedObject: &testDecodable{ + Other: "test", + Value: 1, + }, + }, + + // runtime.VersionedObjects are decoded + { + data: []byte(`{"value":1,"Other":"test"}`), + into: &runtime.VersionedObjects{Objects: []runtime.Object{}}, + creater: &mockCreater{obj: &testDecodable{}}, + typer: &mockTyper{gvk: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}}, + defaultGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + expectedObject: &runtime.VersionedObjects{ + Objects: []runtime.Object{ + &testDecodable{ + Other: "test", + Value: 1, + }, + }, + }, + }, + // runtime.VersionedObjects with an object are decoded into + { + data: []byte(`{"Other":"test"}`), + into: &runtime.VersionedObjects{Objects: []runtime.Object{&testDecodable{Value: 2}}}, + typer: &mockTyper{gvk: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}}, + expectedGVK: &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, + expectedObject: &runtime.VersionedObjects{ + Objects: []runtime.Object{ + &testDecodable{ + Other: "test", + Value: 2, + }, + }, + }, + }, + } + + for i, test := range testCases { + var s runtime.Serializer + if test.yaml { + s = json.NewYAMLSerializer(json.DefaultMetaFactory, test.creater, test.typer) + } else { + s = json.NewSerializer(json.DefaultMetaFactory, test.creater, test.typer, test.pretty) + } + obj, gvk, err := s.Decode([]byte(test.data), test.defaultGVK, test.into) + + if !reflect.DeepEqual(test.expectedGVK, gvk) { + t.Errorf("%d: unexpected GVK: %v", i, gvk) + } + + switch { + case err == nil && test.errFn != nil: + t.Errorf("%d: failed: %v", i, err) + continue + case err != nil && test.errFn == nil: + t.Errorf("%d: failed: %v", i, err) + continue + case err != nil: + if !test.errFn(err) { + t.Errorf("%d: failed: %v", i, err) + } + if obj != nil { + t.Errorf("%d: should have returned nil object", i) + } + continue + } + + if test.into != nil && test.into != obj { + t.Errorf("%d: expected into to be returned: %v", i, obj) + continue + } + + if !reflect.DeepEqual(test.expectedObject, obj) { + t.Errorf("%d: unexpected object:\n%s", i, util.ObjectGoPrintSideBySide(test.expectedObject, obj)) + } + } +} + +type mockCreater struct { + apiVersion string + kind string + err error + obj runtime.Object +} + +func (c *mockCreater) New(kind unversioned.GroupVersionKind) (runtime.Object, error) { + c.apiVersion, c.kind = kind.GroupVersion().String(), kind.Kind + return c.obj, c.err +} + +type mockTyper struct { + gvk *unversioned.GroupVersionKind + err error +} + +func (t *mockTyper) ObjectKind(obj runtime.Object) (*unversioned.GroupVersionKind, bool, error) { + return t.gvk, false, t.err +} diff --git a/pkg/runtime/serializer/json/meta.go b/pkg/runtime/serializer/json/meta.go new file mode 100644 index 00000000000..91df105ed6c --- /dev/null +++ b/pkg/runtime/serializer/json/meta.go @@ -0,0 +1,61 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json + +import ( + "encoding/json" + "fmt" + + "k8s.io/kubernetes/pkg/api/unversioned" +) + +// MetaFactory is used to store and retrieve the version and kind +// information for JSON objects in a serializer. +type MetaFactory interface { + // Interpret should return the version and kind of the wire-format of + // the object. + Interpret(data []byte) (*unversioned.GroupVersionKind, error) +} + +// DefaultMetaFactory is a default factory for versioning objects in JSON. The object +// in memory and in the default JSON serialization will use the "kind" and "apiVersion" +// fields. +var DefaultMetaFactory = SimpleMetaFactory{} + +// SimpleMetaFactory provides default methods for retrieving the type and version of objects +// that are identified with an "apiVersion" and "kind" fields in their JSON +// serialization. It may be parameterized with the names of the fields in memory, or an +// optional list of base structs to search for those fields in memory. +type SimpleMetaFactory struct { +} + +// Interpret will return the APIVersion and Kind of the JSON wire-format +// encoding of an object, or an error. +func (SimpleMetaFactory) Interpret(data []byte) (*unversioned.GroupVersionKind, error) { + findKind := struct { + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + }{} + if err := json.Unmarshal(data, &findKind); err != nil { + return nil, fmt.Errorf("couldn't get version/kind; json parse error: %v", err) + } + gv, err := unversioned.ParseGroupVersion(findKind.APIVersion) + if err != nil { + return nil, err + } + return &unversioned.GroupVersionKind{Group: gv.Group, Version: gv.Version, Kind: findKind.Kind}, nil +} diff --git a/pkg/runtime/serializer/json/meta_test.go b/pkg/runtime/serializer/json/meta_test.go new file mode 100644 index 00000000000..4b6351286f7 --- /dev/null +++ b/pkg/runtime/serializer/json/meta_test.go @@ -0,0 +1,45 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json + +import "testing" + +func TestSimpleMetaFactoryInterpret(t *testing.T) { + factory := SimpleMetaFactory{} + gvk, err := factory.Interpret([]byte(`{"apiVersion":"1","kind":"object"}`)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if gvk.Version != "1" || gvk.Kind != "object" { + t.Errorf("unexpected interpret: %#v", gvk) + } + + // no kind or version + gvk, err = factory.Interpret([]byte(`{}`)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if gvk.Version != "" || gvk.Kind != "" { + t.Errorf("unexpected interpret: %#v", gvk) + } + + // unparsable + gvk, err = factory.Interpret([]byte(`{`)) + if err == nil { + t.Errorf("unexpected non-error") + } +} diff --git a/pkg/runtime/serializer/protobuf/doc.go b/pkg/runtime/serializer/protobuf/doc.go deleted file mode 100644 index 3fec7197aff..00000000000 --- a/pkg/runtime/serializer/protobuf/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package protobuf handles serializing API objects to and from wire formats. -package protobuf diff --git a/pkg/runtime/serializer/recognizer/recognizer.go b/pkg/runtime/serializer/recognizer/recognizer.go new file mode 100644 index 00000000000..14a2cb3e841 --- /dev/null +++ b/pkg/runtime/serializer/recognizer/recognizer.go @@ -0,0 +1,79 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recognizer + +import ( + "bytes" + "fmt" + "io" + + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/runtime" +) + +type RecognizingDecoder interface { + runtime.Decoder + RecognizesData(peek io.Reader) (bool, error) +} + +func NewDecoder(decoders ...runtime.Decoder) runtime.Decoder { + recognizing, blind := []RecognizingDecoder{}, []runtime.Decoder{} + for _, d := range decoders { + if r, ok := d.(RecognizingDecoder); ok { + recognizing = append(recognizing, r) + } else { + blind = append(blind, d) + } + } + return &decoder{ + recognizing: recognizing, + blind: blind, + } +} + +type decoder struct { + recognizing []RecognizingDecoder + blind []runtime.Decoder +} + +func (d *decoder) Decode(data []byte, gvk *unversioned.GroupVersionKind, into runtime.Object) (runtime.Object, *unversioned.GroupVersionKind, error) { + var lastErr error + for _, r := range d.recognizing { + buf := bytes.NewBuffer(data) + ok, err := r.RecognizesData(buf) + if err != nil { + lastErr = err + continue + } + if !ok { + continue + } + return r.Decode(data, gvk, into) + } + for _, d := range d.blind { + out, actual, err := d.Decode(data, gvk, into) + if err != nil { + lastErr = err + continue + } + return out, actual, nil + } + if lastErr == nil { + lastErr = fmt.Errorf("no serialization format matched the provided data") + } + return nil, nil, lastErr +} diff --git a/pkg/runtime/serializer/recognizer/recognizer_test.go b/pkg/runtime/serializer/recognizer/recognizer_test.go new file mode 100644 index 00000000000..c83b87aa4dd --- /dev/null +++ b/pkg/runtime/serializer/recognizer/recognizer_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recognizer + +import ( + "testing" + + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/runtime/serializer/json" +) + +type A struct{} + +func (A) GetObjectKind() unversioned.ObjectKind { return unversioned.EmptyObjectKind } + +func TestRecognizer(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypes(unversioned.GroupVersion{Version: "v1"}, &A{}) + d := NewDecoder( + json.NewSerializer(json.DefaultMetaFactory, s, runtime.ObjectTyperToTyper(s), false), + json.NewYAMLSerializer(json.DefaultMetaFactory, s, runtime.ObjectTyperToTyper(s)), + ) + out, _, err := d.Decode([]byte(` +kind: A +apiVersion: v1 +`), nil, nil) + if err != nil { + t.Fatal(err) + } + t.Logf("%#v", out) + + out, _, err = d.Decode([]byte(` +{ + "kind":"A", + "apiVersion":"v1" +} +`), nil, nil) + if err != nil { + t.Fatal(err) + } + t.Logf("%#v", out) +} diff --git a/pkg/runtime/serializer/versioning/versioning.go b/pkg/runtime/serializer/versioning/versioning.go new file mode 100644 index 00000000000..1b501369c13 --- /dev/null +++ b/pkg/runtime/serializer/versioning/versioning.go @@ -0,0 +1,243 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package versioning + +import ( + "fmt" + "io" + + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/runtime" +) + +// NewCodecForScheme is a convenience method for callers that are using a scheme. +func NewCodecForScheme( + // TODO: I should be a scheme interface? + scheme *runtime.Scheme, + serializer runtime.Serializer, + encodeVersion []unversioned.GroupVersion, + decodeVersion []unversioned.GroupVersion, +) runtime.Codec { + return NewCodec(serializer, scheme, scheme, scheme, runtime.ObjectTyperToTyper(scheme), encodeVersion, decodeVersion) +} + +// NewCodec takes objects in their internal versions and converts them to external versions before +// serializing them. It assumes the serializer provided to it only deals with external versions. +// This class is also a serializer, but is generally used with a specific version. +func NewCodec( + serializer runtime.Serializer, + convertor runtime.ObjectConvertor, + creater runtime.ObjectCreater, + copier runtime.ObjectCopier, + typer runtime.Typer, + encodeVersion []unversioned.GroupVersion, + decodeVersion []unversioned.GroupVersion, +) runtime.Codec { + internal := &codec{ + serializer: serializer, + convertor: convertor, + creater: creater, + copier: copier, + typer: typer, + } + if encodeVersion != nil { + internal.encodeVersion = make(map[string]unversioned.GroupVersion) + for _, v := range encodeVersion { + internal.encodeVersion[v.Group] = v + } + } + if decodeVersion != nil { + internal.decodeVersion = make(map[string]unversioned.GroupVersion) + for _, v := range decodeVersion { + internal.decodeVersion[v.Group] = v + } + } + + return internal +} + +type codec struct { + serializer runtime.Serializer + convertor runtime.ObjectConvertor + creater runtime.ObjectCreater + copier runtime.ObjectCopier + typer runtime.Typer + + encodeVersion map[string]unversioned.GroupVersion + decodeVersion map[string]unversioned.GroupVersion +} + +// Decode attempts a decode of the object, then tries to convert it to the internal version. If into is provided and the decoding is +// successful, the returned runtime.Object will be the value passed as into. Note that this may bypass conversion if you pass an +// into that matches the serialized version. +func (c *codec) Decode(data []byte, defaultGVK *unversioned.GroupVersionKind, into runtime.Object) (runtime.Object, *unversioned.GroupVersionKind, error) { + versioned, isVersioned := into.(*runtime.VersionedObjects) + if isVersioned { + into = versioned.Last() + } + + obj, gvk, err := c.serializer.Decode(data, defaultGVK, into) + if err != nil { + return nil, gvk, err + } + + // if we specify a target, use generic conversion. + if into != nil { + if into == obj { + if isVersioned { + return versioned, gvk, nil + } + return into, gvk, nil + } + if err := c.convertor.Convert(obj, into); err != nil { + return nil, gvk, err + } + if isVersioned { + versioned.Objects = []runtime.Object{obj, into} + return versioned, gvk, nil + } + return into, gvk, nil + } + + // invoke a version conversion + group := gvk.Group + if defaultGVK != nil { + group = defaultGVK.Group + } + var targetGV unversioned.GroupVersion + if c.decodeVersion == nil { + // convert to internal by default + targetGV.Group = group + targetGV.Version = runtime.APIVersionInternal + } else { + gv, ok := c.decodeVersion[group] + if !ok { + // unknown objects are left in their original version + if isVersioned { + versioned.Objects = []runtime.Object{obj} + return versioned, gvk, nil + } + return obj, gvk, nil + } + targetGV = gv + } + + if gvk.GroupVersion() == targetGV { + if isVersioned { + versioned.Objects = []runtime.Object{obj} + return versioned, gvk, nil + } + return obj, gvk, nil + } + + if isVersioned { + // create a copy, because ConvertToVersion does not guarantee non-mutation of objects + copied, err := c.copier.Copy(obj) + if err != nil { + copied = obj + } + versioned.Objects = []runtime.Object{copied} + } + + // Convert if needed. + out, err := c.convertor.ConvertToVersion(obj, targetGV.String()) + if err != nil { + return nil, gvk, err + } + if isVersioned { + versioned.Objects = append(versioned.Objects, out) + return versioned, gvk, nil + } + return out, gvk, nil +} + +// EncodeToStream ensures the provided object is output in the right scheme. If overrides are specified, when +// encoding the object the first override that matches the object's group is used. Other overrides are ignored. +func (c *codec) EncodeToStream(obj runtime.Object, w io.Writer, overrides ...unversioned.GroupVersion) error { + if _, ok := obj.(*runtime.Unknown); ok { + return c.serializer.EncodeToStream(obj, w, overrides...) + } + gvk, isUnversioned, err := c.typer.ObjectKind(obj) + if err != nil { + return err + } + + if (c.encodeVersion == nil && len(overrides) == 0) || isUnversioned { + old := obj.GetObjectKind().GroupVersionKind() + obj.GetObjectKind().SetGroupVersionKind(gvk) + defer obj.GetObjectKind().SetGroupVersionKind(old) + return c.serializer.EncodeToStream(obj, w, overrides...) + } + + targetGV, ok := c.encodeVersion[gvk.Group] + // use override if provided + for i, override := range overrides { + if override.Group == gvk.Group { + ok = true + targetGV = override + // swap the position of the override + overrides[0], overrides[i] = targetGV, overrides[0] + break + } + } + + // attempt a conversion to the sole encode version + if !ok && len(c.encodeVersion) == 1 { + ok = true + for _, v := range c.encodeVersion { + targetGV = v + } + // ensure the target override is first + overrides = promoteOrPrependGroupVersion(targetGV, overrides) + } + + // if no fallback is available, error + if !ok { + return fmt.Errorf("the codec does not recognize group %q for kind %q and cannot encode it", gvk.Group, gvk.Kind) + } + + // Perform a conversion if necessary + if gvk.GroupVersion() != targetGV { + out, err := c.convertor.ConvertToVersion(obj, targetGV.String()) + if err != nil { + if ok { + return err + } + } else { + obj = out + } + } else { + old := obj.GetObjectKind().GroupVersionKind() + defer obj.GetObjectKind().SetGroupVersionKind(old) + obj.GetObjectKind().SetGroupVersionKind(&unversioned.GroupVersionKind{Group: targetGV.Group, Version: targetGV.Version, Kind: gvk.Kind}) + } + + return c.serializer.EncodeToStream(obj, w, overrides...) +} + +// promoteOrPrependGroupVersion finds the group version in the provided group versions that has the same group as target. +// If the group is found the returned array will have that group version in the first position - if the group is not found +// the returned array will have target in the first position. +func promoteOrPrependGroupVersion(target unversioned.GroupVersion, gvs []unversioned.GroupVersion) []unversioned.GroupVersion { + for i, gv := range gvs { + if gv.Group == target.Group { + gvs[0], gvs[i] = gvs[i], gvs[0] + return gvs + } + } + return append([]unversioned.GroupVersion{target}, gvs...) +} diff --git a/pkg/runtime/serializer/versioning/versioning_test.go b/pkg/runtime/serializer/versioning/versioning_test.go new file mode 100644 index 00000000000..3d7eba27e3c --- /dev/null +++ b/pkg/runtime/serializer/versioning/versioning_test.go @@ -0,0 +1,300 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package versioning + +import ( + "fmt" + "io" + "reflect" + "testing" + + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/util" +) + +type testDecodable struct { + Other string + Value int `json:"value"` + gvk *unversioned.GroupVersionKind +} + +func (d *testDecodable) GetObjectKind() unversioned.ObjectKind { return d } +func (d *testDecodable) SetGroupVersionKind(gvk *unversioned.GroupVersionKind) { d.gvk = gvk } +func (d *testDecodable) GroupVersionKind() *unversioned.GroupVersionKind { return d.gvk } + +func TestDecode(t *testing.T) { + gvk1 := &unversioned.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"} + decodable1 := &testDecodable{} + decodable2 := &testDecodable{} + decodable3 := &testDecodable{} + versionedDecodable1 := &runtime.VersionedObjects{Objects: []runtime.Object{decodable1}} + + testCases := []struct { + serializer runtime.Serializer + convertor runtime.ObjectConvertor + creater runtime.ObjectCreater + copier runtime.ObjectCopier + typer runtime.Typer + yaml bool + pretty bool + + encodes, decodes []unversioned.GroupVersion + + defaultGVK *unversioned.GroupVersionKind + into runtime.Object + + errFn func(error) bool + expectedObject runtime.Object + sameObject runtime.Object + expectedGVK *unversioned.GroupVersionKind + }{ + { + serializer: &mockSerializer{actual: gvk1}, + convertor: &checkConvertor{groupVersion: "other/__internal"}, + expectedGVK: gvk1, + }, + { + serializer: &mockSerializer{actual: gvk1, obj: decodable1}, + convertor: &checkConvertor{in: decodable1, obj: decodable2, groupVersion: "other/__internal"}, + expectedGVK: gvk1, + sameObject: decodable2, + }, + // defaultGVK.Group is allowed to force a conversion to the destination group + { + serializer: &mockSerializer{actual: gvk1, obj: decodable1}, + defaultGVK: &unversioned.GroupVersionKind{Group: "force"}, + convertor: &checkConvertor{in: decodable1, obj: decodable2, groupVersion: "force/__internal"}, + expectedGVK: gvk1, + sameObject: decodable2, + }, + // uses direct conversion for into when objects differ + { + into: decodable3, + serializer: &mockSerializer{actual: gvk1, obj: decodable1}, + convertor: &checkConvertor{in: decodable1, obj: decodable3, directConvert: true}, + expectedGVK: gvk1, + sameObject: decodable3, + }, + { + into: versionedDecodable1, + serializer: &mockSerializer{actual: gvk1, obj: decodable3}, + convertor: &checkConvertor{in: decodable3, obj: decodable1, directConvert: true}, + expectedGVK: gvk1, + sameObject: versionedDecodable1, + }, + // returns directly when serializer returns into + { + into: decodable3, + serializer: &mockSerializer{actual: gvk1, obj: decodable3}, + expectedGVK: gvk1, + sameObject: decodable3, + }, + // returns directly when serializer returns into + { + into: versionedDecodable1, + serializer: &mockSerializer{actual: gvk1, obj: decodable1}, + expectedGVK: gvk1, + sameObject: versionedDecodable1, + }, + + // runtime.VersionedObjects are decoded + { + into: &runtime.VersionedObjects{Objects: []runtime.Object{}}, + + serializer: &mockSerializer{actual: gvk1, obj: decodable1}, + copier: &checkCopy{in: decodable1, obj: decodable1}, + convertor: &checkConvertor{in: decodable1, obj: decodable2, groupVersion: "other/__internal"}, + expectedGVK: gvk1, + expectedObject: &runtime.VersionedObjects{Objects: []runtime.Object{decodable1, decodable2}}, + }, + { + into: &runtime.VersionedObjects{Objects: []runtime.Object{}}, + + serializer: &mockSerializer{actual: gvk1, obj: decodable1}, + copier: &checkCopy{in: decodable1, obj: nil, err: fmt.Errorf("error on copy")}, + convertor: &checkConvertor{in: decodable1, obj: decodable2, groupVersion: "other/__internal"}, + expectedGVK: gvk1, + expectedObject: &runtime.VersionedObjects{Objects: []runtime.Object{decodable1, decodable2}}, + }, + + // decode into the same version as the serialized object + { + decodes: []unversioned.GroupVersion{gvk1.GroupVersion()}, + + serializer: &mockSerializer{actual: gvk1, obj: decodable1}, + expectedGVK: gvk1, + expectedObject: decodable1, + }, + { + into: &runtime.VersionedObjects{Objects: []runtime.Object{}}, + decodes: []unversioned.GroupVersion{gvk1.GroupVersion()}, + + serializer: &mockSerializer{actual: gvk1, obj: decodable1}, + expectedGVK: gvk1, + expectedObject: &runtime.VersionedObjects{Objects: []runtime.Object{decodable1}}, + }, + + // codec with non matching version skips conversion altogether + { + decodes: []unversioned.GroupVersion{{Group: "something", Version: "else"}}, + + serializer: &mockSerializer{actual: gvk1, obj: decodable1}, + expectedGVK: gvk1, + expectedObject: decodable1, + }, + { + into: &runtime.VersionedObjects{Objects: []runtime.Object{}}, + decodes: []unversioned.GroupVersion{{Group: "something", Version: "else"}}, + + serializer: &mockSerializer{actual: gvk1, obj: decodable1}, + expectedGVK: gvk1, + expectedObject: &runtime.VersionedObjects{Objects: []runtime.Object{decodable1}}, + }, + } + + for i, test := range testCases { + t.Logf("%d", i) + s := NewCodec(test.serializer, test.convertor, test.creater, test.copier, test.typer, test.encodes, test.decodes) + obj, gvk, err := s.Decode([]byte(`{}`), test.defaultGVK, test.into) + + if !reflect.DeepEqual(test.expectedGVK, gvk) { + t.Errorf("%d: unexpected GVK: %v", i, gvk) + } + + switch { + case err == nil && test.errFn != nil: + t.Errorf("%d: failed: %v", i, err) + continue + case err != nil && test.errFn == nil: + t.Errorf("%d: failed: %v", i, err) + continue + case err != nil: + if !test.errFn(err) { + t.Errorf("%d: failed: %v", i, err) + } + if obj != nil { + t.Errorf("%d: should have returned nil object", i) + } + continue + } + + if test.into != nil && test.into != obj { + t.Errorf("%d: expected into to be returned: %v", i, obj) + continue + } + + switch { + case test.expectedObject != nil: + if !reflect.DeepEqual(test.expectedObject, obj) { + t.Errorf("%d: unexpected object:\n%s", i, util.ObjectGoPrintSideBySide(test.expectedObject, obj)) + } + case test.sameObject != nil: + if test.sameObject != obj { + t.Errorf("%d: unexpected object:\n%s", i, util.ObjectGoPrintSideBySide(test.sameObject, obj)) + } + case obj != nil: + t.Errorf("%d: unexpected object: %#v", i, obj) + } + } +} + +type checkCopy struct { + in, obj runtime.Object + err error +} + +func (c *checkCopy) Copy(obj runtime.Object) (runtime.Object, error) { + if c.in != nil && c.in != obj { + return nil, fmt.Errorf("unexpected input to copy: %#v", obj) + } + return c.obj, c.err +} + +type checkConvertor struct { + err error + in, obj runtime.Object + groupVersion string + directConvert bool +} + +func (c *checkConvertor) Convert(in, out interface{}) error { + if !c.directConvert { + return fmt.Errorf("unexpected call to Convert") + } + if c.in != nil && c.in != in { + return fmt.Errorf("unexpected in: %s", in) + } + if c.obj != nil && c.obj != out { + return fmt.Errorf("unexpected out: %s", out) + } + return c.err +} +func (c *checkConvertor) ConvertToVersion(in runtime.Object, outVersion string) (out runtime.Object, err error) { + if c.directConvert { + return nil, fmt.Errorf("unexpected call to ConvertToVersion") + } + if c.in != nil && c.in != in { + return nil, fmt.Errorf("unexpected in: %s", in) + } + if c.groupVersion != outVersion { + return nil, fmt.Errorf("unexpected outversion: %s", outVersion) + } + return c.obj, c.err +} +func (c *checkConvertor) ConvertFieldLabel(version, kind, label, value string) (string, string, error) { + return "", "", fmt.Errorf("unexpected call to ConvertFieldLabel") +} + +type mockSerializer struct { + err error + obj runtime.Object + versions []unversioned.GroupVersion + + defaults, actual *unversioned.GroupVersionKind + into runtime.Object +} + +func (s *mockSerializer) Decode(data []byte, defaults *unversioned.GroupVersionKind, into runtime.Object) (runtime.Object, *unversioned.GroupVersionKind, error) { + s.defaults = defaults + s.into = into + return s.obj, s.actual, s.err +} + +func (s *mockSerializer) EncodeToStream(obj runtime.Object, w io.Writer, versions ...unversioned.GroupVersion) error { + s.obj = obj + s.versions = versions + return s.err +} + +type mockCreater struct { + err error + obj runtime.Object +} + +func (c *mockCreater) New(kind unversioned.GroupVersionKind) (runtime.Object, error) { + return c.obj, c.err +} + +type mockTyper struct { + gvk *unversioned.GroupVersionKind + err error +} + +func (t *mockTyper) ObjectKind(obj runtime.Object) (*unversioned.GroupVersionKind, bool, error) { + return t.gvk, false, t.err +} diff --git a/pkg/runtime/serializer/yaml/yaml.go b/pkg/runtime/serializer/yaml/yaml.go new file mode 100644 index 00000000000..637c777be92 --- /dev/null +++ b/pkg/runtime/serializer/yaml/yaml.go @@ -0,0 +1,46 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yaml + +import ( + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/util/yaml" +) + +// yamlSerializer converts YAML passed to the Decoder methods to JSON. +type yamlSerializer struct { + // the nested serializer + runtime.Serializer +} + +// yamlSerializer implements Serializer +var _ runtime.Serializer = yamlSerializer{} + +// NewDecodingSerializer adds YAML decoding support to a serializer that supports JSON. +func NewDecodingSerializer(jsonSerializer runtime.Serializer) runtime.Serializer { + return &yamlSerializer{jsonSerializer} +} + +func (c yamlSerializer) Decode(data []byte, gvk *unversioned.GroupVersionKind, into runtime.Object) (runtime.Object, *unversioned.GroupVersionKind, error) { + out, err := yaml.ToJSON(data) + if err != nil { + return nil, nil, err + } + data = out + return c.Serializer.Decode(data, gvk, into) +} diff --git a/pkg/util/yaml/decoder.go b/pkg/util/yaml/decoder.go index ebe597a598c..5846fc20ff4 100644 --- a/pkg/util/yaml/decoder.go +++ b/pkg/util/yaml/decoder.go @@ -136,7 +136,7 @@ func NewYAMLOrJSONDecoder(r io.Reader, bufferSize int) *YAMLOrJSONDecoder { // provide object, or returns an error. func (d *YAMLOrJSONDecoder) Decode(into interface{}) error { if d.decoder == nil { - buffer, isJSON := guessJSONStream(d.r, d.bufferSize) + buffer, isJSON := GuessJSONStream(d.r, d.bufferSize) if isJSON { glog.V(4).Infof("decoding stream as JSON") d.decoder = json.NewDecoder(buffer) @@ -148,10 +148,10 @@ func (d *YAMLOrJSONDecoder) Decode(into interface{}) error { return d.decoder.Decode(into) } -// guessJSONStream scans the provided reader up to size, looking +// GuessJSONStream scans the provided reader up to size, looking // for an open brace indicating this is JSON. It will return the // bufio.Reader it creates for the consumer. -func guessJSONStream(r io.Reader, size int) (io.Reader, bool) { +func GuessJSONStream(r io.Reader, size int) (io.Reader, bool) { buffer := bufio.NewReaderSize(r, size) b, _ := buffer.Peek(size) return buffer, hasJSONPrefix(b)