Merge pull request #80943 from obitech/yaml_meta

Add a YAML MetaFactory

cmd/kubeadm/app/util/BUILD

@@ -25,6 +25,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/yaml:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
@@ -35,7 +36,6 @@ go_library(
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
-        "//vendor/sigs.k8s.io/yaml:go_default_library",
     ],
 )
 

cmd/kubeadm/app/util/marshal.go

@@ -22,11 +22,11 @@ import (
 	"io"
 
 	"github.com/pkg/errors"
-	"sigs.k8s.io/yaml"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	yamlserializer "k8s.io/apimachinery/pkg/runtime/serializer/yaml"
 	errorsutil "k8s.io/apimachinery/pkg/util/errors"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
 	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
@@ -81,7 +81,6 @@ func SplitYAMLDocuments(yamlBytes []byte) (kubeadmapi.DocumentMap, error) {
 	buf := bytes.NewBuffer(yamlBytes)
 	reader := utilyaml.NewYAMLReader(bufio.NewReader(buf))
 	for {
-		typeMetaInfo := runtime.TypeMeta{}
 		// Read one YAML document at a time, until io.EOF is returned
 		b, err := reader.Read()
 		if err == io.EOF {
@@ -93,31 +92,23 @@ func SplitYAMLDocuments(yamlBytes []byte) (kubeadmapi.DocumentMap, error) {
 			break
 		}
 		// Deserialize the TypeMeta information of this byte slice
-		if err := yaml.Unmarshal(b, &typeMetaInfo); err != nil {
+		gvk, err := yamlserializer.DefaultMetaFactory.Interpret(b)
+		if err != nil {
 			return nil, err
 		}
-		// Require TypeMeta information to be present
-		if len(typeMetaInfo.APIVersion) == 0 || len(typeMetaInfo.Kind) == 0 {
-			errs = append(errs, errors.New("invalid configuration: kind and apiVersion is mandatory information that needs to be specified in all YAML documents"))
-			continue
+		if len(gvk.Group) == 0 || len(gvk.Version) == 0 || len(gvk.Kind) == 0 {
+			return nil, errors.New("invalid configuration: kind and apiVersion is mandatory information that needs to be specified")
 		}
-		// Check whether the kind has been registered before. If it has, throw an error
-		if known := knownKinds[typeMetaInfo.Kind]; known {
-			errs = append(errs, errors.Errorf("invalid configuration: kind %q is specified twice in YAML file", typeMetaInfo.Kind))
-			continue
-		}
-		knownKinds[typeMetaInfo.Kind] = true
 
-		// Build a GroupVersionKind object from the deserialized TypeMeta object
-		gv, err := schema.ParseGroupVersion(typeMetaInfo.APIVersion)
-		if err != nil {
-			errs = append(errs, errors.Wrap(err, "unable to parse apiVersion"))
+		// Check whether the kind has been registered before. If it has, throw an error
+		if known := knownKinds[gvk.Kind]; known {
+			errs = append(errs, errors.Errorf("invalid configuration: kind %q is specified twice in YAML file", gvk.Kind))
 			continue
 		}
-		gvk := gv.WithKind(typeMetaInfo.Kind)
+		knownKinds[gvk.Kind] = true
 
 		// Save the mapping between the gvk and the bytes that object consists of
-		gvkmap[gvk] = b
+		gvkmap[*gvk] = b
 	}
 	if err := errorsutil.NewAggregate(errs); err != nil {
 		return nil, err

staging/src/k8s.io/apimachinery/pkg/runtime/serializer/yaml/BUILD

@@ -4,13 +4,17 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["yaml.go"],
+    srcs = [
+        "meta.go",
+        "yaml.go",
+    ],
     importmap = "k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime/serializer/yaml",
     importpath = "k8s.io/apimachinery/pkg/runtime/serializer/yaml",
     deps = [
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
+        "//vendor/sigs.k8s.io/yaml:go_default_library",
     ],
 )
 
@@ -29,10 +33,14 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = ["yaml_test.go"],
+    srcs = [
+        "meta_test.go",
+        "yaml_test.go",
+    ],
     data = glob(["testdata/**"]),
     embed = [":go_default_library"],
     deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
         "//vendor/sigs.k8s.io/yaml:go_default_library",
     ],

staging/src/k8s.io/apimachinery/pkg/runtime/serializer/yaml/meta.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package yaml
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/yaml"
+)
+
+// DefaultMetaFactory is a default factory for versioning objects in JSON or
+// YAML. The object in memory and in the default serialization will use the
+// "kind" and "apiVersion" fields.
+var DefaultMetaFactory = SimpleMetaFactory{}
+
+// SimpleMetaFactory provides default methods for retrieving the type and version of objects
+// that are identified with an "apiVersion" and "kind" fields in their JSON
+// serialization. It may be parameterized with the names of the fields in memory, or an
+// optional list of base structs to search for those fields in memory.
+type SimpleMetaFactory struct{}
+
+// Interpret will return the APIVersion and Kind of the JSON wire-format
+// encoding of an object, or an error.
+func (SimpleMetaFactory) Interpret(data []byte) (*schema.GroupVersionKind, error) {
+	gvk := runtime.TypeMeta{}
+	if err := yaml.Unmarshal(data, &gvk); err != nil {
+		return nil, fmt.Errorf("could not interpret GroupVersionKind; unmarshal error: %v", err)
+	}
+	gv, err := schema.ParseGroupVersion(gvk.APIVersion)
+	if err != nil {
+		return nil, err
+	}
+	return &schema.GroupVersionKind{Group: gv.Group, Version: gv.Version, Kind: gvk.Kind}, nil
+}

staging/src/k8s.io/apimachinery/pkg/runtime/serializer/yaml/meta_test.go

@@ -0,0 +1,160 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package yaml
+
+import (
+	"reflect"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func TestInterpret(t *testing.T) {
+	testCases := []struct {
+		name     string
+		input    string
+		expected *schema.GroupVersionKind
+		errFn    func(error) bool
+	}{
+		{
+			name: "YAMLSuccessfullyInterpretVK",
+			input: `apiVersion: v1
+kind: Service`,
+			expected: &schema.GroupVersionKind{Version: "v1", Kind: "Service"},
+		},
+		{
+			name: "YAMLSuccessfullyInterpretGVK",
+			input: `apiVersion: core/v2
+kind: Deployment`,
+			expected: &schema.GroupVersionKind{Group: "core", Version: "v2", Kind: "Deployment"},
+		},
+		{
+			name:     "YAMLSuccessfullyInterpretV",
+			input:    `apiVersion: v1`,
+			expected: &schema.GroupVersionKind{Version: "v1"},
+		},
+		{
+			name:     "YAMLSuccessfullyInterpretK",
+			input:    `kind: Service`,
+			expected: &schema.GroupVersionKind{Kind: "Service"},
+		},
+		{
+			name:     "YAMLSuccessfullyInterpretEmptyString",
+			input:    ``,
+			expected: &schema.GroupVersionKind{},
+		},
+		{
+			name:     "YAMLSuccessfullyInterpretEmptyDoc",
+			input:    `---`,
+			expected: &schema.GroupVersionKind{},
+		},
+		{
+			name: "YAMLSuccessfullyInterpretMultiDoc",
+			input: `---
+apiVersion: v1
+kind: Service
+---
+apiVersion: v2
+kind: Deployment`,
+			expected: &schema.GroupVersionKind{Version: "v1", Kind: "Service"},
+		},
+		{
+			name:     "YAMLSuccessfullyInterpretOnlyG",
+			input:    `apiVersion: core/`,
+			expected: &schema.GroupVersionKind{Group: "core"},
+		},
+		{
+			name:     "YAMLSuccessfullyWrongFormat",
+			input:    `foo: bar`,
+			expected: &schema.GroupVersionKind{},
+		},
+		{
+			name:  "YAMLFailInterpretWrongSyntax",
+			input: `foo`,
+			errFn: func(err error) bool { return err != nil },
+		},
+		{
+			name:     "JSONSuccessfullyInterpretVK",
+			input:    `{"apiVersion": "v3", "kind": "DaemonSet"}`,
+			expected: &schema.GroupVersionKind{Version: "v3", Kind: "DaemonSet"},
+		},
+		{
+			name:     "JSONSuccessfullyInterpretGVK",
+			input:    `{"apiVersion": "core/v2", "kind": "Deployment"}`,
+			expected: &schema.GroupVersionKind{Group: "core", Version: "v2", Kind: "Deployment"},
+		},
+		{
+			name:     "JSONSuccessfullyInterpretV",
+			input:    `{"apiVersion": "v1"}`,
+			expected: &schema.GroupVersionKind{Version: "v1"},
+		},
+		{
+			name:     "JSONSuccessfullyInterpretK",
+			input:    `{"kind": "Service"}`,
+			expected: &schema.GroupVersionKind{Kind: "Service"},
+		},
+		{
+			name:     "JSONSuccessfullyInterpretEmptyString",
+			input:    ``,
+			expected: &schema.GroupVersionKind{},
+		},
+		{
+			name:     "JSONSuccessfullyInterpretEmptyObject",
+			input:    `{}`,
+			expected: &schema.GroupVersionKind{},
+		},
+		{
+			name: "JSONSuccessfullyInterpretMultiDoc",
+			input: `{"apiVersion": "v1", "kind": "Service"}, 
+{"apiVersion": "v2", "kind": "Deployment"}`,
+			expected: &schema.GroupVersionKind{Version: "v1", Kind: "Service"},
+		},
+		{
+			name:     "JSONSuccessfullyWrongFormat",
+			input:    `{"foo": "bar"}`,
+			expected: &schema.GroupVersionKind{},
+		},
+		{
+			name:  "JSONFailInterpretArray",
+			input: `[]`,
+			errFn: func(err error) bool { return err != nil },
+		},
+		{
+			name:  "JSONFailInterpretWrongSyntax",
+			input: `{"foo"`,
+			errFn: func(err error) bool { return err != nil },
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			actual, err := DefaultMetaFactory.Interpret([]byte(test.input))
+			switch {
+			case test.errFn != nil:
+				if !test.errFn(err) {
+					t.Errorf("unexpected error: %v", err)
+				}
+			case err != nil:
+				t.Errorf("unexpected error: %v", err)
+			case !reflect.DeepEqual(test.expected, actual):
+				t.Errorf("outcome mismatch -- expected: %#v, actual: %#v",
+					test.expected, actual,
+				)
+			}
+		})
+	}
+}

vendor/modules.txt

@@ -1239,6 +1239,7 @@ k8s.io/apimachinery/pkg/runtime/serializer/protobuf
 k8s.io/apimachinery/pkg/runtime/serializer/recognizer
 k8s.io/apimachinery/pkg/runtime/serializer/streaming
 k8s.io/apimachinery/pkg/runtime/serializer/versioning
+k8s.io/apimachinery/pkg/runtime/serializer/yaml
 k8s.io/apimachinery/pkg/selection
 k8s.io/apimachinery/pkg/types
 k8s.io/apimachinery/pkg/util/cache