Merge pull request #103564 from kevindelgado/unstr-extr-poc

ExtractItems for unstructured apply configurations

cmd/kubeadm/.import-restrictions

@@ -5,6 +5,7 @@ rules:
   - selectorRegexp: k8s[.]io/kube-openapi/
     allowedPrefixes:
       - k8s.io/kube-openapi/pkg/util/proto
+      - k8s.io/kube-openapi/pkg/schemaconv
   - selectorRegexp: k8s[.]io/kube-proxy/
     allowedPrefixes:
       - k8s.io/kube-proxy/config/v1alpha1

staging/publishing/import-restrictions.yaml

@@ -67,6 +67,7 @@
   - k8s.io/apimachinery
   - k8s.io/client-go
   - k8s.io/klog
+  - k8s.io/kube-openapi
   - k8s.io/utils
 
 # prevent core machinery from taking explicit v1 references unless

staging/src/k8s.io/apimachinery/pkg/util/managedfields/extract.go

@@ -75,6 +75,13 @@ func ExtractInto(object runtime.Object, objectType typed.ParseableType, fieldMan
 	if !ok {
 		return fmt.Errorf("unable to convert managed fields for %s to unstructured, expected map, got %T", fieldManager, u)
 	}
+
+	// set the type meta manually if it doesn't exist to avoid missing kind errors
+	// when decoding from unstructured JSON
+	if _, ok := m["kind"]; !ok && object.GetObjectKind().GroupVersionKind().Kind != "" {
+		m["kind"] = object.GetObjectKind().GroupVersionKind().Kind
+		m["apiVersion"] = object.GetObjectKind().GroupVersionKind().GroupVersion().String()
+	}
 	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(m, applyConfiguration); err != nil {
 		return fmt.Errorf("error extracting into obj from unstructured: %w", err)
 	}

staging/src/k8s.io/apimachinery/pkg/util/managedfields/gvkparser.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package internal
+package managedfields
 
 import (
 	"fmt"

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/typeconverter.go

@@ -22,7 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
+	"k8s.io/apimachinery/pkg/util/managedfields"
 	"k8s.io/kube-openapi/pkg/util/proto"
 	"sigs.k8s.io/structured-merge-diff/v4/typed"
 	"sigs.k8s.io/structured-merge-diff/v4/value"
@@ -65,7 +65,7 @@ func (DeducedTypeConverter) TypedToObject(value *typed.TypedValue) (runtime.Obje
 }
 
 type typeConverter struct {
-	parser *internal.GvkParser
+	parser *managedfields.GvkParser
 }
 
 var _ TypeConverter = &typeConverter{}
@@ -74,7 +74,7 @@ var _ TypeConverter = &typeConverter{}
 // will automatically find the proper version of the object, and the
 // corresponding schema information.
 func NewTypeConverter(models proto.Models, preserveUnknownFields bool) (TypeConverter, error) {
-	parser, err := internal.NewGVKParser(models, preserveUnknownFields)
+	parser, err := managedfields.NewGVKParser(models, preserveUnknownFields)
 	if err != nil {
 		return nil, err
 	}

staging/src/k8s.io/client-go/applyconfigurations/meta/v1/unstructured.go

@@ -0,0 +1,137 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/managedfields"
+	"k8s.io/client-go/discovery"
+	"k8s.io/kube-openapi/pkg/util/proto"
+	"sigs.k8s.io/structured-merge-diff/v4/typed"
+)
+
+// openAPISchemaTTL is how frequently we need to check
+// whether the open API schema has changed or not.
+const openAPISchemaTTL = time.Minute
+
+// UnstructuredExtractor enables extracting the applied configuration state from object for fieldManager into an
+// unstructured object type.
+type UnstructuredExtractor interface {
+	Extract(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error)
+	ExtractStatus(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error)
+}
+
+// gvkParserCache caches the GVKParser in order to prevent from having to repeatedly
+// parse the models from the open API schema when the schema itself changes infrequently.
+type gvkParserCache struct {
+	// discoveryClient is the client for retrieving the openAPI document and checking
+	// whether the document has changed recently
+	discoveryClient discovery.DiscoveryInterface
+	// mu protects the gvkParser
+	mu sync.Mutex
+	// gvkParser retrieves the objectType for a given gvk
+	gvkParser *managedfields.GvkParser
+	// lastChecked is the last time we checked if the openAPI doc has changed.
+	lastChecked time.Time
+}
+
+// regenerateGVKParser builds the parser from the raw OpenAPI schema.
+func regenerateGVKParser(dc discovery.DiscoveryInterface) (*managedfields.GvkParser, error) {
+	doc, err := dc.OpenAPISchema()
+	if err != nil {
+		return nil, err
+	}
+
+	models, err := proto.NewOpenAPIData(doc)
+	if err != nil {
+		return nil, err
+	}
+
+	return managedfields.NewGVKParser(models, false)
+}
+
+// objectTypeForGVK retrieves the typed.ParseableType for a given gvk from the cache
+func (c *gvkParserCache) objectTypeForGVK(gvk schema.GroupVersionKind) (*typed.ParseableType, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	// if the ttl on the openAPISchema has expired,
+	// regenerate the gvk parser
+	if time.Since(c.lastChecked) > openAPISchemaTTL {
+		c.lastChecked = time.Now()
+		parser, err := regenerateGVKParser(c.discoveryClient)
+		if err != nil {
+			return nil, err
+		}
+		c.gvkParser = parser
+	}
+	return c.gvkParser.Type(gvk), nil
+}
+
+type extractor struct {
+	cache *gvkParserCache
+}
+
+// NewUnstructuredExtractor creates the extractor with which you can extract the applied configuration
+// for a given manager from an unstructured object.
+func NewUnstructuredExtractor(dc discovery.DiscoveryInterface) (UnstructuredExtractor, error) {
+	parser, err := regenerateGVKParser(dc)
+	if err != nil {
+		return nil, fmt.Errorf("failed generating initial GVK Parser: %v", err)
+	}
+	return &extractor{
+		cache: &gvkParserCache{
+			gvkParser:       parser,
+			discoveryClient: dc,
+		},
+	}, nil
+}
+
+// Extract extracts the applied configuration owned by fiieldManager from an unstructured object.
+// Note that the apply configuration itself is also an unstructured object.
+func (e *extractor) Extract(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error) {
+	return e.extractUnstructured(object, fieldManager, "")
+}
+
+// ExtractStatus is the same as ExtractUnstructured except
+// that it extracts the status subresource applied configuration.
+// Experimental!
+func (e *extractor) ExtractStatus(object *unstructured.Unstructured, fieldManager string) (*unstructured.Unstructured, error) {
+	return e.extractUnstructured(object, fieldManager, "status")
+}
+
+func (e *extractor) extractUnstructured(object *unstructured.Unstructured, fieldManager string, subresource string) (*unstructured.Unstructured, error) {
+	gvk := object.GetObjectKind().GroupVersionKind()
+	objectType, err := e.cache.objectTypeForGVK(gvk)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch the objectType: %v", err)
+	}
+	result := &unstructured.Unstructured{}
+	err = managedfields.ExtractInto(object, *objectType, fieldManager, result, subresource)
+	if err != nil {
+		return nil, fmt.Errorf("failed calling ExtractInto for unstructured: %v", err)
+	}
+	result.SetName(object.GetName())
+	result.SetNamespace(object.GetNamespace())
+	result.SetKind(object.GetKind())
+	result.SetAPIVersion(object.GetAPIVersion())
+	return result, nil
+}

staging/src/k8s.io/client-go/go.mod

@@ -33,6 +33,7 @@ require (
 	k8s.io/api v0.0.0
 	k8s.io/apimachinery v0.0.0
 	k8s.io/klog/v2 v2.9.0
+	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e
 	k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.2
 	sigs.k8s.io/yaml v1.2.0

staging/src/k8s.io/component-base/go.sum

@@ -100,6 +100,7 @@ github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -266,14 +267,17 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -653,6 +657,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

test/integration/client/dynamic_client_test.go

@@ -18,6 +18,7 @@ package client
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"reflect"
 	"testing"
@@ -29,8 +30,11 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
+	metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	clientscheme "k8s.io/client-go/kubernetes/scheme"
@@ -203,6 +207,75 @@ func TestDynamicClientWatch(t *testing.T) {
 	}
 }
 
+func TestUnstructuredExtract(t *testing.T) {
+	result := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins", "ServiceAccount"}, framework.SharedEtcd())
+	defer result.TearDownFn()
+
+	dynamicClient, err := dynamic.NewForConfig(result.ClientConfig)
+	if err != nil {
+		t.Fatalf("unexpected error creating dynamic client: %v", err)
+	}
+
+	resource := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
+
+	// Apply an unstructured with the dynamic client
+	name := "test-pod"
+	pod := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "v1",
+			"kind":       "Pod",
+			"metadata": map[string]interface{}{
+				"name": name,
+				// namespace will always get set by extract,
+				// so we add it here (even though it's optional)
+				// to ensure what we apply equals what we extract.
+				"namespace": "default",
+			},
+			"spec": map[string]interface{}{
+				"containers": []interface{}{
+					map[string]interface{}{
+						"name":  "test",
+						"image": "test-image",
+					},
+				},
+			},
+		},
+	}
+	mgr := "testManager"
+	podData, err := json.Marshal(pod)
+	if err != nil {
+		t.Fatalf("failed to marshal pod into bytes: %v", err)
+	}
+
+	// apply the unstructured object to the cluster
+	actual, err := dynamicClient.Resource(resource).Namespace("default").Patch(
+		context.TODO(),
+		name,
+		types.ApplyPatchType,
+		podData,
+		metav1.PatchOptions{FieldManager: mgr})
+	if err != nil {
+		t.Fatalf("unexpected error when creating pod: %v", err)
+	}
+
+	// extract the object
+	discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(result.ClientConfig)
+	extractor, err := metav1ac.NewUnstructuredExtractor(discoveryClient)
+	if err != nil {
+		t.Fatalf("unexpected error when constructing extrator: %v", err)
+	}
+	extracted, err := extractor.Extract(actual, mgr)
+	if err != nil {
+		t.Fatalf("unexpected error when extracting: %v", err)
+	}
+
+	// confirm that the extracted object equals the applied object
+	if !reflect.DeepEqual(pod, extracted) {
+		t.Fatalf("extracted pod doesn't equal applied pod. wanted:\n %v\n, got:\n %v\n", pod, extracted)
+	}
+
+}
+
 func unstructuredToPod(obj *unstructured.Unstructured) (*v1.Pod, error) {
 	json, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
 	if err != nil {