From c364b639fb48e13a5b0925309b00226d3758999c Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Mon, 22 Aug 2022 16:42:18 -0700 Subject: [PATCH 01/16] add function to upgrade managedfields CSA to SSA Kubernetes-commit: 27cd307e23df3a2d508d52ff10ac8f46bf3bcea3 --- util/csaupgrade/upgrade.go | 134 ++++++++++++++++++++++++++++++++ util/csaupgrade/upgrade_test.go | 121 ++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 util/csaupgrade/upgrade.go create mode 100644 util/csaupgrade/upgrade_test.go diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go new file mode 100644 index 00000000..a4ea0879 --- /dev/null +++ b/util/csaupgrade/upgrade.go @@ -0,0 +1,134 @@ +package csaupgrade + +import ( + "encoding/json" + "fmt" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" + "sigs.k8s.io/structured-merge-diff/v4/fieldpath" +) + +// Upgrades the Manager information for fields managed with CSA +// Prepares fields owned by `csaManager` for 'Update' operations for use now +// with the given `ssaManager` for `Apply` operations +// +// csaManager - Name of FieldManager formerly used for `Update` operations +// ssaManager - Name of FieldManager formerly used for `Apply` operations +// subResource - Name of subresource used for api calls or empty string for main resource +func UpgradeManagedFields( + obj runtime.Object, + csaManager string, + ssaManager string, + subResource string, +) (runtime.Object, error) { + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, fmt.Errorf("error accessing object metadata: %w", err) + } + + managed, error := fieldmanager.DecodeManagedFields(accessor.GetManagedFields()) + if error != nil { + return nil, fmt.Errorf("failed to decode managed fields: %w", error) + } + // If SSA manager exists: + // find CSA manager of same version, union. discard the rest + // Else SSA manager does not exist: + // find most recent CSA manager. convert to Apply operation + + ssaIdentifier, err := fieldmanager.BuildManagerIdentifier(&metav1.ManagedFieldsEntry{ + Manager: ssaManager, + Operation: metav1.ManagedFieldsOperationApply, + Subresource: subResource, + }) + if err != nil { + return nil, fmt.Errorf("failed to build manager identifier for ssa manager") + } + + ssaMan, ssaExists := managed.Fields()[ssaIdentifier] + + // Collect all relevant CSA managers before operating on them + csaManagers := map[string]fieldpath.VersionedSet{} + for name, entry := range managed.Fields() { + if entry.Applied() { + // Not interested in SSA managed fields entries + continue + } + + // Manager string is a JSON representation of encoded entry + // Pull manager name and subresource from it + encodedVersionedSet := &metav1.ManagedFieldsEntry{} + err = json.Unmarshal([]byte(name), encodedVersionedSet) + if err != nil { + return nil, fmt.Errorf("error unmarshalling manager identifier %v: %v", name, err) + } + + if encodedVersionedSet.Manager != csaManager || + encodedVersionedSet.Subresource != subResource { + continue + } + + csaManagers[name] = entry + } + + if len(csaManagers) == 0 { + return obj, nil + } + + if ssaExists { + for name, entry := range csaManagers { + if entry.APIVersion() == ssaMan.APIVersion() { + // Merge entries if they are compatible versions + ssaMan = fieldpath.NewVersionedSet( + ssaMan.Set().Union(entry.Set()), + entry.APIVersion(), + true, + ) + managed.Fields()[ssaIdentifier] = ssaMan + } + + // Discard entry in all cases: + // if it has the wrong version we discard since managed fields versions + // cannot be converted + // if it has the correct version its fields were moved into the + // ssaManager's fieldSet + delete(managed.Fields(), name) + } + } else { + // Loop through sorted CSA managers. Take the first one we care about + firstName := "" + for _, entry := range accessor.GetManagedFields() { + if entry.Manager == csaManager && + entry.Subresource == subResource && + entry.Operation == metav1.ManagedFieldsOperationUpdate { + + if len(firstName) == 0 { + ident, err := fieldmanager.BuildManagerIdentifier(&entry) + if err != nil { + return nil, fmt.Errorf("failed to build manager identifier: %w", err) + } + + firstName = ident + break + } + } + } + + managed.Fields()[ssaIdentifier] = csaManagers[firstName] + + for name := range csaManagers { + delete(managed.Fields(), name) + } + } + + now := metav1.Now() + managed.Times()[ssaIdentifier] = &now + + copied := obj.DeepCopyObject() + if err := fieldmanager.EncodeObjectManagedFields(copied, managed); err != nil { + return nil, err + } + return copied, nil +} diff --git a/util/csaupgrade/upgrade_test.go b/util/csaupgrade/upgrade_test.go new file mode 100644 index 00000000..4113bfa7 --- /dev/null +++ b/util/csaupgrade/upgrade_test.go @@ -0,0 +1,121 @@ +package csaupgrade_test + +import ( + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/conversion" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/util/csaupgrade" +) + +var avoidTimestampEqualities = func() conversion.Equalities { + var eqs = equality.Semantic.Copy() + + err := eqs.AddFunc( + func(a, b metav1.ManagedFieldsEntry) bool { + // Two objects' managed fields are equivalent if, ignoring timestamp, + // the objects are deeply equal. + a.Time = nil + b.Time = nil + return reflect.DeepEqual(a, b) + }, + ) + + if err != nil { + panic(err) + } + + return eqs +}() + +func TestUpgradeCSA(t *testing.T) { + // Initial object has managed fields from using CSA + originalYAML := []byte(` +apiVersion: v1 +data: + key: value + legacy: unused +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + creationTimestamp: "2022-08-22T23:08:23Z" + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + name: test + namespace: default + resourceVersion: "12502" + uid: 1f186675-58e6-4d7b-8bc5-7ece581e3013 +`) + initialObject := unstructured.Unstructured{} + err := yaml.Unmarshal(originalYAML, &initialObject) + if err != nil { + t.Fatal(err) + } + + upgraded, err := csaupgrade.UpgradeManagedFields(&initialObject, "kubectl-client-side-apply", "kubectl", "") + if err != nil { + t.Fatal(err) + } + + expectedYAML := []byte(` +apiVersion: v1 +data: + key: value + legacy: unused +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + creationTimestamp: "2022-08-22T23:08:23Z" + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl + operation: Apply + time: "2022-08-22T23:08:23Z" + name: test + namespace: default + resourceVersion: "12502" + uid: 1f186675-58e6-4d7b-8bc5-7ece581e3013 +`) + + expectedObject := unstructured.Unstructured{} + err = yaml.Unmarshal(expectedYAML, &expectedObject) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(&expectedObject, upgraded) { + t.Fatal(cmp.Diff(&expectedObject, upgraded)) + } +} From 90ef078c30dd765348a70fe514efdd436054e59c Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Tue, 23 Aug 2022 16:12:47 -0700 Subject: [PATCH 02/16] dont expose internal methods in implementatoin Kubernetes-commit: fe2b5d00f2c1f16636638acd10b8d640e6de22c9 --- util/csaupgrade/upgrade.go | 199 +++++++++++++++++++++---------------- 1 file changed, 111 insertions(+), 88 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index a4ea0879..e082f30e 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -1,13 +1,12 @@ package csaupgrade import ( - "encoding/json" + "bytes" "fmt" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" "sigs.k8s.io/structured-merge-diff/v4/fieldpath" ) @@ -20,8 +19,8 @@ import ( // subResource - Name of subresource used for api calls or empty string for main resource func UpgradeManagedFields( obj runtime.Object, - csaManager string, - ssaManager string, + csaManagerName string, + ssaManagerName string, subResource string, ) (runtime.Object, error) { accessor, err := meta.Accessor(obj) @@ -29,106 +28,130 @@ func UpgradeManagedFields( return nil, fmt.Errorf("error accessing object metadata: %w", err) } - managed, error := fieldmanager.DecodeManagedFields(accessor.GetManagedFields()) - if error != nil { - return nil, fmt.Errorf("failed to decode managed fields: %w", error) - } - // If SSA manager exists: - // find CSA manager of same version, union. discard the rest - // Else SSA manager does not exist: - // find most recent CSA manager. convert to Apply operation + // Create managed fields clone since we modify the values + var managedFields []metav1.ManagedFieldsEntry + managedFields = append(managedFields, accessor.GetManagedFields()...) - ssaIdentifier, err := fieldmanager.BuildManagerIdentifier(&metav1.ManagedFieldsEntry{ - Manager: ssaManager, - Operation: metav1.ManagedFieldsOperationApply, - Subresource: subResource, - }) - if err != nil { - return nil, fmt.Errorf("failed to build manager identifier for ssa manager") - } + // Locate SSA manager + ssaManagerIndex, ssaManagerExists := findFirstIndex(managedFields, + func(entry metav1.ManagedFieldsEntry) bool { + return entry.Manager == ssaManagerName && + entry.Operation == metav1.ManagedFieldsOperationApply && + entry.Subresource == "" + }) - ssaMan, ssaExists := managed.Fields()[ssaIdentifier] + if ssaManagerExists { + ssaManager := managedFields[ssaManagerIndex] - // Collect all relevant CSA managers before operating on them - csaManagers := map[string]fieldpath.VersionedSet{} - for name, entry := range managed.Fields() { - if entry.Applied() { - // Not interested in SSA managed fields entries - continue - } + // find Update manager of same APIVersion, union ssa fields with it. + // discard all other Update managers of the same name + csaManagerIndex, csaManagerExists := findFirstIndex(managedFields, + func(entry metav1.ManagedFieldsEntry) bool { + return entry.Manager == csaManagerName && + entry.Operation == metav1.ManagedFieldsOperationUpdate && + entry.Subresource == "" && + entry.APIVersion == ssaManager.APIVersion + }) - // Manager string is a JSON representation of encoded entry - // Pull manager name and subresource from it - encodedVersionedSet := &metav1.ManagedFieldsEntry{} - err = json.Unmarshal([]byte(name), encodedVersionedSet) - if err != nil { - return nil, fmt.Errorf("error unmarshalling manager identifier %v: %v", name, err) - } + if csaManagerExists { + csaManager := managedFields[csaManagerIndex] - if encodedVersionedSet.Manager != csaManager || - encodedVersionedSet.Subresource != subResource { - continue - } - - csaManagers[name] = entry - } - - if len(csaManagers) == 0 { - return obj, nil - } - - if ssaExists { - for name, entry := range csaManagers { - if entry.APIVersion() == ssaMan.APIVersion() { - // Merge entries if they are compatible versions - ssaMan = fieldpath.NewVersionedSet( - ssaMan.Set().Union(entry.Set()), - entry.APIVersion(), - true, - ) - managed.Fields()[ssaIdentifier] = ssaMan + // Union the csa manager with the existing SSA manager + ssaFieldSet, err := fieldsToSet(*ssaManager.FieldsV1) + if err != nil { + return nil, fmt.Errorf("failed to convert fields to set: %w", err) } - // Discard entry in all cases: - // if it has the wrong version we discard since managed fields versions - // cannot be converted - // if it has the correct version its fields were moved into the - // ssaManager's fieldSet - delete(managed.Fields(), name) + csaFieldSet, err := fieldsToSet(*csaManager.FieldsV1) + if err != nil { + return nil, fmt.Errorf("failed to convert fields to set: %w", err) + } + + combinedFieldSet := ssaFieldSet.Union(&csaFieldSet) + combinedFieldSetEncoded, err := setToFields(*combinedFieldSet) + if err != nil { + return nil, fmt.Errorf("failed to encode field set: %w", err) + } + + managedFields[ssaManagerIndex].FieldsV1 = &combinedFieldSetEncoded } } else { - // Loop through sorted CSA managers. Take the first one we care about - firstName := "" - for _, entry := range accessor.GetManagedFields() { - if entry.Manager == csaManager && - entry.Subresource == subResource && - entry.Operation == metav1.ManagedFieldsOperationUpdate { + // SSA manager does not exist. Find the most recent matching CSA manager, + // convert it to an SSA manager. + // + // (find first index, since managed fields are sorted so that most recent is + // first in the list) + csaManagerIndex, csaManagerExists := findFirstIndex(managedFields, func(entry metav1.ManagedFieldsEntry) bool { + return entry.Manager == csaManagerName && entry.Operation == metav1.ManagedFieldsOperationUpdate && entry.Subresource == "" + }) - if len(firstName) == 0 { - ident, err := fieldmanager.BuildManagerIdentifier(&entry) - if err != nil { - return nil, fmt.Errorf("failed to build manager identifier: %w", err) - } - - firstName = ident - break - } - } + if !csaManagerExists { + // There are no CSA managers that need to be converted. Nothing to do + // Return early + return obj, nil } - managed.Fields()[ssaIdentifier] = csaManagers[firstName] - - for name := range csaManagers { - delete(managed.Fields(), name) - } + // Convert the entry to apply operation + managedFields[csaManagerIndex].Operation = metav1.ManagedFieldsOperationApply + managedFields[csaManagerIndex].Manager = ssaManagerName } - now := metav1.Now() - managed.Times()[ssaIdentifier] = &now + // Create version of managed fields which has no CSA managers with the given name + filteredManagers := filter(managedFields, func(entry metav1.ManagedFieldsEntry) bool { + return !(entry.Manager == csaManagerName && + entry.Operation == metav1.ManagedFieldsOperationUpdate && + entry.Subresource == "") + }) copied := obj.DeepCopyObject() - if err := fieldmanager.EncodeObjectManagedFields(copied, managed); err != nil { - return nil, err + copiedAccessor, err := meta.Accessor(copied) + if err != nil { + return nil, fmt.Errorf("failed to get meta accessor for copied object: %w", err) } + copiedAccessor.SetManagedFields(filteredManagers) return copied, nil } + +func findFirstIndex[T any]( + collection []T, + predicate func(T) bool, +) (int, bool) { + for idx, entry := range collection { + if predicate(entry) { + return idx, true + } + } + + return -1, false +} + +func filter[T any]( + collection []T, + predicate func(T) bool, +) []T { + result := make([]T, 0, len(collection)) + + for _, value := range collection { + if predicate(value) { + result = append(result, value) + } + } + + if len(result) == 0 { + return nil + } + + return result +} + +// FieldsToSet creates a set paths from an input trie of fields +func fieldsToSet(f metav1.FieldsV1) (s fieldpath.Set, err error) { + err = s.FromJSON(bytes.NewReader(f.Raw)) + return s, err +} + +// SetToFields creates a trie of fields from an input set of paths +func setToFields(s fieldpath.Set) (f metav1.FieldsV1, err error) { + f.Raw, err = s.ToJSON() + return f, err +} From aa892ab1ac984c69f82b9417fedba9d07411b60c Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Tue, 23 Aug 2022 16:17:27 -0700 Subject: [PATCH 03/16] remove unused code Kubernetes-commit: f495d7656ca79e2d5cae03b37977e75369352645 --- util/csaupgrade/upgrade_test.go | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/util/csaupgrade/upgrade_test.go b/util/csaupgrade/upgrade_test.go index 4113bfa7..81b7f3f1 100644 --- a/util/csaupgrade/upgrade_test.go +++ b/util/csaupgrade/upgrade_test.go @@ -5,34 +5,11 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "k8s.io/apimachinery/pkg/api/equality" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/conversion" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/util/csaupgrade" ) -var avoidTimestampEqualities = func() conversion.Equalities { - var eqs = equality.Semantic.Copy() - - err := eqs.AddFunc( - func(a, b metav1.ManagedFieldsEntry) bool { - // Two objects' managed fields are equivalent if, ignoring timestamp, - // the objects are deeply equal. - a.Time = nil - b.Time = nil - return reflect.DeepEqual(a, b) - }, - ) - - if err != nil { - panic(err) - } - - return eqs -}() - func TestUpgradeCSA(t *testing.T) { // Initial object has managed fields from using CSA originalYAML := []byte(` From 049ba69f2f96ddcde5db1b1fc2be3a491645b2b4 Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Wed, 24 Aug 2022 10:03:16 -0700 Subject: [PATCH 04/16] expose FieldsToSet and SetToFields Kubernetes-commit: a338fff435c35a4da2556a8f4c4ab059c98e909c --- util/csaupgrade/upgrade.go | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index e082f30e..33330358 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -1,13 +1,12 @@ package csaupgrade import ( - "bytes" "fmt" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/structured-merge-diff/v4/fieldpath" + "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" ) // Upgrades the Manager information for fields managed with CSA @@ -57,18 +56,18 @@ func UpgradeManagedFields( csaManager := managedFields[csaManagerIndex] // Union the csa manager with the existing SSA manager - ssaFieldSet, err := fieldsToSet(*ssaManager.FieldsV1) + ssaFieldSet, err := fieldmanager.FieldsToSet(*ssaManager.FieldsV1) if err != nil { return nil, fmt.Errorf("failed to convert fields to set: %w", err) } - csaFieldSet, err := fieldsToSet(*csaManager.FieldsV1) + csaFieldSet, err := fieldmanager.FieldsToSet(*csaManager.FieldsV1) if err != nil { return nil, fmt.Errorf("failed to convert fields to set: %w", err) } combinedFieldSet := ssaFieldSet.Union(&csaFieldSet) - combinedFieldSetEncoded, err := setToFields(*combinedFieldSet) + combinedFieldSetEncoded, err := fieldmanager.SetToFields(*combinedFieldSet) if err != nil { return nil, fmt.Errorf("failed to encode field set: %w", err) } @@ -143,15 +142,3 @@ func filter[T any]( return result } - -// FieldsToSet creates a set paths from an input trie of fields -func fieldsToSet(f metav1.FieldsV1) (s fieldpath.Set, err error) { - err = s.FromJSON(bytes.NewReader(f.Raw)) - return s, err -} - -// SetToFields creates a trie of fields from an input set of paths -func setToFields(s fieldpath.Set) (f metav1.FieldsV1, err error) { - f.Raw, err = s.ToJSON() - return f, err -} From ced85a85218389116cdc486bf4670139a17b197f Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Wed, 24 Aug 2022 10:12:41 -0700 Subject: [PATCH 05/16] update godoc Kubernetes-commit: f94ef92f2ac44fb07bc8d75d5f250b757b343c11 --- util/csaupgrade/upgrade.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index 33330358..afd0391c 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -11,7 +11,16 @@ import ( // Upgrades the Manager information for fields managed with CSA // Prepares fields owned by `csaManager` for 'Update' operations for use now -// with the given `ssaManager` for `Apply` operations +// with the given `ssaManager` for `Apply` operations. +// +// Caveats: +// 1. This operation is not reversible. Information about which fields the client +// owned will be lost in this operation. +// 2. Supports being performed either before or after initial server-side apply. +// 3. Client-side apply tends to own more fields (including fields that are defaulted), +// this will possibly remove this defaults, they will be re-defaulted, that's fine. +// 4. Care must be taken to not overwrite the managed fields on the server if they +// have changed during the RMW operation. // // csaManager - Name of FieldManager formerly used for `Update` operations // ssaManager - Name of FieldManager formerly used for `Apply` operations From 46dc22f46a36a2b1920c96b29d5a09b08d7467ab Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Wed, 24 Aug 2022 10:24:05 -0700 Subject: [PATCH 06/16] clean up test Kubernetes-commit: 8f6d7f1fa13db3d7a3354a13f014e1ce7acabda0 --- util/csaupgrade/upgrade.go | 3 +- util/csaupgrade/upgrade_test.go | 78 +++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index afd0391c..372155a3 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -20,7 +20,7 @@ import ( // 3. Client-side apply tends to own more fields (including fields that are defaulted), // this will possibly remove this defaults, they will be re-defaulted, that's fine. // 4. Care must be taken to not overwrite the managed fields on the server if they -// have changed during the RMW operation. +// have changed before sending a patch. // // csaManager - Name of FieldManager formerly used for `Update` operations // ssaManager - Name of FieldManager formerly used for `Apply` operations @@ -29,7 +29,6 @@ func UpgradeManagedFields( obj runtime.Object, csaManagerName string, ssaManagerName string, - subResource string, ) (runtime.Object, error) { accessor, err := meta.Accessor(obj) if err != nil { diff --git a/util/csaupgrade/upgrade_test.go b/util/csaupgrade/upgrade_test.go index 81b7f3f1..8f9a2e74 100644 --- a/util/csaupgrade/upgrade_test.go +++ b/util/csaupgrade/upgrade_test.go @@ -11,12 +11,19 @@ import ( ) func TestUpgradeCSA(t *testing.T) { - // Initial object has managed fields from using CSA - originalYAML := []byte(` + + cases := []struct { + CSAManager string + SSAManager string + OriginalObject []byte + ExpectedObject []byte + }{ + { + CSAManager: "kubectl-client-side-apply", + SSAManager: "kubectl", + OriginalObject: []byte(` apiVersion: v1 -data: - key: value - legacy: unused +data: {} kind: ConfigMap metadata: annotations: @@ -40,25 +47,10 @@ metadata: time: "2022-08-22T23:08:23Z" name: test namespace: default - resourceVersion: "12502" - uid: 1f186675-58e6-4d7b-8bc5-7ece581e3013 -`) - initialObject := unstructured.Unstructured{} - err := yaml.Unmarshal(originalYAML, &initialObject) - if err != nil { - t.Fatal(err) - } - - upgraded, err := csaupgrade.UpgradeManagedFields(&initialObject, "kubectl-client-side-apply", "kubectl", "") - if err != nil { - t.Fatal(err) - } - - expectedYAML := []byte(` +`), + ExpectedObject: []byte(` apiVersion: v1 -data: - key: value - legacy: unused +data: {} kind: ConfigMap metadata: annotations: @@ -81,18 +73,36 @@ metadata: operation: Apply time: "2022-08-22T23:08:23Z" name: test - namespace: default - resourceVersion: "12502" - uid: 1f186675-58e6-4d7b-8bc5-7ece581e3013 -`) - - expectedObject := unstructured.Unstructured{} - err = yaml.Unmarshal(expectedYAML, &expectedObject) - if err != nil { - t.Fatal(err) + namespace: default +`), + }, } - if !reflect.DeepEqual(&expectedObject, upgraded) { - t.Fatal(cmp.Diff(&expectedObject, upgraded)) + for _, testCase := range cases { + initialObject := unstructured.Unstructured{} + err := yaml.Unmarshal(testCase.OriginalObject, &initialObject) + if err != nil { + t.Fatal(err) + } + + upgraded, err := csaupgrade.UpgradeManagedFields( + &initialObject, + testCase.CSAManager, + testCase.SSAManager, + ) + + if err != nil { + t.Fatal(err) + } + + expectedObject := unstructured.Unstructured{} + err = yaml.Unmarshal(testCase.ExpectedObject, &expectedObject) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(&expectedObject, upgraded) { + t.Fatal(cmp.Diff(&expectedObject, upgraded)) + } } } From efe378914a01cb4f3fa6174341690ff50bab4f0b Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Wed, 24 Aug 2022 11:51:30 -0700 Subject: [PATCH 07/16] add more test cases Kubernetes-commit: 42c960497a4737d8147d40d436a00be81ec4248d --- util/csaupgrade/upgrade_test.go | 418 ++++++++++++++++++++++++++++++-- 1 file changed, 397 insertions(+), 21 deletions(-) diff --git a/util/csaupgrade/upgrade_test.go b/util/csaupgrade/upgrade_test.go index 8f9a2e74..26bd0318 100644 --- a/util/csaupgrade/upgrade_test.go +++ b/util/csaupgrade/upgrade_test.go @@ -13,12 +13,17 @@ import ( func TestUpgradeCSA(t *testing.T) { cases := []struct { + Name string CSAManager string SSAManager string OriginalObject []byte ExpectedObject []byte }{ { + // Case where there is a CSA entry with the given name, but no SSA entry + // is found. Expect that the CSA entry is converted to an SSA entry + // and renamed. + Name: "csa-basic-direct-conversion", CSAManager: "kubectl-client-side-apply", SSAManager: "kubectl", OriginalObject: []byte(` @@ -74,35 +79,406 @@ metadata: time: "2022-08-22T23:08:23Z" name: test namespace: default +`), + }, + { + // This is the case when kubectl --server-side is used for the first time + // Server creates duplicate managed fields entry - one for Update and another + // for Apply. Expect entries to be merged into one entry, which is unchanged + // from initial SSA. + Name: "csa-combine-with-ssa-duplicate-keys", + CSAManager: "kubectl-client-side-apply", + SSAManager: "kubectl", + OriginalObject: []byte(` +apiVersion: v1 +data: {} +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + creationTimestamp: "2022-08-22T23:08:23Z" + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl + operation: Apply + time: "2022-08-23T23:08:23Z" + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + name: test + namespace: default +`), + ExpectedObject: []byte(` +apiVersion: v1 +data: {} +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + creationTimestamp: "2022-08-22T23:08:23Z" + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl + operation: Apply + time: "2022-08-23T23:08:23Z" + name: test + namespace: default +`), + }, + { + // This is the case when kubectl --server-side is used for the first time, + // but then a key is removed. A bug would take place where key is left in + // CSA entry but no longer present in SSA entry, so it would not be pruned. + // This shows that upgrading such an object results in correct behavior next + // time SSA applier + // Expect final object to have all keys from both entries + Name: "csa-combine-with-ssa-additional-keys", + CSAManager: "kubectl-client-side-apply", + SSAManager: "kubectl", + OriginalObject: []byte(` +apiVersion: v1 +data: {} +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + creationTimestamp: "2022-08-22T23:08:23Z" + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl + operation: Apply + time: "2022-08-23T23:08:23Z" + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + name: test + namespace: default +`), + ExpectedObject: []byte(` +apiVersion: v1 +data: {} +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + creationTimestamp: "2022-08-22T23:08:23Z" + managedFields: + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl + operation: Apply + time: "2022-08-23T23:08:23Z" + name: test + namespace: default +`), + }, + { + // Case when there are multiple CSA versions on the object which do not + // match the version from the apply entry. Shows they are tossed away + // without being merged. + Name: "csa-no-applicable-version", + CSAManager: "kubectl-client-side-apply", + SSAManager: "kubectl", + OriginalObject: []byte(` +apiVersion: v1 +data: {} +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + creationTimestamp: "2022-08-22T23:08:23Z" + managedFields: + - apiVersion: v5 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl + operation: Apply + time: "2022-08-23T23:08:23Z" + - apiVersion: v1 + fieldsType: FieldsV1 + fieldsV1: + f:data: + f:key2: {} + f:metadata: + f:annotations: + f:hello2: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + - apiVersion: v2 + fieldsType: FieldsV1 + fieldsV1: + f:data: + f:key3: {} + f:metadata: + f:annotations: + f:hello3: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + - apiVersion: v3 + fieldsType: FieldsV1 + fieldsV1: + f:data: + f:key4: {} + f:metadata: + f:annotations: + f:hello3: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + - apiVersion: v4 + fieldsType: FieldsV1 + fieldsV1: + f:data: + f:key5: {} + f:metadata: + f:annotations: + f:hello4: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + name: test + namespace: default +`), + ExpectedObject: []byte(` +apiVersion: v1 +data: {} +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + creationTimestamp: "2022-08-22T23:08:23Z" + managedFields: + - apiVersion: v5 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl + operation: Apply + time: "2022-08-23T23:08:23Z" + name: test + namespace: default +`), + }, + { + // Case when there are multiple CSA versions on the object which do not + // match the version from the apply entry, and one which does. + // Shows that CSA entry with matching version is unioned into the SSA entry. + Name: "csa-single-applicable-version", + CSAManager: "kubectl-client-side-apply", + SSAManager: "kubectl", + OriginalObject: []byte(` +apiVersion: v1 +data: {} +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + creationTimestamp: "2022-08-22T23:08:23Z" + managedFields: + - apiVersion: v5 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl + operation: Apply + time: "2022-08-23T23:08:23Z" + - apiVersion: v5 + fieldsType: FieldsV1 + fieldsV1: + f:data: + f:key2: {} + f:metadata: + f:annotations: + f:hello2: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + - apiVersion: v2 + fieldsType: FieldsV1 + fieldsV1: + f:data: + f:key3: {} + f:metadata: + f:annotations: + f:hello3: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + - apiVersion: v3 + fieldsType: FieldsV1 + fieldsV1: + f:data: + f:key4: {} + f:metadata: + f:annotations: + f:hello4: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + - apiVersion: v4 + fieldsType: FieldsV1 + fieldsV1: + f:data: + f:key5: {} + f:metadata: + f:annotations: + f:hello5: {} + manager: kubectl-client-side-apply + operation: Update + time: "2022-08-22T23:08:23Z" + name: test + namespace: default +`), + ExpectedObject: []byte(` +apiVersion: v1 +data: {} +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + creationTimestamp: "2022-08-22T23:08:23Z" + managedFields: + - apiVersion: v5 + fieldsType: FieldsV1 + fieldsV1: + f:data: + .: {} + f:key: {} + f:key2: {} + f:legacy: {} + f:metadata: + f:annotations: + .: {} + f:hello2: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + manager: kubectl + operation: Apply + time: "2022-08-23T23:08:23Z" + name: test + namespace: default `), }, } for _, testCase := range cases { - initialObject := unstructured.Unstructured{} - err := yaml.Unmarshal(testCase.OriginalObject, &initialObject) - if err != nil { - t.Fatal(err) - } + t.Run(testCase.Name, func(t *testing.T) { + initialObject := unstructured.Unstructured{} + err := yaml.Unmarshal(testCase.OriginalObject, &initialObject) + if err != nil { + t.Fatal(err) + } - upgraded, err := csaupgrade.UpgradeManagedFields( - &initialObject, - testCase.CSAManager, - testCase.SSAManager, - ) + upgraded, err := csaupgrade.UpgradeManagedFields( + &initialObject, + testCase.CSAManager, + testCase.SSAManager, + ) - if err != nil { - t.Fatal(err) - } + if err != nil { + t.Fatal(err) + } - expectedObject := unstructured.Unstructured{} - err = yaml.Unmarshal(testCase.ExpectedObject, &expectedObject) - if err != nil { - t.Fatal(err) - } + expectedObject := unstructured.Unstructured{} + err = yaml.Unmarshal(testCase.ExpectedObject, &expectedObject) + if err != nil { + t.Fatal(err) + } - if !reflect.DeepEqual(&expectedObject, upgraded) { - t.Fatal(cmp.Diff(&expectedObject, upgraded)) - } + if !reflect.DeepEqual(&expectedObject, upgraded) { + t.Fatal(cmp.Diff(&expectedObject, upgraded)) + } + }) } } From 089614c43e2ac786759872746e90619f49aa5b71 Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Wed, 24 Aug 2022 12:31:57 -0700 Subject: [PATCH 08/16] remove last applied configuration information Kubernetes-commit: a7fe0f0283765e4970211f7227602e2caa4b3a57 --- util/csaupgrade/upgrade.go | 69 +++++++++++++++++++++++++-------- util/csaupgrade/upgrade_test.go | 39 +++++-------------- 2 files changed, 63 insertions(+), 45 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index 372155a3..d76eb682 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -7,8 +7,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" + "sigs.k8s.io/structured-merge-diff/v4/fieldpath" ) +const csaAnnotationName = "kubectl.kubernetes.io/last-applied-configuration" + +var csaAnnotationFieldSet = fieldpath.NewSet(fieldpath.MakePathOrDie("metadata", "annotations", csaAnnotationName)) + // Upgrades the Manager information for fields managed with CSA // Prepares fields owned by `csaManager` for 'Update' operations for use now // with the given `ssaManager` for `Apply` operations. @@ -60,37 +65,48 @@ func UpgradeManagedFields( entry.APIVersion == ssaManager.APIVersion }) + ssaFieldSet, err := fieldmanager.FieldsToSet(*ssaManager.FieldsV1) + if err != nil { + return nil, fmt.Errorf("failed to convert fields to set: %w", err) + } + + combinedFieldSet := &ssaFieldSet + + // Union the csa manager with the existing SSA manager if csaManagerExists { csaManager := managedFields[csaManagerIndex] - // Union the csa manager with the existing SSA manager - ssaFieldSet, err := fieldmanager.FieldsToSet(*ssaManager.FieldsV1) - if err != nil { - return nil, fmt.Errorf("failed to convert fields to set: %w", err) - } - csaFieldSet, err := fieldmanager.FieldsToSet(*csaManager.FieldsV1) if err != nil { return nil, fmt.Errorf("failed to convert fields to set: %w", err) } - combinedFieldSet := ssaFieldSet.Union(&csaFieldSet) - combinedFieldSetEncoded, err := fieldmanager.SetToFields(*combinedFieldSet) - if err != nil { - return nil, fmt.Errorf("failed to encode field set: %w", err) - } - - managedFields[ssaManagerIndex].FieldsV1 = &combinedFieldSetEncoded + combinedFieldSet = combinedFieldSet.Union(&csaFieldSet) } + + // Ensure that the resultant fieldset does not include the + // last applied annotation + combinedFieldSet = combinedFieldSet.Difference(csaAnnotationFieldSet) + + combinedFieldSetEncoded, err := fieldmanager.SetToFields(*combinedFieldSet) + if err != nil { + return nil, fmt.Errorf("failed to encode field set: %w", err) + } + + managedFields[ssaManagerIndex].FieldsV1 = &combinedFieldSetEncoded + } else { // SSA manager does not exist. Find the most recent matching CSA manager, // convert it to an SSA manager. // // (find first index, since managed fields are sorted so that most recent is // first in the list) - csaManagerIndex, csaManagerExists := findFirstIndex(managedFields, func(entry metav1.ManagedFieldsEntry) bool { - return entry.Manager == csaManagerName && entry.Operation == metav1.ManagedFieldsOperationUpdate && entry.Subresource == "" - }) + csaManagerIndex, csaManagerExists := findFirstIndex(managedFields, + func(entry metav1.ManagedFieldsEntry) bool { + return entry.Manager == csaManagerName && + entry.Operation == metav1.ManagedFieldsOperationUpdate && + entry.Subresource == "" + }) if !csaManagerExists { // There are no CSA managers that need to be converted. Nothing to do @@ -98,7 +114,22 @@ func UpgradeManagedFields( return obj, nil } + csaManager := managedFields[csaManagerIndex] + csaFieldSet, err := fieldmanager.FieldsToSet(*csaManager.FieldsV1) + if err != nil { + return nil, fmt.Errorf("failed to convert fields to set: %w", err) + } + + // Remove last applied configuration from owned fields, if necessary + csaFieldSet = *csaFieldSet.Difference(csaAnnotationFieldSet) + + csaFieldSetEncoded, err := fieldmanager.SetToFields(csaFieldSet) + if err != nil { + return nil, fmt.Errorf("failed to encode field set: %w", err) + } + // Convert the entry to apply operation + managedFields[csaManagerIndex].FieldsV1 = &csaFieldSetEncoded managedFields[csaManagerIndex].Operation = metav1.ManagedFieldsOperationApply managedFields[csaManagerIndex].Manager = ssaManagerName } @@ -116,6 +147,12 @@ func UpgradeManagedFields( return nil, fmt.Errorf("failed to get meta accessor for copied object: %w", err) } copiedAccessor.SetManagedFields(filteredManagers) + + // Wipe out CSA annotation if it exists + annotations := copiedAccessor.GetAnnotations() + delete(annotations, csaAnnotationName) + copiedAccessor.SetAnnotations(annotations) + return copied, nil } diff --git a/util/csaupgrade/upgrade_test.go b/util/csaupgrade/upgrade_test.go index 26bd0318..5af49e74 100644 --- a/util/csaupgrade/upgrade_test.go +++ b/util/csaupgrade/upgrade_test.go @@ -58,9 +58,7 @@ apiVersion: v1 data: {} kind: ConfigMap metadata: - annotations: - kubectl.kubernetes.io/last-applied-configuration: | - {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + annotations: {} creationTimestamp: "2022-08-22T23:08:23Z" managedFields: - apiVersion: v1 @@ -71,9 +69,7 @@ metadata: f:key: {} f:legacy: {} f:metadata: - f:annotations: - .: {} - f:kubectl.kubernetes.io/last-applied-configuration: {} + f:annotations: {} manager: kubectl operation: Apply time: "2022-08-22T23:08:23Z" @@ -135,9 +131,7 @@ apiVersion: v1 data: {} kind: ConfigMap metadata: - annotations: - kubectl.kubernetes.io/last-applied-configuration: | - {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + annotations: {} creationTimestamp: "2022-08-22T23:08:23Z" managedFields: - apiVersion: v1 @@ -148,9 +142,7 @@ metadata: f:key: {} f:legacy: {} f:metadata: - f:annotations: - .: {} - f:kubectl.kubernetes.io/last-applied-configuration: {} + f:annotations: {} manager: kubectl operation: Apply time: "2022-08-23T23:08:23Z" @@ -164,7 +156,7 @@ metadata: // CSA entry but no longer present in SSA entry, so it would not be pruned. // This shows that upgrading such an object results in correct behavior next // time SSA applier - // Expect final object to have all keys from both entries + // Expect final object to have unioned keys from both entries Name: "csa-combine-with-ssa-additional-keys", CSAManager: "kubectl-client-side-apply", SSAManager: "kubectl", @@ -213,9 +205,7 @@ apiVersion: v1 data: {} kind: ConfigMap metadata: - annotations: - kubectl.kubernetes.io/last-applied-configuration: | - {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + annotations: {} creationTimestamp: "2022-08-22T23:08:23Z" managedFields: - apiVersion: v1 @@ -226,9 +216,7 @@ metadata: f:key: {} f:legacy: {} f:metadata: - f:annotations: - .: {} - f:kubectl.kubernetes.io/last-applied-configuration: {} + f:annotations: {} manager: kubectl operation: Apply time: "2022-08-23T23:08:23Z" @@ -319,9 +307,7 @@ apiVersion: v1 data: {} kind: ConfigMap metadata: - annotations: - kubectl.kubernetes.io/last-applied-configuration: | - {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + annotations: {} creationTimestamp: "2022-08-22T23:08:23Z" managedFields: - apiVersion: v5 @@ -332,9 +318,7 @@ metadata: f:key: {} f:legacy: {} f:metadata: - f:annotations: - .: {} - f:kubectl.kubernetes.io/last-applied-configuration: {} + f:annotations: {} manager: kubectl operation: Apply time: "2022-08-23T23:08:23Z" @@ -425,9 +409,7 @@ apiVersion: v1 data: {} kind: ConfigMap metadata: - annotations: - kubectl.kubernetes.io/last-applied-configuration: | - {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}} + annotations: {} creationTimestamp: "2022-08-22T23:08:23Z" managedFields: - apiVersion: v5 @@ -442,7 +424,6 @@ metadata: f:annotations: .: {} f:hello2: {} - f:kubectl.kubernetes.io/last-applied-configuration: {} manager: kubectl operation: Apply time: "2022-08-23T23:08:23Z" From cfaca90a30293a53b9701700fb6b2654f74b78a3 Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Tue, 30 Aug 2022 13:21:29 -0700 Subject: [PATCH 09/16] address comments Kubernetes-commit: e8d8eb4381f96ec4923a801116050ee998de4887 --- util/csaupgrade/upgrade.go | 39 ++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index d76eb682..88b7d70f 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -1,12 +1,12 @@ package csaupgrade import ( + "bytes" "fmt" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" "sigs.k8s.io/structured-merge-diff/v4/fieldpath" ) @@ -14,10 +14,14 @@ const csaAnnotationName = "kubectl.kubernetes.io/last-applied-configuration" var csaAnnotationFieldSet = fieldpath.NewSet(fieldpath.MakePathOrDie("metadata", "annotations", csaAnnotationName)) -// Upgrades the Manager information for fields managed with CSA +// Upgrades the Manager information for fields managed with client-side-apply (CSA) // Prepares fields owned by `csaManager` for 'Update' operations for use now // with the given `ssaManager` for `Apply` operations. // +// This transformation should be performed on an object if it has been previously +// managed using client-side-apply to prepare it for future use with +// server-side-apply. +// // Caveats: // 1. This operation is not reversible. Information about which fields the client // owned will be lost in this operation. @@ -27,9 +31,11 @@ var csaAnnotationFieldSet = fieldpath.NewSet(fieldpath.MakePathOrDie("metadata", // 4. Care must be taken to not overwrite the managed fields on the server if they // have changed before sending a patch. // -// csaManager - Name of FieldManager formerly used for `Update` operations -// ssaManager - Name of FieldManager formerly used for `Apply` operations -// subResource - Name of subresource used for api calls or empty string for main resource +// obj - Target of the operation which has been managed with CSA in the past +// csaManagerName - Name of FieldManager formerly used for `Update` operations +// ssaManagerName - Name of FieldManager formerly used for `Apply` operations +// Returns: a copy of the `obj` paramter with its managed fields modified so that +// it may be used with server-side apply in the future. func UpgradeManagedFields( obj runtime.Object, csaManagerName string, @@ -65,7 +71,7 @@ func UpgradeManagedFields( entry.APIVersion == ssaManager.APIVersion }) - ssaFieldSet, err := fieldmanager.FieldsToSet(*ssaManager.FieldsV1) + ssaFieldSet, err := fieldsToSet(*ssaManager.FieldsV1) if err != nil { return nil, fmt.Errorf("failed to convert fields to set: %w", err) } @@ -76,7 +82,7 @@ func UpgradeManagedFields( if csaManagerExists { csaManager := managedFields[csaManagerIndex] - csaFieldSet, err := fieldmanager.FieldsToSet(*csaManager.FieldsV1) + csaFieldSet, err := fieldsToSet(*csaManager.FieldsV1) if err != nil { return nil, fmt.Errorf("failed to convert fields to set: %w", err) } @@ -88,7 +94,7 @@ func UpgradeManagedFields( // last applied annotation combinedFieldSet = combinedFieldSet.Difference(csaAnnotationFieldSet) - combinedFieldSetEncoded, err := fieldmanager.SetToFields(*combinedFieldSet) + combinedFieldSetEncoded, err := setToFields(*combinedFieldSet) if err != nil { return nil, fmt.Errorf("failed to encode field set: %w", err) } @@ -115,7 +121,7 @@ func UpgradeManagedFields( } csaManager := managedFields[csaManagerIndex] - csaFieldSet, err := fieldmanager.FieldsToSet(*csaManager.FieldsV1) + csaFieldSet, err := fieldsToSet(*csaManager.FieldsV1) if err != nil { return nil, fmt.Errorf("failed to convert fields to set: %w", err) } @@ -123,7 +129,7 @@ func UpgradeManagedFields( // Remove last applied configuration from owned fields, if necessary csaFieldSet = *csaFieldSet.Difference(csaAnnotationFieldSet) - csaFieldSetEncoded, err := fieldmanager.SetToFields(csaFieldSet) + csaFieldSetEncoded, err := setToFields(csaFieldSet) if err != nil { return nil, fmt.Errorf("failed to encode field set: %w", err) } @@ -187,3 +193,16 @@ func filter[T any]( return result } + +// Included from fieldmanager.internal to avoid dependency cycle +// FieldsToSet creates a set paths from an input trie of fields +func fieldsToSet(f metav1.FieldsV1) (s fieldpath.Set, err error) { + err = s.FromJSON(bytes.NewReader(f.Raw)) + return s, err +} + +// SetToFields creates a trie of fields from an input set of paths +func setToFields(s fieldpath.Set) (f metav1.FieldsV1, err error) { + f.Raw, err = s.ToJSON() + return f, err +} From 675ca931801ec5d0af7c0dd52c76225d53f7d144 Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Tue, 30 Aug 2022 15:46:43 -0700 Subject: [PATCH 10/16] refactor if statement Kubernetes-commit: f7defeecce9c9e32d180deb2a2c67e063a10265f --- util/csaupgrade/upgrade.go | 115 +++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 56 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index 88b7d70f..a3f0af2d 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -19,7 +19,7 @@ var csaAnnotationFieldSet = fieldpath.NewSet(fieldpath.MakePathOrDie("metadata", // with the given `ssaManager` for `Apply` operations. // // This transformation should be performed on an object if it has been previously -// managed using client-side-apply to prepare it for future use with +// managed using client-side-apply to prepare it for future use with // server-side-apply. // // Caveats: @@ -59,48 +59,11 @@ func UpgradeManagedFields( }) if ssaManagerExists { - ssaManager := managedFields[ssaManagerIndex] - - // find Update manager of same APIVersion, union ssa fields with it. - // discard all other Update managers of the same name - csaManagerIndex, csaManagerExists := findFirstIndex(managedFields, - func(entry metav1.ManagedFieldsEntry) bool { - return entry.Manager == csaManagerName && - entry.Operation == metav1.ManagedFieldsOperationUpdate && - entry.Subresource == "" && - entry.APIVersion == ssaManager.APIVersion - }) - - ssaFieldSet, err := fieldsToSet(*ssaManager.FieldsV1) + err = unionManagerIntoIndex(managedFields, ssaManagerIndex, csaManagerName) if err != nil { - return nil, fmt.Errorf("failed to convert fields to set: %w", err) + return nil, err } - combinedFieldSet := &ssaFieldSet - - // Union the csa manager with the existing SSA manager - if csaManagerExists { - csaManager := managedFields[csaManagerIndex] - - csaFieldSet, err := fieldsToSet(*csaManager.FieldsV1) - if err != nil { - return nil, fmt.Errorf("failed to convert fields to set: %w", err) - } - - combinedFieldSet = combinedFieldSet.Union(&csaFieldSet) - } - - // Ensure that the resultant fieldset does not include the - // last applied annotation - combinedFieldSet = combinedFieldSet.Difference(csaAnnotationFieldSet) - - combinedFieldSetEncoded, err := setToFields(*combinedFieldSet) - if err != nil { - return nil, fmt.Errorf("failed to encode field set: %w", err) - } - - managedFields[ssaManagerIndex].FieldsV1 = &combinedFieldSetEncoded - } else { // SSA manager does not exist. Find the most recent matching CSA manager, // convert it to an SSA manager. @@ -120,24 +83,14 @@ func UpgradeManagedFields( return obj, nil } - csaManager := managedFields[csaManagerIndex] - csaFieldSet, err := fieldsToSet(*csaManager.FieldsV1) - if err != nil { - return nil, fmt.Errorf("failed to convert fields to set: %w", err) - } - - // Remove last applied configuration from owned fields, if necessary - csaFieldSet = *csaFieldSet.Difference(csaAnnotationFieldSet) - - csaFieldSetEncoded, err := setToFields(csaFieldSet) - if err != nil { - return nil, fmt.Errorf("failed to encode field set: %w", err) - } - - // Convert the entry to apply operation - managedFields[csaManagerIndex].FieldsV1 = &csaFieldSetEncoded + // Convert CSA manager into SSA manager managedFields[csaManagerIndex].Operation = metav1.ManagedFieldsOperationApply managedFields[csaManagerIndex].Manager = ssaManagerName + + err = unionManagerIntoIndex(managedFields, csaManagerIndex, csaManagerName) + if err != nil { + return nil, err + } } // Create version of managed fields which has no CSA managers with the given name @@ -162,6 +115,56 @@ func UpgradeManagedFields( return copied, nil } +// Locates an Update manager entry named `csaManagerName` with the same APIVersion +// as the manager at the targetIndex. Unions both manager's fields together +// into the manager specified by `targetIndex`. No other managers are modified. +func unionManagerIntoIndex(entries []metav1.ManagedFieldsEntry, targetIndex int, csaManagerName string) error { + ssaManager := entries[targetIndex] + + // find Update manager of same APIVersion, union ssa fields with it. + // discard all other Update managers of the same name + csaManagerIndex, csaManagerExists := findFirstIndex(entries, + func(entry metav1.ManagedFieldsEntry) bool { + return entry.Manager == csaManagerName && + entry.Operation == metav1.ManagedFieldsOperationUpdate && + entry.Subresource == "" && + entry.APIVersion == ssaManager.APIVersion + }) + + targetFieldSet, err := fieldsToSet(*ssaManager.FieldsV1) + if err != nil { + return fmt.Errorf("failed to convert fields to set: %w", err) + } + + combinedFieldSet := &targetFieldSet + + // Union the csa manager with the existing SSA manager. Do nothing if + // there was no good candidate found + if csaManagerExists { + csaManager := entries[csaManagerIndex] + + csaFieldSet, err := fieldsToSet(*csaManager.FieldsV1) + if err != nil { + return fmt.Errorf("failed to convert fields to set: %w", err) + } + + combinedFieldSet = combinedFieldSet.Union(&csaFieldSet) + } + + // Ensure that the resultant fieldset does not include the + // last applied annotation + combinedFieldSet = combinedFieldSet.Difference(csaAnnotationFieldSet) + + // Encode the fields back to the serialized format + combinedFieldSetEncoded, err := setToFields(*combinedFieldSet) + if err != nil { + return fmt.Errorf("failed to encode field set: %w", err) + } + + entries[targetIndex].FieldsV1 = &combinedFieldSetEncoded + return nil +} + func findFirstIndex[T any]( collection []T, predicate func(T) bool, From 703d15eff61aa5650c4781ffaf69c1e9fc196e01 Mon Sep 17 00:00:00 2001 From: Alex Zielenski Date: Fri, 2 Sep 2022 17:34:07 -0700 Subject: [PATCH 11/16] Update staging/src/k8s.io/client-go/util/csaupgrade/upgrade.go Co-authored-by: Antoine Pelisse Kubernetes-commit: 5caffef694b50168f546d7758a26de832302b541 --- util/csaupgrade/upgrade.go | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index a3f0af2d..e199a949 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -51,46 +51,39 @@ func UpgradeManagedFields( managedFields = append(managedFields, accessor.GetManagedFields()...) // Locate SSA manager - ssaManagerIndex, ssaManagerExists := findFirstIndex(managedFields, + replaceIndex, managerExists := findFirstIndex(managedFields, func(entry metav1.ManagedFieldsEntry) bool { return entry.Manager == ssaManagerName && entry.Operation == metav1.ManagedFieldsOperationApply && entry.Subresource == "" }) - if ssaManagerExists { - err = unionManagerIntoIndex(managedFields, ssaManagerIndex, csaManagerName) - if err != nil { - return nil, err - } - - } else { + if !managerExists { // SSA manager does not exist. Find the most recent matching CSA manager, // convert it to an SSA manager. // // (find first index, since managed fields are sorted so that most recent is // first in the list) - csaManagerIndex, csaManagerExists := findFirstIndex(managedFields, + replaceIndex, managerExists = findFirstIndex(managedFields, func(entry metav1.ManagedFieldsEntry) bool { return entry.Manager == csaManagerName && entry.Operation == metav1.ManagedFieldsOperationUpdate && entry.Subresource == "" }) - if !csaManagerExists { + if !managerExists { // There are no CSA managers that need to be converted. Nothing to do // Return early return obj, nil } // Convert CSA manager into SSA manager - managedFields[csaManagerIndex].Operation = metav1.ManagedFieldsOperationApply - managedFields[csaManagerIndex].Manager = ssaManagerName - - err = unionManagerIntoIndex(managedFields, csaManagerIndex, csaManagerName) - if err != nil { - return nil, err - } + managedFields[replaceIndex].Operation = metav1.ManagedFieldsOperationApply + managedFields[replaceIndex].Manager = ssaManagerName + } + err = unionManagerIntoIndex(managedFields, replaceIndex, csaManagerName) + if err != nil { + return nil, err } // Create version of managed fields which has no CSA managers with the given name From 9aa7c11460f0dad787b9250954825493926395fc Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Fri, 2 Sep 2022 17:38:29 -0700 Subject: [PATCH 12/16] remove fieldsv1 from upgrade body Kubernetes-commit: 5c9534b9cfdcfa9625153051f0859e101abb97a9 --- util/csaupgrade/upgrade.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index e199a949..a182d829 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -124,7 +124,7 @@ func unionManagerIntoIndex(entries []metav1.ManagedFieldsEntry, targetIndex int, entry.APIVersion == ssaManager.APIVersion }) - targetFieldSet, err := fieldsToSet(*ssaManager.FieldsV1) + targetFieldSet, err := decodeManagedFieldsEntrySet(ssaManager) if err != nil { return fmt.Errorf("failed to convert fields to set: %w", err) } @@ -136,7 +136,7 @@ func unionManagerIntoIndex(entries []metav1.ManagedFieldsEntry, targetIndex int, if csaManagerExists { csaManager := entries[csaManagerIndex] - csaFieldSet, err := fieldsToSet(*csaManager.FieldsV1) + csaFieldSet, err := decodeManagedFieldsEntrySet(csaManager) if err != nil { return fmt.Errorf("failed to convert fields to set: %w", err) } @@ -149,12 +149,11 @@ func unionManagerIntoIndex(entries []metav1.ManagedFieldsEntry, targetIndex int, combinedFieldSet = combinedFieldSet.Difference(csaAnnotationFieldSet) // Encode the fields back to the serialized format - combinedFieldSetEncoded, err := setToFields(*combinedFieldSet) + err = encodeManagedFieldsEntrySet(&entries[targetIndex], *combinedFieldSet) if err != nil { return fmt.Errorf("failed to encode field set: %w", err) } - entries[targetIndex].FieldsV1 = &combinedFieldSetEncoded return nil } @@ -192,13 +191,13 @@ func filter[T any]( // Included from fieldmanager.internal to avoid dependency cycle // FieldsToSet creates a set paths from an input trie of fields -func fieldsToSet(f metav1.FieldsV1) (s fieldpath.Set, err error) { - err = s.FromJSON(bytes.NewReader(f.Raw)) +func decodeManagedFieldsEntrySet(f metav1.ManagedFieldsEntry) (s fieldpath.Set, err error) { + err = s.FromJSON(bytes.NewReader(f.FieldsV1.Raw)) return s, err } // SetToFields creates a trie of fields from an input set of paths -func setToFields(s fieldpath.Set) (f metav1.FieldsV1, err error) { - f.Raw, err = s.ToJSON() - return f, err +func encodeManagedFieldsEntrySet(f *metav1.ManagedFieldsEntry, s fieldpath.Set) (err error) { + f.FieldsV1.Raw, err = s.ToJSON() + return err } From 7634f2e0020944e426457a02ab6cf6c89d9d4825 Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Fri, 2 Sep 2022 17:40:34 -0700 Subject: [PATCH 13/16] make upgrade modify input instead of deep copying Kubernetes-commit: adcb5ec3d43908a916fd1ae75af121568f26b80d --- util/csaupgrade/upgrade.go | 26 +++++++++++--------------- util/csaupgrade/upgrade_test.go | 5 +++-- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index a182d829..62a18a90 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -40,10 +40,10 @@ func UpgradeManagedFields( obj runtime.Object, csaManagerName string, ssaManagerName string, -) (runtime.Object, error) { +) error { accessor, err := meta.Accessor(obj) if err != nil { - return nil, fmt.Errorf("error accessing object metadata: %w", err) + return fmt.Errorf("error accessing object metadata: %w", err) } // Create managed fields clone since we modify the values @@ -74,7 +74,7 @@ func UpgradeManagedFields( if !managerExists { // There are no CSA managers that need to be converted. Nothing to do // Return early - return obj, nil + return nil } // Convert CSA manager into SSA manager @@ -83,7 +83,7 @@ func UpgradeManagedFields( } err = unionManagerIntoIndex(managedFields, replaceIndex, csaManagerName) if err != nil { - return nil, err + return err } // Create version of managed fields which has no CSA managers with the given name @@ -93,19 +93,15 @@ func UpgradeManagedFields( entry.Subresource == "") }) - copied := obj.DeepCopyObject() - copiedAccessor, err := meta.Accessor(copied) - if err != nil { - return nil, fmt.Errorf("failed to get meta accessor for copied object: %w", err) - } - copiedAccessor.SetManagedFields(filteredManagers) - - // Wipe out CSA annotation if it exists - annotations := copiedAccessor.GetAnnotations() + // Wipe out last-applied-configuration annotation if it exists + annotations := accessor.GetAnnotations() delete(annotations, csaAnnotationName) - copiedAccessor.SetAnnotations(annotations) - return copied, nil + // Commit changes to object + accessor.SetAnnotations(annotations) + accessor.SetManagedFields(filteredManagers) + + return nil } // Locates an Update manager entry named `csaManagerName` with the same APIVersion diff --git a/util/csaupgrade/upgrade_test.go b/util/csaupgrade/upgrade_test.go index 5af49e74..db5ff913 100644 --- a/util/csaupgrade/upgrade_test.go +++ b/util/csaupgrade/upgrade_test.go @@ -441,8 +441,9 @@ metadata: t.Fatal(err) } - upgraded, err := csaupgrade.UpgradeManagedFields( - &initialObject, + upgraded := initialObject.DeepCopy() + err = csaupgrade.UpgradeManagedFields( + upgraded, testCase.CSAManager, testCase.SSAManager, ) From dac0826f18c85481a6f56e7c83848a28e09ebb24 Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Fri, 2 Sep 2022 17:42:08 -0700 Subject: [PATCH 14/16] remove inaccurate comment Kubernetes-commit: 52fcd20b578b67234a9ac84b5abf1cd2b3d06ea4 --- util/csaupgrade/upgrade.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index 62a18a90..1fc07d67 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -34,8 +34,6 @@ var csaAnnotationFieldSet = fieldpath.NewSet(fieldpath.MakePathOrDie("metadata", // obj - Target of the operation which has been managed with CSA in the past // csaManagerName - Name of FieldManager formerly used for `Update` operations // ssaManagerName - Name of FieldManager formerly used for `Apply` operations -// Returns: a copy of the `obj` paramter with its managed fields modified so that -// it may be used with server-side apply in the future. func UpgradeManagedFields( obj runtime.Object, csaManagerName string, From 2efbeaf56ed503c418d09381bcb1582a60a5ffd8 Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Tue, 6 Sep 2022 19:51:47 -0700 Subject: [PATCH 15/16] add boilerplate Kubernetes-commit: 3528fd7e47f585c144b13c0c2b75bc4e432987c6 --- util/csaupgrade/upgrade.go | 16 ++++++++++++++++ util/csaupgrade/upgrade_test.go | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index 1fc07d67..734178d5 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -1,3 +1,19 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package csaupgrade import ( diff --git a/util/csaupgrade/upgrade_test.go b/util/csaupgrade/upgrade_test.go index db5ff913..e4e8885b 100644 --- a/util/csaupgrade/upgrade_test.go +++ b/util/csaupgrade/upgrade_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package csaupgrade_test import ( From 00d892f4470a703ad543932dcf313e1b1e0e7921 Mon Sep 17 00:00:00 2001 From: Alexander Zielenski <351783+alexzielenski@users.noreply.github.com> Date: Tue, 6 Sep 2022 20:58:50 -0700 Subject: [PATCH 16/16] correct spacing Kubernetes-commit: dbff8e117c8d4aeccac6ea211dfa334e30621727 --- util/csaupgrade/upgrade.go | 2 +- util/csaupgrade/upgrade_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/util/csaupgrade/upgrade.go b/util/csaupgrade/upgrade.go index 734178d5..40cb4984 100644 --- a/util/csaupgrade/upgrade.go +++ b/util/csaupgrade/upgrade.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/util/csaupgrade/upgrade_test.go b/util/csaupgrade/upgrade_test.go index e4e8885b..e333fed0 100644 --- a/util/csaupgrade/upgrade_test.go +++ b/util/csaupgrade/upgrade_test.go @@ -1,11 +1,11 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,