From 7c515cd1cb6ca5a2dbf30a5836720a95405ed244 Mon Sep 17 00:00:00 2001 From: HirazawaUi <695097494plus@gmail.com> Date: Wed, 18 Dec 2024 23:27:41 +0800 Subject: [PATCH] remove kubeadm.alpha.kubernetes.io/cri-socket annotation when kubeadm upgrade --- .../cmd/phases/upgrade/apply/uploadconfig.go | 4 ++++ .../app/cmd/phases/upgrade/kubeletconfig.go | 8 +++++++ cmd/kubeadm/app/phases/patchnode/patchnode.go | 22 +++++++++++++++++-- cmd/kubeadm/app/util/apiclient/dryrun.go | 2 +- test/e2e_kubeadm/nodes_test.go | 15 +++++++++++-- 5 files changed, 46 insertions(+), 5 deletions(-) diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go index 4842b16719d..137f0849a92 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go @@ -113,6 +113,10 @@ func runUploadKubeletConfig(c workflow.RunData) error { if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { return errors.Wrap(err, "error writing CRISocket for this node") } + } else { + if err := patchnodephase.RemoveCRISocketAnnotation(client, cfg.NodeRegistration.Name); err != nil { + return err + } } return nil diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/kubeletconfig.go b/cmd/kubeadm/app/cmd/phases/upgrade/kubeletconfig.go index 540174ee6be..dcf15c9d906 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/kubeletconfig.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/kubeletconfig.go @@ -25,6 +25,8 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/features" + patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" ) @@ -69,6 +71,12 @@ func runKubeletConfigPhase(c workflow.RunData) error { return err } + if features.Enabled(data.InitCfg().ClusterConfiguration.FeatureGates, features.NodeLocalCRISocket) { + if err := patchnodephase.RemoveCRISocketAnnotation(data.Client(), data.InitCfg().NodeRegistration.Name); err != nil { + return err + } + } + fmt.Println("[upgrade/kubelet-config] The kubelet configuration for this node was successfully upgraded!") return nil } diff --git a/cmd/kubeadm/app/phases/patchnode/patchnode.go b/cmd/kubeadm/app/phases/patchnode/patchnode.go index 195c753d8b2..6bf6ac90af2 100644 --- a/cmd/kubeadm/app/phases/patchnode/patchnode.go +++ b/cmd/kubeadm/app/phases/patchnode/patchnode.go @@ -17,6 +17,8 @@ limitations under the License. package patchnode import ( + "github.com/pkg/errors" + "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -27,8 +29,7 @@ import ( // AnnotateCRISocket annotates the node with the given crisocket func AnnotateCRISocket(client clientset.Interface, nodeName string, criSocket string) error { - - klog.V(1).Infof("[patchnode] Uploading the CRI Socket information %q to the Node API object %q as an annotation\n", criSocket, nodeName) + klog.V(1).Infof("[patchnode] Uploading the CRI socket %q to Node %q as an annotation", criSocket, nodeName) return apiclient.PatchNode(client, nodeName, func(n *v1.Node) { annotateNodeWithCRISocket(n, criSocket) @@ -41,3 +42,20 @@ func annotateNodeWithCRISocket(n *v1.Node, criSocket string) { } n.ObjectMeta.Annotations[constants.AnnotationKubeadmCRISocket] = criSocket } + +// RemoveCRISocketAnnotation removes the crisocket annotation from a node. +func RemoveCRISocketAnnotation(client clientset.Interface, nodeName string) error { + klog.V(1).Infof("[patchnode] Removing the CRI socket annotation from Node %q", nodeName) + + if err := apiclient.PatchNode(client, nodeName, removeNodeCRISocketAnnotation); err != nil { + return errors.Wrapf(err, "could not remove the CRI socket annotation from Node %q", nodeName) + } + return nil +} + +func removeNodeCRISocketAnnotation(n *v1.Node) { + if n.ObjectMeta.Annotations == nil { + return + } + delete(n.ObjectMeta.Annotations, constants.AnnotationKubeadmCRISocket) +} diff --git a/cmd/kubeadm/app/util/apiclient/dryrun.go b/cmd/kubeadm/app/util/apiclient/dryrun.go index 29c5ec256f3..1bb4caa8281 100644 --- a/cmd/kubeadm/app/util/apiclient/dryrun.go +++ b/cmd/kubeadm/app/util/apiclient/dryrun.go @@ -568,7 +568,7 @@ func getNode(name string) *corev1.Node { "kubernetes.io/hostname": name, }, Annotations: map[string]string{ - "kubeadm.alpha.kubernetes.io/cri-socket": "dry-run-cri-socket", + constants.AnnotationKubeadmCRISocket: "dry-run-cri-socket", }, }, } diff --git a/test/e2e_kubeadm/nodes_test.go b/test/e2e_kubeadm/nodes_test.go index 21c5d793267..81e0d1459e6 100644 --- a/test/e2e_kubeadm/nodes_test.go +++ b/test/e2e_kubeadm/nodes_test.go @@ -52,11 +52,22 @@ var _ = Describe("nodes", func() { List(ctx, metav1.ListOptions{}) framework.ExpectNoError(err, "error reading nodes") + var nodeLocalCRISocketEnabled bool + cc := getClusterConfiguration(f.ClientSet) + if _, ok := cc["featureGates"]; ok { + fgCC := cc["featureGates"].(map[interface{}]interface{}) + if fg, ok := fgCC["NodeLocalCRISocket"]; ok { + nodeLocalCRISocketEnabled = fg.(bool) + } + } + // Checks that the nodes have the CRI socket annotation // and that it is prefixed with a URL scheme for _, node := range nodes.Items { - gomega.Expect(node.Annotations).To(gomega.HaveKey(nodesCRISocketAnnotation)) - gomega.Expect(node.Annotations[nodesCRISocketAnnotation]).To(gomega.HavePrefix("unix://")) + if !nodeLocalCRISocketEnabled { + gomega.Expect(node.Annotations).To(gomega.HaveKey(nodesCRISocketAnnotation)) + gomega.Expect(node.Annotations[nodesCRISocketAnnotation]).To(gomega.HavePrefix("unix://")) + } } })