From 207556e057d4cdb9b0fad35aef0733ec0479e4f0 Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Wed, 29 Dec 2021 01:59:07 +0200 Subject: [PATCH] kubeadm: make "upgrade node" include URL scheme in socket paths The CRI socket that kubeadm writes as an annotation on a particular Node object can include an endpoint that does not have an URL scheme. This is undesired as long term the kubelet can stop allowing endpoints without URL scheme. For control plane nodes "kubeadm upgrade apply" takes the locally defaulted / populated NodeRegistration and refreshes the CRI socket in PerformPostUpgradeTasks. But for secondary nodes "kubeadm upgrade node" does not. Adapt "upgrade node" to fetch the NodeRegistration for this node and fix the CRI socket missing URL scheme if needed in the Node annotation. --- .../app/cmd/phases/upgrade/node/data.go | 1 + .../cmd/phases/upgrade/node/kubeletconfig.go | 35 +++++++++++++++++++ cmd/kubeadm/app/cmd/upgrade/node.go | 7 ++++ cmd/kubeadm/app/util/config/cluster.go | 12 +++---- cmd/kubeadm/app/util/config/cluster_test.go | 4 +-- 5 files changed, 51 insertions(+), 8 deletions(-) diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go index b4f24861eb0..c6ab05a970f 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go @@ -34,4 +34,5 @@ type Data interface { Client() clientset.Interface IgnorePreflightErrors() sets.String PatchesDir() string + KubeConfigPath() string } diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go index d4602fea94d..04ddeff3ff0 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go @@ -20,15 +20,22 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/pkg/errors" + "k8s.io/klog/v2" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" ) @@ -87,6 +94,34 @@ func runKubeletConfigPhase() func(c workflow.RunData) error { return nil } + // Handle a missing URL scheme in the Node CRI socket. + // Older versions of kubeadm tolerate CRI sockets without URL schemes (/var/run/foo without unix://). + // During "upgrade node" for worker nodes the cfg.NodeRegistration would be left empty. + // This requires to call GetNodeRegistration on demand and fetch the node name and CRI socket. + // If the NodeRegistration (nro) contains a socket without a URL scheme, update it. + // + // TODO: this workaround can be removed in 1.25 once all user node sockets have a URL scheme: + // https://github.com/kubernetes/kubeadm/issues/2426 + var nro *kubeadmapi.NodeRegistrationOptions + var missingURLScheme bool + if !dryRun { + if err := configutil.GetNodeRegistration(data.KubeConfigPath(), data.Client(), nro); err != nil { + return errors.Wrap(err, "could not retrieve the node registration options for this node") + } + missingURLScheme = strings.HasPrefix(nro.CRISocket, kubeadmapiv1.DefaultContainerRuntimeURLScheme) + } + if missingURLScheme { + if !dryRun { + newSocket := kubeadmapiv1.DefaultContainerRuntimeURLScheme + "://" + nro.CRISocket + klog.V(2).Infof("ensuring that Node %q has a CRI socket annotation with URL scheme %q", nro.Name, newSocket) + if err := patchnodephase.AnnotateCRISocket(data.Client(), nro.Name, newSocket); err != nil { + return errors.Wrapf(err, "error updating the CRI socket for Node %q", nro.Name) + } + } else { + fmt.Println("[dryrun] would update the node CRI socket path to include an URL scheme") + } + } + fmt.Println("[upgrade] The configuration for this node was successfully updated!") fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.") return nil diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index 08b4bf07c46..7072312a0b8 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -61,6 +61,7 @@ type nodeData struct { client clientset.Interface patchesDir string ignorePreflightErrors sets.String + kubeConfigPath string } // newCmdNode returns the cobra command for `kubeadm upgrade node` @@ -159,6 +160,7 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions) (*node isControlPlaneNode: isControlPlaneNode, patchesDir: options.patchesDir, ignorePreflightErrors: ignorePreflightErrorsSet, + kubeConfigPath: options.kubeConfigPath, }, nil } @@ -201,3 +203,8 @@ func (d *nodeData) PatchesDir() string { func (d *nodeData) IgnorePreflightErrors() sets.String { return d.ignorePreflightErrors } + +// KubeconfigPath returns the path to the user kubeconfig file. +func (d *nodeData) KubeConfigPath() string { + return d.kubeConfigPath +} diff --git a/cmd/kubeadm/app/util/config/cluster.go b/cmd/kubeadm/app/util/config/cluster.go index 730728b374c..242d82940b9 100644 --- a/cmd/kubeadm/app/util/config/cluster.go +++ b/cmd/kubeadm/app/util/config/cluster.go @@ -97,7 +97,8 @@ func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Inte // get nodes specific information as well if !newControlPlane { // gets the nodeRegistration for the current from the node object - if err := getNodeRegistration(kubeconfigDir, client, &initcfg.NodeRegistration); err != nil { + kubeconfigFile := filepath.Join(kubeconfigDir, constants.KubeletKubeConfigFileName) + if err := GetNodeRegistration(kubeconfigFile, client, &initcfg.NodeRegistration); err != nil { return nil, errors.Wrap(err, "failed to get node registration") } // gets the APIEndpoint for the current node @@ -117,10 +118,10 @@ func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Inte return initcfg, nil } -// getNodeRegistration returns the nodeRegistration for the current node -func getNodeRegistration(kubeconfigDir string, client clientset.Interface, nodeRegistration *kubeadmapi.NodeRegistrationOptions) error { +// GetNodeRegistration returns the nodeRegistration for the current node +func GetNodeRegistration(kubeconfigFile string, client clientset.Interface, nodeRegistration *kubeadmapi.NodeRegistrationOptions) error { // gets the name of the current node - nodeName, err := getNodeNameFromKubeletConfig(kubeconfigDir) + nodeName, err := getNodeNameFromKubeletConfig(kubeconfigFile) if err != nil { return errors.Wrap(err, "failed to get node name from kubelet config") } @@ -149,9 +150,8 @@ func getNodeRegistration(kubeconfigDir string, client clientset.Interface, nodeR // getNodeNameFromKubeletConfig gets the node name from a kubelet config file // TODO: in future we want to switch to a more canonical way for doing this e.g. by having this // information in the local kubelet config.yaml -func getNodeNameFromKubeletConfig(kubeconfigDir string) (string, error) { +func getNodeNameFromKubeletConfig(fileName string) (string, error) { // loads the kubelet.conf file - fileName := filepath.Join(kubeconfigDir, constants.KubeletKubeConfigFileName) config, err := clientcmd.LoadFromFile(fileName) if err != nil { return "", err diff --git a/cmd/kubeadm/app/util/config/cluster_test.go b/cmd/kubeadm/app/util/config/cluster_test.go index e1736401736..019997d3f94 100644 --- a/cmd/kubeadm/app/util/config/cluster_test.go +++ b/cmd/kubeadm/app/util/config/cluster_test.go @@ -261,7 +261,7 @@ func TestGetNodeNameFromKubeletConfig(t *testing.T) { return } - name, err := getNodeNameFromKubeletConfig(tmpdir) + name, err := getNodeNameFromKubeletConfig(kubeconfigPath) if rt.expectedError != (err != nil) { t.Errorf("unexpected return err from getNodeRegistration: %v", err) return @@ -338,7 +338,7 @@ func TestGetNodeRegistration(t *testing.T) { } cfg := &kubeadmapi.InitConfiguration{} - err = getNodeRegistration(tmpdir, client, &cfg.NodeRegistration) + err = GetNodeRegistration(cfgPath, client, &cfg.NodeRegistration) if rt.expectedError != (err != nil) { t.Errorf("unexpected return err from getNodeRegistration: %v", err) return