From 241c1c7737bff852b0d47cc879453305761f87d9 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Fri, 28 Jun 2024 12:57:45 +0200 Subject: [PATCH 1/3] kubeadm: feature gate ControlPlaneKubeletLocalMode on init --- cmd/kubeadm/app/cmd/phases/init/kubeconfig.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/kubeadm/app/cmd/phases/init/kubeconfig.go b/cmd/kubeadm/app/cmd/phases/init/kubeconfig.go index 77432a13354..7e255b29fbb 100644 --- a/cmd/kubeadm/app/cmd/phases/init/kubeconfig.go +++ b/cmd/kubeadm/app/cmd/phases/init/kubeconfig.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) @@ -157,7 +158,16 @@ func runKubeConfigFile(kubeConfigFileName string) func(workflow.RunData) error { cfg.CertificatesDir = data.CertificateWriteDir() defer func() { cfg.CertificatesDir = data.CertificateDir() }() + initConfiguration := data.Cfg().DeepCopy() + + if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) { + if kubeConfigFileName == kubeadmconstants.KubeletKubeConfigFileName { + // Unset the ControlPlaneEndpoint so the creation falls back to the LocalAPIEndpoint for the kubelet's kubeconfig. + initConfiguration.ControlPlaneEndpoint = "" + } + } + // creates the KubeConfig file (or use existing) - return kubeconfigphase.CreateKubeConfigFile(kubeConfigFileName, data.KubeConfigDir(), data.Cfg()) + return kubeconfigphase.CreateKubeConfigFile(kubeConfigFileName, data.KubeConfigDir(), initConfiguration) } } From e51b55b45110d7276cbe65ce789c37e2515880a4 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Fri, 28 Jun 2024 13:31:26 +0200 Subject: [PATCH 2/3] kubeadm: feature gate ControlPlaneKubeletLocalMode on upgrade --- .../cmd/phases/upgrade/node/controlplane.go | 7 +++ cmd/kubeadm/app/phases/upgrade/postupgrade.go | 58 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go index cbdca3ec1c6..330d89b9823 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go @@ -24,6 +24,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) @@ -82,6 +83,12 @@ func runControlPlane() func(c workflow.RunData) error { return errors.Wrap(err, "failed to perform addons upgrade") } + if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) { + if err := upgrade.UpdateKubeletLocalMode(cfg, dryRun); err != nil { + return errors.Wrap(err, "failed to update kubelet local mode") + } + } + fmt.Println("[upgrade] The control plane instance for this node was successfully updated!") return nil diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 02e7bcca618..b7d15f41b93 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -31,10 +31,12 @@ import ( errorsutil "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" @@ -109,6 +111,12 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon errs = append(errs, err) } + if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) { + if err := UpdateKubeletLocalMode(cfg, dryRun); err != nil { + return errors.Wrap(err, "failed to update kubelet local mode") + } + } + return errorsutil.NewAggregate(errs) } @@ -281,3 +289,53 @@ func GetKubeletDir(dryRun bool) (string, error) { } return kubeadmconstants.KubeletRunDirectory, nil } + +func UpdateKubeletLocalMode(cfg *kubeadmapi.InitConfiguration, dryRun bool) error { + // TODO(chrischdi): how to get the correct dir? kubeadm init has a flag to change the location + dir := kubeadmconstants.KubernetesDir + + kubeletKubeConfigFilePath := filepath.Join(dir, kubeadmconstants.KubeletKubeConfigFileName) + + if _, err := os.Stat(kubeletKubeConfigFilePath); err != nil { + if os.IsNotExist(err) { + // TODO(chrischdi): should we print a warning or even return the error? + return nil + } + return err + } + + config, err := clientcmd.LoadFromFile(kubeletKubeConfigFilePath) + if err != nil { + return err + } + + configContext, ok := config.Contexts[config.CurrentContext] + if !ok { + return errors.Errorf("cannot find cluster for active context in kubeconfig %q", kubeletKubeConfigFilePath) + } + + localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint) + if err != nil { + return err + } + + // Skip changing kubeconfig file if LocalAPIEndpoint is already set. + if config.Clusters[configContext.Cluster].Server == localAPIEndpoint { + return nil + } + + if dryRun { + fmt.Printf("[dryrun] Would change the server url from %q to %q in %s and try to restart kubelet\n", localAPIEndpoint, config.Clusters[configContext.Cluster].Server, kubeletKubeConfigFilePath) + return nil + } + + config.Clusters[configContext.Cluster].Server = localAPIEndpoint + + if err := clientcmd.WriteToFile(*config, kubeletKubeConfigFilePath); err != nil { + return err + } + + kubeletphase.TryRestartKubelet() + + return nil +} From 8871513c1b64cae321552abfe9a3a90969637560 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Fri, 28 Jun 2024 17:11:49 +0200 Subject: [PATCH 3/3] review fixes --- cmd/kubeadm/app/phases/upgrade/postupgrade.go | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index b7d15f41b93..f6cb20d8e70 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -290,15 +290,16 @@ func GetKubeletDir(dryRun bool) (string, error) { return kubeadmconstants.KubeletRunDirectory, nil } +// UpdateKubeletLocalMode changes the Server URL in the kubelets kubeconfig to the local API endpoint if it is currently +// set to the ControlPlaneEndpoint. +// TODO: remove this function once kubeletKubeConfigFilePath goes GA and is hardcoded to enabled by default: +// https://github.com/kubernetes/kubeadm/issues/2271 func UpdateKubeletLocalMode(cfg *kubeadmapi.InitConfiguration, dryRun bool) error { - // TODO(chrischdi): how to get the correct dir? kubeadm init has a flag to change the location - dir := kubeadmconstants.KubernetesDir - - kubeletKubeConfigFilePath := filepath.Join(dir, kubeadmconstants.KubeletKubeConfigFileName) + kubeletKubeConfigFilePath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName) if _, err := os.Stat(kubeletKubeConfigFilePath); err != nil { if os.IsNotExist(err) { - // TODO(chrischdi): should we print a warning or even return the error? + klog.V(2).Infof("Could not mutate the Server URL in %s: %v", kubeletKubeConfigFilePath, err) return nil } return err @@ -319,16 +320,23 @@ func UpdateKubeletLocalMode(cfg *kubeadmapi.InitConfiguration, dryRun bool) erro return err } - // Skip changing kubeconfig file if LocalAPIEndpoint is already set. - if config.Clusters[configContext.Cluster].Server == localAPIEndpoint { + controlPlaneAPIEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint) + if err != nil { + return err + } + + // Skip changing kubeconfig file if Server does not match the ControlPlaneEndoint. + if config.Clusters[configContext.Cluster].Server != controlPlaneAPIEndpoint || controlPlaneAPIEndpoint == localAPIEndpoint { + klog.V(2).Infof("Skipping update of the Server URL in %s, because it's already not equal to %q or already matches the localAPIEndpoint", kubeletKubeConfigFilePath, cfg.ControlPlaneEndpoint) return nil } if dryRun { - fmt.Printf("[dryrun] Would change the server url from %q to %q in %s and try to restart kubelet\n", localAPIEndpoint, config.Clusters[configContext.Cluster].Server, kubeletKubeConfigFilePath) + fmt.Printf("[dryrun] Would change the Server URL from %q to %q in %s and try to restart kubelet\n", config.Clusters[configContext.Cluster].Server, localAPIEndpoint, kubeletKubeConfigFilePath) return nil } + klog.V(1).Infof("Changing the Server URL from %q to %q in %s", config.Clusters[configContext.Cluster].Server, localAPIEndpoint, kubeletKubeConfigFilePath) config.Clusters[configContext.Cluster].Server = localAPIEndpoint if err := clientcmd.WriteToFile(*config, kubeletKubeConfigFilePath); err != nil {