Merge pull request #129956 from chrischdi/pr-kubeadm-cp-local-mode-fixes

kubeadm: Promote ControlPlaneKubeletLocalMode feature gate to beta - second attempt

cmd/kubeadm/app/cmd/join.go

@@ -609,7 +609,9 @@ func (j *joinData) Client() (clientset.Interface, error) {
 			AppendReactor(dryRun.GetKubeadmConfigReactor()).
 			AppendReactor(dryRun.GetKubeadmCertsReactor()).
 			AppendReactor(dryRun.GetKubeProxyConfigReactor()).
-			AppendReactor(dryRun.GetKubeletConfigReactor())
+			AppendReactor(dryRun.GetKubeletConfigReactor()).
+			AppendReactor(dryRun.GetNodeReactor()).
+			AppendReactor(dryRun.PatchNodeReactor())
 
 		j.client = dryRun.FakeClient()
 		return j.client, nil

cmd/kubeadm/app/cmd/phases/join/kubelet.go

@@ -293,44 +293,55 @@ func runKubeletWaitBootstrapPhase(c workflow.RunData) (returnErr error) {
 		return err
 	}
 
-	bootstrapKubeConfigFile := filepath.Join(data.KubeConfigDir(), kubeadmconstants.KubeletBootstrapKubeConfigFileName)
+	var client clientset.Interface
-	// Deletes the bootstrapKubeConfigFile, so the credential used for TLS bootstrap is removed from disk
-	defer func() {
-		_ = os.Remove(bootstrapKubeConfigFile)
-	}()
 
-	// Apply patches to the in-memory kubelet configuration so that any configuration changes like kubelet healthz
+	if data.DryRun() {
-	// address and port options are respected during the wait below. WriteConfigToDisk already applied patches to
+		fmt.Println("[kubelet-wait] Would wait for the kubelet to be bootstrapped")
-	// the kubelet.yaml written to disk. This should be done after WriteConfigToDisk because both use the same config
-	// in memory and we don't want patches to be applied two times to the config that is written to disk.
-	if err := kubeletphase.ApplyPatchesToConfig(&initCfg.ClusterConfiguration, data.PatchesDir()); err != nil {
-		return errors.Wrap(err, "could not apply patches to the in-memory kubelet configuration")
-	}
 
-	// Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf
+		// Use the dry-run client.
-	// Wait for the kubelet to create the /etc/kubernetes/kubelet.conf kubeconfig file. If this process
+		if client, err = data.Client(); err != nil {
-	// times out, display a somewhat user-friendly message.
+			return errors.Wrap(err, "could not get client for dry-run")
-	waiter := apiclient.NewKubeWaiter(nil, 0, os.Stdout)
+		}
-	waiter.SetTimeout(cfg.Timeouts.KubeletHealthCheck.Duration)
+	} else {
-	kubeletConfig := initCfg.ClusterConfiguration.ComponentConfigs[componentconfigs.KubeletGroup].Get()
+		bootstrapKubeConfigFile := filepath.Join(data.KubeConfigDir(), kubeadmconstants.KubeletBootstrapKubeConfigFileName)
-	kubeletConfigTyped, ok := kubeletConfig.(*kubeletconfig.KubeletConfiguration)
+		// Deletes the bootstrapKubeConfigFile, so the credential used for TLS bootstrap is removed from disk
-	if !ok {
+		defer func() {
-		return errors.New("could not convert the KubeletConfiguration to a typed object")
+			_ = os.Remove(bootstrapKubeConfigFile)
-	}
+		}()
-	if err := waiter.WaitForKubelet(kubeletConfigTyped.HealthzBindAddress, *kubeletConfigTyped.HealthzPort); err != nil {
-		fmt.Printf(kubeadmJoinFailMsg, err)
-		return err
-	}
 
-	if err := waitForTLSBootstrappedClient(cfg.Timeouts.TLSBootstrap.Duration); err != nil {
+		// Apply patches to the in-memory kubelet configuration so that any configuration changes like kubelet healthz
-		fmt.Printf(kubeadmJoinFailMsg, err)
+		// address and port options are respected during the wait below. WriteConfigToDisk already applied patches to
-		return err
+		// the kubelet.yaml written to disk. This should be done after WriteConfigToDisk because both use the same config
-	}
+		// in memory and we don't want patches to be applied two times to the config that is written to disk.
+		if err := kubeletphase.ApplyPatchesToConfig(&initCfg.ClusterConfiguration, data.PatchesDir()); err != nil {
+			return errors.Wrap(err, "could not apply patches to the in-memory kubelet configuration")
+		}
 
-	// When we know the /etc/kubernetes/kubelet.conf file is available, get the client
+		// Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf
-	client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath())
+		// Wait for the kubelet to create the /etc/kubernetes/kubelet.conf kubeconfig file. If this process
-	if err != nil {
+		// times out, display a somewhat user-friendly message.
-		return err
+		waiter := apiclient.NewKubeWaiter(nil, 0, os.Stdout)
+		waiter.SetTimeout(cfg.Timeouts.KubeletHealthCheck.Duration)
+		kubeletConfig := initCfg.ClusterConfiguration.ComponentConfigs[componentconfigs.KubeletGroup].Get()
+		kubeletConfigTyped, ok := kubeletConfig.(*kubeletconfig.KubeletConfiguration)
+		if !ok {
+			return errors.New("could not convert the KubeletConfiguration to a typed object")
+		}
+		if err := waiter.WaitForKubelet(kubeletConfigTyped.HealthzBindAddress, *kubeletConfigTyped.HealthzPort); err != nil {
+			fmt.Printf(kubeadmJoinFailMsg, err)
+			return err
+		}
+
+		if err := waitForTLSBootstrappedClient(cfg.Timeouts.TLSBootstrap.Duration); err != nil {
+			fmt.Printf(kubeadmJoinFailMsg, err)
+			return err
+		}
+
+		// When we know the /etc/kubernetes/kubelet.conf file is available, get the client
+		client, err = kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath())
+		if err != nil {
+			return err
+		}
 	}
 
 	if !features.Enabled(initCfg.ClusterConfiguration.FeatureGates, features.NodeLocalCRISocket) {

cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeconfig.go

@@ -24,7 +24,6 @@ import (
 
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
-	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 )
 
@@ -53,13 +52,11 @@ func runKubeconfig() func(c workflow.RunData) error {
 
 		cfg := data.InitCfg()
 
-		if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
+		if err := upgrade.UpdateKubeletKubeconfigServer(cfg, data.DryRun()); err != nil {
-			if err := upgrade.UpdateKubeletLocalMode(cfg, data.DryRun()); err != nil {
+			return errors.Wrap(err, "failed to update kubelet local mode")
-				return errors.Wrap(err, "failed to update kubelet local mode")
-			}
 		}
 
-		fmt.Println("[upgrad/kubeconfig] The kubeconfig files for this node were successfully upgraded!")
+		fmt.Println("[upgrade/kubeconfig] The kubeconfig files for this node were successfully upgraded!")
 
 		return nil
 	}

cmd/kubeadm/app/cmd/phases/upgrade/node/kubeconfig.go

@@ -24,7 +24,6 @@ import (
 
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
-	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 )
 
@@ -60,10 +59,8 @@ func runKubeconfig() func(c workflow.RunData) error {
 		// Otherwise, retrieve all the info required for kubeconfig upgrade.
 		cfg := data.InitCfg()
 
-		if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
+		if err := upgrade.UpdateKubeletKubeconfigServer(cfg, data.DryRun()); err != nil {
-			if err := upgrade.UpdateKubeletLocalMode(cfg, data.DryRun()); err != nil {
+			return errors.Wrap(err, "failed to update kubelet local mode")
-				return errors.Wrap(err, "failed to update kubelet local mode")
-			}
 		}
 
 		fmt.Println("[upgrade/kubeconfig] The kubeconfig files for this node were successfully upgraded!")

cmd/kubeadm/app/features/features.go

@@ -36,7 +36,7 @@ const (
 	RootlessControlPlane = "RootlessControlPlane"
 	// WaitForAllControlPlaneComponents is expected to be alpha in v1.30
 	WaitForAllControlPlaneComponents = "WaitForAllControlPlaneComponents"
-	// ControlPlaneKubeletLocalMode is expected to be in alpha in v1.31, beta in v1.32
+	// ControlPlaneKubeletLocalMode is expected to be in alpha in v1.31, beta in v1.33
 	ControlPlaneKubeletLocalMode = "ControlPlaneKubeletLocalMode"
 	// NodeLocalCRISocket is expected to be in alpha in v1.32, beta in v1.33, ga in v1.35
 	NodeLocalCRISocket = "NodeLocalCRISocket"
@@ -54,7 +54,7 @@ var InitFeatureGates = FeatureList{
 			" Once UserNamespacesSupport graduates to GA, kubeadm will start using it and RootlessControlPlane will be removed.",
 	},
 	WaitForAllControlPlaneComponents: {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}},
-	ControlPlaneKubeletLocalMode:     {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
+	ControlPlaneKubeletLocalMode:     {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}},
 	NodeLocalCRISocket:               {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
 }
 

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -193,11 +193,11 @@ func WriteKubeletConfigFiles(cfg *kubeadmapi.InitConfiguration, kubeletConfigDir
 	return nil
 }
 
-// UpdateKubeletLocalMode changes the Server URL in the kubelets kubeconfig to the local API endpoint if it is currently
+// UpdateKubeletKubeconfigServer changes the Server URL in the kubelets kubeconfig if necessary depending on the
-// set to the ControlPlaneEndpoint.
+// ControlPlaneKubeletLocalMode feature gate.
-// TODO: remove this function once kubeletKubeConfigFilePath goes GA and is hardcoded to enabled by default:
+// TODO: remove this function once ControlPlaneKubeletLocalMode goes GA and is hardcoded to be enabled by default:
 // https://github.com/kubernetes/kubeadm/issues/2271
-func UpdateKubeletLocalMode(cfg *kubeadmapi.InitConfiguration, dryRun bool) error {
+func UpdateKubeletKubeconfigServer(cfg *kubeadmapi.InitConfiguration, dryRun bool) error {
 	kubeletKubeConfigFilePath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
 
 	if _, err := os.Stat(kubeletKubeConfigFilePath); err != nil {
@@ -228,19 +228,30 @@ func UpdateKubeletLocalMode(cfg *kubeadmapi.InitConfiguration, dryRun bool) erro
 		return err
 	}
 
-	// Skip changing kubeconfig file if Server does not match the ControlPlaneEndpoint.
+	var targetServer string
-	if config.Clusters[configContext.Cluster].Server != controlPlaneAPIEndpoint || controlPlaneAPIEndpoint == localAPIEndpoint {
+	if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
-		klog.V(2).Infof("Skipping update of the Server URL in %s, because it's already not equal to %q or already matches the localAPIEndpoint", kubeletKubeConfigFilePath, cfg.ControlPlaneEndpoint)
+		// Skip changing kubeconfig file if Server does not match the ControlPlaneEndpoint.
-		return nil
+		if config.Clusters[configContext.Cluster].Server != controlPlaneAPIEndpoint || controlPlaneAPIEndpoint == localAPIEndpoint {
+			klog.V(2).Infof("Skipping update of the Server URL in %s, because it's already not equal to %q or already matches the localAPIEndpoint", kubeletKubeConfigFilePath, controlPlaneAPIEndpoint)
+			return nil
+		}
+		targetServer = localAPIEndpoint
+	} else {
+		// Skip changing kubeconfig file if Server does not match the localAPIEndpoint.
+		if config.Clusters[configContext.Cluster].Server != localAPIEndpoint {
+			klog.V(2).Infof("Skipping update of the Server URL in %s, because it already matches the controlPlaneAPIEndpoint", kubeletKubeConfigFilePath)
+			return nil
+		}
+		targetServer = controlPlaneAPIEndpoint
 	}
 
 	if dryRun {
-		fmt.Printf("[dryrun] Would change the Server URL from %q to %q in %s and try to restart kubelet\n", config.Clusters[configContext.Cluster].Server, localAPIEndpoint, kubeletKubeConfigFilePath)
+		fmt.Printf("[dryrun] Would change the Server URL from %q to %q in %s and try to restart kubelet\n", config.Clusters[configContext.Cluster].Server, targetServer, kubeletKubeConfigFilePath)
 		return nil
 	}
 
-	klog.V(1).Infof("Changing the Server URL from %q to %q in %s", config.Clusters[configContext.Cluster].Server, localAPIEndpoint, kubeletKubeConfigFilePath)
+	klog.V(1).Infof("Changing the Server URL from %q to %q in %s", config.Clusters[configContext.Cluster].Server, targetServer, kubeletKubeConfigFilePath)
-	config.Clusters[configContext.Cluster].Server = localAPIEndpoint
+	config.Clusters[configContext.Cluster].Server = targetServer
 
 	if err := clientcmd.WriteToFile(*config, kubeletKubeConfigFilePath); err != nil {
 		return err