Merge pull request #84118 from neolit123/1.17-kubeadm-add-kubelet-post-phase

kubeadm: enable kubelet client certificate rotation on primary CP nodes

cmd/kubeadm/app/cmd/init.go

@@ -63,21 +63,21 @@ var (
 		{{if .ControlPlaneEndpoint -}}
 		{{if .UploadCerts -}}
 		You can now join any number of the control-plane node running the following command on each as root:
-					  
+
 		  {{.joinControlPlaneCommand}}
-					
+
 		Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
-		As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use 
+		As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
 		"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
-		  
+
 		{{else -}}
-		You can now join any number of control-plane nodes by copying certificate authorities 
+		You can now join any number of control-plane nodes by copying certificate authorities
 		and service account keys on each node and then running the following as root:
-				  
+
-		  {{.joinControlPlaneCommand}}	  
+		  {{.joinControlPlaneCommand}}
-		  
+
 		{{end}}{{end}}Then you can join any number of worker nodes by running the following on each as root:
-						  
+
 		{{.joinWorkerCommand}}
 		`)))
 )
@@ -182,6 +182,7 @@ func NewCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command {
 	initRunner.AppendPhase(phases.NewUploadCertsPhase())
 	initRunner.AppendPhase(phases.NewMarkControlPlanePhase())
 	initRunner.AppendPhase(phases.NewBootstrapTokenPhase())
+	initRunner.AppendPhase(phases.NewKubeletFinalizePhase())
 	initRunner.AppendPhase(phases.NewAddonPhase())
 
 	// sets the data builder function, that will be used by the runner

cmd/kubeadm/app/cmd/phases/init/BUILD

@@ -11,6 +11,7 @@ go_library(
         "etcd.go",
         "kubeconfig.go",
         "kubelet.go",
+        "kubeletfinalize.go",
         "markcontrolplane.go",
         "preflight.go",
         "uploadcerts.go",
@@ -47,6 +48,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//vendor/github.com/lithammer/dedent:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",

cmd/kubeadm/app/cmd/phases/init/kubelet.go

@@ -17,6 +17,8 @@ limitations under the License.
 package phases
 
 import (
+	"fmt"
+
 	"github.com/pkg/errors"
 	"k8s.io/klog"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
@@ -76,7 +78,7 @@ func runKubeletStart(c workflow.RunData) error {
 
 	// Try to start the kubelet service in case it's inactive
 	if !data.DryRun() {
-		klog.V(1).Infoln("Starting the kubelet")
+		fmt.Println("[kubelet-start] Starting the kubelet")
 		kubeletphase.TryStartKubelet()
 	}
 

cmd/kubeadm/app/cmd/phases/init/kubeletfinalize.go

@@ -0,0 +1,136 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package phases
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	clientcmd "k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
+)
+
+var (
+	kubeletFinalizePhaseExample = cmdutil.Examples(`
+		# Updates settings relevant to the kubelet after TLS bootstrap"
+		kubeadm init phase kubelet-finalize all --config
+		`)
+)
+
+// NewKubeletFinalizePhase creates a kubeadm workflow phase that updates settings
+// relevant to the kubelet after TLS bootstrap.
+func NewKubeletFinalizePhase() workflow.Phase {
+	return workflow.Phase{
+		Name:    "kubelet-finalize",
+		Short:   "Updates settings relevant to the kubelet after TLS bootstrap",
+		Example: kubeletFinalizePhaseExample,
+		Phases: []workflow.Phase{
+			{
+				Name:           "all",
+				Short:          "Run all kubelet-finalize phases",
+				InheritFlags:   []string{options.CfgPath, options.CertificatesDir},
+				Example:        kubeletFinalizePhaseExample,
+				RunAllSiblings: true,
+			},
+			{
+				Name:         "experimental-cert-rotation",
+				Short:        "Enable kubelet client certificate rotation",
+				InheritFlags: []string{options.CfgPath, options.CertificatesDir},
+				Run:          runKubeletFinalizeCertRotation,
+			},
+		},
+	}
+}
+
+// runKubeletFinalizeCertRotation detects if the kubelet certificate rotation is enabled
+// and updates the kubelet.conf file to point to a rotatable certificate and key for the
+// Node user.
+func runKubeletFinalizeCertRotation(c workflow.RunData) error {
+	data, ok := c.(InitData)
+	if !ok {
+		return errors.New("kubelet-finalize phase invoked with an invalid data struct")
+	}
+
+	// Check if the user has added the kubelet --cert-dir flag.
+	// If yes, use that path, else use the kubeadm provided value.
+	cfg := data.Cfg()
+	pkiPath := filepath.Join(data.KubeletDir(), "pki")
+	val, ok := cfg.NodeRegistration.KubeletExtraArgs["cert-dir"]
+	if ok {
+		pkiPath = val
+	}
+
+	// Check for the existence of the kubelet-client-current.pem file in the kubelet certificate directory.
+	rotate := false
+	pemPath := filepath.Join(pkiPath, "kubelet-client-current.pem")
+	if _, err := os.Stat(pemPath); err == nil {
+		klog.V(1).Infof("[kubelet-finalize] Assuming that kubelet client certificate rotation is enabled: found %q", pemPath)
+		rotate = true
+	} else {
+		klog.V(1).Infof("[kubelet-finalize] Assuming that kubelet client certificate rotation is disabled: %v", err)
+	}
+
+	// Exit early if rotation is disabled.
+	if !rotate {
+		return nil
+	}
+
+	kubeconfigPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
+	fmt.Printf("[kubelet-finalize] Updating %q to point to a rotatable kubelet client certificate and key\n", kubeconfigPath)
+
+	// Exit early if dry-running is enabled.
+	if data.DryRun() {
+		return nil
+	}
+
+	// Load the kubeconfig from disk.
+	kubeconfig, err := clientcmd.LoadFromFile(kubeconfigPath)
+	if err != nil {
+		return errors.Wrapf(err, "could not load %q", kubeconfigPath)
+	}
+
+	// Perform basic validation. The errors here can only happen if the kubelet.conf was corrupted.
+	userName := fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name)
+	info, ok := kubeconfig.AuthInfos[userName]
+	if !ok {
+		return errors.Errorf("the file %q does not contain authentication for user %q", kubeconfigPath, cfg.NodeRegistration.Name)
+	}
+
+	// Update the client certificate and key of the node authorizer to point to the PEM symbolic link.
+	info.ClientKeyData = []byte{}
+	info.ClientCertificateData = []byte{}
+	info.ClientKey = pemPath
+	info.ClientCertificate = pemPath
+
+	// Writes the kubeconfig back to disk.
+	if err = clientcmd.WriteToFile(*kubeconfig, kubeconfigPath); err != nil {
+		return errors.Wrapf(err, "failed to serialize %q", kubeconfigPath)
+	}
+
+	// Restart the kubelet.
+	klog.V(1).Info("[kubelet-finalize] Restarting the kubelet to enable client certificate rotation")
+	kubeletphase.TryRestartKubelet()
+
+	return nil
+}

cmd/kubeadm/app/cmd/phases/join/kubelet.go

@@ -147,7 +147,7 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) {
 	}
 
 	// Try to start the kubelet service in case it's inactive
-	klog.V(1).Infoln("[kubelet-start] Starting the kubelet")
+	fmt.Println("[kubelet-start] Starting the kubelet")
 	kubeletphase.TryStartKubelet()
 
 	// Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf

cmd/kubeadm/app/phases/kubelet/kubelet.go

@@ -35,7 +35,6 @@ func TryStartKubelet() {
 		fmt.Println("[kubelet-start] couldn't detect a kubelet service, can't make sure the kubelet is running properly.")
 	}
 
-	fmt.Println("[kubelet-start] Activating the kubelet service")
 	// This runs "systemctl daemon-reload && systemctl restart kubelet"
 	if err := initSystem.ServiceRestart("kubelet"); err != nil {
 		fmt.Printf("[kubelet-start] WARNING: unable to start the kubelet service: [%v]\n", err)
@@ -61,3 +60,22 @@ func TryStopKubelet() {
 		fmt.Printf("[kubelet-start] WARNING: unable to stop the kubelet service momentarily: [%v]\n", err)
 	}
 }
+
+// TryRestartKubelet attempts to restart the kubelet service
+func TryRestartKubelet() {
+	// If we notice that the kubelet service is inactive, try to start it
+	initSystem, err := initsystem.GetInitSystem()
+	if err != nil {
+		fmt.Println("[kubelet-start] no supported init system detected, won't make sure the kubelet not running for a short period of time while setting up configuration for it.")
+		return
+	}
+
+	if !initSystem.ServiceExists("kubelet") {
+		fmt.Println("[kubelet-start] couldn't detect a kubelet service, can't make sure the kubelet not running for a short period of time while setting up configuration for it.")
+	}
+
+	// This runs "systemctl daemon-reload && systemctl stop kubelet"
+	if err := initSystem.ServiceRestart("kubelet"); err != nil {
+		fmt.Printf("[kubelet-start] WARNING: unable to restart the kubelet service momentarily: [%v]\n", err)
+	}
+}