Add restart components to custom certs

cluster/hosts.go

@@ -118,9 +118,13 @@ func (c *Cluster) InvertIndexHosts() error {
 	return nil
 }
 
-func (c *Cluster) SetUpHosts(ctx context.Context, rotateCerts bool) error {
+func (c *Cluster) SetUpHosts(ctx context.Context, flags ExternalFlags) error {
 	if c.AuthnStrategies[AuthnX509Provider] {
 		log.Infof(ctx, "[certificates] Deploying kubernetes certificates to Cluster nodes")
+		forceDeploy := false
+		if flags.CustomCerts || c.RancherKubernetesEngineConfig.RotateCertificates != nil {
+			forceDeploy = true
+		}
 		hostList := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts)
 		var errgrp errgroup.Group
 
@@ -129,7 +133,7 @@ func (c *Cluster) SetUpHosts(ctx context.Context, rotateCerts bool) error {
 			errgrp.Go(func() error {
 				var errList []error
 				for host := range hostsQueue {
-					err := pki.DeployCertificatesOnPlaneHost(ctx, host.(*hosts.Host), c.RancherKubernetesEngineConfig, c.Certificates, c.SystemImages.CertDownloader, c.PrivateRegistriesMap, rotateCerts)
+					err := pki.DeployCertificatesOnPlaneHost(ctx, host.(*hosts.Host), c.RancherKubernetesEngineConfig, c.Certificates, c.SystemImages.CertDownloader, c.PrivateRegistriesMap, forceDeploy)
 					if err != nil {
 						errList = append(errList, err)
 					}

cluster/reconcile.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/k8s"
 	"github.com/rancher/rke/log"
@@ -47,6 +48,12 @@ func ReconcileCluster(ctx context.Context, kubeCluster, currentCluster *Cluster,
 	if err := reconcileControl(ctx, currentCluster, kubeCluster, kubeClient); err != nil {
 		return err
 	}
+	if flags.CustomCerts {
+		if err := restartComponentsWhenCertChanges(ctx, currentCluster, kubeCluster); err != nil {
+			return err
+		}
+	}
+
 	log.Infof(ctx, "[reconcile] Reconciled cluster state successfully")
 	return nil
 }
@@ -243,3 +250,75 @@ func cleanControlNode(ctx context.Context, kubeCluster, currentCluster *Cluster,
 	}
 	return nil
 }
+
+func restartComponentsWhenCertChanges(ctx context.Context, currentCluster, kubeCluster *Cluster) error {
+	AllCertsMap := map[string]bool{
+		pki.KubeAPICertName:            false,
+		pki.RequestHeaderCACertName:    false,
+		pki.CACertName:                 false,
+		pki.ServiceAccountTokenKeyName: false,
+		pki.APIProxyClientCertName:     false,
+		pki.KubeControllerCertName:     false,
+		pki.KubeSchedulerCertName:      false,
+		pki.KubeProxyCertName:          false,
+		pki.KubeNodeCertName:           false,
+	}
+	checkCertificateChanges(ctx, currentCluster, kubeCluster, AllCertsMap)
+	// check Restart Function
+	allHosts := hosts.GetUniqueHostList(kubeCluster.EtcdHosts, kubeCluster.ControlPlaneHosts, kubeCluster.WorkerHosts)
+	AllCertsFuncMap := map[string][]services.RestartFunc{
+		pki.CACertName:                 []services.RestartFunc{services.RestartKubeAPI, services.RestartKubeController, services.RestartKubelet},
+		pki.KubeAPICertName:            []services.RestartFunc{services.RestartKubeAPI, services.RestartKubeController},
+		pki.RequestHeaderCACertName:    []services.RestartFunc{services.RestartKubeAPI},
+		pki.ServiceAccountTokenKeyName: []services.RestartFunc{services.RestartKubeAPI, services.RestartKubeController},
+		pki.APIProxyClientCertName:     []services.RestartFunc{services.RestartKubeAPI},
+		pki.KubeControllerCertName:     []services.RestartFunc{services.RestartKubeController},
+		pki.KubeSchedulerCertName:      []services.RestartFunc{services.RestartScheduler},
+		pki.KubeProxyCertName:          []services.RestartFunc{services.RestartKubeproxy},
+		pki.KubeNodeCertName:           []services.RestartFunc{services.RestartKubelet},
+	}
+	for certName, changed := range AllCertsMap {
+		if changed {
+			for _, host := range allHosts {
+				runRestartFuncs(ctx, AllCertsFuncMap, certName, host)
+			}
+		}
+	}
+
+	for _, host := range kubeCluster.EtcdHosts {
+		etcdCertName := pki.GetEtcdCrtName(host.Address)
+		certMap := map[string]bool{
+			etcdCertName: false,
+		}
+		checkCertificateChanges(ctx, currentCluster, kubeCluster, certMap)
+		if certMap[etcdCertName] || AllCertsMap[pki.CACertName] {
+			if err := docker.DoRestartContainer(ctx, host.DClient, services.EtcdContainerName, host.HostnameOverride); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func runRestartFuncs(ctx context.Context, certFuncMap map[string][]services.RestartFunc, certName string, host *hosts.Host) error {
+	for _, restartFunc := range certFuncMap[certName] {
+		if err := restartFunc(ctx, host); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func checkCertificateChanges(ctx context.Context, currentCluster, kubeCluster *Cluster, certMap map[string]bool) {
+	for certName := range certMap {
+		if currentCluster.Certificates[certName].CertificatePEM != kubeCluster.Certificates[certName].CertificatePEM {
+			certMap[certName] = true
+			continue
+		}
+		if !(certName == pki.RequestHeaderCACertName || certName == pki.CACertName) {
+			if currentCluster.Certificates[certName].KeyPEM != kubeCluster.Certificates[certName].KeyPEM {
+				certMap[certName] = true
+			}
+		}
+	}
+}

cmd/cert.go

@@ -154,7 +154,7 @@ func rebuildClusterWithRotatedCertificates(ctx context.Context,
 	clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Key))
 	caCrt = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.CACertName].Certificate))
 
-	if err := kubeCluster.SetUpHosts(ctx, true); err != nil {
+	if err := kubeCluster.SetUpHosts(ctx, flags); err != nil {
 		return APIURL, caCrt, clientCert, clientKey, nil, err
 	}
 	// Save new State

cmd/up.go

@@ -161,7 +161,7 @@ func ClusterUp(ctx context.Context, dialersOptions hosts.DialersOptions, flags c
 	caCrt = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.CACertName].Certificate))
 
 	// moved deploying certs before reconcile to remove all unneeded certs generation from reconcile
-	err = kubeCluster.SetUpHosts(ctx, false)
+	err = kubeCluster.SetUpHosts(ctx, flags)
 	if err != nil {
 		return APIURL, caCrt, clientCert, clientKey, nil, err
 	}

pki/deploy.go

@@ -21,13 +21,13 @@ const (
 	StateDeployerContainerName = "cluster-state-deployer"
 )
 
-func DeployCertificatesOnPlaneHost(ctx context.Context, host *hosts.Host, rkeConfig v3.RancherKubernetesEngineConfig, crtMap map[string]CertificatePKI, certDownloaderImage string, prsMap map[string]v3.PrivateRegistry, rotateCerts bool) error {
+func DeployCertificatesOnPlaneHost(ctx context.Context, host *hosts.Host, rkeConfig v3.RancherKubernetesEngineConfig, crtMap map[string]CertificatePKI, certDownloaderImage string, prsMap map[string]v3.PrivateRegistry, forceDeploy bool) error {
 	crtBundle := GenerateRKENodeCerts(ctx, rkeConfig, host.Address, crtMap)
 	env := []string{}
 	for _, crt := range crtBundle {
 		env = append(env, crt.ToEnv()...)
 	}
-	if rotateCerts {
+	if forceDeploy {
 		env = append(env, "FORCE_DEPLOY=true")
 	}
 	return doRunDeployer(ctx, host, env, certDownloaderImage, prsMap)

services/controlplane.go

@@ -93,17 +93,17 @@ func RestartControlPlane(ctx context.Context, controlHosts []*hosts.Host) error
 			for host := range hostsQueue {
 				runHost := host.(*hosts.Host)
 				// restart KubeAPI
-				if err := restartKubeAPI(ctx, runHost); err != nil {
+				if err := RestartKubeAPI(ctx, runHost); err != nil {
 					errList = append(errList, err)
 				}
 
 				// restart KubeController
-				if err := restartKubeController(ctx, runHost); err != nil {
+				if err := RestartKubeController(ctx, runHost); err != nil {
 					errList = append(errList, err)
 				}
 
 				// restart scheduler
-				err := restartScheduler(ctx, runHost)
+				err := RestartScheduler(ctx, runHost)
 				if err != nil {
 					errList = append(errList, err)
 				}

services/kubeapi.go

@@ -25,6 +25,6 @@ func removeKubeAPI(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, KubeAPIContainerName, host.Address)
 }
 
-func restartKubeAPI(ctx context.Context, host *hosts.Host) error {
+func RestartKubeAPI(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRestartContainer(ctx, host.DClient, KubeAPIContainerName, host.Address)
 }

services/kubecontroller.go

@@ -23,6 +23,6 @@ func removeKubeController(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, KubeControllerContainerName, host.Address)
 }
 
-func restartKubeController(ctx context.Context, host *hosts.Host) error {
+func RestartKubeController(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRestartContainer(ctx, host.DClient, KubeControllerContainerName, host.Address)
 }

services/kubelet.go

@@ -24,6 +24,6 @@ func removeKubelet(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, KubeletContainerName, host.Address)
 }
 
-func restartKubelet(ctx context.Context, host *hosts.Host) error {
+func RestartKubelet(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRestartContainer(ctx, host.DClient, KubeletContainerName, host.Address)
 }

services/kubeproxy.go

@@ -23,6 +23,6 @@ func removeKubeproxy(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, KubeproxyContainerName, host.Address)
 }
 
-func restartKubeproxy(ctx context.Context, host *hosts.Host) error {
+func RestartKubeproxy(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRestartContainer(ctx, host.DClient, KubeproxyContainerName, host.Address)
 }

services/proxy.go

@@ -25,6 +25,6 @@ func removeNginxProxy(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, NginxProxyContainerName, host.Address)
 }
 
-func restartNginxProxy(ctx context.Context, host *hosts.Host) error {
+func RestartNginxProxy(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRestartContainer(ctx, host.DClient, NginxProxyContainerName, host.Address)
 }

services/scheduler.go

@@ -23,6 +23,6 @@ func removeScheduler(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, SchedulerContainerName, host.Address)
 }
 
-func restartScheduler(ctx context.Context, host *hosts.Host) error {
+func RestartScheduler(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRestartContainer(ctx, host.DClient, SchedulerContainerName, host.Address)
 }

services/services.go

@@ -47,6 +47,8 @@ const (
 	WorkerThreads = util.WorkerThreads
 )
 
+type RestartFunc func(context.Context, *hosts.Host) error
+
 func runSidekick(ctx context.Context, host *hosts.Host, prsMap map[string]v3.PrivateRegistry, sidecarProcess v3.Process) error {
 	isRunning, err := docker.IsContainerRunning(ctx, host.DClient, host.Address, SidekickContainerName, true)
 	if err != nil {

services/workerplane.go

@@ -109,13 +109,13 @@ func RestartWorkerPlane(ctx context.Context, workerHosts []*hosts.Host) error {
 			var errList []error
 			for host := range hostsQueue {
 				runHost := host.(*hosts.Host)
-				if err := restartKubelet(ctx, runHost); err != nil {
+				if err := RestartKubelet(ctx, runHost); err != nil {
 					errList = append(errList, err)
 				}
-				if err := restartKubeproxy(ctx, runHost); err != nil {
+				if err := RestartKubeproxy(ctx, runHost); err != nil {
 					errList = append(errList, err)
 				}
-				if err := restartNginxProxy(ctx, runHost); err != nil {
+				if err := RestartNginxProxy(ctx, runHost); err != nil {
 					errList = append(errList, err)
 				}
 			}