Add Rotate certificates command to rke

cluster/addons.go

@@ -28,6 +28,9 @@ const (
 	IngressAddonJobName            = "rke-ingress-controller-deploy-job"
 	IngressAddonDeleteJobName      = "rke-ingress-controller-delete-job"
 	MetricsServerAddonResourceName = "rke-metrics-addon"
+	NginxIngressAddonAppName       = "ingress-nginx"
+	KubeDNSAddonAppName            = "kube-dns"
+	KubeDNSAutoscalerAppName       = "kube-dns-autoscaler"
 )
 
 type ingressOptions struct {

cluster/certificates.go

@@ -71,7 +71,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
 					if err := rebuildLocalAdminConfig(ctx, kubeCluster); err != nil {
 						return err
 					}
-					kubeCluster.Certificates, err = regenerateAPICertificate(kubeCluster, kubeCluster.Certificates)
+					err = pki.GenerateKubeAPICertificate(ctx, kubeCluster.Certificates, kubeCluster.RancherKubernetesEngineConfig, "", "")
 					if err != nil {
 						return fmt.Errorf("Failed to regenerate KubeAPI certificate %v", err)
 					}
@@ -129,6 +129,7 @@ func getClusterCerts(ctx context.Context, kubeClient *kubernetes.Clientset, etcd
 		pki.KubeAdminCertName,
 		pki.APIProxyClientCertName,
 		pki.RequestHeaderCACertName,
+		pki.ServiceAccountTokenKeyName,
 	}
 
 	for _, etcdHost := range etcdHosts {
@@ -141,14 +142,16 @@ func getClusterCerts(ctx context.Context, kubeClient *kubernetes.Clientset, etcd
 		secret, err := k8s.GetSecret(kubeClient, certName)
 		if err != nil && !strings.HasPrefix(certName, "kube-etcd") &&
 			!strings.Contains(certName, pki.RequestHeaderCACertName) &&
-			!strings.Contains(certName, pki.APIProxyClientCertName) {
+			!strings.Contains(certName, pki.APIProxyClientCertName) &&
+			!strings.Contains(certName, pki.ServiceAccountTokenKeyName) {
 			return nil, err
 		}
 		// If I can't find an etcd, requestheader, or proxy client cert, I will not fail and will create it later.
 		if (secret == nil || secret.Data == nil) &&
 			(strings.HasPrefix(certName, "kube-etcd") ||
 				strings.Contains(certName, pki.RequestHeaderCACertName) ||
-				strings.Contains(certName, pki.APIProxyClientCertName)) {
+				strings.Contains(certName, pki.APIProxyClientCertName) ||
+				strings.Contains(certName, pki.ServiceAccountTokenKeyName)) {
 			certMap[certName] = pki.CertificatePKI{}
 			continue
 		}
@@ -346,7 +349,7 @@ func (c *Cluster) getBackupHosts() []*hosts.Host {
 
 func regenerateAPIAggregationCerts(c *Cluster, certificates map[string]pki.CertificatePKI) (map[string]pki.CertificatePKI, error) {
 	logrus.Debugf("[certificates] Regenerating Kubernetes API server aggregation layer requestheader client CA certificates")
-	requestHeaderCACrt, requestHeaderCAKey, err := pki.GenerateCACertAndKey(pki.RequestHeaderCACertName)
+	requestHeaderCACrt, requestHeaderCAKey, err := pki.GenerateCACertAndKey(pki.RequestHeaderCACertName, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -361,3 +364,54 @@ func regenerateAPIAggregationCerts(c *Cluster, certificates map[string]pki.Certi
 	certificates[pki.APIProxyClientCertName] = pki.ToCertObject(pki.APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey)
 	return certificates, nil
 }
+
+func RotateRKECertificates(ctx context.Context, c *Cluster, configPath, configDir string, components []string, rotateCACerts bool) error {
+	var (
+		serviceAccountTokenKey string
+	)
+	componentsCertsFuncMap := map[string]pki.GenFunc{
+		services.KubeAPIContainerName:        pki.GenerateKubeAPICertificate,
+		services.KubeControllerContainerName: pki.GenerateKubeControllerCertificate,
+		services.SchedulerContainerName:      pki.GenerateKubeSchedulerCertificate,
+		services.KubeproxyContainerName:      pki.GenerateKubeProxyCertificate,
+		services.KubeletContainerName:        pki.GenerateKubeNodeCertificate,
+		services.EtcdContainerName:           pki.GenerateEtcdCertificates,
+	}
+	if rotateCACerts {
+		// rotate CA cert and RequestHeader CA cert
+		if err := pki.GenerateRKECACerts(ctx, c.Certificates, configPath, configDir); err != nil {
+			return err
+		}
+		components = nil
+	}
+	for _, k8sComponent := range components {
+		genFunc := componentsCertsFuncMap[k8sComponent]
+		if genFunc != nil {
+			if err := genFunc(ctx, c.Certificates, c.RancherKubernetesEngineConfig, configPath, configDir); err != nil {
+				return err
+			}
+		}
+	}
+	if len(components) == 0 {
+		// do not rotate service account token
+		if c.Certificates[pki.ServiceAccountTokenKeyName].Key != nil {
+			serviceAccountTokenKey = string(cert.EncodePrivateKeyPEM(c.Certificates[pki.ServiceAccountTokenKeyName].Key))
+		}
+		if err := pki.GenerateRKEServicesCerts(ctx, c.Certificates, c.RancherKubernetesEngineConfig, configPath, configDir); err != nil {
+			return err
+		}
+		if serviceAccountTokenKey != "" {
+			privateKey, err := cert.ParsePrivateKeyPEM([]byte(serviceAccountTokenKey))
+			if err != nil {
+				return err
+			}
+			c.Certificates[pki.ServiceAccountTokenKeyName] = pki.ToCertObject(
+				pki.ServiceAccountTokenKeyName,
+				pki.ServiceAccountTokenKeyName,
+				"",
+				c.Certificates[pki.ServiceAccountTokenKeyName].Certificate,
+				privateKey.(*rsa.PrivateKey))
+		}
+	}
+	return nil
+}

cluster/cluster.go

@@ -67,6 +67,10 @@ const (
 	WorkerPlane                = "workerPlan"
 	EtcdPlane                  = "etcd"
 
+	KubeAppLabel = "k8s-app"
+	AppLabel     = "app"
+	NameLabel    = "name"
+
 	WorkerThreads = util.WorkerThreads
 )
 
@@ -458,3 +462,47 @@ func ConfigureCluster(
 	}
 	return nil
 }
+
+func RestartClusterPods(ctx context.Context, kubeCluster *Cluster) error {
+	log.Infof(ctx, "Restarting network, ingress, and metrics pods")
+	// this will remove the pods created by RKE and let the controller creates them again
+	kubeClient, err := k8s.NewClient(kubeCluster.LocalKubeConfigPath, kubeCluster.K8sWrapTransport)
+	if err != nil {
+		return fmt.Errorf("Failed to initialize new kubernetes client: %v", err)
+	}
+	labelsList := []string{
+		fmt.Sprintf("%s=%s", KubeAppLabel, CalicoNetworkPlugin),
+		fmt.Sprintf("%s=%s", KubeAppLabel, FlannelNetworkPlugin),
+		fmt.Sprintf("%s=%s", KubeAppLabel, CanalNetworkPlugin),
+		fmt.Sprintf("%s=%s", NameLabel, WeaveNetworkPlugin),
+		fmt.Sprintf("%s=%s", AppLabel, NginxIngressAddonAppName),
+		fmt.Sprintf("%s=%s", KubeAppLabel, DefaultMonitoringProvider),
+		fmt.Sprintf("%s=%s", KubeAppLabel, KubeDNSAddonAppName),
+		fmt.Sprintf("%s=%s", KubeAppLabel, KubeDNSAutoscalerAppName),
+	}
+	var errgrp errgroup.Group
+	labelQueue := util.GetObjectQueue(labelsList)
+	for w := 0; w < services.WorkerThreads; w++ {
+		errgrp.Go(func() error {
+			var errList []error
+			for label := range labelQueue {
+				runLabel := label.(string)
+				// list pods to be deleted
+				pods, err := k8s.ListPodsByLabel(kubeClient, runLabel)
+				if err != nil {
+					errList = append(errList, err)
+				}
+				// delete pods
+				err = k8s.DeletePods(kubeClient, pods)
+				if err != nil {
+					errList = append(errList, err)
+				}
+			}
+			return util.ErrList(errList)
+		})
+	}
+	if err := errgrp.Wait(); err != nil {
+		return err
+	}
+	return nil
+}

cluster/hosts.go

@@ -116,7 +116,7 @@ func (c *Cluster) InvertIndexHosts() error {
 	return nil
 }
 
-func (c *Cluster) SetUpHosts(ctx context.Context) error {
+func (c *Cluster) SetUpHosts(ctx context.Context, rotateCerts bool) error {
 	if c.Authentication.Strategy == X509AuthenticationProvider {
 		log.Infof(ctx, "[certificates] Deploying kubernetes certificates to Cluster nodes")
 		hostList := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts)
@@ -127,7 +127,7 @@ func (c *Cluster) SetUpHosts(ctx context.Context) error {
 			errgrp.Go(func() error {
 				var errList []error
 				for host := range hostsQueue {
-					err := pki.DeployCertificatesOnPlaneHost(ctx, host.(*hosts.Host), c.RancherKubernetesEngineConfig, c.Certificates, c.SystemImages.CertDownloader, c.PrivateRegistriesMap)
+					err := pki.DeployCertificatesOnPlaneHost(ctx, host.(*hosts.Host), c.RancherKubernetesEngineConfig, c.Certificates, c.SystemImages.CertDownloader, c.PrivateRegistriesMap, rotateCerts)
 					if err != nil {
 						errList = append(errList, err)
 					}

cluster/plan.go

@@ -134,7 +134,7 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process {
 		"tls-private-key-file":               pki.GetKeyPath(pki.KubeAPICertName),
 		"kubelet-client-certificate":         pki.GetCertPath(pki.KubeAPICertName),
 		"kubelet-client-key":                 pki.GetKeyPath(pki.KubeAPICertName),
-		"service-account-key-file":           pki.GetKeyPath(pki.KubeAPICertName),
+		"service-account-key-file":           pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
 		"etcd-cafile":                        etcdCAClientCert,
 		"etcd-certfile":                      etcdClientCert,
 		"etcd-keyfile":                       etcdClientKey,
@@ -249,7 +249,7 @@ func (c *Cluster) BuildKubeControllerProcess(prefixPath string) v3.Process {
 		"allocate-node-cidrs":              "true",
 		"cluster-cidr":                     c.ClusterCIDR,
 		"service-cluster-ip-range":         c.Services.KubeController.ServiceClusterIPRange,
-		"service-account-private-key-file": pki.GetKeyPath(pki.KubeAPICertName),
+		"service-account-private-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
 		"root-ca-file":                     pki.GetCertPath(pki.CACertName),
 	}
 	if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {

cluster/reconcile.go

@@ -47,6 +47,12 @@ func ReconcileCluster(ctx context.Context, kubeCluster, currentCluster *Cluster,
 	if err := reconcileControl(ctx, currentCluster, kubeCluster, kubeClient); err != nil {
 		return err
 	}
+	// Handle service account token key issue
+	kubeAPICert := currentCluster.Certificates[pki.KubeAPICertName]
+	if currentCluster.Certificates[pki.ServiceAccountTokenKeyName].Key == nil {
+		log.Infof(ctx, "[certificates] Creating service account token key")
+		currentCluster.Certificates[pki.ServiceAccountTokenKeyName] = pki.ToCertObject(pki.ServiceAccountTokenKeyName, pki.ServiceAccountTokenKeyName, "", kubeAPICert.Certificate, kubeAPICert.Key)
+	}
 	log.Infof(ctx, "[reconcile] Reconciled cluster state successfully")
 	return nil
 }

cmd/cert.go

@@ -0,0 +1,141 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/rancher/rke/cluster"
+	"github.com/rancher/rke/hosts"
+	"github.com/rancher/rke/k8s"
+	"github.com/rancher/rke/log"
+	"github.com/rancher/rke/pki"
+	"github.com/rancher/rke/services"
+	"github.com/rancher/types/apis/management.cattle.io/v3"
+	"github.com/urfave/cli"
+)
+
+func CertificateCommand() cli.Command {
+	return cli.Command{
+		Name:  "cert",
+		Usage: "Certificates management for RKE cluster",
+		Subcommands: cli.Commands{
+			cli.Command{
+				Name:   "rotate",
+				Usage:  "Rotate RKE cluster certificates",
+				Action: rotateRKECertificatesFromCli,
+				Flags: []cli.Flag{
+					cli.StringFlag{
+						Name:   "config",
+						Usage:  "Specify an alternate cluster YAML file",
+						Value:  pki.ClusterConfig,
+						EnvVar: "RKE_CONFIG",
+					},
+					cli.StringSliceFlag{
+						Name: "service",
+						Usage: fmt.Sprintf("Specify a k8s service to rotate certs, (allowed values: %s, %s, %s, %s, %s, %s)",
+							services.KubeAPIContainerName,
+							services.KubeControllerContainerName,
+							services.SchedulerContainerName,
+							services.KubeletContainerName,
+							services.KubeproxyContainerName,
+							services.EtcdContainerName,
+						),
+					},
+					cli.BoolFlag{
+						Name:  "rotate-ca",
+						Usage: "Rotate all certificates including CA certs",
+					},
+				},
+			},
+		},
+	}
+}
+
+func rotateRKECertificatesFromCli(ctx *cli.Context) error {
+	k8sComponent := ctx.StringSlice("service")
+	rotateCACert := ctx.Bool("rotate-ca")
+	clusterFile, filePath, err := resolveClusterFile(ctx)
+	if err != nil {
+		return fmt.Errorf("Failed to resolve cluster file: %v", err)
+	}
+	clusterFilePath = filePath
+
+	rkeConfig, err := cluster.ParseConfig(clusterFile)
+	if err != nil {
+		return fmt.Errorf("Failed to parse cluster file: %v", err)
+	}
+	rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig)
+	if err != nil {
+		return err
+	}
+
+	return RotateRKECertificates(context.Background(), rkeConfig, nil, nil, nil, false, "", k8sComponent, rotateCACert)
+}
+
+func showRKECertificatesFromCli(ctx *cli.Context) error {
+	return nil
+}
+
+func RotateRKECertificates(
+	ctx context.Context,
+	rkeConfig *v3.RancherKubernetesEngineConfig,
+	dockerDialerFactory, localConnDialerFactory hosts.DialerFactory,
+	k8sWrapTransport k8s.WrapTransport,
+	local bool, configDir string, components []string, rotateCACerts bool) error {
+
+	log.Infof(ctx, "Rotating Kubernetes cluster certificates")
+	kubeCluster, err := cluster.ParseCluster(ctx, rkeConfig, clusterFilePath, configDir, dockerDialerFactory, localConnDialerFactory, k8sWrapTransport)
+	if err != nil {
+		return err
+	}
+
+	if err := kubeCluster.TunnelHosts(ctx, local); err != nil {
+		return err
+	}
+
+	currentCluster, err := kubeCluster.GetClusterState(ctx)
+	if err != nil {
+		return err
+	}
+
+	if err := cluster.SetUpAuthentication(ctx, kubeCluster, currentCluster); err != nil {
+		return err
+	}
+
+	if err := cluster.RotateRKECertificates(ctx, kubeCluster, clusterFilePath, configDir, components, rotateCACerts); err != nil {
+		return err
+	}
+
+	if err := kubeCluster.SetUpHosts(ctx, true); err != nil {
+		return err
+	}
+	// Restarting Kubernetes components
+	servicesMap := make(map[string]bool)
+	for _, component := range components {
+		servicesMap[component] = true
+	}
+
+	if len(components) == 0 || rotateCACerts || servicesMap[services.EtcdContainerName] {
+		if err := services.RestartEtcdPlane(ctx, kubeCluster.EtcdHosts); err != nil {
+			return err
+		}
+	}
+
+	if err := services.RestartControlPlane(ctx, kubeCluster.ControlPlaneHosts); err != nil {
+		return err
+	}
+
+	allHosts := hosts.GetUniqueHostList(kubeCluster.EtcdHosts, kubeCluster.ControlPlaneHosts, kubeCluster.WorkerHosts)
+	if err := services.RestartWorkerPlane(ctx, allHosts); err != nil {
+		return err
+	}
+
+	if err := kubeCluster.SaveClusterState(ctx, &kubeCluster.RancherKubernetesEngineConfig); err != nil {
+		return err
+	}
+
+	if rotateCACerts {
+		return cluster.RestartClusterPods(ctx, kubeCluster)
+	}
+	return nil
+}

cmd/up.go

@@ -109,7 +109,7 @@ func ClusterUp(
 	if len(kubeCluster.ControlPlaneHosts) > 0 {
 		APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443")
 	}
-	err = kubeCluster.SetUpHosts(ctx)
+	err = kubeCluster.SetUpHosts(ctx, false)
 	if err != nil {
 		return APIURL, caCrt, clientCert, clientKey, nil, err
 	}

docker/docker.go

@@ -28,7 +28,7 @@ import (
 const (
 	DockerRegistryURL = "docker.io"
 	// RestartTimeout in seconds
-	RestartTimeout = 30
+	RestartTimeout = 5
 	// StopTimeout in seconds
 	StopTimeout = 5
 )
@@ -230,6 +230,14 @@ func RemoveContainer(ctx context.Context, dClient *client.Client, hostname strin
 	return nil
 }
 
+func RestartContainer(ctx context.Context, dClient *client.Client, hostname, containerName string) error {
+	restartTimeout := RestartTimeout * time.Second
+	err := dClient.ContainerRestart(ctx, containerName, &restartTimeout)
+	if err != nil {
+		return fmt.Errorf("Can't restart Docker container [%s] for host [%s]: %v", containerName, hostname, err)
+	}
+	return nil
+}
 func StopContainer(ctx context.Context, dClient *client.Client, hostname string, containerName string) error {
 	// define the stop timeout
 	stopTimeoutDuration := StopTimeout * time.Second
@@ -455,3 +463,23 @@ func GetKubeletDockerConfig(prsMap map[string]v3.PrivateRegistry) (string, error
 	}
 	return string(cfg), nil
 }
+
+func DoRestartContainer(ctx context.Context, dClient *client.Client, containerName, hostname string) error {
+	logrus.Debugf("[restart/%s] Checking if container is running on host [%s]", containerName, hostname)
+	// not using the wrapper to check if the error is a NotFound error
+	_, err := dClient.ContainerInspect(ctx, containerName)
+	if err != nil {
+		if client.IsErrNotFound(err) {
+			logrus.Debugf("[restart/%s] Container doesn't exist on host [%s]", containerName, hostname)
+			return nil
+		}
+		return err
+	}
+	logrus.Debugf("[restart/%s] Restarting container on host [%s]", containerName, hostname)
+	err = RestartContainer(ctx, dClient, hostname, containerName)
+	if err != nil {
+		return err
+	}
+	log.Infof(ctx, "[restart/%s] Successfully restarted container on host [%s]", containerName, hostname)
+	return nil
+}

k8s/pod.go

@@ -0,0 +1,24 @@
+package k8s
+
+import (
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func DeletePods(k8sClient *kubernetes.Clientset, podList *v1.PodList) error {
+	for _, pod := range podList.Items {
+		if err := k8sClient.CoreV1().Pods(pod.Namespace).Delete(pod.Name, &metav1.DeleteOptions{}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func ListPodsByLabel(k8sClient *kubernetes.Clientset, label string) (*v1.PodList, error) {
+	pods, err := k8sClient.CoreV1().Pods("").List(metav1.ListOptions{LabelSelector: label})
+	if err != nil {
+		return nil, err
+	}
+	return pods, nil
+}

main.go

@@ -42,6 +42,7 @@ func mainErr() error {
 		cmd.VersionCommand(),
 		cmd.ConfigCommand(),
 		cmd.EtcdCommand(),
+		cmd.CertificateCommand(),
 	}
 	app.Flags = []cli.Flag{
 		cli.BoolFlag{

pki/constants.go

@@ -1,5 +1,7 @@
 package pki
 
+import "time"
+
 const (
 	CertPathPrefix          = "/etc/kubernetes/ssl/"
 	CertificatesServiceName = "certificates"
@@ -12,17 +14,18 @@ const (
 	ClusterStateEnv         = "CLUSTER_STATE"
 	BundleCertPath          = "/backup/pki.bundle.tar.gz"
 
-	CACertName              = "kube-ca"
-	RequestHeaderCACertName = "kube-apiserver-requestheader-ca"
-	KubeAPICertName         = "kube-apiserver"
-	KubeControllerCertName  = "kube-controller-manager"
-	KubeSchedulerCertName   = "kube-scheduler"
-	KubeProxyCertName       = "kube-proxy"
-	KubeNodeCertName        = "kube-node"
-	EtcdCertName            = "kube-etcd"
-	EtcdClientCACertName    = "kube-etcd-client-ca"
-	EtcdClientCertName      = "kube-etcd-client"
-	APIProxyClientCertName  = "kube-apiserver-proxy-client"
+	CACertName                 = "kube-ca"
+	RequestHeaderCACertName    = "kube-apiserver-requestheader-ca"
+	KubeAPICertName            = "kube-apiserver"
+	KubeControllerCertName     = "kube-controller-manager"
+	KubeSchedulerCertName      = "kube-scheduler"
+	KubeProxyCertName          = "kube-proxy"
+	KubeNodeCertName           = "kube-node"
+	EtcdCertName               = "kube-etcd"
+	EtcdClientCACertName       = "kube-etcd-client-ca"
+	EtcdClientCertName         = "kube-etcd-client"
+	APIProxyClientCertName     = "kube-apiserver-proxy-client"
+	ServiceAccountTokenKeyName = "kube-service-account-token"
 
 	KubeNodeCommonName       = "system:node"
 	KubeNodeOrganizationName = "system:nodes"
@@ -30,4 +33,5 @@ const (
 	KubeAdminCertName         = "kube-admin"
 	KubeAdminOrganizationName = "system:masters"
 	KubeAdminConfigPrefix     = "kube_config_"
+	duration365d              = time.Hour * 24 * 365
 )

pki/deploy.go

@@ -24,12 +24,15 @@ const (
 	StateDeployerContainerName = "cluster-state-deployer"
 )
 
-func DeployCertificatesOnPlaneHost(ctx context.Context, host *hosts.Host, rkeConfig v3.RancherKubernetesEngineConfig, crtMap map[string]CertificatePKI, certDownloaderImage string, prsMap map[string]v3.PrivateRegistry) error {
+func DeployCertificatesOnPlaneHost(ctx context.Context, host *hosts.Host, rkeConfig v3.RancherKubernetesEngineConfig, crtMap map[string]CertificatePKI, certDownloaderImage string, prsMap map[string]v3.PrivateRegistry, rotateCerts bool) error {
 	crtBundle := GenerateRKENodeCerts(ctx, rkeConfig, host.Address, crtMap)
 	env := []string{}
 	for _, crt := range crtBundle {
 		env = append(env, crt.ToEnv()...)
 	}
+	if rotateCerts {
+		env = append(env, "FORCE_DEPLOY=true")
+	}
 	return doRunDeployer(ctx, host, env, certDownloaderImage, prsMap)
 }
 
@@ -153,15 +156,16 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho
 	tmpCerts := make(map[string]CertificatePKI)
 
 	crtList := map[string]bool{
-		CACertName:              false,
-		KubeAPICertName:         false,
-		KubeControllerCertName:  true,
-		KubeSchedulerCertName:   true,
-		KubeProxyCertName:       true,
-		KubeNodeCertName:        true,
-		KubeAdminCertName:       false,
-		RequestHeaderCACertName: false,
-		APIProxyClientCertName:  false,
+		CACertName:                 false,
+		KubeAPICertName:            false,
+		KubeControllerCertName:     true,
+		KubeSchedulerCertName:      true,
+		KubeProxyCertName:          true,
+		KubeNodeCertName:           true,
+		KubeAdminCertName:          false,
+		RequestHeaderCACertName:    false,
+		APIProxyClientCertName:     false,
+		ServiceAccountTokenKeyName: false,
 	}
 
 	for _, etcdHost := range extraHosts {
@@ -175,7 +179,8 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho
 		// I will only exit with an error if it's not a not-found-error and this is not an etcd certificate
 		if err != nil && (!strings.HasPrefix(certName, "kube-etcd") &&
 			!strings.Contains(certName, APIProxyClientCertName) &&
-			!strings.Contains(certName, RequestHeaderCACertName)) {
+			!strings.Contains(certName, RequestHeaderCACertName) &&
+			!strings.Contains(certName, ServiceAccountTokenKeyName)) {
 			// IsErrNotFound doesn't catch this because it's a custom error
 			if isFileNotFoundErr(err) {
 				return nil, nil
@@ -186,7 +191,8 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho
 		// If I can't find an etcd or api aggregator cert, I will not fail and will create it later.
 		if crt == "" && (strings.HasPrefix(certName, "kube-etcd") ||
 			strings.Contains(certName, APIProxyClientCertName) ||
-			strings.Contains(certName, RequestHeaderCACertName)) {
+			strings.Contains(certName, RequestHeaderCACertName) ||
+			strings.Contains(certName, ServiceAccountTokenKeyName)) {
 			tmpCerts[certName] = CertificatePKI{}
 			continue
 		}
@@ -247,45 +253,3 @@ func FetchFileFromHost(ctx context.Context, filePath, image string, host *hosts.
 
 	return file, nil
 }
-
-func getTempPath(s string) string {
-	return TempCertPath + path.Base(s)
-}
-
-func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string, extraHosts []*hosts.Host) map[string]CertificatePKI {
-	certs := make(map[string]CertificatePKI)
-	// CACert
-	certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key)
-	// KubeAPI
-	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key)
-	// kubeController
-	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key)
-	// KubeScheduler
-	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key)
-	// KubeProxy
-	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key)
-	// KubeNode
-	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key)
-	// KubeAdmin
-	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key)
-	kubeAdminCertObj.Config = tmpCerts[KubeAdminCertName].Config
-	kubeAdminCertObj.ConfigPath = localConfigPath
-	certs[KubeAdminCertName] = kubeAdminCertObj
-	// etcd
-	for _, host := range extraHosts {
-		etcdName := GetEtcdCrtName(host.InternalAddress)
-		etcdCrt, etcdKey := tmpCerts[etcdName].Certificate, tmpCerts[etcdName].Key
-		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
-	}
-
-	return certs
-}
-
-func isFileNotFoundErr(e error) bool {
-	if strings.Contains(e.Error(), "no such file or directory") ||
-		strings.Contains(e.Error(), "Could not find the file") ||
-		strings.Contains(e.Error(), "No such container:path:") {
-		return true
-	}
-	return false
-}

pki/pki.go

@@ -15,7 +15,6 @@ import (
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/log"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
-	"k8s.io/client-go/util/cert"
 )
 
 type CertificatePKI struct {
@@ -33,6 +32,8 @@ type CertificatePKI struct {
 	ConfigPath    string
 }
 
+type GenFunc func(context.Context, map[string]CertificatePKI, v3.RancherKubernetesEngineConfig, string, string) error
+
 const (
 	etcdRole            = "etcd"
 	controlRole         = "controlplane"
@@ -42,131 +43,14 @@ const (
 
 func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) (map[string]CertificatePKI, error) {
 	certs := make(map[string]CertificatePKI)
-	// generate CA certificate and key
-	log.Infof(ctx, "[certificates] Generating CA kubernetes certificates")
-	caCrt, caKey, err := GenerateCACertAndKey(CACertName)
-	if err != nil {
-		return nil, err
+	// generate RKE CA certificates
+	if err := GenerateRKECACerts(ctx, certs, configPath, configDir); err != nil {
+		return certs, err
 	}
-	certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey)
-
-	// generate API certificate and key
-	log.Infof(ctx, "[certificates] Generating Kubernetes API server certificates")
-	kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange)
-	clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain
-	cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole)
-	etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to get Kubernetes Service IP: %v", err)
+	// Generating certificates for kubernetes components
+	if err := GenerateRKEServicesCerts(ctx, certs, rkeConfig, configPath, configDir); err != nil {
+		return certs, err
 	}
-	kubeAPIAltNames := GetAltNames(cpHosts, clusterDomain, kubernetesServiceIP, rkeConfig.Authentication.SANs)
-	kubeAPICrt, kubeAPIKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, KubeAPICertName, kubeAPIAltNames, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey)
-
-	// generate Kube controller-manager certificate and key
-	log.Infof(ctx, "[certificates] Generating Kube Controller certificates")
-	kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey)
-
-	// generate Kube scheduler certificate and key
-	log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates")
-	kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey)
-
-	// generate Kube Proxy certificate and key
-	log.Infof(ctx, "[certificates] Generating Kube Proxy certificates")
-	kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey)
-
-	log.Infof(ctx, "[certificates] Generating Node certificate")
-	nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, nil, []string{KubeNodeOrganizationName})
-	if err != nil {
-		return nil, err
-	}
-	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey)
-
-	// generate Admin certificate and key
-	log.Infof(ctx, "[certificates] Generating admin certificates and kubeconfig")
-	if len(configPath) == 0 {
-		configPath = ClusterConfig
-	}
-	localKubeConfigPath := GetLocalKubeConfig(configPath, configDir)
-	kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, nil, []string{KubeAdminOrganizationName})
-	if err != nil {
-		return nil, err
-	}
-	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey)
-	if len(cpHosts) > 0 {
-		kubeAdminConfig := GetKubeConfigX509WithData(
-			"https://"+cpHosts[0].Address+":6443",
-			rkeConfig.ClusterName,
-			KubeAdminCertName,
-			string(cert.EncodeCertPEM(caCrt)),
-			string(cert.EncodeCertPEM(kubeAdminCrt)),
-			string(cert.EncodePrivateKeyPEM(kubeAdminKey)))
-		kubeAdminCertObj.Config = kubeAdminConfig
-		kubeAdminCertObj.ConfigPath = localKubeConfigPath
-	} else {
-		kubeAdminCertObj.Config = ""
-	}
-	certs[KubeAdminCertName] = kubeAdminCertObj
-	// generate etcd certificate and key
-	if len(rkeConfig.Services.Etcd.ExternalURLs) > 0 {
-		clientCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.Cert))
-		if err != nil {
-			return nil, err
-		}
-		clientKey, err := cert.ParsePrivateKeyPEM([]byte(rkeConfig.Services.Etcd.Key))
-		if err != nil {
-			return nil, err
-		}
-		certs[EtcdClientCertName] = ToCertObject(EtcdClientCertName, "", "", clientCert[0], clientKey.(*rsa.PrivateKey))
-
-		caCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.CACert))
-		if err != nil {
-			return nil, err
-		}
-		certs[EtcdClientCACertName] = ToCertObject(EtcdClientCACertName, "", "", caCert[0], nil)
-	}
-	etcdAltNames := GetAltNames(etcdHosts, clusterDomain, kubernetesServiceIP, []string{})
-	for _, host := range etcdHosts {
-		log.Infof(ctx, "[certificates] Generating etcd-%s certificate and key", host.InternalAddress)
-		etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil)
-		if err != nil {
-			return nil, err
-		}
-		etcdName := GetEtcdCrtName(host.InternalAddress)
-		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
-	}
-
-	// generate request header client CA certificate and key
-	log.Infof(ctx, "[certificates] Generating Kubernetes API server aggregation layer requestheader client CA certificates")
-	requestHeaderCACrt, requestHeaderCAKey, err := GenerateCACertAndKey(RequestHeaderCACertName)
-	if err != nil {
-		return nil, err
-	}
-	certs[RequestHeaderCACertName] = ToCertObject(RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey)
-
-	//generate API server proxy client key and certs
-	log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client certificates")
-	apiserverProxyClientCrt, apiserverProxyClientKey, err := GenerateSignedCertAndKey(requestHeaderCACrt, requestHeaderCAKey, true, APIProxyClientCertName, nil, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-	certs[APIProxyClientCertName] = ToCertObject(APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey)
-
 	return certs, nil
 }
 

pki/services.go

@@ -0,0 +1,234 @@
+package pki
+
+import (
+	"context"
+	"crypto/rsa"
+	"fmt"
+
+	"github.com/rancher/rke/hosts"
+	"github.com/rancher/rke/log"
+	"github.com/rancher/types/apis/management.cattle.io/v3"
+	"k8s.io/client-go/util/cert"
+)
+
+func GenerateKubeAPICertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	// generate API certificate and key
+	log.Infof(ctx, "[certificates] Generating Kubernetes API server certificates")
+	var privateAPIKey *rsa.PrivateKey
+	caCrt := certs[CACertName].Certificate
+	caKey := certs[CACertName].Key
+	kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange)
+	if err != nil {
+		return fmt.Errorf("Failed to get Kubernetes Service IP: %v", err)
+	}
+	clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain
+	cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole)
+	kubeAPIAltNames := GetAltNames(cpHosts, clusterDomain, kubernetesServiceIP, rkeConfig.Authentication.SANs)
+	// handle rotation on old clusters
+	if certs[ServiceAccountTokenKeyName].Key == nil {
+		privateAPIKey = certs[KubeAPICertName].Key
+	}
+	kubeAPICrt, kubeAPIKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, KubeAPICertName, kubeAPIAltNames, privateAPIKey, nil)
+	if err != nil {
+		return err
+	}
+	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey)
+	return nil
+}
+
+func GenerateKubeControllerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	// generate Kube controller-manager certificate and key
+	log.Infof(ctx, "[certificates] Generating Kube Controller certificates")
+	caCrt := certs[CACertName].Certificate
+	caKey := certs[CACertName].Key
+	kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, nil, nil)
+	if err != nil {
+		return err
+	}
+	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey)
+	return nil
+}
+
+func GenerateKubeSchedulerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	// generate Kube scheduler certificate and key
+	log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates")
+	caCrt := certs[CACertName].Certificate
+	caKey := certs[CACertName].Key
+	kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, nil, nil)
+	if err != nil {
+		return err
+	}
+	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey)
+	return nil
+}
+
+func GenerateKubeProxyCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	// generate Kube Proxy certificate and key
+	log.Infof(ctx, "[certificates] Generating Kube Proxy certificates")
+	caCrt := certs[CACertName].Certificate
+	caKey := certs[CACertName].Key
+	kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, nil, nil)
+	if err != nil {
+		return err
+	}
+	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey)
+	return nil
+}
+
+func GenerateKubeNodeCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	// generate kubelet certificate
+	log.Infof(ctx, "[certificates] Generating Node certificate")
+	caCrt := certs[CACertName].Certificate
+	caKey := certs[CACertName].Key
+	nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, nil, []string{KubeNodeOrganizationName})
+	if err != nil {
+		return err
+	}
+	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey)
+	return nil
+}
+
+func GenerateKubeAdminCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	// generate Admin certificate and key
+	log.Infof(ctx, "[certificates] Generating admin certificates and kubeconfig")
+	caCrt := certs[CACertName].Certificate
+	caKey := certs[CACertName].Key
+	cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole)
+	if len(configPath) == 0 {
+		configPath = ClusterConfig
+	}
+	localKubeConfigPath := GetLocalKubeConfig(configPath, configDir)
+	kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, nil, []string{KubeAdminOrganizationName})
+	if err != nil {
+		return err
+	}
+	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey)
+	if len(cpHosts) > 0 {
+		kubeAdminConfig := GetKubeConfigX509WithData(
+			"https://"+cpHosts[0].Address+":6443",
+			rkeConfig.ClusterName,
+			KubeAdminCertName,
+			string(cert.EncodeCertPEM(caCrt)),
+			string(cert.EncodeCertPEM(kubeAdminCrt)),
+			string(cert.EncodePrivateKeyPEM(kubeAdminKey)))
+		kubeAdminCertObj.Config = kubeAdminConfig
+		kubeAdminCertObj.ConfigPath = localKubeConfigPath
+	} else {
+		kubeAdminCertObj.Config = ""
+	}
+	certs[KubeAdminCertName] = kubeAdminCertObj
+	return nil
+}
+
+func GenerateAPIProxyClientCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	//generate API server proxy client key and certs
+	log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client certificates")
+	caCrt := certs[RequestHeaderCACertName].Certificate
+	caKey := certs[RequestHeaderCACertName].Key
+	apiserverProxyClientCrt, apiserverProxyClientKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, APIProxyClientCertName, nil, nil, nil)
+	if err != nil {
+		return err
+	}
+	certs[APIProxyClientCertName] = ToCertObject(APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey)
+	return nil
+}
+
+func GenerateExternalEtcdCertificates(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	clientCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.Cert))
+	if err != nil {
+		return err
+	}
+	clientKey, err := cert.ParsePrivateKeyPEM([]byte(rkeConfig.Services.Etcd.Key))
+	if err != nil {
+		return err
+	}
+	certs[EtcdClientCertName] = ToCertObject(EtcdClientCertName, "", "", clientCert[0], clientKey.(*rsa.PrivateKey))
+
+	caCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.CACert))
+	if err != nil {
+		return err
+	}
+	certs[EtcdClientCACertName] = ToCertObject(EtcdClientCACertName, "", "", caCert[0], nil)
+	return nil
+}
+
+func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	caCrt := certs[CACertName].Certificate
+	caKey := certs[CACertName].Key
+	kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange)
+	if err != nil {
+		return fmt.Errorf("Failed to get Kubernetes Service IP: %v", err)
+	}
+	clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain
+	etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole)
+	etcdAltNames := GetAltNames(etcdHosts, clusterDomain, kubernetesServiceIP, []string{})
+	for _, host := range etcdHosts {
+		log.Infof(ctx, "[certificates] Generating etcd-%s certificate and key", host.InternalAddress)
+		etcdName := GetEtcdCrtName(host.InternalAddress)
+		etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil)
+		if err != nil {
+			return err
+		}
+		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
+	}
+	return nil
+}
+
+func GenerateServiceTokenKey(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	// generate service account token key
+	var privateAPIKey *rsa.PrivateKey
+	caCrt := certs[CACertName].Certificate
+	caKey := certs[CACertName].Key
+	// handle rotation on old clusters
+	if certs[ServiceAccountTokenKeyName].Key == nil {
+		privateAPIKey = certs[KubeAPICertName].Key
+	}
+	tokenCrt, tokenKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, ServiceAccountTokenKeyName, nil, privateAPIKey, nil)
+	if err != nil {
+		return fmt.Errorf("Failed to generate private key for service account token: %v", err)
+	}
+	certs[ServiceAccountTokenKeyName] = ToCertObject(ServiceAccountTokenKeyName, ServiceAccountTokenKeyName, "", tokenCrt, tokenKey)
+	return nil
+}
+
+func GenerateRKECACerts(ctx context.Context, certs map[string]CertificatePKI, configPath, configDir string) error {
+	// generate kubernetes CA certificate and key
+	log.Infof(ctx, "[certificates] Generating CA kubernetes certificates")
+	caCrt, caKey, err := GenerateCACertAndKey(CACertName, certs[CACertName].Key)
+	if err != nil {
+		return err
+	}
+	certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey)
+
+	// generate request header client CA certificate and key
+	log.Infof(ctx, "[certificates] Generating Kubernetes API server aggregation layer requestheader client CA certificates")
+	requestHeaderCACrt, requestHeaderCAKey, err := GenerateCACertAndKey(RequestHeaderCACertName, nil)
+	if err != nil {
+		return err
+	}
+	certs[RequestHeaderCACertName] = ToCertObject(RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey)
+	return nil
+}
+
+func GenerateRKEServicesCerts(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) error {
+	RKECerts := []GenFunc{
+		GenerateKubeAPICertificate,
+		GenerateServiceTokenKey,
+		GenerateKubeControllerCertificate,
+		GenerateKubeSchedulerCertificate,
+		GenerateKubeProxyCertificate,
+		GenerateKubeNodeCertificate,
+		GenerateKubeAdminCertificate,
+		GenerateAPIProxyClientCertificate,
+		GenerateEtcdCertificates,
+	}
+	for _, gen := range RKECerts {
+		if err := gen(ctx, certs, rkeConfig, configPath, configDir); err != nil {
+			return err
+		}
+	}
+	if len(rkeConfig.Services.Etcd.ExternalURLs) > 0 {
+		return GenerateExternalEtcdCertificates(ctx, certs, rkeConfig, configPath, configDir)
+	}
+	return nil
+}

pki/util.go

@@ -1,12 +1,19 @@
 package pki
 
 import (
+	cryptorand "crypto/rand"
 	"crypto/rsa"
 	"crypto/x509"
+	"crypto/x509/pkix"
+	"errors"
 	"fmt"
+	"math"
+	"math/big"
 	"net"
+	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
@@ -44,17 +51,21 @@ func GenerateSignedCertAndKey(
 		Usages:       usages,
 		AltNames:     *altNames,
 	}
-	clientCert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey)
+	clientCert, err := newSignedCert(caConfig, rootKey, caCrt, caKey)
 	if err != nil {
 		return nil, nil, fmt.Errorf("Failed to generate %s certificate: %v", commonName, err)
 	}
 	return clientCert, rootKey, nil
 }
 
-func GenerateCACertAndKey(commonName string) (*x509.Certificate, *rsa.PrivateKey, error) {
-	rootKey, err := cert.NewPrivateKey()
-	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err)
+func GenerateCACertAndKey(commonName string, privateKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
+	var err error
+	rootKey := privateKey
+	if rootKey == nil {
+		rootKey, err = cert.NewPrivateKey()
+		if err != nil {
+			return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err)
+		}
 	}
 	caConfig := cert.Config{
 		CommonName: commonName,
@@ -197,7 +208,7 @@ func ToCertObject(componentName, commonName, ouName string, cert *x509.Certifica
 	path := GetCertPath(componentName)
 	keyPath := GetKeyPath(componentName)
 
-	if componentName != CACertName && componentName != KubeAPICertName && !strings.Contains(componentName, EtcdCertName) {
+	if componentName != CACertName && componentName != KubeAPICertName && !strings.Contains(componentName, EtcdCertName) && componentName != ServiceAccountTokenKeyName {
 		config = getKubeConfigX509("https://127.0.0.1:6443", "local", componentName, caCertPath, path, keyPath)
 		configPath = GetConfigPath(componentName)
 		configEnvName = getConfigEnvFromEnv(envName)
@@ -227,6 +238,7 @@ func getControlCertKeys() []string {
 	return []string{
 		CACertName,
 		KubeAPICertName,
+		ServiceAccountTokenKeyName,
 		KubeControllerCertName,
 		KubeSchedulerCertName,
 		KubeProxyCertName,
@@ -293,3 +305,78 @@ func strKeyToEnv(crtName, key string) string {
 	envName := getEnvFromName(crtName)
 	return fmt.Sprintf("%s=%s", getKeyEnvFromEnv(envName), key)
 }
+
+func getTempPath(s string) string {
+	return TempCertPath + path.Base(s)
+}
+
+func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string, extraHosts []*hosts.Host) map[string]CertificatePKI {
+	certs := make(map[string]CertificatePKI)
+	// CACert
+	certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key)
+	// KubeAPI
+	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key)
+	// kubeController
+	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key)
+	// KubeScheduler
+	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key)
+	// KubeProxy
+	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key)
+	// KubeNode
+	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key)
+	// KubeAdmin
+	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key)
+	kubeAdminCertObj.Config = tmpCerts[KubeAdminCertName].Config
+	kubeAdminCertObj.ConfigPath = localConfigPath
+	certs[KubeAdminCertName] = kubeAdminCertObj
+	// etcd
+	for _, host := range extraHosts {
+		etcdName := GetEtcdCrtName(host.InternalAddress)
+		etcdCrt, etcdKey := tmpCerts[etcdName].Certificate, tmpCerts[etcdName].Key
+		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
+	}
+
+	return certs
+}
+
+// Overriding k8s.io/client-go/util/cert.NewSignedCert function to extend the expiration date to 10 years instead of 1 year
+func newSignedCert(cfg cert.Config, key *rsa.PrivateKey, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, error) {
+	serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64))
+	if err != nil {
+		return nil, err
+	}
+	if len(cfg.CommonName) == 0 {
+		return nil, errors.New("must specify a CommonName")
+	}
+	if len(cfg.Usages) == 0 {
+		return nil, errors.New("must specify at least one ExtKeyUsage")
+	}
+
+	certTmpl := x509.Certificate{
+		Subject: pkix.Name{
+			CommonName:   cfg.CommonName,
+			Organization: cfg.Organization,
+		},
+		DNSNames:     cfg.AltNames.DNSNames,
+		IPAddresses:  cfg.AltNames.IPs,
+		SerialNumber: serial,
+		NotBefore:    caCert.NotBefore,
+		NotAfter:     time.Now().Add(duration365d * 10).UTC(),
+		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  cfg.Usages,
+	}
+	certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
+	if err != nil {
+		return nil, err
+	}
+	return x509.ParseCertificate(certDERBytes)
+}
+
+func isFileNotFoundErr(e error) bool {
+	if strings.Contains(e.Error(), "no such file or directory") ||
+		strings.Contains(e.Error(), "Could not find the file") ||
+		strings.Contains(e.Error(), "No such container:path:") {
+		return true
+	}
+	return false
+}

services/controlplane.go

@@ -82,6 +82,42 @@ func RemoveControlPlane(ctx context.Context, controlHosts []*hosts.Host, force b
 	return nil
 }
 
+func RestartControlPlane(ctx context.Context, controlHosts []*hosts.Host) error {
+	log.Infof(ctx, "[%s] Restarting the Controller Plane..", ControlRole)
+	var errgrp errgroup.Group
+
+	hostsQueue := util.GetObjectQueue(controlHosts)
+	for w := 0; w < WorkerThreads; w++ {
+		errgrp.Go(func() error {
+			var errList []error
+			for host := range hostsQueue {
+				runHost := host.(*hosts.Host)
+				// restart KubeAPI
+				if err := restartKubeAPI(ctx, runHost); err != nil {
+					errList = append(errList, err)
+				}
+
+				// restart KubeController
+				if err := restartKubeController(ctx, runHost); err != nil {
+					errList = append(errList, err)
+				}
+
+				// restart scheduler
+				err := restartScheduler(ctx, runHost)
+				if err != nil {
+					errList = append(errList, err)
+				}
+			}
+			return util.ErrList(errList)
+		})
+	}
+	if err := errgrp.Wait(); err != nil {
+		return err
+	}
+	log.Infof(ctx, "[%s] Successfully restarted Controller Plane..", ControlRole)
+	return nil
+}
+
 func doDeployControlHost(ctx context.Context, host *hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, alpineImage string, certMap map[string]pki.CertificatePKI) error {
 	if host.IsWorker {
 		if err := removeNginxProxy(ctx, host); err != nil {

services/etcd.go

@@ -72,6 +72,30 @@ func RunEtcdPlane(
 	return nil
 }
 
+func RestartEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host) error {
+	log.Infof(ctx, "[%s] Restarting up etcd plane..", ETCDRole)
+	var errgrp errgroup.Group
+
+	hostsQueue := util.GetObjectQueue(etcdHosts)
+	for w := 0; w < WorkerThreads; w++ {
+		errgrp.Go(func() error {
+			var errList []error
+			for host := range hostsQueue {
+				runHost := host.(*hosts.Host)
+				if err := docker.DoRestartContainer(ctx, runHost.DClient, EtcdContainerName, runHost.Address); err != nil {
+					errList = append(errList, err)
+				}
+			}
+			return util.ErrList(errList)
+		})
+	}
+	if err := errgrp.Wait(); err != nil {
+		return err
+	}
+	log.Infof(ctx, "[%s] Successfully restarted etcd plane..", ETCDRole)
+	return nil
+}
+
 func RemoveEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, force bool) error {
 	log.Infof(ctx, "[%s] Tearing down etcd plane..", ETCDRole)
 

services/kubeapi.go

@@ -24,3 +24,7 @@ func runKubeAPI(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, p
 func removeKubeAPI(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, KubeAPIContainerName, host.Address)
 }
+
+func restartKubeAPI(ctx context.Context, host *hosts.Host) error {
+	return docker.DoRestartContainer(ctx, host.DClient, KubeAPIContainerName, host.Address)
+}

services/kubecontroller.go

@@ -22,3 +22,7 @@ func runKubeController(ctx context.Context, host *hosts.Host, df hosts.DialerFac
 func removeKubeController(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, KubeControllerContainerName, host.Address)
 }
+
+func restartKubeController(ctx context.Context, host *hosts.Host) error {
+	return docker.DoRestartContainer(ctx, host.DClient, KubeControllerContainerName, host.Address)
+}

services/kubelet.go

@@ -23,3 +23,7 @@ func runKubelet(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, p
 func removeKubelet(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, KubeletContainerName, host.Address)
 }
+
+func restartKubelet(ctx context.Context, host *hosts.Host) error {
+	return docker.DoRestartContainer(ctx, host.DClient, KubeletContainerName, host.Address)
+}

services/kubeproxy.go

@@ -22,3 +22,7 @@ func runKubeproxy(ctx context.Context, host *hosts.Host, df hosts.DialerFactory,
 func removeKubeproxy(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, KubeproxyContainerName, host.Address)
 }
+
+func restartKubeproxy(ctx context.Context, host *hosts.Host) error {
+	return docker.DoRestartContainer(ctx, host.DClient, KubeproxyContainerName, host.Address)
+}

services/proxy.go

@@ -24,3 +24,7 @@ func runNginxProxy(ctx context.Context, host *hosts.Host, prsMap map[string]v3.P
 func removeNginxProxy(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, NginxProxyContainerName, host.Address)
 }
+
+func restartNginxProxy(ctx context.Context, host *hosts.Host) error {
+	return docker.DoRestartContainer(ctx, host.DClient, NginxProxyContainerName, host.Address)
+}

services/scheduler.go

@@ -22,3 +22,7 @@ func runScheduler(ctx context.Context, host *hosts.Host, df hosts.DialerFactory,
 func removeScheduler(ctx context.Context, host *hosts.Host) error {
 	return docker.DoRemoveContainer(ctx, host.DClient, SchedulerContainerName, host.Address)
 }
+
+func restartScheduler(ctx context.Context, host *hosts.Host) error {
+	return docker.DoRestartContainer(ctx, host.DClient, SchedulerContainerName, host.Address)
+}

services/workerplane.go

@@ -99,6 +99,37 @@ func RemoveWorkerPlane(ctx context.Context, workerHosts []*hosts.Host, force boo
 	return nil
 }
 
+func RestartWorkerPlane(ctx context.Context, workerHosts []*hosts.Host) error {
+	log.Infof(ctx, "[%s] Restarting Worker Plane..", WorkerRole)
+	var errgrp errgroup.Group
+
+	hostsQueue := util.GetObjectQueue(workerHosts)
+	for w := 0; w < WorkerThreads; w++ {
+		errgrp.Go(func() error {
+			var errList []error
+			for host := range hostsQueue {
+				runHost := host.(*hosts.Host)
+				if err := restartKubelet(ctx, runHost); err != nil {
+					errList = append(errList, err)
+				}
+				if err := restartKubeproxy(ctx, runHost); err != nil {
+					errList = append(errList, err)
+				}
+				if err := restartNginxProxy(ctx, runHost); err != nil {
+					errList = append(errList, err)
+				}
+			}
+			return util.ErrList(errList)
+		})
+	}
+	if err := errgrp.Wait(); err != nil {
+		return err
+	}
+	log.Infof(ctx, "[%s] Successfully restarted Worker Plane..", WorkerRole)
+
+	return nil
+}
+
 func doDeployWorkerPlane(ctx context.Context, host *hosts.Host,
 	localConnDialerFactory hosts.DialerFactory,
 	prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, certMap map[string]pki.CertificatePKI, alpineImage string) error {