Add generic generation for pki objects and etcd TLS

cluster/certificates.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/k8s"
 	"github.com/rancher/rke/log"
 	"github.com/rancher/rke/pki"
@@ -21,7 +22,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
 			kubeCluster.Certificates = currentCluster.Certificates
 		} else {
 			log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", kubeCluster.EtcdHosts[0].Address)
-			kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.SystemImages[AplineImage], kubeCluster.LocalKubeConfigPath)
+			kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, kubeCluster.EtcdHosts[0], kubeCluster.SystemImages[AplineImage], kubeCluster.LocalKubeConfigPath)
 			if err != nil {
 				return err
 			}
@@ -33,7 +34,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
 
 			kubeCluster.Certificates, err = pki.StartCertificatesGeneration(ctx,
 				kubeCluster.ControlPlaneHosts,
-				kubeCluster.WorkerHosts,
+				kubeCluster.EtcdHosts,
 				kubeCluster.ClusterDomain,
 				kubeCluster.LocalKubeConfigPath,
 				kubeCluster.KubernetesServiceIP)
@@ -41,7 +42,7 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
 				return fmt.Errorf("Failed to generate Kubernetes certificates: %v", err)
 			}
 			log.Infof(ctx, "[certificates] Temporarily saving certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address)
-			if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage]); err != nil {
+			if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts, kubeCluster.EtcdHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage], pki.TempCertPath); err != nil {
 				return err
 			}
 			log.Infof(ctx, "[certificates] Saved certs to etcd host [%s]", kubeCluster.EtcdHosts[0].Address)
@@ -56,32 +57,31 @@ func regenerateAPICertificate(c *Cluster, certificates map[string]pki.Certificat
 	caCrt := certificates[pki.CACertName].Certificate
 	caKey := certificates[pki.CACertName].Key
 	kubeAPIKey := certificates[pki.KubeAPICertName].Key
-	kubeAPICert, err := pki.GenerateCertWithKey(pki.KubeAPICertName, kubeAPIKey, caCrt, caKey, kubeAPIAltNames)
+	kubeAPICert, _, err := pki.GenerateSignedCertAndKey(caCrt, caKey, true, pki.KubeAPICertName, kubeAPIAltNames, kubeAPIKey, nil)
 	if err != nil {
 		return nil, err
 	}
-	certificates[pki.KubeAPICertName] = pki.CertificatePKI{
+	certificates[pki.KubeAPICertName] = pki.ToCertObject(pki.KubeAPICertName, "", "", kubeAPICert, kubeAPIKey)
-		Certificate:   kubeAPICert,
-		Key:           kubeAPIKey,
-		Config:        certificates[pki.KubeAPICertName].Config,
-		EnvName:       certificates[pki.KubeAPICertName].EnvName,
-		ConfigEnvName: certificates[pki.KubeAPICertName].ConfigEnvName,
-		KeyEnvName:    certificates[pki.KubeAPICertName].KeyEnvName,
-	}
 	return certificates, nil
 }
 
-func getClusterCerts(ctx context.Context, kubeClient *kubernetes.Clientset) (map[string]pki.CertificatePKI, error) {
+func getClusterCerts(ctx context.Context, kubeClient *kubernetes.Clientset, etcdHosts []*hosts.Host) (map[string]pki.CertificatePKI, error) {
 	log.Infof(ctx, "[certificates] Getting Cluster certificates from Kubernetes")
 	certificatesNames := []string{
 		pki.CACertName,
 		pki.KubeAPICertName,
-		pki.KubeNodeName,
+		pki.KubeNodeCertName,
-		pki.KubeProxyName,
+		pki.KubeProxyCertName,
-		pki.KubeControllerName,
+		pki.KubeControllerCertName,
-		pki.KubeSchedulerName,
+		pki.KubeSchedulerCertName,
-		pki.KubeAdminCommonName,
+		pki.KubeAdminCertName,
 	}
+
+	for _, etcdHost := range etcdHosts {
+		etcdName := pki.GetEtcdCrtName(etcdHost.InternalAddress)
+		certificatesNames = append(certificatesNames, etcdName)
+	}
+
 	certMap := make(map[string]pki.CertificatePKI)
 	for _, certName := range certificatesNames {
 		secret, err := k8s.GetSecret(kubeClient, certName)

cluster/cluster.go

@@ -221,7 +221,7 @@ func GetLocalKubeConfig(configPath, configDir string) string {
 func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error {
 	log.Infof(ctx, "[reconcile] Rebuilding and updating local kube config")
 	var workingConfig, newConfig string
-	currentKubeConfig := kubeCluster.Certificates[pki.KubeAdminCommonName]
+	currentKubeConfig := kubeCluster.Certificates[pki.KubeAdminCertName]
 	caCrt := kubeCluster.Certificates[pki.CACertName].Certificate
 	for _, cpHost := range kubeCluster.ControlPlaneHosts {
 		if (currentKubeConfig == pki.CertificatePKI{}) {
@@ -232,7 +232,7 @@ func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error {
 			caData := string(cert.EncodeCertPEM(caCrt))
 			crtData := string(cert.EncodeCertPEM(currentKubeConfig.Certificate))
 			keyData := string(cert.EncodePrivateKeyPEM(currentKubeConfig.Key))
-			newConfig = pki.GetKubeConfigX509WithData(kubeURL, pki.KubeAdminCommonName, caData, crtData, keyData)
+			newConfig = pki.GetKubeConfigX509WithData(kubeURL, pki.KubeAdminCertName, caData, crtData, keyData)
 		}
 		if err := pki.DeployAdminConfig(ctx, newConfig, kubeCluster.LocalKubeConfigPath); err != nil {
 			return fmt.Errorf("Failed to redeploy local admin config with new host")
@@ -244,7 +244,7 @@ func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error {
 		}
 	}
 	currentKubeConfig.Config = workingConfig
-	kubeCluster.Certificates[pki.KubeAdminCommonName] = currentKubeConfig
+	kubeCluster.Certificates[pki.KubeAdminCertName] = currentKubeConfig
 	return nil
 }
 
@@ -271,7 +271,7 @@ func getLocalAdminConfigWithNewAddress(localConfigPath, cpAddress string) string
 	config.Host = fmt.Sprintf("https://%s:6443", cpAddress)
 	return pki.GetKubeConfigX509WithData(
 		"https://"+cpAddress+":6443",
-		pki.KubeAdminCommonName,
+		pki.KubeAdminCertName,
 		string(config.CAData),
 		string(config.CertData),
 		string(config.KeyData))

cluster/hosts.go

@@ -72,22 +72,18 @@ func (c *Cluster) InvertIndexHosts() error {
 func (c *Cluster) SetUpHosts(ctx context.Context) error {
 	if c.Authentication.Strategy == X509AuthenticationProvider {
 		log.Infof(ctx, "[certificates] Deploying kubernetes certificates to Cluster nodes")
-		err := pki.DeployCertificatesOnMasters(ctx, c.ControlPlaneHosts, c.Certificates, c.SystemImages[CertDownloaderImage])
+		if err := pki.DeployCertificatesOnMasters(ctx, c.ControlPlaneHosts, c.Certificates, c.SystemImages[CertDownloaderImage]); err != nil {
-		if err != nil {
 			return err
 		}
-		err = pki.DeployCertificatesOnWorkers(ctx, c.WorkerHosts, c.Certificates, c.SystemImages[CertDownloaderImage])
+		if err := pki.DeployCertificatesOnWorkers(ctx, c.WorkerHosts, c.Certificates, c.SystemImages[CertDownloaderImage]); err != nil {
-		if err != nil {
 			return err
 		}
-		// Deploying worker certs on etcd hosts as well
+		// Deploying etcd certificates
-		err = pki.DeployCertificatesOnWorkers(ctx, c.EtcdHosts, c.Certificates, c.SystemImages[CertDownloaderImage])
+		if err := pki.DeployCertificatesOnEtcd(ctx, c.EtcdHosts, c.Certificates, c.SystemImages[CertDownloaderImage]); err != nil {
-		if err != nil {
 			return err
 		}
 
-		err = pki.DeployAdminConfig(ctx, c.Certificates[pki.KubeAdminCommonName].Config, c.LocalKubeConfigPath)
+		if err := pki.DeployAdminConfig(ctx, c.Certificates[pki.KubeAdminCertName].Config, c.LocalKubeConfigPath); err != nil {
-		if err != nil {
 			return err
 		}
 		log.Infof(ctx, "[certificates] Successfully deployed kubernetes certificates to Cluster nodes")

cluster/network.go

@@ -8,6 +8,8 @@ import (
 	"net"
 	"strings"
 
+	b64 "encoding/base64"
+
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/go-connections/nat"
 	"github.com/rancher/rke/docker"
@@ -18,6 +20,7 @@ import (
 	"github.com/rancher/rke/templates"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
+	"k8s.io/client-go/util/cert"
 )
 
 const (
@@ -66,10 +69,14 @@ const (
 	APIRoot = "APIRoot"
 	// kubernetes client certificates and kubeconfig paths
 
-	ClientCert = "ClientCert"
+	ClientCert     = "ClientCert"
-	ClientKey  = "ClientKey"
+	ClientCertPath = "ClientCertPath"
-	ClientCA   = "ClientCA"
+	ClientKey      = "ClientKey"
-	KubeCfg    = "KubeCfg"
+	ClientKeyPath  = "ClientKeyPath"
+	ClientCA       = "ClientCA"
+	ClientCAPath   = "ClientCAPath"
+
+	KubeCfg = "KubeCfg"
 
 	ClusterCIDR = "ClusterCIDR"
 	// Images key names
@@ -120,13 +127,20 @@ func (c *Cluster) doFlannelDeploy(ctx context.Context) error {
 }
 
 func (c *Cluster) doCalicoDeploy(ctx context.Context) error {
+	clientCert := b64.StdEncoding.EncodeToString(cert.EncodeCertPEM(c.Certificates[pki.KubeNodeCertName].Certificate))
+	clientkey := b64.StdEncoding.EncodeToString(cert.EncodePrivateKeyPEM(c.Certificates[pki.KubeNodeCertName].Key))
+	clientConfig := pki.GetConfigPath(pki.KubeNodeCertName)
+	caCert := b64.StdEncoding.EncodeToString(cert.EncodeCertPEM(c.Certificates[pki.CACertName].Certificate))
 	calicoConfig := map[string]string{
 		EtcdEndpoints:    services.GetEtcdConnString(c.EtcdHosts),
 		APIRoot:          "https://127.0.0.1:6443",
-		ClientCert:       pki.KubeNodeCertPath,
+		ClientCert:       clientCert,
-		ClientKey:        pki.KubeNodeKeyPath,
+		ClientCertPath:   pki.GetCertPath(pki.KubeNodeCertName),
-		ClientCA:         pki.CACertPath,
+		ClientKey:        clientkey,
-		KubeCfg:          pki.KubeNodeConfigPath,
+		ClientKeyPath:    pki.GetKeyPath(pki.KubeNodeCertName),
+		ClientCA:         caCert,
+		ClientCAPath:     pki.GetCertPath(pki.CACertName),
+		KubeCfg:          clientConfig,
 		ClusterCIDR:      c.ClusterCIDR,
 		CNIImage:         c.Network.Options[CalicoCNIImage],
 		NodeImage:        c.Network.Options[CalicoNodeImage],
@@ -143,12 +157,13 @@ func (c *Cluster) doCalicoDeploy(ctx context.Context) error {
 }
 
 func (c *Cluster) doCanalDeploy(ctx context.Context) error {
+	clientConfig := pki.GetConfigPath(pki.KubeNodeCertName)
 	canalConfig := map[string]string{
-		ClientCert:      pki.KubeNodeCertPath,
+		ClientCertPath:  pki.GetCertPath(pki.KubeNodeCertName),
 		APIRoot:         "https://127.0.0.1:6443",
-		ClientKey:       pki.KubeNodeKeyPath,
+		ClientKeyPath:   pki.GetKeyPath(pki.KubeNodeCertName),
-		ClientCA:        pki.CACertPath,
+		ClientCAPath:    pki.GetCertPath(pki.CACertName),
-		KubeCfg:         pki.KubeNodeConfigPath,
+		KubeCfg:         clientConfig,
 		ClusterCIDR:     c.ClusterCIDR,
 		NodeImage:       c.Network.Options[CanalNodeImage],
 		CNIImage:        c.Network.Options[CanalCNIImage],

cluster/reconcile.go

@@ -7,9 +7,11 @@ import (
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/k8s"
 	"github.com/rancher/rke/log"
+	"github.com/rancher/rke/pki"
 	"github.com/rancher/rke/services"
 	"github.com/sirupsen/logrus"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/cert"
 )
 
 const (
@@ -155,10 +157,14 @@ func reconcileHost(ctx context.Context, toDeleteHost *hosts.Host, worker, etcd b
 }
 
 func reconcileEtcd(ctx context.Context, currentCluster, kubeCluster *Cluster, kubeClient *kubernetes.Clientset) error {
-	log.Infof(ctx, "[reconcile] Check etcd hosts to be deleted")
+	logrus.Infof("[reconcile] Check etcd hosts to be deleted")
+	// get tls for the first current etcd host
+	clientCert := cert.EncodeCertPEM(currentCluster.Certificates[pki.KubeNodeCertName].Certificate)
+	clientkey := cert.EncodePrivateKeyPEM(currentCluster.Certificates[pki.KubeNodeCertName].Key)
+
 	etcdToDelete := hosts.GetToDeleteHosts(currentCluster.EtcdHosts, kubeCluster.EtcdHosts)
 	for _, etcdHost := range etcdToDelete {
-		if err := services.RemoveEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory); err != nil {
+		if err := services.RemoveEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory, clientCert, clientkey); err != nil {
 			log.Warnf(ctx, "[reconcile] %v", err)
 			continue
 		}
@@ -174,15 +180,34 @@ func reconcileEtcd(ctx context.Context, currentCluster, kubeCluster *Cluster, ku
 	}
 	log.Infof(ctx, "[reconcile] Check etcd hosts to be added")
 	etcdToAdd := hosts.GetToAddHosts(currentCluster.EtcdHosts, kubeCluster.EtcdHosts)
+	crtMap := currentCluster.Certificates
+	var err error
 	for _, etcdHost := range etcdToAdd {
 		etcdHost.ToAddEtcdMember = true
+		// Generate new certificate for the new etcd member
+		crtMap, err = pki.RegenerateEtcdCertificate(
+			ctx,
+			crtMap,
+			etcdHost,
+			kubeCluster.EtcdHosts,
+			kubeCluster.ClusterDomain,
+			kubeCluster.KubernetesServiceIP)
+		if err != nil {
+			return err
+		}
 	}
+	currentCluster.Certificates = crtMap
 	for _, etcdHost := range etcdToAdd {
-		if err := services.AddEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory); err != nil {
+		// deploy certificates on new etcd host
+		if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.EtcdHosts, etcdHost, currentCluster.Certificates, kubeCluster.SystemImages[CertDownloaderImage], pki.CertPathPrefix); err != nil {
+			return err
+		}
+
+		if err := services.AddEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory, clientCert, clientkey); err != nil {
 			return err
 		}
 		etcdHost.ToAddEtcdMember = false
-		if err := services.ReloadEtcdCluster(ctx, kubeCluster.EtcdHosts, kubeCluster.Services.Etcd, currentCluster.LocalConnDialerFactory); err != nil {
+		if err := services.ReloadEtcdCluster(ctx, kubeCluster.EtcdHosts, kubeCluster.Services.Etcd, currentCluster.LocalConnDialerFactory, clientCert, clientkey); err != nil {
 			return err
 		}
 	}

cluster/state.go

@@ -58,7 +58,10 @@ func (c *Cluster) GetClusterState(ctx context.Context) (*Cluster, error) {
 		currentCluster = getStateFromKubernetes(ctx, c.KubeClient, c.LocalKubeConfigPath)
 		// Get previous kubernetes certificates
 		if currentCluster != nil {
-			currentCluster.Certificates, err = getClusterCerts(ctx, c.KubeClient)
+			if err := currentCluster.InvertIndexHosts(); err != nil {
+				return nil, fmt.Errorf("Failed to classify hosts from fetched cluster: %v", err)
+			}
+			currentCluster.Certificates, err = getClusterCerts(ctx, c.KubeClient, currentCluster.EtcdHosts)
 			currentCluster.DockerDialerFactory = c.DockerDialerFactory
 			if err != nil {
 				return nil, fmt.Errorf("Failed to Get Kubernetes certificates: %v", err)
@@ -66,9 +69,6 @@ func (c *Cluster) GetClusterState(ctx context.Context) (*Cluster, error) {
 			// setting cluster defaults for the fetched cluster as well
 			currentCluster.setClusterDefaults(ctx)
 
-			if err := currentCluster.InvertIndexHosts(); err != nil {
-				return nil, fmt.Errorf("Failed to classify hosts from fetched cluster: %v", err)
-			}
 			currentCluster.Certificates, err = regenerateAPICertificate(c, currentCluster.Certificates)
 			if err != nil {
 				return nil, fmt.Errorf("Failed to regenerate KubeAPI certificate %v", err)

cmd/up.go

@@ -110,8 +110,8 @@ func ClusterUp(
 
 	APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443")
 	caCrt = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.CACertName].Certificate))
-	clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCommonName].Certificate))
+	clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Certificate))
-	clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCommonName].Key))
+	clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Key))
 
 	log.Infof(ctx, "Finished building Kubernetes cluster successfully")
 	return APIURL, caCrt, clientCert, clientKey, nil

pki/constants.go

@@ -1,65 +1,25 @@
 package pki
 
 const (
+	CertPathPrefix          = "/etc/kubernetes/ssl/"
 	CertificatesServiceName = "certificates"
 	CrtDownloaderContainer  = "cert-deployer"
 	CertFetcherContainer    = "cert-fetcher"
 	CertificatesSecretName  = "k8s-certs"
 	TempCertPath            = "/etc/kubernetes/.tmp/"
 
-	CACertName    = "kube-ca"
+	CACertName             = "kube-ca"
-	CACertENVName = "KUBE_CA"
+	KubeAPICertName        = "kube-apiserver"
-	CAKeyENVName  = "KUBE_CA_KEY"
+	KubeControllerCertName = "kube-controller-manager"
-	CACertPath    = "/etc/kubernetes/ssl/kube-ca.pem"
+	KubeSchedulerCertName  = "kube-scheduler"
-	CAKeyPath     = "/etc/kubernetes/ssl/kube-ca-key.pem"
+	KubeProxyCertName      = "kube-proxy"
+	KubeNodeCertName       = "kube-node"
+	EtcdCertName           = "kube-etcd"
 
-	KubeAPICertName    = "kube-apiserver"
-	KubeAPICertENVName = "KUBE_API"
-	KubeAPIKeyENVName  = "KUBE_API_KEY"
-	KubeAPICertPath    = "/etc/kubernetes/ssl/kube-api.pem"
-	KubeAPIKeyPath     = "/etc/kubernetes/ssl/kube-api-key.pem"
-
-	KubeControllerName          = "kube-controller-manager"
-	KubeControllerCommonName    = "system:kube-controller-manager"
-	KubeControllerCertENVName   = "KUBE_CONTROLLER_MANAGER"
-	KubeControllerKeyENVName    = "KUBE_CONTROLLER_MANAGER_KEY"
-	KubeControllerConfigENVName = "KUBECFG_CONTROLLER_MANAGER"
-	KubeControllerCertPath      = "/etc/kubernetes/ssl/kube-controller-manager.pem"
-	KubeControllerKeyPath       = "/etc/kubernetes/ssl/kube-controller-manager-key.pem"
-	KubeControllerConfigPath    = "/etc/kubernetes/ssl/kubecfg-controller-manager.yaml"
-
-	KubeSchedulerName          = "kube-scheduler"
-	KubeSchedulerCommonName    = "system:kube-scheduler"
-	KubeSchedulerCertENVName   = "KUBE_SCHEDULER"
-	KubeSchedulerKeyENVName    = "KUBE_SCHEDULER_KEY"
-	KubeSchedulerConfigENVName = "KUBECFG_SCHEDULER"
-	KubeSchedulerCertPath      = "/etc/kubernetes/ssl/kube-scheduler.pem"
-	KubeSchedulerKeyPath       = "/etc/kubernetes/ssl/kube-scheduler-key.pem"
-	KubeSchedulerConfigPath    = "/etc/kubernetes/ssl/kubecfg-scheduler.yaml"
-
-	KubeProxyName          = "kube-proxy"
-	KubeProxyCommonName    = "system:kube-proxy"
-	KubeProxyCertENVName   = "KUBE_PROXY"
-	KubeProxyKeyENVName    = "KUBE_PROXY_KEY"
-	KubeProxyConfigENVName = "KUBECFG_KUBE_PROXY"
-	KubeProxyCertPath      = "/etc/kubernetes/ssl/kube-proxy.pem"
-	KubeProxyKeyPath       = "/etc/kubernetes/ssl/kube-proxy-key.pem"
-	KubeProxyConfigPath    = "/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml"
-
-	KubeNodeName             = "kube-node"
 	KubeNodeCommonName       = "system:node"
 	KubeNodeOrganizationName = "system:nodes"
-	KubeNodeCertENVName      = "KUBE_NODE"
-	KubeNodeKeyENVName       = "KUBE_NODE_KEY"
-	KubeNodeConfigENVName    = "KUBECFG_KUBE_NODE"
-	KubeNodeCertPath         = "/etc/kubernetes/ssl/kube-node.pem"
-	KubeNodeKeyPath          = "/etc/kubernetes/ssl/kube-node-key.pem"
-	KubeNodeConfigPath       = "/etc/kubernetes/ssl/kubecfg-kube-node.yaml"
 
-	KubeAdminCommonName       = "kube-admin"
+	KubeAdminCertName         = "kube-admin"
 	KubeAdminOrganizationName = "system:masters"
 	KubeAdminConfigPrefix     = ".kube_config_"
-	KubeAdminConfigENVName    = "KUBECFG_ADMIN"
-	KubeAdminCertEnvName      = "KUBE_ADMIN"
-	KubeAdminKeyEnvName       = "KUBE_ADMIN_KEY"
 )

pki/deploy.go

@@ -24,10 +24,10 @@ func DeployCertificatesOnMasters(ctx context.Context, cpHosts []*hosts.Host, crt
 	crtList := []string{
 		CACertName,
 		KubeAPICertName,
-		KubeControllerName,
+		KubeControllerCertName,
-		KubeSchedulerName,
+		KubeSchedulerCertName,
-		KubeProxyName,
+		KubeProxyCertName,
-		KubeNodeName,
+		KubeNodeCertName,
 	}
 	env := []string{}
 	for _, crtName := range crtList {
@@ -48,8 +48,8 @@ func DeployCertificatesOnWorkers(ctx context.Context, workerHosts []*hosts.Host,
 	// list of certificates that should be deployed on the workers
 	crtList := []string{
 		CACertName,
-		KubeProxyName,
+		KubeProxyCertName,
-		KubeNodeName,
+		KubeNodeCertName,
 	}
 	env := []string{}
 	for _, crtName := range crtList {
@@ -66,6 +66,31 @@ func DeployCertificatesOnWorkers(ctx context.Context, workerHosts []*hosts.Host,
 	return nil
 }
 
+func DeployCertificatesOnEtcd(ctx context.Context, etcdHosts []*hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage string) error {
+	// list of certificates that should be deployed on the etcd
+	crtList := []string{
+		CACertName,
+		KubeProxyCertName,
+		KubeNodeCertName,
+	}
+	for _, host := range etcdHosts {
+		crtList = append(crtList, GetEtcdCrtName(host.InternalAddress))
+	}
+	env := []string{}
+	for _, crtName := range crtList {
+		c := crtMap[crtName]
+		env = append(env, c.ToEnv()...)
+	}
+
+	for i := range etcdHosts {
+		err := doRunDeployer(ctx, etcdHosts[i], env, certDownloaderImage)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func doRunDeployer(ctx context.Context, host *hosts.Host, containerEnv []string, certDownloaderImage string) error {
 	// remove existing container. Only way it's still here is if previous deployment failed
 	isRunning := false
@@ -135,19 +160,22 @@ func RemoveAdminConfig(ctx context.Context, localConfigPath string) {
 	log.Infof(ctx, "Local admin Kubeconfig removed successfully")
 }
 
-func DeployCertificatesOnHost(ctx context.Context, host *hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage string) error {
+func DeployCertificatesOnHost(ctx context.Context, extraHosts []*hosts.Host, host *hosts.Host, crtMap map[string]CertificatePKI, certDownloaderImage, certPath string) error {
 	crtList := []string{
 		CACertName,
 		KubeAPICertName,
-		KubeControllerName,
+		KubeControllerCertName,
-		KubeSchedulerName,
+		KubeSchedulerCertName,
-		KubeProxyName,
+		KubeProxyCertName,
-		KubeNodeName,
+		KubeNodeCertName,
-		KubeAdminCommonName,
+		KubeAdminCertName,
+	}
+	for _, host := range extraHosts {
+		// Deploy etcd certificates
+		crtList = append(crtList, GetEtcdCrtName(host.InternalAddress))
 	}
-
 	env := []string{
-		"CRTS_DEPLOY_PATH=" + TempCertPath,
+		"CRTS_DEPLOY_PATH=" + certPath,
 	}
 	for _, crtName := range crtList {
 		c := crtMap[crtName]
@@ -157,24 +185,27 @@ func DeployCertificatesOnHost(ctx context.Context, host *hosts.Host, crtMap map[
 	return doRunDeployer(ctx, host, env, certDownloaderImage)
 }
 
-func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, localConfigPath string) (map[string]CertificatePKI, error) {
+func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, host *hosts.Host, image, localConfigPath string) (map[string]CertificatePKI, error) {
 	// rebuilding the certificates. This should look better after refactoring pki
 	tmpCerts := make(map[string]CertificatePKI)
 
-	certEnvMap := map[string][]string{
+	crtList := map[string]bool{
-		CACertName:          []string{CACertPath, CAKeyPath},
+		CACertName:             false,
-		KubeAPICertName:     []string{KubeAPICertPath, KubeAPIKeyPath},
+		KubeAPICertName:        false,
-		KubeControllerName:  []string{KubeControllerCertPath, KubeControllerKeyPath, KubeControllerConfigPath},
+		KubeControllerCertName: true,
-		KubeSchedulerName:   []string{KubeSchedulerCertPath, KubeSchedulerKeyPath, KubeSchedulerConfigPath},
+		KubeSchedulerCertName:  true,
-		KubeProxyName:       []string{KubeProxyCertPath, KubeProxyKeyPath, KubeProxyConfigPath},
+		KubeProxyCertName:      true,
-		KubeNodeName:        []string{KubeNodeCertPath, KubeNodeKeyPath, KubeNodeConfigPath},
+		KubeNodeCertName:       true,
-		KubeAdminCommonName: []string{"kube-admin.pem", "kube-admin-key.pem", "kubecfg-admin.yaml"},
+		KubeAdminCertName:      true,
+	}
+	for _, etcdHost := range extraHosts {
+		// Fetch etcd certificates
+		crtList[GetEtcdCrtName(etcdHost.InternalAddress)] = false
 	}
-	// get files from hosts
 
-	for crtName, certEnv := range certEnvMap {
+	for certName, config := range crtList {
 		certificate := CertificatePKI{}
-		crt, err := fetchFileFromHost(ctx, getTempPath(certEnv[0]), image, host)
+		crt, err := fetchFileFromHost(ctx, GetCertTempPath(certName), image, host)
 		if err != nil {
 			if strings.Contains(err.Error(), "no such file or directory") ||
 				strings.Contains(err.Error(), "Could not find the file") {
@@ -182,10 +213,10 @@ func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, loc
 			}
 			return nil, err
 		}
-		key, err := fetchFileFromHost(ctx, getTempPath(certEnv[1]), image, host)
+		key, err := fetchFileFromHost(ctx, GetKeyTempPath(certName), image, host)
 
-		if len(certEnv) > 2 {
+		if config {
-			config, err := fetchFileFromHost(ctx, getTempPath(certEnv[2]), image, host)
+			config, err := fetchFileFromHost(ctx, GetConfigTempPath(certName), image, host)
 			if err != nil {
 				return nil, err
 			}
@@ -201,14 +232,14 @@ func FetchCertificatesFromHost(ctx context.Context, host *hosts.Host, image, loc
 		}
 		certificate.Certificate = parsedCert[0]
 		certificate.Key = parsedKey.(*rsa.PrivateKey)
-		tmpCerts[crtName] = certificate
+		tmpCerts[certName] = certificate
-		logrus.Debugf("[certificates] Recovered certificate: %s", crtName)
+		logrus.Debugf("[certificates] Recovered certificate: %s", certName)
 	}
 
 	if err := docker.RemoveContainer(ctx, host.DClient, host.Address, CertFetcherContainer); err != nil {
 		return nil, err
 	}
-	return populateCertMap(tmpCerts, localConfigPath), nil
+	return populateCertMap(tmpCerts, localConfigPath, extraHosts), nil
 
 }
 
@@ -244,96 +275,31 @@ func getTempPath(s string) string {
 	return TempCertPath + path.Base(s)
 }
 
-func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string) map[string]CertificatePKI {
+func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string, extraHosts []*hosts.Host) map[string]CertificatePKI {
 	certs := make(map[string]CertificatePKI)
-	//CACert
+	// CACert
-	certs[CACertName] = CertificatePKI{
+	certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key)
-		Certificate: tmpCerts[CACertName].Certificate,
+	// KubeAPI
-		Key:         tmpCerts[CACertName].Key,
+	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key)
-		Name:        CACertName,
+	// kubeController
-		EnvName:     CACertENVName,
+	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key)
-		KeyEnvName:  CAKeyENVName,
+	// KubeScheduler
-		Path:        CACertPath,
+	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key)
-		KeyPath:     CAKeyPath,
-	}
-	//KubeAPI
-	certs[KubeAPICertName] = CertificatePKI{
-		Certificate: tmpCerts[KubeAPICertName].Certificate,
-		Key:         tmpCerts[KubeAPICertName].Key,
-		Name:        KubeAPICertName,
-		EnvName:     KubeAPICertENVName,
-		KeyEnvName:  KubeAPIKeyENVName,
-		Path:        KubeAPICertPath,
-		KeyPath:     KubeAPIKeyPath,
-	}
-	//kubeController
-	certs[KubeControllerName] = CertificatePKI{
-		Certificate:   tmpCerts[KubeControllerName].Certificate,
-		Key:           tmpCerts[KubeControllerName].Key,
-		Config:        tmpCerts[KubeControllerName].Config,
-		Name:          KubeControllerName,
-		CommonName:    KubeControllerCommonName,
-		EnvName:       KubeControllerCertENVName,
-		KeyEnvName:    KubeControllerKeyENVName,
-		Path:          KubeControllerCertPath,
-		KeyPath:       KubeControllerKeyPath,
-		ConfigEnvName: KubeControllerConfigENVName,
-		ConfigPath:    KubeControllerConfigPath,
-	}
-	//KubeScheduler
-	certs[KubeSchedulerName] = CertificatePKI{
-		Certificate:   tmpCerts[KubeSchedulerName].Certificate,
-		Key:           tmpCerts[KubeSchedulerName].Key,
-		Config:        tmpCerts[KubeSchedulerName].Config,
-		Name:          KubeSchedulerName,
-		CommonName:    KubeSchedulerCommonName,
-		EnvName:       KubeSchedulerCertENVName,
-		KeyEnvName:    KubeSchedulerKeyENVName,
-		Path:          KubeSchedulerCertPath,
-		KeyPath:       KubeSchedulerKeyPath,
-		ConfigEnvName: KubeSchedulerConfigENVName,
-		ConfigPath:    KubeSchedulerConfigPath,
-	}
 	// KubeProxy
-	certs[KubeProxyName] = CertificatePKI{
+	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key)
-		Certificate:   tmpCerts[KubeProxyName].Certificate,
-		Key:           tmpCerts[KubeProxyName].Key,
-		Config:        tmpCerts[KubeProxyName].Config,
-		Name:          KubeProxyName,
-		CommonName:    KubeProxyCommonName,
-		EnvName:       KubeProxyCertENVName,
-		Path:          KubeProxyCertPath,
-		KeyEnvName:    KubeProxyKeyENVName,
-		KeyPath:       KubeProxyKeyPath,
-		ConfigEnvName: KubeProxyConfigENVName,
-		ConfigPath:    KubeProxyConfigPath,
-	}
 	// KubeNode
-	certs[KubeNodeName] = CertificatePKI{
+	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key)
-		Certificate:   tmpCerts[KubeNodeName].Certificate,
+	// KubeAdmin
-		Key:           tmpCerts[KubeNodeName].Key,
+	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key)
-		Config:        tmpCerts[KubeNodeName].Config,
+	kubeAdminCertObj.Config = tmpCerts[KubeAdminCertName].Config
-		Name:          KubeNodeName,
+	kubeAdminCertObj.ConfigPath = localConfigPath
-		CommonName:    KubeNodeCommonName,
+	certs[KubeAdminCertName] = kubeAdminCertObj
-		OUName:        KubeNodeOrganizationName,
+	// etcd
-		EnvName:       KubeNodeCertENVName,
+	for _, host := range extraHosts {
-		KeyEnvName:    KubeNodeKeyENVName,
+		etcdName := GetEtcdCrtName(host.InternalAddress)
-		Path:          KubeNodeCertPath,
+		etcdCrt, etcdKey := tmpCerts[etcdName].Certificate, tmpCerts[etcdName].Key
-		KeyPath:       KubeNodeKeyPath,
+		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
-		ConfigEnvName: KubeNodeConfigENVName,
-		ConfigPath:    KubeNodeCommonName,
 	}
 
-	certs[KubeAdminCommonName] = CertificatePKI{
-		Certificate:   tmpCerts[KubeAdminCommonName].Certificate,
-		Key:           tmpCerts[KubeAdminCommonName].Key,
-		Config:        tmpCerts[KubeAdminCommonName].Config,
-		CommonName:    KubeAdminCommonName,
-		OUName:        KubeAdminOrganizationName,
-		ConfigEnvName: KubeAdminConfigENVName,
-		ConfigPath:    localConfigPath,
-		EnvName:       KubeAdminCertEnvName,
-		KeyEnvName:    KubeAdminKeyEnvName,
-	}
 	return certs
 }

pki/pki.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/rsa"
 	"crypto/x509"
-	"fmt"
 	"net"
 
 	"github.com/rancher/rke/hosts"
@@ -29,16 +28,16 @@ type CertificatePKI struct {
 }
 
 // StartCertificatesGeneration ...
-func StartCertificatesGeneration(ctx context.Context, cpHosts []*hosts.Host, workerHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) {
+func StartCertificatesGeneration(ctx context.Context, cpHosts, etcdHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) {
-	log.Infof(ctx, "[certificates] Generating kubernetes certificates")
+	logrus.Infof("[certificates] Generating kubernetes certificates")
-	certs, err := generateCerts(ctx, cpHosts, clusterDomain, localConfigPath, KubernetesServiceIP)
+	certs, err := generateCerts(ctx, cpHosts, etcdHosts, clusterDomain, localConfigPath, KubernetesServiceIP)
 	if err != nil {
 		return nil, err
 	}
 	return certs, nil
 }
 
-func generateCerts(ctx context.Context, cpHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) {
+func generateCerts(ctx context.Context, cpHosts, etcdHosts []*hosts.Host, clusterDomain, localConfigPath string, KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) {
 	certs := make(map[string]CertificatePKI)
 	// generate CA certificate and key
 	log.Infof(ctx, "[certificates] Generating CA kubernetes certificates")
@@ -46,270 +45,100 @@ func generateCerts(ctx context.Context, cpHosts []*hosts.Host, clusterDomain, lo
 	if err != nil {
 		return nil, err
 	}
-	logrus.Debugf("[certificates] CA Certificate: %s", string(cert.EncodeCertPEM(caCrt)))
+	certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey)
-	certs[CACertName] = CertificatePKI{
-		Certificate: caCrt,
-		Key:         caKey,
-		Name:        CACertName,
-		EnvName:     CACertENVName,
-		KeyEnvName:  CAKeyENVName,
-		Path:        CACertPath,
-		KeyPath:     CAKeyPath,
-	}
 
 	// generate API certificate and key
 	log.Infof(ctx, "[certificates] Generating Kubernetes API server certificates")
 	kubeAPIAltNames := GetAltNames(cpHosts, clusterDomain, KubernetesServiceIP)
-	kubeAPICrt, kubeAPIKey, err := GenerateKubeAPICertAndKey(caCrt, caKey, kubeAPIAltNames)
+	kubeAPICrt, kubeAPIKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, KubeAPICertName, kubeAPIAltNames, nil, nil)
 	if err != nil {
 		return nil, err
 	}
-	logrus.Debugf("[certificates] Kube API Certificate: %s", string(cert.EncodeCertPEM(kubeAPICrt)))
+	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey)
-	certs[KubeAPICertName] = CertificatePKI{
-		Certificate: kubeAPICrt,
-		Key:         kubeAPIKey,
-		Name:        KubeAPICertName,
-		EnvName:     KubeAPICertENVName,
-		KeyEnvName:  KubeAPIKeyENVName,
-		Path:        KubeAPICertPath,
-		KeyPath:     KubeAPIKeyPath,
-	}
 
 	// generate Kube controller-manager certificate and key
 	log.Infof(ctx, "[certificates] Generating Kube Controller certificates")
-	kubeControllerCrt, kubeControllerKey, err := generateClientCertAndKey(caCrt, caKey, KubeControllerCommonName, []string{})
+	kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, nil, nil)
 	if err != nil {
 		return nil, err
 	}
-	logrus.Debugf("[certificates] Kube Controller Certificate: %s", string(cert.EncodeCertPEM(kubeControllerCrt)))
+	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey)
-	certs[KubeControllerName] = CertificatePKI{
-		Certificate:   kubeControllerCrt,
-		Key:           kubeControllerKey,
-		Config:        getKubeConfigX509("https://127.0.0.1:6443", KubeControllerName, CACertPath, KubeControllerCertPath, KubeControllerKeyPath),
-		Name:          KubeControllerName,
-		CommonName:    KubeControllerCommonName,
-		EnvName:       KubeControllerCertENVName,
-		KeyEnvName:    KubeControllerKeyENVName,
-		Path:          KubeControllerCertPath,
-		KeyPath:       KubeControllerKeyPath,
-		ConfigEnvName: KubeControllerConfigENVName,
-		ConfigPath:    KubeControllerConfigPath,
-	}
 
 	// generate Kube scheduler certificate and key
 	log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates")
-	kubeSchedulerCrt, kubeSchedulerKey, err := generateClientCertAndKey(caCrt, caKey, KubeSchedulerCommonName, []string{})
+	kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, nil, nil)
 	if err != nil {
 		return nil, err
 	}
-	logrus.Debugf("[certificates] Kube Scheduler Certificate: %s", string(cert.EncodeCertPEM(kubeSchedulerCrt)))
+	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey)
-	certs[KubeSchedulerName] = CertificatePKI{
-		Certificate:   kubeSchedulerCrt,
-		Key:           kubeSchedulerKey,
-		Config:        getKubeConfigX509("https://127.0.0.1:6443", KubeSchedulerName, CACertPath, KubeSchedulerCertPath, KubeSchedulerKeyPath),
-		Name:          KubeSchedulerName,
-		CommonName:    KubeSchedulerCommonName,
-		EnvName:       KubeSchedulerCertENVName,
-		KeyEnvName:    KubeSchedulerKeyENVName,
-		Path:          KubeSchedulerCertPath,
-		KeyPath:       KubeSchedulerKeyPath,
-		ConfigEnvName: KubeSchedulerConfigENVName,
-		ConfigPath:    KubeSchedulerConfigPath,
-	}
 
 	// generate Kube Proxy certificate and key
 	log.Infof(ctx, "[certificates] Generating Kube Proxy certificates")
-	kubeProxyCrt, kubeProxyKey, err := generateClientCertAndKey(caCrt, caKey, KubeProxyCommonName, []string{})
+	kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, nil, nil)
 	if err != nil {
 		return nil, err
 	}
-	logrus.Debugf("[certificates] Kube Proxy Certificate: %s", string(cert.EncodeCertPEM(kubeProxyCrt)))
+	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey)
-	certs[KubeProxyName] = CertificatePKI{
-		Certificate:   kubeProxyCrt,
-		Key:           kubeProxyKey,
-		Config:        getKubeConfigX509("https://127.0.0.1:6443", KubeProxyName, CACertPath, KubeProxyCertPath, KubeProxyKeyPath),
-		Name:          KubeProxyName,
-		CommonName:    KubeProxyCommonName,
-		EnvName:       KubeProxyCertENVName,
-		Path:          KubeProxyCertPath,
-		KeyEnvName:    KubeProxyKeyENVName,
-		KeyPath:       KubeProxyKeyPath,
-		ConfigEnvName: KubeProxyConfigENVName,
-		ConfigPath:    KubeProxyConfigPath,
-	}
 
 	// generate Kubelet certificate and key
 	log.Infof(ctx, "[certificates] Generating Node certificate")
-	nodeCrt, nodeKey, err := generateClientCertAndKey(caCrt, caKey, KubeNodeCommonName, []string{KubeNodeOrganizationName})
+	nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, nil, []string{KubeNodeOrganizationName})
 	if err != nil {
 		return nil, err
 	}
-	logrus.Debugf("[certificates] Node Certificate: %s", string(cert.EncodeCertPEM(kubeProxyCrt)))
+	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey)
-	certs[KubeNodeName] = CertificatePKI{
+
-		Certificate:   nodeCrt,
+	// generate Admin certificate and key
-		Key:           nodeKey,
+	logrus.Infof("[certificates] Generating admin certificates and kubeconfig")
-		Config:        getKubeConfigX509("https://127.0.0.1:6443", KubeNodeName, CACertPath, KubeNodeCertPath, KubeNodeKeyPath),
+	kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, nil, []string{KubeAdminOrganizationName})
-		Name:          KubeNodeName,
-		CommonName:    KubeNodeCommonName,
-		OUName:        KubeNodeOrganizationName,
-		EnvName:       KubeNodeCertENVName,
-		KeyEnvName:    KubeNodeKeyENVName,
-		Path:          KubeNodeCertPath,
-		KeyPath:       KubeNodeKeyPath,
-		ConfigEnvName: KubeNodeConfigENVName,
-		ConfigPath:    KubeNodeCommonName,
-	}
-	log.Infof(ctx, "[certificates] Generating admin certificates and kubeconfig")
-	kubeAdminCrt, kubeAdminKey, err := generateClientCertAndKey(caCrt, caKey, KubeAdminCommonName, []string{KubeAdminOrganizationName})
 	if err != nil {
 		return nil, err
 	}
-	logrus.Debugf("[certificates] Admin Certificate: %s", string(cert.EncodeCertPEM(kubeAdminCrt)))
+	kubeAdminConfig := GetKubeConfigX509WithData(
-	certs[KubeAdminCommonName] = CertificatePKI{
+		"https://"+cpHosts[0].Address+":6443",
-		Certificate: kubeAdminCrt,
+		KubeAdminCertName,
-		Key:         kubeAdminKey,
+		string(cert.EncodeCertPEM(caCrt)),
-		Config: GetKubeConfigX509WithData(
+		string(cert.EncodeCertPEM(kubeAdminCrt)),
-			"https://"+cpHosts[0].Address+":6443",
+		string(cert.EncodePrivateKeyPEM(kubeAdminKey)))
-			KubeAdminCommonName,
+
-			string(cert.EncodeCertPEM(caCrt)),
+	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey)
-			string(cert.EncodeCertPEM(kubeAdminCrt)),
+	kubeAdminCertObj.Config = kubeAdminConfig
-			string(cert.EncodePrivateKeyPEM(kubeAdminKey))),
+	kubeAdminCertObj.ConfigPath = localConfigPath
-		CommonName:    KubeAdminCommonName,
+	certs[KubeAdminCertName] = kubeAdminCertObj
-		OUName:        KubeAdminOrganizationName,
+
-		ConfigEnvName: KubeAdminConfigENVName,
+	etcdAltNames := GetAltNames(etcdHosts, clusterDomain, KubernetesServiceIP)
-		ConfigPath:    localConfigPath,
+	for _, host := range etcdHosts {
-		EnvName:       KubeAdminCertEnvName,
+		logrus.Infof("[certificates] Generating etcd-%s certificate and key", host.InternalAddress)
-		KeyEnvName:    KubeAdminKeyEnvName,
+		etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil)
+		if err != nil {
+			return nil, err
+		}
+		etcdName := GetEtcdCrtName(host.InternalAddress)
+		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
 	}
+
 	return certs, nil
 }
 
-func generateClientCertAndKey(caCrt *x509.Certificate, caKey *rsa.PrivateKey, commonName string, orgs []string) (*x509.Certificate, *rsa.PrivateKey, error) {
+func RegenerateEtcdCertificate(
-	rootKey, err := cert.NewPrivateKey()
+	ctx context.Context,
+	crtMap map[string]CertificatePKI,
+	etcdHost *hosts.Host,
+	etcdHosts []*hosts.Host,
+	clusterDomain string,
+	KubernetesServiceIP net.IP) (map[string]CertificatePKI, error) {
+
+	log.Infof(ctx, "[certificates] Regenerating new etcd-%s certificate and key", etcdHost.InternalAddress)
+	caCrt := crtMap[CACertName].Certificate
+	caKey := crtMap[CACertName].Key
+	etcdAltNames := GetAltNames(etcdHosts, clusterDomain, KubernetesServiceIP)
+
+	etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to generate private key for %s certificate: %v", commonName, err)
+		return nil, err
 	}
-	caConfig := cert.Config{
+	etcdName := GetEtcdCrtName(etcdHost.InternalAddress)
-		CommonName:   commonName,
+	crtMap[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
-		Organization: orgs,
+	log.Infof(ctx, "[certificates] Successfully generated new etcd-%s certificate and key", etcdHost.InternalAddress)
-		Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	return crtMap, nil
-	}
-	clientCert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey)
-	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to generate %s certificate: %v", commonName, err)
-	}
-
-	return clientCert, rootKey, nil
-}
-
-func GenerateKubeAPICertAndKey(caCrt *x509.Certificate, caKey *rsa.PrivateKey, altNames *cert.AltNames) (*x509.Certificate, *rsa.PrivateKey, error) {
-	rootKey, err := cert.NewPrivateKey()
-	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to generate private key for kube-apiserver certificate: %v", err)
-	}
-	caConfig := cert.Config{
-		CommonName: KubeAPICertName,
-		Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
-		AltNames:   *altNames,
-	}
-	kubeCACert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey)
-	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to generate kube-apiserver certificate: %v", err)
-	}
-
-	return kubeCACert, rootKey, nil
-}
-
-func GenerateCertWithKey(commonName string, key *rsa.PrivateKey, caCrt *x509.Certificate, caKey *rsa.PrivateKey, altNames *cert.AltNames) (*x509.Certificate, error) {
-	caConfig := cert.Config{
-		CommonName: commonName,
-		Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth,
-			x509.ExtKeyUsageServerAuth},
-		AltNames: *altNames,
-	}
-	cert, err := cert.NewSignedCert(caConfig, key, caCrt, caKey)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to generate certificate with existing key: %v", err)
-	}
-	return cert, nil
-}
-
-func generateCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
-	rootKey, err := cert.NewPrivateKey()
-	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err)
-	}
-	caConfig := cert.Config{
-		CommonName: CACertName,
-	}
-	kubeCACert, err := cert.NewSelfSignedCACert(caConfig, rootKey)
-	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to generate CA certificate: %v", err)
-	}
-
-	return kubeCACert, rootKey, nil
-}
-
-func GetAltNames(cpHosts []*hosts.Host, clusterDomain string, KubernetesServiceIP net.IP) *cert.AltNames {
-	ips := []net.IP{}
-	dnsNames := []string{}
-	for _, host := range cpHosts {
-		// Check if node address is a valid IP
-		if nodeIP := net.ParseIP(host.Address); nodeIP != nil {
-			ips = append(ips, nodeIP)
-		} else {
-			dnsNames = append(dnsNames, host.Address)
-		}
-
-		// Check if node internal address is a valid IP
-		if len(host.InternalAddress) != 0 && host.InternalAddress != host.Address {
-			if internalIP := net.ParseIP(host.InternalAddress); internalIP != nil {
-				ips = append(ips, internalIP)
-			} else {
-				dnsNames = append(dnsNames, host.InternalAddress)
-			}
-		}
-		// Add hostname to the ALT dns names
-		if len(host.HostnameOverride) != 0 && host.HostnameOverride != host.Address {
-			dnsNames = append(dnsNames, host.HostnameOverride)
-		}
-	}
-	ips = append(ips, net.ParseIP("127.0.0.1"))
-	ips = append(ips, KubernetesServiceIP)
-	dnsNames = append(dnsNames, []string{
-		"localhost",
-		"kubernetes",
-		"kubernetes.default",
-		"kubernetes.default.svc",
-		"kubernetes.default.svc." + clusterDomain,
-	}...)
-	return &cert.AltNames{
-		IPs:      ips,
-		DNSNames: dnsNames,
-	}
-}
-
-func (c *CertificatePKI) ToEnv() []string {
-	env := []string{
-		c.CertToEnv(),
-		c.KeyToEnv(),
-	}
-	if c.Config != "" {
-		env = append(env, c.ConfigToEnv())
-	}
-	return env
-}
-
-func (c *CertificatePKI) CertToEnv() string {
-	encodedCrt := cert.EncodeCertPEM(c.Certificate)
-	return fmt.Sprintf("%s=%s", c.EnvName, string(encodedCrt))
-}
-
-func (c *CertificatePKI) KeyToEnv() string {
-	encodedKey := cert.EncodePrivateKeyPEM(c.Key)
-	return fmt.Sprintf("%s=%s", c.KeyEnvName, string(encodedKey))
-}
-
-func (c *CertificatePKI) ConfigToEnv() string {
-	return fmt.Sprintf("%s=%s", c.ConfigEnvName, c.Config)
 }

pki/pki_test.go

@@ -38,11 +38,11 @@ func TestPKI(t *testing.T) {
 
 	certificatesToVerify := []string{
 		KubeAPICertName,
-		KubeNodeName,
+		KubeNodeCertName,
-		KubeProxyName,
+		KubeProxyCertName,
-		KubeControllerName,
+		KubeControllerCertName,
-		KubeSchedulerName,
+		KubeSchedulerCertName,
-		KubeAdminCommonName,
+		KubeAdminCertName,
 	}
 	opts := x509.VerifyOptions{
 		Roots:     roots,

pki/util.go

@@ -0,0 +1,209 @@
+package pki
+
+import (
+	"crypto/rsa"
+	"crypto/x509"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/rancher/rke/hosts"
+	"k8s.io/client-go/util/cert"
+)
+
+func GenerateSignedCertAndKey(
+	caCrt *x509.Certificate,
+	caKey *rsa.PrivateKey,
+	serverCrt bool,
+	commonName string,
+	altNames *cert.AltNames,
+	reusedKey *rsa.PrivateKey,
+	orgs []string) (*x509.Certificate, *rsa.PrivateKey, error) {
+	// Generate a generic signed certificate
+	var rootKey *rsa.PrivateKey
+	var err error
+	rootKey = reusedKey
+	if reusedKey == nil {
+		rootKey, err = cert.NewPrivateKey()
+		if err != nil {
+			return nil, nil, fmt.Errorf("Failed to generate private key for %s certificate: %v", commonName, err)
+		}
+	}
+	usages := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
+	if serverCrt {
+		usages = append(usages, x509.ExtKeyUsageServerAuth)
+	}
+	if altNames == nil {
+		altNames = &cert.AltNames{}
+	}
+	caConfig := cert.Config{
+		CommonName:   commonName,
+		Organization: orgs,
+		Usages:       usages,
+		AltNames:     *altNames,
+	}
+	clientCert, err := cert.NewSignedCert(caConfig, rootKey, caCrt, caKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to generate %s certificate: %v", commonName, err)
+	}
+	return clientCert, rootKey, nil
+}
+
+func generateCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
+	rootKey, err := cert.NewPrivateKey()
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err)
+	}
+	caConfig := cert.Config{
+		CommonName: CACertName,
+	}
+	kubeCACert, err := cert.NewSelfSignedCACert(caConfig, rootKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to generate CA certificate: %v", err)
+	}
+
+	return kubeCACert, rootKey, nil
+}
+
+func GetAltNames(cpHosts []*hosts.Host, clusterDomain string, KubernetesServiceIP net.IP) *cert.AltNames {
+	ips := []net.IP{}
+	dnsNames := []string{}
+	for _, host := range cpHosts {
+		// Check if node address is a valid IP
+		if nodeIP := net.ParseIP(host.Address); nodeIP != nil {
+			ips = append(ips, nodeIP)
+		} else {
+			dnsNames = append(dnsNames, host.Address)
+		}
+
+		// Check if node internal address is a valid IP
+		if len(host.InternalAddress) != 0 && host.InternalAddress != host.Address {
+			if internalIP := net.ParseIP(host.InternalAddress); internalIP != nil {
+				ips = append(ips, internalIP)
+			} else {
+				dnsNames = append(dnsNames, host.InternalAddress)
+			}
+		}
+		// Add hostname to the ALT dns names
+		if len(host.HostnameOverride) != 0 && host.HostnameOverride != host.Address {
+			dnsNames = append(dnsNames, host.HostnameOverride)
+		}
+	}
+	ips = append(ips, net.ParseIP("127.0.0.1"))
+	ips = append(ips, KubernetesServiceIP)
+	dnsNames = append(dnsNames, []string{
+		"localhost",
+		"kubernetes",
+		"kubernetes.default",
+		"kubernetes.default.svc",
+		"kubernetes.default.svc." + clusterDomain,
+	}...)
+	return &cert.AltNames{
+		IPs:      ips,
+		DNSNames: dnsNames,
+	}
+}
+
+func (c *CertificatePKI) ToEnv() []string {
+	env := []string{
+		c.CertToEnv(),
+		c.KeyToEnv(),
+	}
+	if c.Config != "" {
+		env = append(env, c.ConfigToEnv())
+	}
+	return env
+}
+
+func (c *CertificatePKI) CertToEnv() string {
+	encodedCrt := cert.EncodeCertPEM(c.Certificate)
+	return fmt.Sprintf("%s=%s", c.EnvName, string(encodedCrt))
+}
+
+func (c *CertificatePKI) KeyToEnv() string {
+	encodedKey := cert.EncodePrivateKeyPEM(c.Key)
+	return fmt.Sprintf("%s=%s", c.KeyEnvName, string(encodedKey))
+}
+
+func (c *CertificatePKI) ConfigToEnv() string {
+	return fmt.Sprintf("%s=%s", c.ConfigEnvName, c.Config)
+}
+
+func getEnvFromName(name string) string {
+	return strings.Replace(strings.ToUpper(name), "-", "_", -1)
+}
+
+func getKeyEnvFromEnv(env string) string {
+	return fmt.Sprintf("%s_KEY", env)
+}
+
+func getConfigEnvFromEnv(env string) string {
+	return fmt.Sprintf("KUBECFG_%s", env)
+}
+
+func GetEtcdCrtName(address string) string {
+	newAddress := strings.Replace(address, ".", "-", -1)
+	return fmt.Sprintf("%s-%s", EtcdCertName, newAddress)
+}
+
+func GetCertPath(name string) string {
+	return fmt.Sprintf("%s%s.pem", CertPathPrefix, name)
+}
+
+func GetKeyPath(name string) string {
+	return fmt.Sprintf("%s%s-key.pem", CertPathPrefix, name)
+}
+
+func GetConfigPath(name string) string {
+	return fmt.Sprintf("%skubecfg-%s.yaml", CertPathPrefix, name)
+}
+
+func GetCertTempPath(name string) string {
+	return fmt.Sprintf("%s%s.pem", TempCertPath, name)
+}
+
+func GetKeyTempPath(name string) string {
+	return fmt.Sprintf("%s%s-key.pem", TempCertPath, name)
+}
+
+func GetConfigTempPath(name string) string {
+	return fmt.Sprintf("%skubecfg-%s.yaml", TempCertPath, name)
+}
+
+func ToCertObject(componentName, commonName, ouName string, cert *x509.Certificate, key *rsa.PrivateKey) CertificatePKI {
+	var config, configPath, configEnvName string
+	if len(commonName) == 0 {
+		commonName = getDefaultCN(componentName)
+	}
+
+	envName := getEnvFromName(componentName)
+	keyEnvName := getKeyEnvFromEnv(envName)
+	caCertPath := GetCertPath(CACertName)
+	path := GetCertPath(componentName)
+	keyPath := GetKeyPath(componentName)
+
+	if componentName != CACertName && componentName != KubeAPICertName && !strings.Contains(componentName, EtcdCertName) {
+		config = getKubeConfigX509("https://127.0.0.1:6443", componentName, caCertPath, path, keyPath)
+		configPath = GetConfigPath(componentName)
+		configEnvName = getConfigEnvFromEnv(envName)
+	}
+
+	return CertificatePKI{
+		Certificate:   cert,
+		Key:           key,
+		Config:        config,
+		Name:          componentName,
+		CommonName:    commonName,
+		OUName:        ouName,
+		EnvName:       envName,
+		KeyEnvName:    keyEnvName,
+		ConfigEnvName: configEnvName,
+		Path:          path,
+		KeyPath:       keyPath,
+		ConfigPath:    configPath,
+	}
+}
+
+func getDefaultCN(name string) string {
+	return fmt.Sprintf("system:%s", name)
+}

services/etcd.go

@@ -11,6 +11,7 @@ import (
 	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/log"
+	"github.com/rancher/rke/pki"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
 	"github.com/sirupsen/logrus"
 )
@@ -19,7 +20,8 @@ func RunEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, etcdService v3.E
 	log.Infof(ctx, "[%s] Building up Etcd Plane..", ETCDRole)
 	initCluster := getEtcdInitialCluster(etcdHosts)
 	for _, host := range etcdHosts {
-		imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster)
+		nodeName := pki.GetEtcdCrtName(host.InternalAddress)
+		imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster, nodeName)
 		err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, EtcdContainerName, host.Address, ETCDRole)
 		if err != nil {
 			return err
@@ -57,7 +59,7 @@ func RemoveEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, force bool) e
 	return nil
 }
 
-func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster string) (*container.Config, *container.HostConfig) {
+func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster, nodeName string) (*container.Config, *container.HostConfig) {
 	clusterState := "new"
 	if host.ExistingEtcdCluster {
 		clusterState = "existing"
@@ -67,18 +69,29 @@ func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster s
 		Cmd: []string{"/usr/local/bin/etcd",
 			"--name=etcd-" + host.HostnameOverride,
 			"--data-dir=/etcd-data",
-			"--advertise-client-urls=http://" + host.InternalAddress + ":2379,http://" + host.InternalAddress + ":4001",
+			"--advertise-client-urls=https://" + host.InternalAddress + ":2379,https://" + host.InternalAddress + ":4001",
-			"--listen-client-urls=http://0.0.0.0:2379",
+			"--listen-client-urls=https://0.0.0.0:2379",
-			"--initial-advertise-peer-urls=http://" + host.InternalAddress + ":2380",
+			"--initial-advertise-peer-urls=https://" + host.InternalAddress + ":2380",
-			"--listen-peer-urls=http://0.0.0.0:2380",
+			"--listen-peer-urls=https://0.0.0.0:2380",
 			"--initial-cluster-token=etcd-cluster-1",
 			"--initial-cluster=" + initCluster,
-			"--initial-cluster-state=" + clusterState},
+			"--initial-cluster-state=" + clusterState,
+			"--peer-client-cert-auth",
+			"--client-cert-auth",
+			"--trusted-ca-file=" + pki.GetCertPath(pki.CACertName),
+			"--peer-trusted-ca-file=" + pki.GetCertPath(pki.CACertName),
+			"--cert-file=" + pki.GetCertPath(nodeName),
+			"--key-file=" + pki.GetKeyPath(nodeName),
+			"--peer-cert-file=" + pki.GetCertPath(nodeName),
+			"--peer-key-file=" + pki.GetKeyPath(nodeName),
+		},
 	}
 	hostCfg := &container.HostConfig{
 		RestartPolicy: container.RestartPolicy{Name: "always"},
 		Binds: []string{
-			"/var/lib/etcd:/etcd-data"},
+			"/var/lib/etcd:/etcd-data",
+			"/etc/kubernetes:/etc/kubernetes",
+		},
 		NetworkMode: "host",
 	}
 	for arg, value := range etcdService.ExtraArgs {
@@ -89,12 +102,12 @@ func buildEtcdConfig(host *hosts.Host, etcdService v3.ETCDService, initCluster s
 	return imageCfg, hostCfg
 }
 
-func AddEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory) error {
+func AddEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte) error {
 	log.Infof(ctx, "[add/%s] Adding member [etcd-%s] to etcd cluster", ETCDRole, etcdHost.HostnameOverride)
-	peerURL := fmt.Sprintf("http://%s:2380", etcdHost.InternalAddress)
+	peerURL := fmt.Sprintf("https://%s:2380", etcdHost.InternalAddress)
 	added := false
 	for _, host := range etcdHosts {
-		etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory)
+		etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory, cert, key)
 		if err != nil {
 			logrus.Debugf("Failed to create etcd client for host [%s]: %v", host.Address, err)
 			continue
@@ -114,12 +127,12 @@ func AddEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts
 	return nil
 }
 
-func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory) error {
+func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte) error {
 	log.Infof(ctx, "[remove/%s] Removing member [etcd-%s] from etcd cluster", ETCDRole, etcdHost.HostnameOverride)
 	var mID string
 	removed := false
 	for _, host := range etcdHosts {
-		etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory)
+		etcdClient, err := getEtcdClient(ctx, host, localConnDialerFactory, cert, key)
 		if err != nil {
 			logrus.Debugf("Failed to create etcd client for host [%s]: %v", host.Address, err)
 			continue
@@ -150,7 +163,7 @@ func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*ho
 	return nil
 }
 
-func ReloadEtcdCluster(ctx context.Context, etcdHosts []*hosts.Host, etcdService v3.ETCDService, localConnDialerFactory hosts.DialerFactory) error {
+func ReloadEtcdCluster(ctx context.Context, etcdHosts []*hosts.Host, etcdService v3.ETCDService, localConnDialerFactory hosts.DialerFactory, cert, key []byte) error {
 	readyEtcdHosts := []*hosts.Host{}
 	for _, host := range etcdHosts {
 		if !host.ToAddEtcdMember {
@@ -160,16 +173,20 @@ func ReloadEtcdCluster(ctx context.Context, etcdHosts []*hosts.Host, etcdService
 	}
 	initCluster := getEtcdInitialCluster(readyEtcdHosts)
 	for _, host := range readyEtcdHosts {
-		imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster)
+		imageCfg, hostCfg := buildEtcdConfig(host, etcdService, initCluster, pki.GetEtcdCrtName(host.InternalAddress))
 		if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, EtcdContainerName, host.Address, ETCDRole); err != nil {
 			return err
 		}
 	}
 	time.Sleep(10 * time.Second)
+	var healthy bool
 	for _, host := range readyEtcdHosts {
-		if healthy := isEtcdHealthy(ctx, localConnDialerFactory, host); healthy {
+		if healthy = isEtcdHealthy(ctx, localConnDialerFactory, host, cert, key); healthy {
 			break
 		}
 	}
+	if !healthy {
+		return fmt.Errorf("[etcd] Etcd Cluster is not healthy")
+	}
 	return nil
 }

services/etcd_test.go

@@ -5,11 +5,12 @@ import (
 	"testing"
 
 	"github.com/rancher/rke/hosts"
+	"github.com/rancher/rke/pki"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
 )
 
 const (
-	TestInitEtcdClusterString = "etcd-etcd1=http://1.1.1.1:2380,etcd-etcd2=http://2.2.2.2:2380"
+	TestInitEtcdClusterString = "etcd-etcd1=https://1.1.1.1:2380,etcd-etcd2=https://2.2.2.2:2380"
 	TestEtcdImage             = "etcd/etcdImage:latest"
 	TestEtcdNamePrefix        = "--name=etcd-"
 	TestEtcdVolumeBind        = "/var/lib/etcd:/etcd-data"
@@ -46,7 +47,8 @@ func TestEtcdConfig(t *testing.T) {
 	assertEqual(t, initCluster, TestInitEtcdClusterString, "")
 
 	for _, host := range etcdHosts {
-		imageCfg, hostCfg := buildEtcdConfig(host, etcdService, TestInitEtcdClusterString)
+		nodeName := pki.GetEtcdCrtName(host.InternalAddress)
+		imageCfg, hostCfg := buildEtcdConfig(host, etcdService, TestInitEtcdClusterString, nodeName)
 		assertEqual(t, isStringInSlice(TestEtcdNamePrefix+host.HostnameOverride, imageCfg.Cmd), true,
 			fmt.Sprintf("Failed to find [%s] in Etcd command", TestEtcdNamePrefix+host.HostnameOverride))
 		assertEqual(t, TestEtcdImage, imageCfg.Image,

services/etcd_util.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -14,36 +15,48 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func getEtcdClient(ctx context.Context, etcdHost *hosts.Host, localConnDialerFactory hosts.DialerFactory) (etcdclient.Client, error) {
+func getEtcdClient(ctx context.Context, etcdHost *hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte) (etcdclient.Client, error) {
 	dialer, err := getEtcdDialer(localConnDialerFactory, etcdHost)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to create a dialer for host [%s]: %v", etcdHost.Address, err)
 	}
+	tlsConfig, err := getEtcdTLSConfig(cert, key)
+	if err != nil {
+		return nil, err
+	}
 
 	var DefaultEtcdTransport etcdclient.CancelableTransport = &http.Transport{
-		Dial: dialer,
+		Dial:                dialer,
+		TLSClientConfig:     tlsConfig,
+		TLSHandshakeTimeout: 10 * time.Second,
 	}
 
 	cfg := etcdclient.Config{
-		Endpoints: []string{"http://127.0.0.1:2379"},
+		Endpoints: []string{"https://127.0.0.1:2379"},
 		Transport: DefaultEtcdTransport,
 	}
 
 	return etcdclient.New(cfg)
 }
 
-func isEtcdHealthy(ctx context.Context, localConnDialerFactory hosts.DialerFactory, host *hosts.Host) bool {
+func isEtcdHealthy(ctx context.Context, localConnDialerFactory hosts.DialerFactory, host *hosts.Host, cert, key []byte) bool {
 	logrus.Debugf("[etcd] Check etcd cluster health")
 	for i := 0; i < 3; i++ {
 		dialer, err := getEtcdDialer(localConnDialerFactory, host)
 		if err != nil {
-			logrus.Debugf("Failed to create a dialer for host [%s]: %v", host.Address, err)
+			return false
-			time.Sleep(5 * time.Second)
-			continue
 		}
+		tlsConfig, err := getEtcdTLSConfig(cert, key)
+		if err != nil {
+			logrus.Debugf("[etcd] Failed to create etcd tls config for host [%s]: %v", host.Address, err)
+			return false
+		}
+
 		hc := http.Client{
 			Transport: &http.Transport{
-				Dial: dialer,
+				Dial:                dialer,
+				TLSClientConfig:     tlsConfig,
+				TLSHandshakeTimeout: 10 * time.Second,
 			},
 		}
 		healthy, err := getHealthEtcd(hc, host)
@@ -62,7 +75,7 @@ func isEtcdHealthy(ctx context.Context, localConnDialerFactory hosts.DialerFacto
 
 func getHealthEtcd(hc http.Client, host *hosts.Host) (string, error) {
 	healthy := struct{ Health string }{}
-	resp, err := hc.Get("http://127.0.0.1:2379/health")
+	resp, err := hc.Get("https://127.0.0.1:2379/health")
 	if err != nil {
 		return healthy.Health, fmt.Errorf("Failed to get /health for host [%s]: %v", host.Address, err)
 	}
@@ -80,7 +93,7 @@ func getHealthEtcd(hc http.Client, host *hosts.Host) (string, error) {
 func getEtcdInitialCluster(hosts []*hosts.Host) string {
 	initialCluster := ""
 	for i, host := range hosts {
-		initialCluster += fmt.Sprintf("etcd-%s=http://%s:2380", host.HostnameOverride, host.InternalAddress)
+		initialCluster += fmt.Sprintf("etcd-%s=https://%s:2380", host.HostnameOverride, host.InternalAddress)
 		if i < (len(hosts) - 1) {
 			initialCluster += ","
 		}
@@ -102,10 +115,27 @@ func getEtcdDialer(localConnDialerFactory hosts.DialerFactory, etcdHost *hosts.H
 func GetEtcdConnString(hosts []*hosts.Host) string {
 	connString := ""
 	for i, host := range hosts {
-		connString += "http://" + host.InternalAddress + ":2379"
+		connString += "https://" + host.InternalAddress + ":2379"
 		if i < (len(hosts) - 1) {
 			connString += ","
 		}
 	}
 	return connString
 }
+
+func getEtcdTLSConfig(certificate, key []byte) (*tls.Config, error) {
+	// get tls config
+	x509Pair, err := tls.X509KeyPair([]byte(certificate), []byte(key))
+	if err != nil {
+		return nil, err
+
+	}
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: true,
+		Certificates:       []tls.Certificate{x509Pair},
+	}
+	if err != nil {
+		return nil, err
+	}
+	return tlsConfig, nil
+}

services/kubeapi.go

@@ -41,10 +41,13 @@ func buildKubeAPIConfig(host *hosts.Host, kubeAPIService v3.KubeAPIService, etcd
 			"--runtime-config=batch/v2alpha1",
 			"--runtime-config=authentication.k8s.io/v1beta1=true",
 			"--storage-backend=etcd3",
-			"--client-ca-file=" + pki.CACertPath,
+			"--client-ca-file=" + pki.GetCertPath(pki.CACertName),
-			"--tls-cert-file=" + pki.KubeAPICertPath,
+			"--tls-cert-file=" + pki.GetCertPath(pki.KubeAPICertName),
-			"--tls-private-key-file=" + pki.KubeAPIKeyPath,
+			"--tls-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
-			"--service-account-key-file=" + pki.KubeAPIKeyPath},
+			"--service-account-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
+			"--etcd-cafile=" + pki.GetCertPath(pki.CACertName),
+			"--etcd-certfile=" + pki.GetCertPath(pki.KubeAPICertName),
+			"--etcd-keyfile=" + pki.GetKeyPath(pki.KubeAPICertName)},
 	}
 	imageCfg.Cmd = append(imageCfg.Cmd, "--etcd-servers="+etcdConnString)
 

services/kubeapi_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 const (
-	TestEtcdConnString      = "http://1.1.1.1:2379,http://2.2.2.2:2379"
+	TestEtcdConnString      = "https://1.1.1.1:2379,https://2.2.2.2:2379"
 	TestKubeAPIImage        = "rancher/k8s:latest"
 	TestInsecureBindAddress = "--insecure-bind-address=127.0.0.1"
 	TestKubeAPIVolumeBind   = "/etc/kubernetes:/etc/kubernetes"

services/kubecontroller.go

@@ -31,7 +31,7 @@ func buildKubeControllerConfig(kubeControllerService v3.KubeControllerService, a
 			"--address=0.0.0.0",
 			"--cloud-provider=",
 			"--leader-elect=true",
-			"--kubeconfig=" + pki.KubeControllerConfigPath,
+			"--kubeconfig=" + pki.GetConfigPath(pki.KubeControllerCertName),
 			"--enable-hostpath-provisioner=false",
 			"--node-monitor-grace-period=40s",
 			"--pod-eviction-timeout=5m0s",
@@ -39,8 +39,8 @@ func buildKubeControllerConfig(kubeControllerService v3.KubeControllerService, a
 			"--allocate-node-cidrs=true",
 			"--cluster-cidr=" + kubeControllerService.ClusterCIDR,
 			"--service-cluster-ip-range=" + kubeControllerService.ServiceClusterIPRange,
-			"--service-account-private-key-file=" + pki.KubeAPIKeyPath,
+			"--service-account-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
-			"--root-ca-file=" + pki.CACertPath,
+			"--root-ca-file=" + pki.GetCertPath(pki.CACertName),
 		},
 	}
 	if authorizationMode == RBACAuthorizationMode {

services/kubelet.go

@@ -42,7 +42,7 @@ func buildKubeletConfig(host *hosts.Host, kubeletService v3.KubeletService, unsc
 			"--resolv-conf=/etc/resolv.conf",
 			"--allow-privileged=true",
 			"--cloud-provider=",
-			"--kubeconfig=" + pki.KubeNodeConfigPath,
+			"--kubeconfig=" + pki.GetConfigPath(pki.KubeNodeCertName),
 			"--require-kubeconfig=True",
 		},
 	}

services/kubeproxy.go

@@ -30,7 +30,7 @@ func buildKubeproxyConfig(host *hosts.Host, kubeproxyService v3.KubeproxyService
 			"kube-proxy",
 			"--v=2",
 			"--healthz-bind-address=0.0.0.0",
-			"--kubeconfig=" + pki.KubeProxyConfigPath,
+			"--kubeconfig=" + pki.GetConfigPath(pki.KubeProxyCertName),
 		},
 	}
 	hostCfg := &container.HostConfig{

services/scheduler.go

@@ -31,7 +31,7 @@ func buildSchedulerConfig(host *hosts.Host, schedulerService v3.SchedulerService
 			"--leader-elect=true",
 			"--v=2",
 			"--address=0.0.0.0",
-			"--kubeconfig=" + pki.KubeSchedulerConfigPath,
+			"--kubeconfig=" + pki.GetConfigPath(pki.KubeSchedulerCertName),
 		},
 	}
 	hostCfg := &container.HostConfig{

templates/calico.go

@@ -104,9 +104,9 @@ data:
         {
             "type": "calico",
             "etcd_endpoints": "{{.EtcdEndpoints}}",
-            "etcd_key_file": "",
+            "etcd_key_file": "{{.ClientKeyPath}}",
-            "etcd_cert_file": "",
+            "etcd_cert_file": "{{.ClientCertPath}}",
-            "etcd_ca_cert_file": "",
+            "etcd_ca_cert_file": "{{.ClientCAPath}}",
             "log_level": "info",
             "mtu": 1500,
             "ipam": {
@@ -115,9 +115,9 @@ data:
             "policy": {
                 "type": "k8s",
                 "k8s_api_root": "{{.APIRoot}}",
-                "k8s_client_certificate": "{{.ClientCert}}",
+                "k8s_client_certificate": "{{.ClientCertPath}}",
-                "k8s_client_key": "{{.ClientKey}}",
+                "k8s_client_key": "{{.ClientKeyPath}}",
-                "k8s_certificate_authority": "{{.ClientCA}}"
+                "k8s_certificate_authority": "{{.ClientCAPath}}"
             },
             "kubernetes": {
                 "kubeconfig": "{{.KubeCfg}}"
@@ -131,16 +131,12 @@ data:
       ]
     }
 
-  # If you're using TLS enabled etcd uncomment the following.
+  etcd_ca: "/calico-secrets/etcd-ca"
-  # You must also populate the Secret below with these files.
+  etcd_cert: "/calico-secrets/etcd-cert"
-  etcd_ca: ""   # "/calico-secrets/etcd-ca"
+  etcd_key: "/calico-secrets/etcd-key"
-  etcd_cert: "" # "/calico-secrets/etcd-cert"
-  etcd_key: ""  # "/calico-secrets/etcd-key"
 
 ---
 
-# The following contains k8s Secrets for use with a TLS enabled etcd cluster.
-# For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/
 apiVersion: v1
 kind: Secret
 type: Opaque
@@ -148,13 +144,9 @@ metadata:
   name: calico-etcd-secrets
   namespace: kube-system
 data:
-  # Populate the following files with etcd TLS configuration if desired, but leave blank if
+  etcd-key: {{.ClientKey}}
-  # not using TLS for etcd.
+  etcd-cert: {{.ClientCert}}
-  # This self-hosted install expects three files with the following names.  The values
+  etcd-ca:  {{.ClientCA}}
-  # should be base64 encoded strings of the entire contents of each file.
-  # etcd-key: null
-  # etcd-cert: null
-  # etcd-ca: null
 
 ---
 

templates/canal.go

@@ -156,9 +156,9 @@ data:
                 "policy": {
                     "type": "k8s",
                     "k8s_api_root": "{{.APIRoot}}",
-                    "k8s_client_certificate": "{{.ClientCert}}",
+                    "k8s_client_certificate": "{{.ClientCertPath}}",
-                    "k8s_client_key": "{{.ClientKey}}",
+                    "k8s_client_key": "{{.ClientKeyPath}}",
-                    "k8s_certificate_authority": "{{.ClientCA}}"
+                    "k8s_certificate_authority": "{{.ClientCAPath}}"
                 },
                 "kubernetes": {
                     "kubeconfig": "{{.KubeCfg}}"