Adding csr generation and custom certs

cluster/certificates.go

@@ -33,7 +33,7 @@ func regenerateAPICertificate(c *Cluster, certificates map[string]pki.Certificat
 	if err != nil {
 		return nil, err
 	}
-	certificates[pki.KubeAPICertName] = pki.ToCertObject(pki.KubeAPICertName, "", "", kubeAPICert, kubeAPIKey)
+	certificates[pki.KubeAPICertName] = pki.ToCertObject(pki.KubeAPICertName, "", "", kubeAPICert, kubeAPIKey, nil)
 	return certificates, nil
 }
 
@@ -114,7 +114,7 @@ func GetClusterCertsFromKubernetes(ctx context.Context, kubeCluster *Cluster) (m
 	kubeAPICert := certMap[pki.KubeAPICertName]
 	if certMap[pki.ServiceAccountTokenKeyName].Key == nil {
 		log.Infof(ctx, "[certificates] Creating service account token key")
-		certMap[pki.ServiceAccountTokenKeyName] = pki.ToCertObject(pki.ServiceAccountTokenKeyName, pki.ServiceAccountTokenKeyName, "", kubeAPICert.Certificate, kubeAPICert.Key)
+		certMap[pki.ServiceAccountTokenKeyName] = pki.ToCertObject(pki.ServiceAccountTokenKeyName, pki.ServiceAccountTokenKeyName, "", kubeAPICert.Certificate, kubeAPICert.Key, nil)
 	}
 	log.Infof(ctx, "[certificates] Successfully fetched Cluster certificates from Kubernetes")
 	return certMap, nil
@@ -137,7 +137,7 @@ func regenerateAPIAggregationCerts(c *Cluster, certificates map[string]pki.Certi
 	if err != nil {
 		return nil, err
 	}
-	certificates[pki.RequestHeaderCACertName] = pki.ToCertObject(pki.RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey)
+	certificates[pki.RequestHeaderCACertName] = pki.ToCertObject(pki.RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey, nil)
 
 	//generate API server proxy client key and certs
 	logrus.Debugf("[certificates] Regenerating Kubernetes API server proxy client certificates")
@@ -145,7 +145,7 @@ func regenerateAPIAggregationCerts(c *Cluster, certificates map[string]pki.Certi
 	if err != nil {
 		return nil, err
 	}
-	certificates[pki.APIProxyClientCertName] = pki.ToCertObject(pki.APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey)
+	certificates[pki.APIProxyClientCertName] = pki.ToCertObject(pki.APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey, nil)
 	return certificates, nil
 }
 
@@ -196,7 +196,7 @@ func RotateRKECertificates(ctx context.Context, c *Cluster, flags ExternalFlags,
 				pki.ServiceAccountTokenKeyName,
 				"",
 				c.Certificates[pki.ServiceAccountTokenKeyName].Certificate,
-				privateKey.(*rsa.PrivateKey))
+				privateKey.(*rsa.PrivateKey), nil)
 		}
 	}
 	clusterState.DesiredState.CertificatesBundle = c.Certificates

cluster/cluster.go

@@ -34,6 +34,7 @@ type Cluster struct {
 	CloudConfigFile                  string
 	ControlPlaneHosts                []*hosts.Host
 	Certificates                     map[string]pki.CertificatePKI
+	CertificateDir                   string
 	ClusterDomain                    string
 	ClusterCIDR                      string
 	ClusterDNSServer                 string
@@ -49,8 +50,8 @@ type Cluster struct {
 	LocalConnDialerFactory           hosts.DialerFactory
 	PrivateRegistriesMap             map[string]v3.PrivateRegistry
 	StateFilePath                    string
-	UseKubectlDeploy                 bool
 	UpdateWorkersOnly                bool
+	UseKubectlDeploy                 bool
 	v3.RancherKubernetesEngineConfig `yaml:",inline"`
 	WorkerHosts                      []*hosts.Host
 }
@@ -152,16 +153,19 @@ func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngin
 		ConfigPath:                    flags.ClusterFilePath,
 		ConfigDir:                     flags.ConfigDir,
 		DinD:                          flags.DinD,
+		CertificateDir:                flags.CertificateDir,
 		StateFilePath:                 GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir),
 		PrivateRegistriesMap:          make(map[string]v3.PrivateRegistry),
 	}
 	if len(c.ConfigPath) == 0 {
 		c.ConfigPath = pki.ClusterConfig
 	}
-
+	// set kube_config, state file, and certificate dir
-	// set kube_config and state file
 	c.LocalKubeConfigPath = pki.GetLocalKubeConfig(c.ConfigPath, c.ConfigDir)
 	c.StateFilePath = GetStateFilePath(c.ConfigPath, c.ConfigDir)
+	if len(c.CertificateDir) == 0 {
+		c.CertificateDir = GetCertificateDirPath(c.ConfigPath, c.ConfigDir)
+	}
 
 	// Setting cluster Defaults
 	c.setClusterDefaults(ctx)

cluster/defaults.go

@@ -52,10 +52,13 @@ const (
 )
 
 type ExternalFlags struct {
-	ConfigDir        string
+	CertificateDir   string
 	ClusterFilePath  string
 	DinD             bool
+	ConfigDir        string
+	CustomCerts      bool
 	DisablePortCheck bool
+	GenerateCSR      bool
 	Local            bool
 	UpdateOnly       bool
 }

cluster/state.go

@@ -22,6 +22,7 @@ import (
 
 const (
 	stateFileExt = ".rkestate"
+	certDirExt   = "_certs"
 )
 
 type FullState struct {
@@ -150,6 +151,20 @@ func RebuildState(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConf
 		},
 	}
 
+	if flags.CustomCerts {
+		certBundle, err := pki.ReadCertsAndKeysFromDir(flags.CertificateDir)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to read certificates from dir [%s]: %v", flags.CertificateDir, err)
+		}
+		// make sure all custom certs are included
+		if err := pki.ValidateBundleContent(rkeConfig, certBundle, flags.ClusterFilePath, flags.ConfigDir); err != nil {
+			return nil, fmt.Errorf("Failed to validates certificates from dir [%s]: %v", flags.CertificateDir, err)
+		}
+		newState.DesiredState.CertificatesBundle = certBundle
+		newState.CurrentState = oldState.CurrentState
+		return newState, nil
+	}
+
 	// Rebuilding the certificates of the desired state
 	if oldState.DesiredState.CertificatesBundle == nil {
 		// Get the certificate Bundle
@@ -161,15 +176,7 @@ func RebuildState(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConf
 	} else {
 		// Regenerating etcd certificates for any new etcd nodes
 		pkiCertBundle := oldState.DesiredState.CertificatesBundle
-		if err := pki.GenerateEtcdCertificates(ctx, pkiCertBundle, *rkeConfig, "", "", false); err != nil {
+		if err := pki.GenerateRKEServicesCerts(ctx, pkiCertBundle, *rkeConfig, flags.ClusterFilePath, flags.ConfigDir, false); err != nil {
-			return nil, err
-		}
-		// Regenerating kubeapi certificates for any new kubeapi nodes
-		if err := pki.GenerateKubeAPICertificate(ctx, pkiCertBundle, *rkeConfig, "", "", false); err != nil {
-			return nil, err
-		}
-		// Regenerating kubeadmin certificates/config
-		if err := pki.GenerateKubeAdminCertificate(ctx, pkiCertBundle, *rkeConfig, flags.ClusterFilePath, flags.ConfigDir, false); err != nil {
 			return nil, err
 		}
 		newState.DesiredState.CertificatesBundle = pkiCertBundle
@@ -206,6 +213,21 @@ func GetStateFilePath(configPath, configDir string) string {
 	return trimmedName + stateFileExt
 }
 
+func GetCertificateDirPath(configPath, configDir string) string {
+	if configPath == "" {
+		configPath = pki.ClusterConfig
+	}
+	baseDir := filepath.Dir(configPath)
+	if len(configDir) > 0 {
+		baseDir = filepath.Dir(configDir)
+	}
+	fileName := filepath.Base(configPath)
+	baseDir += "/"
+	fullPath := fmt.Sprintf("%s%s", baseDir, fileName)
+	trimmedName := strings.TrimSuffix(fullPath, filepath.Ext(fullPath))
+	return trimmedName + certDirExt
+}
+
 func ReadStateFile(ctx context.Context, statePath string) (*FullState, error) {
 	rkeFullState := &FullState{}
 	fp, err := filepath.Abs(statePath)

cmd/cert.go

@@ -47,6 +47,23 @@ func CertificateCommand() cli.Command {
 					},
 				},
 			},
+			cli.Command{
+				Name:   "generate-csr",
+				Usage:  "Generate certificate sign requests for k8s components",
+				Action: generateCSRFromCli,
+				Flags: []cli.Flag{
+					cli.StringFlag{
+						Name:   "config",
+						Usage:  "Specify an alternate cluster YAML file",
+						Value:  pki.ClusterConfig,
+						EnvVar: "RKE_CONFIG",
+					},
+					cli.StringFlag{
+						Name:  "cert-dir",
+						Usage: "Specify a certificate dir path",
+					},
+				},
+			},
 		},
 	}
 }
@@ -81,6 +98,28 @@ func rotateRKECertificatesFromCli(ctx *cli.Context) error {
 	return err
 }
 
+func generateCSRFromCli(ctx *cli.Context) error {
+	clusterFile, filePath, err := resolveClusterFile(ctx)
+	if err != nil {
+		return fmt.Errorf("Failed to resolve cluster file: %v", err)
+	}
+
+	rkeConfig, err := cluster.ParseConfig(clusterFile)
+	if err != nil {
+		return fmt.Errorf("Failed to parse cluster file: %v", err)
+	}
+	rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig)
+	if err != nil {
+		return err
+	}
+	// setting up the flags
+	externalFlags := cluster.GetExternalFlags(false, false, false, "", filePath)
+	externalFlags.CertificateDir = ctx.String("cert-dir")
+	externalFlags.CustomCerts = ctx.Bool("custom-certs")
+
+	return GenerateRKECSRs(context.Background(), rkeConfig, externalFlags)
+}
+
 func showRKECertificatesFromCli(ctx *cli.Context) error {
 	return nil
 }
@@ -167,3 +206,27 @@ func rotateRKECertificates(ctx context.Context, kubeCluster *cluster.Cluster, fl
 	}
 	return rkeFullState, nil
 }
+
+func GenerateRKECSRs(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, flags cluster.ExternalFlags) error {
+	log.Infof(ctx, "Generating Kubernetes cluster CSR certificates")
+	if len(flags.CertificateDir) == 0 {
+		flags.CertificateDir = cluster.GetCertificateDirPath(flags.ClusterFilePath, flags.ConfigDir)
+	}
+
+	certBundle, err := pki.ReadCSRsAndKeysFromDir(flags.CertificateDir)
+	if err != nil {
+		return err
+	}
+
+	// initialze the cluster object from the config file
+	kubeCluster, err := cluster.InitClusterObject(ctx, rkeConfig, flags)
+	if err != nil {
+		return err
+	}
+
+	// Generating csrs for kubernetes components
+	if err := pki.GenerateRKEServicesCSRs(ctx, certBundle, kubeCluster.RancherKubernetesEngineConfig); err != nil {
+		return err
+	}
+	return pki.WriteCertificates(kubeCluster.CertificateDir, certBundle)
+}

cmd/common.go

@@ -67,6 +67,9 @@ func ClusterInit(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfi
 	log.Infof(ctx, "Initiating Kubernetes cluster")
 	var fullState *cluster.FullState
 	stateFilePath := cluster.GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir)
+	if len(flags.CertificateDir) == 0 {
+		flags.CertificateDir = cluster.GetCertificateDirPath(flags.ClusterFilePath, flags.ConfigDir)
+	}
 	rkeFullState, _ := cluster.ReadStateFile(ctx, stateFilePath)
 
 	kubeCluster, err := cluster.InitClusterObject(ctx, rkeConfig, flags)

cmd/up.go

@@ -52,6 +52,14 @@ func UpCommand() cli.Command {
 			Name:  "init",
 			Usage: "Initiate RKE cluster",
 		},
+		cli.StringFlag{
+			Name:  "cert-dir",
+			Usage: "Specify a certificate dir path",
+		},
+		cli.BoolFlag{
+			Name:  "custom-certs",
+			Usage: "Use custom certificates from a cert dir",
+		},
 	}
 
 	upFlags = append(upFlags, commonFlags...)
@@ -257,7 +265,9 @@ func clusterUpFromCli(ctx *cli.Context) error {
 	disablePortCheck := ctx.Bool("disable-port-check")
 	// setting up the flags
 	flags := cluster.GetExternalFlags(false, updateOnly, disablePortCheck, "", filePath)
-
+	// Custom certificates and certificate dir flags
+	flags.CertificateDir = ctx.String("cert-dir")
+	flags.CustomCerts = ctx.Bool("custom-certs")
 	if ctx.Bool("init") {
 		return ClusterInit(context.Background(), rkeConfig, hosts.DialersOptions{}, flags)
 	}

pki/pki.go

@@ -20,8 +20,10 @@ import (
 type CertificatePKI struct {
 	Certificate    *x509.Certificate        `json:"-"`
 	Key            *rsa.PrivateKey          `json:"-"`
+	CSR            *x509.CertificateRequest `json:"-"`
 	CertificatePEM string                   `json:"certificatePEM"`
 	KeyPEM         string                   `json:"keyPEM"`
+	CSRPEM         string                   `json:"-"`
 	Config         string                   `json:"config"`
 	Name           string                   `json:"name"`
 	CommonName     string                   `json:"commonName"`
@@ -35,6 +37,7 @@ type CertificatePKI struct {
 }
 
 type GenFunc func(context.Context, map[string]CertificatePKI, v3.RancherKubernetesEngineConfig, string, string, bool) error
+type CSRFunc func(context.Context, map[string]CertificatePKI, v3.RancherKubernetesEngineConfig) error
 
 const (
 	etcdRole            = "etcd"
@@ -110,7 +113,7 @@ func RegenerateEtcdCertificate(
 		return nil, err
 	}
 	etcdName := GetEtcdCrtName(etcdHost.InternalAddress)
-	crtMap[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
+	crtMap[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey, nil)
 	log.Infof(ctx, "[certificates] Successfully generated new etcd-%s certificate and key", etcdHost.InternalAddress)
 	return crtMap, nil
 }

pki/services.go

@@ -34,65 +34,166 @@ func GenerateKubeAPICertificate(ctx context.Context, certs map[string]Certificat
 	if err != nil {
 		return err
 	}
-	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey)
+	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICrt, kubeAPIKey, nil)
 	// handle service account tokens in old clusters
 	apiCert := certs[KubeAPICertName]
 	if certs[ServiceAccountTokenKeyName].Key == nil {
 		log.Infof(ctx, "[certificates] Generating Service account token key")
-		certs[ServiceAccountTokenKeyName] = ToCertObject(ServiceAccountTokenKeyName, ServiceAccountTokenKeyName, "", apiCert.Certificate, apiCert.Key)
+		certs[ServiceAccountTokenKeyName] = ToCertObject(ServiceAccountTokenKeyName, ServiceAccountTokenKeyName, "", apiCert.Certificate, apiCert.Key, nil)
 	}
 	return nil
 }
 
+func GenerateKubeAPICSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error {
+	// generate API csr and key
+	kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange)
+	if err != nil {
+		return fmt.Errorf("Failed to get Kubernetes Service IP: %v", err)
+	}
+	clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain
+	cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole)
+	kubeAPIAltNames := GetAltNames(cpHosts, clusterDomain, kubernetesServiceIP, rkeConfig.Authentication.SANs)
+	kubeAPICert := certs[KubeAPICertName].Certificate
+	oldKubeAPICSR := certs[KubeAPICertName].CSR
+	if oldKubeAPICSR != nil &&
+		reflect.DeepEqual(kubeAPIAltNames.DNSNames, oldKubeAPICSR.DNSNames) &&
+		deepEqualIPsAltNames(kubeAPIAltNames.IPs, oldKubeAPICSR.IPAddresses) {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Kubernetes API server csr")
+	kubeAPICSR, kubeAPIKey, err := GenerateCertSigningRequestAndKey(true, KubeAPICertName, kubeAPIAltNames, certs[KubeAPICertName].Key, nil)
+	if err != nil {
+		return err
+	}
+	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", kubeAPICert, kubeAPIKey, kubeAPICSR)
+	return nil
+}
+
 func GenerateKubeControllerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error {
 	// generate Kube controller-manager certificate and key
-	log.Infof(ctx, "[certificates] Generating Kube Controller certificates")
 	caCrt := certs[CACertName].Certificate
 	caKey := certs[CACertName].Key
-	kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, nil, nil)
+	if certs[KubeControllerCertName].Certificate != nil {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Kube Controller certificates")
+	kubeControllerCrt, kubeControllerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeControllerCertName), nil, certs[KubeControllerCertName].Key, nil)
 	if err != nil {
 		return err
 	}
-	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey)
+	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey, nil)
+	return nil
+}
+
+func GenerateKubeControllerCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error {
+	// generate Kube controller-manager csr and key
+	kubeControllerCrt := certs[KubeControllerCertName].Certificate
+	kubeControllerCSRPEM := certs[KubeControllerCertName].CSRPEM
+	if kubeControllerCSRPEM != "" {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Kube Controller csr")
+	kubeControllerCSR, kubeControllerKey, err := GenerateCertSigningRequestAndKey(false, getDefaultCN(KubeControllerCertName), nil, certs[KubeControllerCertName].Key, nil)
+	if err != nil {
+		return err
+	}
+	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", kubeControllerCrt, kubeControllerKey, kubeControllerCSR)
 	return nil
 }
 
 func GenerateKubeSchedulerCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error {
 	// generate Kube scheduler certificate and key
-	log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates")
 	caCrt := certs[CACertName].Certificate
 	caKey := certs[CACertName].Key
-	kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, nil, nil)
+	if certs[KubeSchedulerCertName].Certificate != nil {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Kube Scheduler certificates")
+	kubeSchedulerCrt, kubeSchedulerKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeSchedulerCertName), nil, certs[KubeSchedulerCertName].Key, nil)
 	if err != nil {
 		return err
 	}
-	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey)
+	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey, nil)
+	return nil
+}
+
+func GenerateKubeSchedulerCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error {
+	// generate Kube scheduler csr and key
+	kubeSchedulerCrt := certs[KubeSchedulerCertName].Certificate
+	kubeSchedulerCSRPEM := certs[KubeSchedulerCertName].CSRPEM
+	if kubeSchedulerCSRPEM != "" {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Kube Scheduler csr")
+	kubeSchedulerCSR, kubeSchedulerKey, err := GenerateCertSigningRequestAndKey(false, getDefaultCN(KubeSchedulerCertName), nil, certs[KubeSchedulerCertName].Key, nil)
+	if err != nil {
+		return err
+	}
+	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", kubeSchedulerCrt, kubeSchedulerKey, kubeSchedulerCSR)
 	return nil
 }
 
 func GenerateKubeProxyCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error {
 	// generate Kube Proxy certificate and key
-	log.Infof(ctx, "[certificates] Generating Kube Proxy certificates")
 	caCrt := certs[CACertName].Certificate
 	caKey := certs[CACertName].Key
-	kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, nil, nil)
+	if certs[KubeProxyCertName].Certificate != nil {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Kube Proxy certificates")
+	kubeProxyCrt, kubeProxyKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, getDefaultCN(KubeProxyCertName), nil, certs[KubeProxyCertName].Key, nil)
 	if err != nil {
 		return err
 	}
-	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey)
+	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey, nil)
+	return nil
+}
+
+func GenerateKubeProxyCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error {
+	// generate Kube Proxy csr and key
+	kubeProxyCrt := certs[KubeProxyCertName].Certificate
+	kubeProxyCSRPEM := certs[KubeProxyCertName].CSRPEM
+	if kubeProxyCSRPEM != "" {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Kube Proxy csr")
+	kubeProxyCSR, kubeProxyKey, err := GenerateCertSigningRequestAndKey(false, getDefaultCN(KubeProxyCertName), nil, certs[KubeProxyCertName].Key, nil)
+	if err != nil {
+		return err
+	}
+	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey, kubeProxyCSR)
 	return nil
 }
 
 func GenerateKubeNodeCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error {
 	// generate kubelet certificate
-	log.Infof(ctx, "[certificates] Generating Node certificate")
 	caCrt := certs[CACertName].Certificate
 	caKey := certs[CACertName].Key
-	nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, nil, []string{KubeNodeOrganizationName})
+	if certs[KubeNodeCertName].Certificate != nil {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Node certificate")
+	nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, certs[KubeNodeCertName].Key, []string{KubeNodeOrganizationName})
 	if err != nil {
 		return err
 	}
-	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey)
+	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey, nil)
+	return nil
+}
+
+func GenerateKubeNodeCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error {
+	// generate kubelet csr and key
+	nodeCrt := certs[KubeNodeCertName].Certificate
+	nodeCSRPEM := certs[KubeNodeCertName].CSRPEM
+	if nodeCSRPEM != "" {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Node csr and key")
+	nodeCSR, nodeKey, err := GenerateCertSigningRequestAndKey(false, KubeNodeCommonName, nil, certs[KubeNodeCertName].Key, []string{KubeNodeOrganizationName})
+	if err != nil {
+		return err
+	}
+	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, nodeCrt, nodeKey, nodeCSR)
 	return nil
 }
 
@@ -106,11 +207,11 @@ func GenerateKubeAdminCertificate(ctx context.Context, certs map[string]Certific
 		configPath = ClusterConfig
 	}
 	localKubeConfigPath := GetLocalKubeConfig(configPath, configDir)
-	kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, nil, []string{KubeAdminOrganizationName})
+	kubeAdminCrt, kubeAdminKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeAdminCertName, nil, certs[KubeAdminCertName].Key, []string{KubeAdminOrganizationName})
 	if err != nil {
 		return err
 	}
-	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey)
+	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey, nil)
 	if len(cpHosts) > 0 {
 		kubeAdminConfig := GetKubeConfigX509WithData(
 			"https://"+cpHosts[0].Address+":6443",
@@ -128,16 +229,52 @@ func GenerateKubeAdminCertificate(ctx context.Context, certs map[string]Certific
 	return nil
 }
 
-func GenerateAPIProxyClientCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error {
+func GenerateKubeAdminCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error {
-	//generate API server proxy client key and certs
+	// generate Admin certificate and key
-	log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client certificates")
+	kubeAdminCrt := certs[KubeAdminCertName].Certificate
-	caCrt := certs[RequestHeaderCACertName].Certificate
+	kubeAdminCSRPEM := certs[KubeAdminCertName].CSRPEM
-	caKey := certs[RequestHeaderCACertName].Key
+	if kubeAdminCSRPEM != "" {
-	apiserverProxyClientCrt, apiserverProxyClientKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, APIProxyClientCertName, nil, nil, nil)
+		return nil
+	}
+	kubeAdminCSR, kubeAdminKey, err := GenerateCertSigningRequestAndKey(false, KubeAdminCertName, nil, certs[KubeAdminCertName].Key, []string{KubeAdminOrganizationName})
 	if err != nil {
 		return err
 	}
-	certs[APIProxyClientCertName] = ToCertObject(APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey)
+	log.Infof(ctx, "[certificates] Generating admin csr and kubeconfig")
+	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey, kubeAdminCSR)
+	certs[KubeAdminCertName] = kubeAdminCertObj
+	return nil
+}
+
+func GenerateAPIProxyClientCertificate(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error {
+	//generate API server proxy client key and certs
+	caCrt := certs[RequestHeaderCACertName].Certificate
+	caKey := certs[RequestHeaderCACertName].Key
+	if certs[APIProxyClientCertName].Certificate != nil {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client certificates")
+	apiserverProxyClientCrt, apiserverProxyClientKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, APIProxyClientCertName, nil, certs[APIProxyClientCertName].Key, nil)
+	if err != nil {
+		return err
+	}
+	certs[APIProxyClientCertName] = ToCertObject(APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey, nil)
+	return nil
+}
+
+func GenerateAPIProxyClientCSR(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error {
+	//generate API server proxy client key and certs
+	apiserverProxyClientCrt := certs[APIProxyClientCertName].Certificate
+	apiserverProxyClientCSRPEM := certs[APIProxyClientCertName].CSRPEM
+	if apiserverProxyClientCSRPEM != "" {
+		return nil
+	}
+	log.Infof(ctx, "[certificates] Generating Kubernetes API server proxy client csr")
+	apiserverProxyClientCSR, apiserverProxyClientKey, err := GenerateCertSigningRequestAndKey(true, APIProxyClientCertName, nil, certs[APIProxyClientCertName].Key, nil)
+	if err != nil {
+		return err
+	}
+	certs[APIProxyClientCertName] = ToCertObject(APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey, apiserverProxyClientCSR)
 	return nil
 }
 
@@ -150,13 +287,13 @@ func GenerateExternalEtcdCertificates(ctx context.Context, certs map[string]Cert
 	if err != nil {
 		return err
 	}
-	certs[EtcdClientCertName] = ToCertObject(EtcdClientCertName, "", "", clientCert[0], clientKey.(*rsa.PrivateKey))
+	certs[EtcdClientCertName] = ToCertObject(EtcdClientCertName, "", "", clientCert[0], clientKey.(*rsa.PrivateKey), nil)
 
 	caCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.CACert))
 	if err != nil {
 		return err
 	}
-	certs[EtcdClientCACertName] = ToCertObject(EtcdClientCACertName, "", "", caCert[0], nil)
+	certs[EtcdClientCACertName] = ToCertObject(EtcdClientCACertName, "", "", caCert[0], nil, nil)
 	return nil
 }
 
@@ -176,20 +313,48 @@ func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificateP
 			continue
 		}
 		log.Infof(ctx, "[certificates] Generating etcd-%s certificate and key", host.InternalAddress)
-		etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, nil, nil)
+		etcdCrt, etcdKey, err := GenerateSignedCertAndKey(caCrt, caKey, true, EtcdCertName, etcdAltNames, certs[etcdName].Key, nil)
 		if err != nil {
 			return err
 		}
-		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
+		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey, nil)
+	}
+	return nil
+}
+
+func GenerateEtcdCSRs(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error {
+	kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange)
+	if err != nil {
+		return fmt.Errorf("Failed to get Kubernetes Service IP: %v", err)
+	}
+	clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain
+	etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole)
+	etcdAltNames := GetAltNames(etcdHosts, clusterDomain, kubernetesServiceIP, []string{})
+	for _, host := range etcdHosts {
+		etcdName := GetEtcdCrtName(host.InternalAddress)
+		etcdCrt := certs[etcdName].Certificate
+		etcdCSRPEM := certs[etcdName].CSRPEM
+		if etcdCSRPEM != "" {
+			return nil
+		}
+		log.Infof(ctx, "[certificates] Generating etcd-%s csr and key", host.InternalAddress)
+		etcdCSR, etcdKey, err := GenerateCertSigningRequestAndKey(true, EtcdCertName, etcdAltNames, certs[etcdName].Key, nil)
+		if err != nil {
+			return err
+		}
+		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey, etcdCSR)
 	}
 	return nil
 }
 
 func GenerateServiceTokenKey(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string, rotate bool) error {
 	// generate service account token key
-	var privateAPIKey *rsa.PrivateKey
+	privateAPIKey := certs[ServiceAccountTokenKeyName].Key
 	caCrt := certs[CACertName].Certificate
 	caKey := certs[CACertName].Key
+	if certs[ServiceAccountTokenKeyName].Certificate != nil {
+		return nil
+	}
 	// handle rotation on old clusters
 	if certs[ServiceAccountTokenKeyName].Key == nil {
 		privateAPIKey = certs[KubeAPICertName].Key
@@ -198,7 +363,7 @@ func GenerateServiceTokenKey(ctx context.Context, certs map[string]CertificatePK
 	if err != nil {
 		return fmt.Errorf("Failed to generate private key for service account token: %v", err)
 	}
-	certs[ServiceAccountTokenKeyName] = ToCertObject(ServiceAccountTokenKeyName, ServiceAccountTokenKeyName, "", tokenCrt, tokenKey)
+	certs[ServiceAccountTokenKeyName] = ToCertObject(ServiceAccountTokenKeyName, ServiceAccountTokenKeyName, "", tokenCrt, tokenKey, nil)
 	return nil
 }
 
@@ -209,7 +374,7 @@ func GenerateRKECACerts(ctx context.Context, certs map[string]CertificatePKI, co
 	if err != nil {
 		return err
 	}
-	certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey)
+	certs[CACertName] = ToCertObject(CACertName, "", "", caCrt, caKey, nil)
 
 	// generate request header client CA certificate and key
 	log.Infof(ctx, "[certificates] Generating Kubernetes API server aggregation layer requestheader client CA certificates")
@@ -217,7 +382,7 @@ func GenerateRKECACerts(ctx context.Context, certs map[string]CertificatePKI, co
 	if err != nil {
 		return err
 	}
-	certs[RequestHeaderCACertName] = ToCertObject(RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey)
+	certs[RequestHeaderCACertName] = ToCertObject(RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey, nil)
 	return nil
 }
 
@@ -243,3 +408,22 @@ func GenerateRKEServicesCerts(ctx context.Context, certs map[string]CertificateP
 	}
 	return nil
 }
+
+func GenerateRKEServicesCSRs(ctx context.Context, certs map[string]CertificatePKI, rkeConfig v3.RancherKubernetesEngineConfig) error {
+	RKECerts := []CSRFunc{
+		GenerateKubeAPICSR,
+		GenerateKubeControllerCSR,
+		GenerateKubeSchedulerCSR,
+		GenerateKubeProxyCSR,
+		GenerateKubeNodeCSR,
+		GenerateKubeAdminCSR,
+		GenerateAPIProxyClientCSR,
+		GenerateEtcdCSRs,
+	}
+	for _, csr := range RKECerts {
+		if err := csr(ctx, certs, rkeConfig); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pki/util.go

@@ -5,11 +5,14 @@ import (
 	"crypto/rsa"
 	"crypto/x509"
 	"crypto/x509/pkix"
+	"encoding/pem"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"math"
 	"math/big"
 	"net"
+	"os"
 	"path"
 	"path/filepath"
 	"reflect"
@@ -18,6 +21,7 @@ import (
 
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
+	"github.com/sirupsen/logrus"
 	"k8s.io/client-go/util/cert"
 )
 
@@ -59,6 +63,43 @@ func GenerateSignedCertAndKey(
 	return clientCert, rootKey, nil
 }
 
+func GenerateCertSigningRequestAndKey(
+	serverCrt bool,
+	commonName string,
+	altNames *cert.AltNames,
+	reusedKey *rsa.PrivateKey,
+	orgs []string) ([]byte, *rsa.PrivateKey, error) {
+	// Generate a generic signed certificate
+	var rootKey *rsa.PrivateKey
+	var err error
+	rootKey = reusedKey
+	if reusedKey == nil {
+		rootKey, err = cert.NewPrivateKey()
+		if err != nil {
+			return nil, nil, fmt.Errorf("Failed to generate private key for %s certificate: %v", commonName, err)
+		}
+	}
+	usages := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
+	if serverCrt {
+		usages = append(usages, x509.ExtKeyUsageServerAuth)
+	}
+	if altNames == nil {
+		altNames = &cert.AltNames{}
+	}
+	caConfig := cert.Config{
+		CommonName:   commonName,
+		Organization: orgs,
+		Usages:       usages,
+		AltNames:     *altNames,
+	}
+	clientCSR, err := newCertSigningRequest(caConfig, rootKey)
+
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to generate %s certificate: %v", commonName, err)
+	}
+	return clientCSR, rootKey, nil
+}
+
 func GenerateCACertAndKey(commonName string, privateKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
 	var err error
 	rootKey := privateKey
@@ -197,8 +238,10 @@ func GetConfigTempPath(name string) string {
 	return fmt.Sprintf("%skubecfg-%s.yaml", TempCertPath, name)
 }
 
-func ToCertObject(componentName, commonName, ouName string, certificate *x509.Certificate, key *rsa.PrivateKey) CertificatePKI {
+func ToCertObject(componentName, commonName, ouName string, certificate *x509.Certificate, key *rsa.PrivateKey, csrASN1 []byte) CertificatePKI {
-	var config, configPath, configEnvName string
+	var config, configPath, configEnvName, certificatePEM, keyPEM string
+	var csr *x509.CertificateRequest
+	var csrPEM []byte
 	if len(commonName) == 0 {
 		commonName = getDefaultCN(componentName)
 	}
@@ -208,8 +251,18 @@ func ToCertObject(componentName, commonName, ouName string, certificate *x509.Ce
 	caCertPath := GetCertPath(CACertName)
 	path := GetCertPath(componentName)
 	keyPath := GetKeyPath(componentName)
-	certificatePEM := string(cert.EncodeCertPEM(certificate))
+	if certificate != nil {
-	keyPEM := string(cert.EncodePrivateKeyPEM(key))
+		certificatePEM = string(cert.EncodeCertPEM(certificate))
+	}
+	if key != nil {
+		keyPEM = string(cert.EncodePrivateKeyPEM(key))
+	}
+	if csrASN1 != nil {
+		csr, _ = x509.ParseCertificateRequest(csrASN1)
+		csrPEM = pem.EncodeToMemory(&pem.Block{
+			Type: "CERTIFICATE REQUEST", Bytes: csrASN1,
+		})
+	}
 
 	if componentName != CACertName && componentName != KubeAPICertName && !strings.Contains(componentName, EtcdCertName) && componentName != ServiceAccountTokenKeyName {
 		config = getKubeConfigX509("https://127.0.0.1:6443", "local", componentName, caCertPath, path, keyPath)
@@ -220,8 +273,10 @@ func ToCertObject(componentName, commonName, ouName string, certificate *x509.Ce
 	return CertificatePKI{
 		Certificate:    certificate,
 		Key:            key,
+		CSR:            csr,
 		CertificatePEM: certificatePEM,
 		KeyPEM:         keyPEM,
+		CSRPEM:         string(csrPEM),
 		Config:         config,
 		Name:           componentName,
 		CommonName:     commonName,
@@ -318,19 +373,19 @@ func getTempPath(s string) string {
 func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string, extraHosts []*hosts.Host) map[string]CertificatePKI {
 	certs := make(map[string]CertificatePKI)
 	// CACert
-	certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key)
+	certs[CACertName] = ToCertObject(CACertName, "", "", tmpCerts[CACertName].Certificate, tmpCerts[CACertName].Key, nil)
 	// KubeAPI
-	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key)
+	certs[KubeAPICertName] = ToCertObject(KubeAPICertName, "", "", tmpCerts[KubeAPICertName].Certificate, tmpCerts[KubeAPICertName].Key, nil)
 	// kubeController
-	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key)
+	certs[KubeControllerCertName] = ToCertObject(KubeControllerCertName, "", "", tmpCerts[KubeControllerCertName].Certificate, tmpCerts[KubeControllerCertName].Key, nil)
 	// KubeScheduler
-	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key)
+	certs[KubeSchedulerCertName] = ToCertObject(KubeSchedulerCertName, "", "", tmpCerts[KubeSchedulerCertName].Certificate, tmpCerts[KubeSchedulerCertName].Key, nil)
 	// KubeProxy
-	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key)
+	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", tmpCerts[KubeProxyCertName].Certificate, tmpCerts[KubeProxyCertName].Key, nil)
 	// KubeNode
-	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key)
+	certs[KubeNodeCertName] = ToCertObject(KubeNodeCertName, KubeNodeCommonName, KubeNodeOrganizationName, tmpCerts[KubeNodeCertName].Certificate, tmpCerts[KubeNodeCertName].Key, nil)
 	// KubeAdmin
-	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key)
+	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, tmpCerts[KubeAdminCertName].Certificate, tmpCerts[KubeAdminCertName].Key, nil)
 	kubeAdminCertObj.Config = tmpCerts[KubeAdminCertName].Config
 	kubeAdminCertObj.ConfigPath = localConfigPath
 	certs[KubeAdminCertName] = kubeAdminCertObj
@@ -338,7 +393,7 @@ func populateCertMap(tmpCerts map[string]CertificatePKI, localConfigPath string,
 	for _, host := range extraHosts {
 		etcdName := GetEtcdCrtName(host.InternalAddress)
 		etcdCrt, etcdKey := tmpCerts[etcdName].Certificate, tmpCerts[etcdName].Key
-		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey)
+		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey, nil)
 	}
 
 	return certs
@@ -377,6 +432,25 @@ func newSignedCert(cfg cert.Config, key *rsa.PrivateKey, caCert *x509.Certificat
 	return x509.ParseCertificate(certDERBytes)
 }
 
+func newCertSigningRequest(cfg cert.Config, key *rsa.PrivateKey) ([]byte, error) {
+	if len(cfg.CommonName) == 0 {
+		return nil, errors.New("must specify a CommonName")
+	}
+	if len(cfg.Usages) == 0 {
+		return nil, errors.New("must specify at least one ExtKeyUsage")
+	}
+
+	certTmpl := x509.CertificateRequest{
+		Subject: pkix.Name{
+			CommonName:   cfg.CommonName,
+			Organization: cfg.Organization,
+		},
+		DNSNames:    cfg.AltNames.DNSNames,
+		IPAddresses: cfg.AltNames.IPs,
+	}
+	return x509.CreateCertificateRequest(cryptorand.Reader, &certTmpl, key)
+}
+
 func isFileNotFoundErr(e error) bool {
 	if strings.Contains(e.Error(), "no such file or directory") ||
 		strings.Contains(e.Error(), "Could not find the file") ||
@@ -400,10 +474,17 @@ func deepEqualIPsAltNames(oldIPs, newIPs []net.IP) bool {
 }
 
 func TransformPEMToObject(in map[string]CertificatePKI) map[string]CertificatePKI {
+	var certificate *x509.Certificate
 	out := map[string]CertificatePKI{}
 	for k, v := range in {
 		certs, _ := cert.ParseCertsPEM([]byte(v.CertificatePEM))
 		key, _ := cert.ParsePrivateKeyPEM([]byte(v.KeyPEM))
+		if len(certs) > 0 {
+			certificate = certs[0]
+		}
+		if key != nil {
+			key = key.(*rsa.PrivateKey)
+		}
 		o := CertificatePKI{
 			ConfigEnvName:  v.ConfigEnvName,
 			Name:           v.Name,
@@ -415,12 +496,248 @@ func TransformPEMToObject(in map[string]CertificatePKI) map[string]CertificatePK
 			KeyEnvName:     v.KeyEnvName,
 			KeyPath:        v.KeyPath,
 			ConfigPath:     v.ConfigPath,
-			Certificate:    certs[0],
+			Certificate:    certificate,
-			Key:            key.(*rsa.PrivateKey),
 			CertificatePEM: v.CertificatePEM,
 			KeyPEM:         v.KeyPEM,
 		}
+		if key != nil {
+			o.Key = key.(*rsa.PrivateKey)
+		}
+
 		out[k] = o
 	}
 	return out
 }
+
+func ReadCSRsAndKeysFromDir(certDir string) (map[string]CertificatePKI, error) {
+	certMap := make(map[string]CertificatePKI)
+	if _, err := os.Stat(certDir); os.IsNotExist(err) {
+		return certMap, nil
+	}
+
+	files, err := ioutil.ReadDir(certDir)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		if strings.Contains(file.Name(), "-csr.pem") {
+			certName := strings.TrimSuffix(file.Name(), "-csr.pem")
+			logrus.Debugf("[certificates] Loading %s csr from directory [%s]", certName, certDir)
+			// fetching csr
+			csrASN1, err := getCSRFromFile(certDir, certName+"-csr.pem")
+			if err != nil {
+				return nil, err
+			}
+			// fetching key
+			key, err := getKeyFromFile(certDir, certName+"-key.pem")
+			if err != nil {
+				return nil, err
+			}
+			certMap[certName] = ToCertObject(certName, getCommonName(certName), getOUName(certName), nil, key, csrASN1)
+		}
+	}
+
+	return certMap, nil
+}
+
+func ReadCertsAndKeysFromDir(certDir string) (map[string]CertificatePKI, error) {
+	certMap := make(map[string]CertificatePKI)
+	if _, err := os.Stat(certDir); os.IsNotExist(err) {
+		return certMap, nil
+	}
+
+	files, err := ioutil.ReadDir(certDir)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		logrus.Debugf("[certificates] reading file %s from directory [%s]", file.Name(), certDir)
+		// fetching cert
+		cert, err := getCertFromFile(certDir, file.Name())
+		if err != nil {
+			continue
+		}
+		// fetching the cert's key
+		certName := strings.TrimSuffix(file.Name(), ".pem")
+		key, err := getKeyFromFile(certDir, certName+"-key.pem")
+		if err != nil {
+			continue
+		}
+		certMap[certName] = ToCertObject(certName, getCommonName(certName), getOUName(certName), cert, key, nil)
+	}
+
+	return certMap, nil
+}
+
+func getCommonName(certName string) string {
+	switch certName {
+	case KubeNodeCertName:
+		return KubeNodeCommonName
+	default:
+		return certName
+	}
+}
+
+func getOUName(certName string) string {
+	switch certName {
+	case KubeNodeCertName:
+		return KubeNodeOrganizationName
+	case KubeAdminCertName:
+		return KubeAdminOrganizationName
+	default:
+		return ""
+	}
+}
+
+func getCertFromFile(certDir string, fileName string) (*x509.Certificate, error) {
+	var certificate *x509.Certificate
+	certPEM, _ := ioutil.ReadFile(filepath.Join(certDir, fileName))
+	if len(certPEM) > 0 {
+		certificates, err := cert.ParseCertsPEM(certPEM)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read certificate [%s]: %v", fileName, err)
+		}
+		certificate = certificates[0]
+	}
+	return certificate, nil
+}
+
+func getKeyFromFile(certDir string, fileName string) (*rsa.PrivateKey, error) {
+	var key *rsa.PrivateKey
+	keyPEM, _ := ioutil.ReadFile(filepath.Join(certDir, fileName))
+	if len(keyPEM) > 0 {
+		keyInterface, err := cert.ParsePrivateKeyPEM(keyPEM)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read key [%s]: %v", fileName, err)
+		}
+		key = keyInterface.(*rsa.PrivateKey)
+	}
+	return key, nil
+}
+
+func getCSRFromFile(certDir string, fileName string) ([]byte, error) {
+	csrPEM, err := ioutil.ReadFile(filepath.Join(certDir, fileName))
+	if err != nil {
+		return nil, fmt.Errorf("failed to read csr [%s]: %v", fileName, err)
+	}
+	csrASN1, _ := pem.Decode(csrPEM)
+	return csrASN1.Bytes, nil
+}
+
+func WriteCertificates(certDirPath string, certBundle map[string]CertificatePKI) error {
+	if _, err := os.Stat(certDirPath); os.IsNotExist(err) {
+		err = os.MkdirAll(certDirPath, 0755)
+		if err != nil {
+			return err
+		}
+	}
+
+	for certName, cert := range certBundle {
+		if cert.CertificatePEM != "" {
+			certificatePath := filepath.Join(certDirPath, certName+".pem")
+			if err := ioutil.WriteFile(certificatePath, []byte(cert.CertificatePEM), 0640); err != nil {
+				return fmt.Errorf("Failed to write certificate to path %v: %v", certificatePath, err)
+			}
+			logrus.Debugf("Successfully Deployed certificate file at [%s]", certificatePath)
+		}
+
+		if cert.KeyPEM != "" {
+			keyPath := filepath.Join(certDirPath, certName+"-key.pem")
+			if err := ioutil.WriteFile(keyPath, []byte(cert.KeyPEM), 0640); err != nil {
+				return fmt.Errorf("Failed to write key to path %v: %v", keyPath, err)
+			}
+			logrus.Debugf("Successfully Deployed key file at [%s]", keyPath)
+		}
+
+		if cert.CSRPEM != "" {
+			csrPath := filepath.Join(certDirPath, certName+"-csr.pem")
+			if err := ioutil.WriteFile(csrPath, []byte(cert.CSRPEM), 0640); err != nil {
+				return fmt.Errorf("Failed to write csr to path %v: %v", csrPath, err)
+			}
+			logrus.Debugf("Successfully Deployed csr file at [%s]", csrPath)
+		}
+	}
+	logrus.Infof("Successfully Deployed certificates at [%s]", certDirPath)
+	return nil
+}
+
+func ValidateBundleContent(rkeConfig *v3.RancherKubernetesEngineConfig, certBundle map[string]CertificatePKI, configPath, configDir string) error {
+	// ensure all needed certs exists
+	// make sure all CA Certs exist
+	if certBundle[CACertName].Certificate == nil {
+		return fmt.Errorf("Failed to find master CA certificate")
+	}
+	if certBundle[RequestHeaderCACertName].Certificate == nil {
+		logrus.Warnf("Failed to find RequestHeader CA certificate, using master CA certificate")
+		certBundle[RequestHeaderCACertName] = ToCertObject(RequestHeaderCACertName, RequestHeaderCACertName, "", certBundle[CACertName].Certificate, nil, nil)
+	}
+	// make sure all components exists
+	ComponentsCerts := []string{
+		KubeAPICertName,
+		KubeControllerCertName,
+		KubeSchedulerCertName,
+		KubeProxyCertName,
+		KubeNodeCertName,
+		KubeAdminCertName,
+		APIProxyClientCertName,
+	}
+	for _, certName := range ComponentsCerts {
+		if certBundle[certName].Certificate == nil || certBundle[certName].Key == nil {
+			return fmt.Errorf("Failed to find [%s] Certificate or Key", certName)
+		}
+	}
+	etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole)
+	for _, host := range etcdHosts {
+		etcdName := GetEtcdCrtName(host.InternalAddress)
+		if certBundle[etcdName].Certificate == nil || certBundle[etcdName].Key == nil {
+			return fmt.Errorf("Failed to find etcd [%s] Certificate or Key", etcdName)
+		}
+	}
+	// Configure kubeconfig
+	cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole)
+	localKubeConfigPath := GetLocalKubeConfig(configPath, configDir)
+	if len(cpHosts) > 0 {
+		kubeAdminCertObj := certBundle[KubeAdminCertName]
+		kubeAdminConfig := GetKubeConfigX509WithData(
+			"https://"+cpHosts[0].Address+":6443",
+			rkeConfig.ClusterName,
+			KubeAdminCertName,
+			string(cert.EncodeCertPEM(certBundle[CACertName].Certificate)),
+			string(cert.EncodeCertPEM(certBundle[KubeAdminCertName].Certificate)),
+			string(cert.EncodePrivateKeyPEM(certBundle[KubeAdminCertName].Key)))
+		kubeAdminCertObj.Config = kubeAdminConfig
+		kubeAdminCertObj.ConfigPath = localKubeConfigPath
+		certBundle[KubeAdminCertName] = kubeAdminCertObj
+	}
+	return validateCAIssuer(rkeConfig, certBundle)
+}
+
+func validateCAIssuer(rkeConfig *v3.RancherKubernetesEngineConfig, certBundle map[string]CertificatePKI) error {
+	// make sure all certs are signed by CA cert
+	caCert := certBundle[CACertName].Certificate
+	ComponentsCerts := []string{
+		KubeAPICertName,
+		KubeControllerCertName,
+		KubeSchedulerCertName,
+		KubeProxyCertName,
+		KubeNodeCertName,
+		KubeAdminCertName,
+	}
+	etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole)
+	for _, host := range etcdHosts {
+		etcdName := GetEtcdCrtName(host.InternalAddress)
+		ComponentsCerts = append(ComponentsCerts, etcdName)
+	}
+	for _, componentCert := range ComponentsCerts {
+		if certBundle[componentCert].Certificate.Issuer.CommonName != caCert.Subject.CommonName {
+			return fmt.Errorf("Component [%s] is not signed by the custom CA certificate", componentCert)
+		}
+	}
+	requestHeaderCACert := certBundle[RequestHeaderCACertName].Certificate
+	if certBundle[APIProxyClientCertName].Certificate.Issuer.CommonName != requestHeaderCACert.Subject.CommonName {
+		return fmt.Errorf("Component [%s] is not signed by the custom Request Header CA certificate", APIProxyClientCertName)
+	}
+	return nil
+}