Merge pull request #353 from galal-hussein/only_etcd

Start cluster with only etcd nodes

cluster/certificates.go

@@ -22,26 +22,42 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
 		if currentCluster != nil {
 			kubeCluster.Certificates = currentCluster.Certificates
 		} else {
-			log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
+			var backupHost *hosts.Host
-			kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, kubeCluster.ControlPlaneHosts[0], kubeCluster.SystemImages.Alpine, kubeCluster.LocalKubeConfigPath, kubeCluster.PrivateRegistriesMap)
+			if len(kubeCluster.Services.Etcd.ExternalURLs) > 0 {
+				backupHost = kubeCluster.ControlPlaneHosts[0]
+			} else {
+				backupHost = kubeCluster.EtcdHosts[0]
+			}
+			log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", backupHost.Address)
+			kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, backupHost, kubeCluster.SystemImages.Alpine, kubeCluster.LocalKubeConfigPath, kubeCluster.PrivateRegistriesMap)
 			if err != nil {
 				return err
 			}
 			if kubeCluster.Certificates != nil {
-				log.Infof(ctx, "[certificates] Certificate backup found on host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
+				log.Infof(ctx, "[certificates] Certificate backup found on host [%s]", backupHost.Address)
+				// this is the case of adding controlplane node on empty cluster with only etcd nodes
+				if kubeCluster.Certificates[pki.KubeAdminCertName].Config == "" && len(kubeCluster.ControlPlaneHosts) > 0 {
+					if err := rebuildLocalAdminConfig(ctx, kubeCluster); err != nil {
+						return err
+					}
+					kubeCluster.Certificates, err = regenerateAPICertificate(kubeCluster, kubeCluster.Certificates)
+					if err != nil {
+						return fmt.Errorf("Failed to regenerate KubeAPI certificate %v", err)
+					}
+				}
 				return nil
 			}
-			log.Infof(ctx, "[certificates] No Certificate backup found on host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
+			log.Infof(ctx, "[certificates] No Certificate backup found on host [%s]", backupHost.Address)
 
 			kubeCluster.Certificates, err = pki.GenerateRKECerts(ctx, kubeCluster.RancherKubernetesEngineConfig, kubeCluster.LocalKubeConfigPath, "")
 			if err != nil {
 				return fmt.Errorf("Failed to generate Kubernetes certificates: %v", err)
 			}
-			log.Infof(ctx, "[certificates] Temporarily saving certs to control host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
+			log.Infof(ctx, "[certificates] Temporarily saving certs to control host [%s]", backupHost.Address)
-			if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.ControlPlaneHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages.CertDownloader, pki.TempCertPath, kubeCluster.PrivateRegistriesMap); err != nil {
+			if err := pki.DeployCertificatesOnHost(ctx, backupHost, kubeCluster.Certificates, kubeCluster.SystemImages.CertDownloader, pki.TempCertPath, kubeCluster.PrivateRegistriesMap); err != nil {
 				return err
 			}
-			log.Infof(ctx, "[certificates] Saved certs to control host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
+			log.Infof(ctx, "[certificates] Saved certs to control host [%s]", backupHost.Address)
 		}
 	}
 	return nil

cluster/cluster.go

@@ -76,9 +76,11 @@ func (c *Cluster) DeployControlPlane(ctx context.Context) error {
 		processMap); err != nil {
 		return fmt.Errorf("[controlPlane] Failed to bring up Control Plane: %v", err)
 	}
-	// Apply Authz configuration after deploying controlplane
+	if len(c.ControlPlaneHosts) > 0 {
-	if err := c.ApplyAuthzResources(ctx); err != nil {
+		// Apply Authz configuration after deploying controlplane
-		return fmt.Errorf("[auths] Failed to apply RBAC resources: %v", err)
+		if err := c.ApplyAuthzResources(ctx); err != nil {
+			return fmt.Errorf("[auths] Failed to apply RBAC resources: %v", err)
+		}
 	}
 	return nil
 }
@@ -163,6 +165,9 @@ func ParseCluster(
 }
 
 func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error {
+	if len(kubeCluster.ControlPlaneHosts) == 0 {
+		return nil
+	}
 	log.Infof(ctx, "[reconcile] Rebuilding and updating local kube config")
 	var workingConfig, newConfig string
 	currentKubeConfig := kubeCluster.Certificates[pki.KubeAdminCertName]
@@ -212,6 +217,9 @@ func getLocalConfigAddress(localConfigPath string) (string, error) {
 
 func getLocalAdminConfigWithNewAddress(localConfigPath, cpAddress string) string {
 	config, _ := clientcmd.BuildConfigFromFlags("", localConfigPath)
+	if config == nil {
+		return ""
+	}
 	config.Host = fmt.Sprintf("https://%s:6443", cpAddress)
 	return pki.GetKubeConfigX509WithData(
 		"https://"+cpAddress+":6443",
@@ -252,21 +260,23 @@ func (c *Cluster) deployAddons(ctx context.Context) error {
 }
 
 func (c *Cluster) SyncLabelsAndTaints(ctx context.Context) error {
-	log.Infof(ctx, "[sync] Syncing nodes Labels and Taints")
+	if len(c.ControlPlaneHosts) > 0 {
-	k8sClient, err := k8s.NewClient(c.LocalKubeConfigPath)
+		log.Infof(ctx, "[sync] Syncing nodes Labels and Taints")
-	if err != nil {
+		k8sClient, err := k8s.NewClient(c.LocalKubeConfigPath)
-		return fmt.Errorf("Failed to initialize new kubernetes client: %v", err)
+		if err != nil {
-	}
+			return fmt.Errorf("Failed to initialize new kubernetes client: %v", err)
-	for _, host := range hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) {
-		if err := k8s.SyncLabels(k8sClient, host.HostnameOverride, host.ToAddLabels, host.ToDelLabels); err != nil {
-			return err
 		}
-		// Taints are not being added by user
+		for _, host := range hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) {
-		if err := k8s.SyncTaints(k8sClient, host.HostnameOverride, host.ToAddTaints, host.ToDelTaints); err != nil {
+			if err := k8s.SyncLabels(k8sClient, host.HostnameOverride, host.ToAddLabels, host.ToDelLabels); err != nil {
-			return err
+				return err
+			}
+			// Taints are not being added by user
+			if err := k8s.SyncTaints(k8sClient, host.HostnameOverride, host.ToAddTaints, host.ToDelTaints); err != nil {
+				return err
+			}
 		}
+		log.Infof(ctx, "[sync] Successfully synced nodes Labels and Taints")
 	}
-	log.Infof(ctx, "[sync] Successfully synced nodes Labels and Taints")
 	return nil
 }
 
@@ -293,13 +303,14 @@ func ConfigureCluster(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
 	if err != nil {
 		return err
 	}
-	kubeCluster.Certificates = crtBundle
+	if len(kubeCluster.ControlPlaneHosts) > 0 {
-	err = kubeCluster.deployNetworkPlugin(ctx)
+		kubeCluster.Certificates = crtBundle
-	if err != nil {
+		if err := kubeCluster.deployNetworkPlugin(ctx); err != nil {
-		return err
+			return err
+		}
+		return kubeCluster.deployAddons(ctx)
 	}
-
+	return nil
-	return kubeCluster.deployAddons(ctx)
 }
 
 func (c *Cluster) getEtcdProcessHostMap(readyEtcdHosts []*hosts.Host) map[*hosts.Host]v3.Process {

cluster/plan.go

@@ -292,6 +292,7 @@ func (c *Cluster) BuildProxyProcess() v3.Process {
 
 	return v3.Process{
 		Env:           Env,
+		Args:          Env,
 		NetworkMode:   "host",
 		RestartPolicy: "always",
 		HealthCheck:   v3.HealthCheck{},

cluster/state.go

@@ -16,19 +16,21 @@ import (
 )
 
 func (c *Cluster) SaveClusterState(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig) error {
-	// Reinitialize kubernetes Client
+	if len(c.ControlPlaneHosts) > 0 {
-	var err error
+		// Reinitialize kubernetes Client
-	c.KubeClient, err = k8s.NewClient(c.LocalKubeConfigPath)
+		var err error
-	if err != nil {
+		c.KubeClient, err = k8s.NewClient(c.LocalKubeConfigPath)
-		return fmt.Errorf("Failed to re-initialize Kubernetes Client: %v", err)
+		if err != nil {
-	}
+			return fmt.Errorf("Failed to re-initialize Kubernetes Client: %v", err)
-	err = saveClusterCerts(ctx, c.KubeClient, c.Certificates)
+		}
-	if err != nil {
+		err = saveClusterCerts(ctx, c.KubeClient, c.Certificates)
-		return fmt.Errorf("[certificates] Failed to Save Kubernetes certificates: %v", err)
+		if err != nil {
-	}
+			return fmt.Errorf("[certificates] Failed to Save Kubernetes certificates: %v", err)
-	err = saveStateToKubernetes(ctx, c.KubeClient, c.LocalKubeConfigPath, rkeConfig)
+		}
-	if err != nil {
+		err = saveStateToKubernetes(ctx, c.KubeClient, c.LocalKubeConfigPath, rkeConfig)
-		return fmt.Errorf("[state] Failed to save configuration state: %v", err)
+		if err != nil {
+			return fmt.Errorf("[state] Failed to save configuration state: %v", err)
+		}
 	}
 	return nil
 }

cluster/validation.go

@@ -9,9 +9,7 @@ import (
 
 func (c *Cluster) ValidateCluster() error {
 	// make sure cluster has at least one controlplane/etcd host
-	if len(c.ControlPlaneHosts) == 0 {
+
-		return fmt.Errorf("Cluster must have at least one control plane host")
-	}
 	if len(c.EtcdHosts) == 0 && len(c.Services.Etcd.ExternalURLs) == 0 {
 		return fmt.Errorf("Cluster must have at least one etcd plane host")
 	}

cmd/up.go

@@ -106,11 +106,12 @@ func ClusterUp(
 	if err != nil {
 		return APIURL, caCrt, clientCert, clientKey, err
 	}
-
+	if len(kubeCluster.ControlPlaneHosts) > 0 {
-	APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443")
+		APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443")
+		clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Certificate))
+		clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Key))
+	}
 	caCrt = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.CACertName].Certificate))
-	clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Certificate))
-	clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Key))
 
 	log.Infof(ctx, "Finished building Kubernetes cluster successfully")
 	return APIURL, caCrt, clientCert, clientKey, nil

hosts/hosts.go

@@ -59,8 +59,8 @@ func (h *Host) CleanUpAll(ctx context.Context, cleanerImage string, prsMap map[s
 }
 
 func (h *Host) CleanUpWorkerHost(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error {
-	if h.IsControl {
+	if h.IsControl || h.IsEtcd {
-		log.Infof(ctx, "[hosts] Host [%s] is already a controlplane host, skipping cleanup.", h.Address)
+		log.Infof(ctx, "[hosts] Host [%s] is already a controlplane or etcd host, skipping cleanup.", h.Address)
 		return nil
 	}
 	toCleanPaths := []string{
@@ -74,8 +74,8 @@ func (h *Host) CleanUpWorkerHost(ctx context.Context, cleanerImage string, prsMa
 }
 
 func (h *Host) CleanUpControlHost(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error {
-	if h.IsWorker {
+	if h.IsWorker || h.IsEtcd {
-		log.Infof(ctx, "[hosts] Host [%s] is already a worker host, skipping cleanup.", h.Address)
+		log.Infof(ctx, "[hosts] Host [%s] is already a worker or etcd host, skipping cleanup.", h.Address)
 		return nil
 	}
 	toCleanPaths := []string{

pki/deploy.go

@@ -80,6 +80,9 @@ func doRunDeployer(ctx context.Context, host *hosts.Host, containerEnv []string,
 }
 
 func DeployAdminConfig(ctx context.Context, kubeConfig, localConfigPath string) error {
+	if len(kubeConfig) == 0 {
+		return nil
+	}
 	logrus.Debugf("Deploying admin Kubeconfig locally: %s", kubeConfig)
 	err := ioutil.WriteFile(localConfigPath, []byte(kubeConfig), 0640)
 	if err != nil {
@@ -120,8 +123,9 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho
 		KubeSchedulerCertName:  true,
 		KubeProxyCertName:      true,
 		KubeNodeCertName:       true,
-		KubeAdminCertName:      true,
+		KubeAdminCertName:      false,
 	}
+
 	for _, etcdHost := range extraHosts {
 		// Fetch etcd certificates
 		crtList[GetEtcdCrtName(etcdHost.InternalAddress)] = false

pki/pki.go

@@ -101,18 +101,20 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
 	if err != nil {
 		return nil, err
 	}
-	kubeAdminConfig := GetKubeConfigX509WithData(
-		"https://"+cpHosts[0].Address+":6443",
-		KubeAdminCertName,
-		string(cert.EncodeCertPEM(caCrt)),
-		string(cert.EncodeCertPEM(kubeAdminCrt)),
-		string(cert.EncodePrivateKeyPEM(kubeAdminKey)))
-
 	kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey)
-	kubeAdminCertObj.Config = kubeAdminConfig
+	if len(cpHosts) > 0 {
-	kubeAdminCertObj.ConfigPath = localKubeConfigPath
+		kubeAdminConfig := GetKubeConfigX509WithData(
+			"https://"+cpHosts[0].Address+":6443",
+			KubeAdminCertName,
+			string(cert.EncodeCertPEM(caCrt)),
+			string(cert.EncodeCertPEM(kubeAdminCrt)),
+			string(cert.EncodePrivateKeyPEM(kubeAdminKey)))
+		kubeAdminCertObj.Config = kubeAdminConfig
+		kubeAdminCertObj.ConfigPath = localKubeConfigPath
+	} else {
+		kubeAdminCertObj.Config = ""
+	}
 	certs[KubeAdminCertName] = kubeAdminCertObj
-
 	// generate etcd certificate and key
 	if len(rkeConfig.Services.Etcd.ExternalURLs) > 0 {
 		clientCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.Cert))