diff --git a/cluster/certificates.go b/cluster/certificates.go index 48835b98..07dbf403 100644 --- a/cluster/certificates.go +++ b/cluster/certificates.go @@ -22,26 +22,42 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust if currentCluster != nil { kubeCluster.Certificates = currentCluster.Certificates } else { - log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", kubeCluster.ControlPlaneHosts[0].Address) - kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, kubeCluster.ControlPlaneHosts[0], kubeCluster.SystemImages.Alpine, kubeCluster.LocalKubeConfigPath, kubeCluster.PrivateRegistriesMap) + var backupHost *hosts.Host + if len(kubeCluster.Services.Etcd.ExternalURLs) > 0 { + backupHost = kubeCluster.ControlPlaneHosts[0] + } else { + backupHost = kubeCluster.EtcdHosts[0] + } + log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", backupHost.Address) + kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, backupHost, kubeCluster.SystemImages.Alpine, kubeCluster.LocalKubeConfigPath, kubeCluster.PrivateRegistriesMap) if err != nil { return err } if kubeCluster.Certificates != nil { - log.Infof(ctx, "[certificates] Certificate backup found on host [%s]", kubeCluster.ControlPlaneHosts[0].Address) + log.Infof(ctx, "[certificates] Certificate backup found on host [%s]", backupHost.Address) + // this is the case of adding controlplane node on empty cluster with only etcd nodes + if kubeCluster.Certificates[pki.KubeAdminCertName].Config == "" && len(kubeCluster.ControlPlaneHosts) > 0 { + if err := rebuildLocalAdminConfig(ctx, kubeCluster); err != nil { + return err + } + kubeCluster.Certificates, err = regenerateAPICertificate(kubeCluster, kubeCluster.Certificates) + if err != nil { + return fmt.Errorf("Failed to regenerate KubeAPI certificate %v", err) + } + } return nil } - log.Infof(ctx, "[certificates] No Certificate backup found on host [%s]", kubeCluster.ControlPlaneHosts[0].Address) + log.Infof(ctx, "[certificates] No Certificate backup found on host [%s]", backupHost.Address) kubeCluster.Certificates, err = pki.GenerateRKECerts(ctx, kubeCluster.RancherKubernetesEngineConfig, kubeCluster.LocalKubeConfigPath, "") if err != nil { return fmt.Errorf("Failed to generate Kubernetes certificates: %v", err) } - log.Infof(ctx, "[certificates] Temporarily saving certs to control host [%s]", kubeCluster.ControlPlaneHosts[0].Address) - if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.ControlPlaneHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages.CertDownloader, pki.TempCertPath, kubeCluster.PrivateRegistriesMap); err != nil { + log.Infof(ctx, "[certificates] Temporarily saving certs to control host [%s]", backupHost.Address) + if err := pki.DeployCertificatesOnHost(ctx, backupHost, kubeCluster.Certificates, kubeCluster.SystemImages.CertDownloader, pki.TempCertPath, kubeCluster.PrivateRegistriesMap); err != nil { return err } - log.Infof(ctx, "[certificates] Saved certs to control host [%s]", kubeCluster.ControlPlaneHosts[0].Address) + log.Infof(ctx, "[certificates] Saved certs to control host [%s]", backupHost.Address) } } return nil diff --git a/cluster/cluster.go b/cluster/cluster.go index ae5e4c01..fa9b3767 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -76,9 +76,11 @@ func (c *Cluster) DeployControlPlane(ctx context.Context) error { processMap); err != nil { return fmt.Errorf("[controlPlane] Failed to bring up Control Plane: %v", err) } - // Apply Authz configuration after deploying controlplane - if err := c.ApplyAuthzResources(ctx); err != nil { - return fmt.Errorf("[auths] Failed to apply RBAC resources: %v", err) + if len(c.ControlPlaneHosts) > 0 { + // Apply Authz configuration after deploying controlplane + if err := c.ApplyAuthzResources(ctx); err != nil { + return fmt.Errorf("[auths] Failed to apply RBAC resources: %v", err) + } } return nil } @@ -163,6 +165,9 @@ func ParseCluster( } func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error { + if len(kubeCluster.ControlPlaneHosts) == 0 { + return nil + } log.Infof(ctx, "[reconcile] Rebuilding and updating local kube config") var workingConfig, newConfig string currentKubeConfig := kubeCluster.Certificates[pki.KubeAdminCertName] @@ -212,6 +217,9 @@ func getLocalConfigAddress(localConfigPath string) (string, error) { func getLocalAdminConfigWithNewAddress(localConfigPath, cpAddress string) string { config, _ := clientcmd.BuildConfigFromFlags("", localConfigPath) + if config == nil { + return "" + } config.Host = fmt.Sprintf("https://%s:6443", cpAddress) return pki.GetKubeConfigX509WithData( "https://"+cpAddress+":6443", @@ -252,21 +260,23 @@ func (c *Cluster) deployAddons(ctx context.Context) error { } func (c *Cluster) SyncLabelsAndTaints(ctx context.Context) error { - log.Infof(ctx, "[sync] Syncing nodes Labels and Taints") - k8sClient, err := k8s.NewClient(c.LocalKubeConfigPath) - if err != nil { - return fmt.Errorf("Failed to initialize new kubernetes client: %v", err) - } - for _, host := range hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) { - if err := k8s.SyncLabels(k8sClient, host.HostnameOverride, host.ToAddLabels, host.ToDelLabels); err != nil { - return err + if len(c.ControlPlaneHosts) > 0 { + log.Infof(ctx, "[sync] Syncing nodes Labels and Taints") + k8sClient, err := k8s.NewClient(c.LocalKubeConfigPath) + if err != nil { + return fmt.Errorf("Failed to initialize new kubernetes client: %v", err) } - // Taints are not being added by user - if err := k8s.SyncTaints(k8sClient, host.HostnameOverride, host.ToAddTaints, host.ToDelTaints); err != nil { - return err + for _, host := range hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) { + if err := k8s.SyncLabels(k8sClient, host.HostnameOverride, host.ToAddLabels, host.ToDelLabels); err != nil { + return err + } + // Taints are not being added by user + if err := k8s.SyncTaints(k8sClient, host.HostnameOverride, host.ToAddTaints, host.ToDelTaints); err != nil { + return err + } } + log.Infof(ctx, "[sync] Successfully synced nodes Labels and Taints") } - log.Infof(ctx, "[sync] Successfully synced nodes Labels and Taints") return nil } @@ -293,13 +303,14 @@ func ConfigureCluster(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC if err != nil { return err } - kubeCluster.Certificates = crtBundle - err = kubeCluster.deployNetworkPlugin(ctx) - if err != nil { - return err + if len(kubeCluster.ControlPlaneHosts) > 0 { + kubeCluster.Certificates = crtBundle + if err := kubeCluster.deployNetworkPlugin(ctx); err != nil { + return err + } + return kubeCluster.deployAddons(ctx) } - - return kubeCluster.deployAddons(ctx) + return nil } func (c *Cluster) getEtcdProcessHostMap(readyEtcdHosts []*hosts.Host) map[*hosts.Host]v3.Process { diff --git a/cluster/plan.go b/cluster/plan.go index 574b8808..bc7aa755 100644 --- a/cluster/plan.go +++ b/cluster/plan.go @@ -292,6 +292,7 @@ func (c *Cluster) BuildProxyProcess() v3.Process { return v3.Process{ Env: Env, + Args: Env, NetworkMode: "host", RestartPolicy: "always", HealthCheck: v3.HealthCheck{}, diff --git a/cluster/state.go b/cluster/state.go index e0203eaf..00bac94d 100644 --- a/cluster/state.go +++ b/cluster/state.go @@ -16,19 +16,21 @@ import ( ) func (c *Cluster) SaveClusterState(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig) error { - // Reinitialize kubernetes Client - var err error - c.KubeClient, err = k8s.NewClient(c.LocalKubeConfigPath) - if err != nil { - return fmt.Errorf("Failed to re-initialize Kubernetes Client: %v", err) - } - err = saveClusterCerts(ctx, c.KubeClient, c.Certificates) - if err != nil { - return fmt.Errorf("[certificates] Failed to Save Kubernetes certificates: %v", err) - } - err = saveStateToKubernetes(ctx, c.KubeClient, c.LocalKubeConfigPath, rkeConfig) - if err != nil { - return fmt.Errorf("[state] Failed to save configuration state: %v", err) + if len(c.ControlPlaneHosts) > 0 { + // Reinitialize kubernetes Client + var err error + c.KubeClient, err = k8s.NewClient(c.LocalKubeConfigPath) + if err != nil { + return fmt.Errorf("Failed to re-initialize Kubernetes Client: %v", err) + } + err = saveClusterCerts(ctx, c.KubeClient, c.Certificates) + if err != nil { + return fmt.Errorf("[certificates] Failed to Save Kubernetes certificates: %v", err) + } + err = saveStateToKubernetes(ctx, c.KubeClient, c.LocalKubeConfigPath, rkeConfig) + if err != nil { + return fmt.Errorf("[state] Failed to save configuration state: %v", err) + } } return nil } diff --git a/cluster/validation.go b/cluster/validation.go index e6966f15..153069eb 100644 --- a/cluster/validation.go +++ b/cluster/validation.go @@ -9,9 +9,7 @@ import ( func (c *Cluster) ValidateCluster() error { // make sure cluster has at least one controlplane/etcd host - if len(c.ControlPlaneHosts) == 0 { - return fmt.Errorf("Cluster must have at least one control plane host") - } + if len(c.EtcdHosts) == 0 && len(c.Services.Etcd.ExternalURLs) == 0 { return fmt.Errorf("Cluster must have at least one etcd plane host") } diff --git a/cmd/up.go b/cmd/up.go index 320db2f0..da2692cf 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -106,11 +106,12 @@ func ClusterUp( if err != nil { return APIURL, caCrt, clientCert, clientKey, err } - - APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443") + if len(kubeCluster.ControlPlaneHosts) > 0 { + APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443") + clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Certificate)) + clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Key)) + } caCrt = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.CACertName].Certificate)) - clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Certificate)) - clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Key)) log.Infof(ctx, "Finished building Kubernetes cluster successfully") return APIURL, caCrt, clientCert, clientKey, nil diff --git a/hosts/hosts.go b/hosts/hosts.go index 5a3bf6f3..ce1a5bd4 100644 --- a/hosts/hosts.go +++ b/hosts/hosts.go @@ -59,8 +59,8 @@ func (h *Host) CleanUpAll(ctx context.Context, cleanerImage string, prsMap map[s } func (h *Host) CleanUpWorkerHost(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error { - if h.IsControl { - log.Infof(ctx, "[hosts] Host [%s] is already a controlplane host, skipping cleanup.", h.Address) + if h.IsControl || h.IsEtcd { + log.Infof(ctx, "[hosts] Host [%s] is already a controlplane or etcd host, skipping cleanup.", h.Address) return nil } toCleanPaths := []string{ @@ -74,8 +74,8 @@ func (h *Host) CleanUpWorkerHost(ctx context.Context, cleanerImage string, prsMa } func (h *Host) CleanUpControlHost(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error { - if h.IsWorker { - log.Infof(ctx, "[hosts] Host [%s] is already a worker host, skipping cleanup.", h.Address) + if h.IsWorker || h.IsEtcd { + log.Infof(ctx, "[hosts] Host [%s] is already a worker or etcd host, skipping cleanup.", h.Address) return nil } toCleanPaths := []string{ diff --git a/pki/deploy.go b/pki/deploy.go index 6f6cccc9..1fb41486 100644 --- a/pki/deploy.go +++ b/pki/deploy.go @@ -80,6 +80,9 @@ func doRunDeployer(ctx context.Context, host *hosts.Host, containerEnv []string, } func DeployAdminConfig(ctx context.Context, kubeConfig, localConfigPath string) error { + if len(kubeConfig) == 0 { + return nil + } logrus.Debugf("Deploying admin Kubeconfig locally: %s", kubeConfig) err := ioutil.WriteFile(localConfigPath, []byte(kubeConfig), 0640) if err != nil { @@ -120,8 +123,9 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho KubeSchedulerCertName: true, KubeProxyCertName: true, KubeNodeCertName: true, - KubeAdminCertName: true, + KubeAdminCertName: false, } + for _, etcdHost := range extraHosts { // Fetch etcd certificates crtList[GetEtcdCrtName(etcdHost.InternalAddress)] = false diff --git a/pki/pki.go b/pki/pki.go index 68e80d43..b33f0c38 100644 --- a/pki/pki.go +++ b/pki/pki.go @@ -101,18 +101,20 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC if err != nil { return nil, err } - kubeAdminConfig := GetKubeConfigX509WithData( - "https://"+cpHosts[0].Address+":6443", - KubeAdminCertName, - string(cert.EncodeCertPEM(caCrt)), - string(cert.EncodeCertPEM(kubeAdminCrt)), - string(cert.EncodePrivateKeyPEM(kubeAdminKey))) - kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey) - kubeAdminCertObj.Config = kubeAdminConfig - kubeAdminCertObj.ConfigPath = localKubeConfigPath + if len(cpHosts) > 0 { + kubeAdminConfig := GetKubeConfigX509WithData( + "https://"+cpHosts[0].Address+":6443", + KubeAdminCertName, + string(cert.EncodeCertPEM(caCrt)), + string(cert.EncodeCertPEM(kubeAdminCrt)), + string(cert.EncodePrivateKeyPEM(kubeAdminKey))) + kubeAdminCertObj.Config = kubeAdminConfig + kubeAdminCertObj.ConfigPath = localKubeConfigPath + } else { + kubeAdminCertObj.Config = "" + } certs[KubeAdminCertName] = kubeAdminCertObj - // generate etcd certificate and key if len(rkeConfig.Services.Etcd.ExternalURLs) > 0 { clientCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.Cert))