rancher/rke
1
0
Fork 0
mirror of https://github.com/rancher/rke.git synced 2025-08-01 15:19:09 +00:00
Code Issues Releases Wiki Activity

Start cluster with only etcd nodes

Browse Source
This commit is contained in:
galal-hussein 2018-02-15 05:25:36 +02:00
parent e4cee4d5ef
commit 11d7aa0e16
9 changed files with 98 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
cluster/certificates.go
View File

@ -22,26 +22,42 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
if currentCluster != nil {
kubeCluster.Certificates = currentCluster.Certificates
} else {
log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, kubeCluster.ControlPlaneHosts[0], kubeCluster.SystemImages.Alpine, kubeCluster.LocalKubeConfigPath, kubeCluster.PrivateRegistriesMap)
var backupHost *hosts.Host
if len(kubeCluster.Services.Etcd.ExternalURLs) > 0 {
backupHost = kubeCluster.ControlPlaneHosts[0]
} else {
backupHost = kubeCluster.EtcdHosts[0]
}
log.Infof(ctx, "[certificates] Attempting to recover certificates from backup on host [%s]", backupHost.Address)
kubeCluster.Certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, backupHost, kubeCluster.SystemImages.Alpine, kubeCluster.LocalKubeConfigPath, kubeCluster.PrivateRegistriesMap)
if err != nil {
return err
}
if kubeCluster.Certificates != nil {
log.Infof(ctx, "[certificates] Certificate backup found on host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
log.Infof(ctx, "[certificates] Certificate backup found on host [%s]", backupHost.Address)
// this is the case of adding controlplane node on empty cluster with only etcd nodes
if kubeCluster.Certificates[pki.KubeAdminCertName].Config == "" && len(kubeCluster.ControlPlaneHosts) > 0 {
if err := rebuildLocalAdminConfig(ctx, kubeCluster); err != nil {
return err
}
kubeCluster.Certificates, err = regenerateAPICertificate(kubeCluster, kubeCluster.Certificates)
if err != nil {
return fmt.Errorf("Failed to regenerate KubeAPI certificate %v", err)
}
}
return nil
}
log.Infof(ctx, "[certificates] No Certificate backup found on host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
log.Infof(ctx, "[certificates] No Certificate backup found on host [%s]", backupHost.Address)
kubeCluster.Certificates, err = pki.GenerateRKECerts(ctx, kubeCluster.RancherKubernetesEngineConfig, kubeCluster.LocalKubeConfigPath, "")
if err != nil {
return fmt.Errorf("Failed to generate Kubernetes certificates: %v", err)
}
log.Infof(ctx, "[certificates] Temporarily saving certs to control host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
if err := pki.DeployCertificatesOnHost(ctx, kubeCluster.ControlPlaneHosts[0], kubeCluster.Certificates, kubeCluster.SystemImages.CertDownloader, pki.TempCertPath, kubeCluster.PrivateRegistriesMap); err != nil {
log.Infof(ctx, "[certificates] Temporarily saving certs to control host [%s]", backupHost.Address)
if err := pki.DeployCertificatesOnHost(ctx, backupHost, kubeCluster.Certificates, kubeCluster.SystemImages.CertDownloader, pki.TempCertPath, kubeCluster.PrivateRegistriesMap); err != nil {
return err
}
log.Infof(ctx, "[certificates] Saved certs to control host [%s]", kubeCluster.ControlPlaneHosts[0].Address)
log.Infof(ctx, "[certificates] Saved certs to control host [%s]", backupHost.Address)
}
}
return nil

53
cluster/cluster.go
View File

@ -76,9 +76,11 @@ func (c *Cluster) DeployControlPlane(ctx context.Context) error {
processMap); err != nil {
return fmt.Errorf("[controlPlane] Failed to bring up Control Plane: %v", err)
}
// Apply Authz configuration after deploying controlplane
if err := c.ApplyAuthzResources(ctx); err != nil {
return fmt.Errorf("[auths] Failed to apply RBAC resources: %v", err)
if len(c.ControlPlaneHosts) > 0 {
// Apply Authz configuration after deploying controlplane
if err := c.ApplyAuthzResources(ctx); err != nil {
return fmt.Errorf("[auths] Failed to apply RBAC resources: %v", err)
}
}
return nil
}
@ -163,6 +165,9 @@ func ParseCluster(
}
func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error {
if len(kubeCluster.ControlPlaneHosts) == 0 {
return nil
}
log.Infof(ctx, "[reconcile] Rebuilding and updating local kube config")
var workingConfig, newConfig string
currentKubeConfig := kubeCluster.Certificates[pki.KubeAdminCertName]
@ -212,6 +217,9 @@ func getLocalConfigAddress(localConfigPath string) (string, error) {
func getLocalAdminConfigWithNewAddress(localConfigPath, cpAddress string) string {
config, _ := clientcmd.BuildConfigFromFlags("", localConfigPath)
if config == nil {
return ""
}
config.Host = fmt.Sprintf("https://%s:6443", cpAddress)
return pki.GetKubeConfigX509WithData(
"https://"+cpAddress+":6443",
@ -252,21 +260,23 @@ func (c *Cluster) deployAddons(ctx context.Context) error {
}
func (c *Cluster) SyncLabelsAndTaints(ctx context.Context) error {
log.Infof(ctx, "[sync] Syncing nodes Labels and Taints")
k8sClient, err := k8s.NewClient(c.LocalKubeConfigPath)
if err != nil {
return fmt.Errorf("Failed to initialize new kubernetes client: %v", err)
}
for _, host := range hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) {
if err := k8s.SyncLabels(k8sClient, host.HostnameOverride, host.ToAddLabels, host.ToDelLabels); err != nil {
return err
if len(c.ControlPlaneHosts) > 0 {
log.Infof(ctx, "[sync] Syncing nodes Labels and Taints")
k8sClient, err := k8s.NewClient(c.LocalKubeConfigPath)
if err != nil {
return fmt.Errorf("Failed to initialize new kubernetes client: %v", err)
}
// Taints are not being added by user
if err := k8s.SyncTaints(k8sClient, host.HostnameOverride, host.ToAddTaints, host.ToDelTaints); err != nil {
return err
for _, host := range hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) {
if err := k8s.SyncLabels(k8sClient, host.HostnameOverride, host.ToAddLabels, host.ToDelLabels); err != nil {
return err
}
// Taints are not being added by user
if err := k8s.SyncTaints(k8sClient, host.HostnameOverride, host.ToAddTaints, host.ToDelTaints); err != nil {
return err
}
}
log.Infof(ctx, "[sync] Successfully synced nodes Labels and Taints")
}
log.Infof(ctx, "[sync] Successfully synced nodes Labels and Taints")
return nil
}
@ -293,13 +303,14 @@ func ConfigureCluster(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
if err != nil {
return err
}
kubeCluster.Certificates = crtBundle
err = kubeCluster.deployNetworkPlugin(ctx)
if err != nil {
return err
if len(kubeCluster.ControlPlaneHosts) > 0 {
kubeCluster.Certificates = crtBundle
if err := kubeCluster.deployNetworkPlugin(ctx); err != nil {
return err
}
return kubeCluster.deployAddons(ctx)
}
return kubeCluster.deployAddons(ctx)
return nil
}
func (c *Cluster) getEtcdProcessHostMap(readyEtcdHosts []*hosts.Host) map[*hosts.Host]v3.Process {

1
cluster/plan.go
View File

@ -292,6 +292,7 @@ func (c *Cluster) BuildProxyProcess() v3.Process {
return v3.Process{
Env: Env,
Args: Env,
NetworkMode: "host",
RestartPolicy: "always",
HealthCheck: v3.HealthCheck{},

28
cluster/state.go
View File

@ -16,19 +16,21 @@ import (
)
func (c *Cluster) SaveClusterState(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig) error {
// Reinitialize kubernetes Client
var err error
c.KubeClient, err = k8s.NewClient(c.LocalKubeConfigPath)
if err != nil {
return fmt.Errorf("Failed to re-initialize Kubernetes Client: %v", err)
}
err = saveClusterCerts(ctx, c.KubeClient, c.Certificates)
if err != nil {
return fmt.Errorf("[certificates] Failed to Save Kubernetes certificates: %v", err)
}
err = saveStateToKubernetes(ctx, c.KubeClient, c.LocalKubeConfigPath, rkeConfig)
if err != nil {
return fmt.Errorf("[state] Failed to save configuration state: %v", err)
if len(c.ControlPlaneHosts) > 0 {
// Reinitialize kubernetes Client
var err error
c.KubeClient, err = k8s.NewClient(c.LocalKubeConfigPath)
if err != nil {
return fmt.Errorf("Failed to re-initialize Kubernetes Client: %v", err)
}
err = saveClusterCerts(ctx, c.KubeClient, c.Certificates)
if err != nil {
return fmt.Errorf("[certificates] Failed to Save Kubernetes certificates: %v", err)
}
err = saveStateToKubernetes(ctx, c.KubeClient, c.LocalKubeConfigPath, rkeConfig)
if err != nil {
return fmt.Errorf("[state] Failed to save configuration state: %v", err)
}
}
return nil
}

4
cluster/validation.go
View File

@ -9,9 +9,7 @@ import (
func (c *Cluster) ValidateCluster() error {
// make sure cluster has at least one controlplane/etcd host
if len(c.ControlPlaneHosts) == 0 {
return fmt.Errorf("Cluster must have at least one control plane host")
}
if len(c.EtcdHosts) == 0 && len(c.Services.Etcd.ExternalURLs) == 0 {
return fmt.Errorf("Cluster must have at least one etcd plane host")
}

9
cmd/up.go
View File

@ -106,11 +106,12 @@ func ClusterUp(
if err != nil {
return APIURL, caCrt, clientCert, clientKey, err
}
APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443")
if len(kubeCluster.ControlPlaneHosts) > 0 {
APIURL = fmt.Sprintf("https://" + kubeCluster.ControlPlaneHosts[0].Address + ":6443")
clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Certificate))
clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Key))
}
caCrt = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.CACertName].Certificate))
clientCert = string(cert.EncodeCertPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Certificate))
clientKey = string(cert.EncodePrivateKeyPEM(kubeCluster.Certificates[pki.KubeAdminCertName].Key))
log.Infof(ctx, "Finished building Kubernetes cluster successfully")
return APIURL, caCrt, clientCert, clientKey, nil

8
hosts/hosts.go
View File

@ -59,8 +59,8 @@ func (h *Host) CleanUpAll(ctx context.Context, cleanerImage string, prsMap map[s
}
func (h *Host) CleanUpWorkerHost(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error {
if h.IsControl {
log.Infof(ctx, "[hosts] Host [%s] is already a controlplane host, skipping cleanup.", h.Address)
if h.IsControl || h.IsEtcd {
log.Infof(ctx, "[hosts] Host [%s] is already a controlplane or etcd host, skipping cleanup.", h.Address)
return nil
}
toCleanPaths := []string{
@ -74,8 +74,8 @@ func (h *Host) CleanUpWorkerHost(ctx context.Context, cleanerImage string, prsMa
}
func (h *Host) CleanUpControlHost(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error {
if h.IsWorker {
log.Infof(ctx, "[hosts] Host [%s] is already a worker host, skipping cleanup.", h.Address)
if h.IsWorker || h.IsEtcd {
log.Infof(ctx, "[hosts] Host [%s] is already a worker or etcd host, skipping cleanup.", h.Address)
return nil
}
toCleanPaths := []string{

6
pki/deploy.go
View File

@ -80,6 +80,9 @@ func doRunDeployer(ctx context.Context, host *hosts.Host, containerEnv []string,
}
func DeployAdminConfig(ctx context.Context, kubeConfig, localConfigPath string) error {
if len(kubeConfig) == 0 {
return nil
}
logrus.Debugf("Deploying admin Kubeconfig locally: %s", kubeConfig)
err := ioutil.WriteFile(localConfigPath, []byte(kubeConfig), 0640)
if err != nil {
@ -120,8 +123,9 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho
KubeSchedulerCertName: true,
KubeProxyCertName: true,
KubeNodeCertName: true,
KubeAdminCertName: true,
KubeAdminCertName: false,
}
for _, etcdHost := range extraHosts {
// Fetch etcd certificates
crtList[GetEtcdCrtName(etcdHost.InternalAddress)] = false

22
pki/pki.go
View File

@ -101,18 +101,20 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
if err != nil {
return nil, err
}
kubeAdminConfig := GetKubeConfigX509WithData(
"https://"+cpHosts[0].Address+":6443",
KubeAdminCertName,
string(cert.EncodeCertPEM(caCrt)),
string(cert.EncodeCertPEM(kubeAdminCrt)),
string(cert.EncodePrivateKeyPEM(kubeAdminKey)))
kubeAdminCertObj := ToCertObject(KubeAdminCertName, KubeAdminCertName, KubeAdminOrganizationName, kubeAdminCrt, kubeAdminKey)
kubeAdminCertObj.Config = kubeAdminConfig
kubeAdminCertObj.ConfigPath = localKubeConfigPath
if len(cpHosts) > 0 {
kubeAdminConfig := GetKubeConfigX509WithData(
"https://"+cpHosts[0].Address+":6443",
KubeAdminCertName,
string(cert.EncodeCertPEM(caCrt)),
string(cert.EncodeCertPEM(kubeAdminCrt)),
string(cert.EncodePrivateKeyPEM(kubeAdminKey)))
kubeAdminCertObj.Config = kubeAdminConfig
kubeAdminCertObj.ConfigPath = localKubeConfigPath
} else {
kubeAdminCertObj.Config = ""
}
certs[KubeAdminCertName] = kubeAdminCertObj
// generate etcd certificate and key
if len(rkeConfig.Services.Etcd.ExternalURLs) > 0 {
clientCert, err := cert.ParseCertsPEM([]byte(rkeConfig.Services.Etcd.Cert))