Merge pull request #548 from galal-hussein/etcd_backup

etcd backup/restore

cluster/certificates.go

@@ -228,3 +228,21 @@ func fetchBackupCertificates(ctx context.Context, backupHosts []*hosts.Host, kub
 	// reporting the last error only.
 	return nil, err
 }
+
+func fetchCertificatesFromEtcd(ctx context.Context, kubeCluster *Cluster) ([]byte, []byte, error) {
+	// Get kubernetes certificates from the etcd hosts
+	certificates := map[string]pki.CertificatePKI{}
+	var err error
+	for _, host := range kubeCluster.EtcdHosts {
+		certificates, err = pki.FetchCertificatesFromHost(ctx, kubeCluster.EtcdHosts, host, kubeCluster.SystemImages.Alpine, kubeCluster.LocalKubeConfigPath, kubeCluster.PrivateRegistriesMap)
+		if certificates != nil {
+			break
+		}
+	}
+	if err != nil || certificates == nil {
+		return nil, nil, fmt.Errorf("Failed to fetch certificates from etcd hosts: %v", err)
+	}
+	clientCert := cert.EncodeCertPEM(certificates[pki.KubeNodeCertName].Certificate)
+	clientkey := cert.EncodePrivateKeyPEM(certificates[pki.KubeNodeCertName].Key)
+	return clientCert, clientkey, nil
+}

cluster/cluster.go

@@ -77,7 +77,12 @@ func (c *Cluster) DeployControlPlane(ctx context.Context) error {
 	if len(c.Services.Etcd.ExternalURLs) > 0 {
 		log.Infof(ctx, "[etcd] External etcd connection string has been specified, skipping etcd plane")
 	} else {
-		if err := services.RunEtcdPlane(ctx, c.EtcdHosts, etcdNodePlanMap, c.LocalConnDialerFactory, c.PrivateRegistriesMap, c.UpdateWorkersOnly, c.SystemImages.Alpine); err != nil {
+		etcdBackup := services.EtcdBackup{
+			Backup:    c.Services.Etcd.Backup,
+			Creation:  c.Services.Etcd.Creation,
+			Retention: c.Services.Etcd.Retention,
+		}
+		if err := services.RunEtcdPlane(ctx, c.EtcdHosts, etcdNodePlanMap, c.LocalConnDialerFactory, c.PrivateRegistriesMap, c.UpdateWorkersOnly, c.SystemImages.Alpine, etcdBackup); err != nil {
 			return fmt.Errorf("[etcd] Failed to bring up Etcd Plane: %v", err)
 		}
 	}

cluster/defaults.go

@@ -27,7 +27,9 @@ const (
 	DefaultNetworkPlugin        = "canal"
 	DefaultNetworkCloudProvider = "none"
 
-	DefaultIngressController = "nginx"
+	DefaultIngressController         = "nginx"
+	DefaultEtcdBackupCreationPeriod  = "5m0s"
+	DefaultEtcdBackupRetentionPeriod = "24h"
 )
 
 func setDefaultIfEmptyMapValue(configMap map[string]string, key string, value string) {
@@ -105,6 +107,8 @@ func (c *Cluster) setClusterServicesDefaults() {
 		&c.Services.Kubelet.Image:                        c.SystemImages.Kubernetes,
 		&c.Services.Kubeproxy.Image:                      c.SystemImages.Kubernetes,
 		&c.Services.Etcd.Image:                           c.SystemImages.Etcd,
+		&c.Services.Etcd.Creation:                        DefaultEtcdBackupCreationPeriod,
+		&c.Services.Etcd.Retention:                       DefaultEtcdBackupRetentionPeriod,
 	}
 	for k, v := range serviceConfigDefaultsMap {
 		setDefaultIfEmpty(k, v)

cluster/etcd.go

@@ -0,0 +1,63 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"path"
+
+	"github.com/rancher/rke/docker"
+	"github.com/rancher/rke/hosts"
+	"github.com/rancher/rke/services"
+	"github.com/rancher/types/apis/management.cattle.io/v3"
+)
+
+func (c *Cluster) BackupEtcd(ctx context.Context, backupName string) error {
+	for _, host := range c.EtcdHosts {
+		if err := services.RunEtcdBackup(ctx, host, c.PrivateRegistriesMap, c.SystemImages.Alpine, c.Services.Etcd.Creation, c.Services.Etcd.Retention, backupName, true); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) RestoreEtcdBackup(ctx context.Context, backupPath string) error {
+	// Stopping all etcd containers
+	for _, host := range c.EtcdHosts {
+		if err := tearDownOldEtcd(ctx, host, c.SystemImages.Alpine, c.PrivateRegistriesMap); err != nil {
+			return err
+		}
+	}
+	// Start restore process on all etcd hosts
+	initCluster := services.GetEtcdInitialCluster(c.EtcdHosts)
+	for _, host := range c.EtcdHosts {
+		if err := services.RestoreEtcdBackup(ctx, host, c.PrivateRegistriesMap, c.SystemImages.Etcd, backupPath, initCluster); err != nil {
+			return fmt.Errorf("[etcd] Failed to restore etcd backup: %v", err)
+		}
+	}
+	// Deploy Etcd Plane
+	etcdNodePlanMap := make(map[string]v3.RKEConfigNodePlan)
+	// Build etcd node plan map
+	for _, etcdHost := range c.EtcdHosts {
+		etcdNodePlanMap[etcdHost.Address] = BuildRKEConfigNodePlan(ctx, c, etcdHost, etcdHost.DockerInfo)
+	}
+	etcdBackup := services.EtcdBackup{
+		Backup:    c.Services.Etcd.Backup,
+		Creation:  c.Services.Etcd.Creation,
+		Retention: c.Services.Etcd.Retention,
+	}
+	if err := services.RunEtcdPlane(ctx, c.EtcdHosts, etcdNodePlanMap, c.LocalConnDialerFactory, c.PrivateRegistriesMap, c.UpdateWorkersOnly, c.SystemImages.Alpine, etcdBackup); err != nil {
+		return fmt.Errorf("[etcd] Failed to bring up Etcd Plane: %v", err)
+	}
+	return nil
+}
+
+func tearDownOldEtcd(ctx context.Context, host *hosts.Host, cleanupImage string, prsMap map[string]v3.PrivateRegistry) error {
+	if err := docker.DoRemoveContainer(ctx, host.DClient, services.EtcdContainerName, host.Address); err != nil {
+		return fmt.Errorf("[etcd] Failed to stop old etcd containers: %v", err)
+	}
+	// cleanup etcd data directory
+	toCleanPaths := []string{
+		path.Join(host.PrefixPath, hosts.ToCleanEtcdDir),
+	}
+	return host.CleanUp(ctx, toCleanPaths, cleanupImage, prsMap)
+}

cluster/plan.go

@@ -593,7 +593,7 @@ func (c *Cluster) BuildEtcdProcess(host *hosts.Host, etcdHosts []*hosts.Host, pr
 	}
 
 	Binds := []string{
-		fmt.Sprintf("%s:/var/lib/rancher/etcd:z", path.Join(prefixPath, "/var/lib/etcd")),
+		fmt.Sprintf("%s:/var/lib/rancher/:z", path.Join(prefixPath, "/var/lib/")),
 		fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(prefixPath, "/etc/kubernetes")),
 	}