Add certificate bundle backup with etcd snapshot-save

cluster/certificates.go

@@ -11,6 +11,7 @@ import (
 	"github.com/rancher/rke/k8s"
 	"github.com/rancher/rke/log"
 	"github.com/rancher/rke/pki"
+	"github.com/rancher/rke/services"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 	"k8s.io/client-go/kubernetes"
@@ -247,3 +248,40 @@ func fetchCertificatesFromEtcd(ctx context.Context, kubeCluster *Cluster) ([]byt
 	clientkey := cert.EncodePrivateKeyPEM(certificates[pki.KubeNodeCertName].Key)
 	return clientCert, clientkey, nil
 }
+
+func (c *Cluster) SaveBackupCertificateBundle(ctx context.Context) error {
+	backupHosts := c.getBackupHosts()
+	var errgrp errgroup.Group
+
+	for _, host := range backupHosts {
+		runHost := host
+		errgrp.Go(func() error {
+			return pki.SaveBackupBundleOnHost(ctx, runHost, c.SystemImages.Alpine, services.EtcdSnapshotPath, c.PrivateRegistriesMap)
+		})
+	}
+	return errgrp.Wait()
+}
+
+func (c *Cluster) ExtractBackupCertificateBundle(ctx context.Context) error {
+	backupHosts := c.getBackupHosts()
+	var errgrp errgroup.Group
+
+	for _, host := range backupHosts {
+		runHost := host
+		errgrp.Go(func() error {
+			return pki.ExtractBackupBundleOnHost(ctx, runHost, c.SystemImages.Alpine, services.EtcdSnapshotPath, c.PrivateRegistriesMap)
+		})
+	}
+	return errgrp.Wait()
+}
+
+func (c *Cluster) getBackupHosts() []*hosts.Host {
+	var backupHosts []*hosts.Host
+	if len(c.Services.Etcd.ExternalURLs) > 0 {
+		backupHosts = c.ControlPlaneHosts
+	} else {
+		// Save certificates on etcd and controlplane hosts
+		backupHosts = hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, nil)
+	}
+	return backupHosts
+}

cmd/etcd.go

@@ -69,6 +69,10 @@ func SnapshotSaveEtcdHosts(
 		return err
 	}
 
+	if err := kubeCluster.SaveBackupCertificateBundle(ctx); err != nil {
+		return err
+	}
+
 	log.Infof(ctx, "Finished saving snapshot [%s] on all etcd hosts", snapshotName)
 	return nil
 }
@@ -91,7 +95,9 @@ func RestoreEtcdSnapshot(
 	if err := kubeCluster.RestoreEtcdSnapshot(ctx, snapshotName); err != nil {
 		return err
 	}
-
+	if err := kubeCluster.ExtractBackupCertificateBundle(ctx); err != nil {
+		return err
+	}
 	log.Infof(ctx, "Finished restoring snapshot [%s] on all etcd hosts", snapshotName)
 	return nil
 }

pki/constants.go

@@ -8,6 +8,7 @@ const (
 	CertificatesSecretName  = "k8s-certs"
 	TempCertPath            = "/etc/kubernetes/.tmp/"
 	ClusterConfig           = "cluster.yml"
+	BundleCertPath          = "/backup/pki.bundle.tar.gz"
 
 	CACertName             = "kube-ca"
 	KubeAPICertName        = "kube-apiserver"

pki/pki.go

@@ -6,7 +6,11 @@ import (
 	"crypto/x509"
 	"fmt"
 	"net"
+	"path"
+	"strings"
 
+	"github.com/docker/docker/api/types/container"
+	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/log"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
@@ -29,9 +33,10 @@ type CertificatePKI struct {
 }
 
 const (
-	etcdRole    = "etcd"
+	etcdRole            = "etcd"
-	controlRole = "controlplane"
+	controlRole         = "controlplane"
-	workerRole  = "worker"
+	workerRole          = "worker"
+	BundleCertContainer = "rke-bundle-cert"
 )
 
 func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) (map[string]CertificatePKI, error) {
@@ -207,3 +212,64 @@ func RegenerateEtcdCertificate(
 	log.Infof(ctx, "[certificates] Successfully generated new etcd-%s certificate and key", etcdHost.InternalAddress)
 	return crtMap, nil
 }
+
+func SaveBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSystemImage, etcdSnapshotPath string, prsMap map[string]v3.PrivateRegistry) error {
+	imageCfg := &container.Config{
+		Cmd: []string{
+			"sh",
+			"-c",
+			fmt.Sprintf("tar czvf %s %s", BundleCertPath, path.Join(host.PrefixPath, TempCertPath)),
+		},
+		Image: alpineSystemImage,
+	}
+	hostCfg := &container.HostConfig{
+
+		Binds: []string{
+			fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
+			fmt.Sprintf("%s:/backup:z", etcdSnapshotPath),
+		},
+		Privileged: true,
+	}
+	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, BundleCertContainer, host.Address, "certificates", prsMap); err != nil {
+		return err
+	}
+	status, err := docker.WaitForContainer(ctx, host.DClient, host.Address, BundleCertContainer)
+	if err != nil {
+		return err
+	}
+	if status != 0 {
+		return fmt.Errorf("Failed to run certificate bundle compress, exit status is: %d", status)
+	}
+	return docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer)
+}
+
+func ExtractBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSystemImage, etcdSnapshotPath string, prsMap map[string]v3.PrivateRegistry) error {
+	fullTempCertPath := path.Join(host.PrefixPath, TempCertPath)
+	imageCfg := &container.Config{
+		Cmd: []string{
+			"sh",
+			"-c",
+			fmt.Sprintf("if [ -f %s ];then tar xzvf %s -C %s --strip-components %d; fi", BundleCertPath, BundleCertPath, fullTempCertPath, len(strings.Split(fullTempCertPath, "/"))-1),
+		},
+		Image: alpineSystemImage,
+	}
+	hostCfg := &container.HostConfig{
+
+		Binds: []string{
+			fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
+			fmt.Sprintf("%s:/backup:z", etcdSnapshotPath),
+		},
+		Privileged: true,
+	}
+	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, BundleCertContainer, host.Address, "certificates", prsMap); err != nil {
+		return err
+	}
+	status, err := docker.WaitForContainer(ctx, host.DClient, host.Address, BundleCertContainer)
+	if err != nil {
+		return err
+	}
+	if status != 0 {
+		return fmt.Errorf("Failed to run certificate bundle extract, exit status is: %d", status)
+	}
+	return docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer)
+}

services/etcd.go

@@ -58,6 +58,9 @@ func RunEtcdPlane(
 			if err := RunEtcdSnapshotSave(ctx, host, prsMap, alpineImage, etcdSnapshot.Creation, etcdSnapshot.Retention, EtcdSnapshotContainerName, false); err != nil {
 				return err
 			}
+			if err := pki.SaveBackupBundleOnHost(ctx, host, alpineImage, EtcdSnapshotPath, prsMap); err != nil {
+				return err
+			}
 		}
 		if err := createLogLink(ctx, host, EtcdContainerName, ETCDRole, alpineImage, prsMap); err != nil {
 			return err