diff --git a/cluster/certificates.go b/cluster/certificates.go index 4eb11803..f51efe72 100644 --- a/cluster/certificates.go +++ b/cluster/certificates.go @@ -11,6 +11,7 @@ import ( "github.com/rancher/rke/k8s" "github.com/rancher/rke/log" "github.com/rancher/rke/pki" + "github.com/rancher/rke/services" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "k8s.io/client-go/kubernetes" @@ -247,3 +248,40 @@ func fetchCertificatesFromEtcd(ctx context.Context, kubeCluster *Cluster) ([]byt clientkey := cert.EncodePrivateKeyPEM(certificates[pki.KubeNodeCertName].Key) return clientCert, clientkey, nil } + +func (c *Cluster) SaveBackupCertificateBundle(ctx context.Context) error { + backupHosts := c.getBackupHosts() + var errgrp errgroup.Group + + for _, host := range backupHosts { + runHost := host + errgrp.Go(func() error { + return pki.SaveBackupBundleOnHost(ctx, runHost, c.SystemImages.Alpine, services.EtcdSnapshotPath, c.PrivateRegistriesMap) + }) + } + return errgrp.Wait() +} + +func (c *Cluster) ExtractBackupCertificateBundle(ctx context.Context) error { + backupHosts := c.getBackupHosts() + var errgrp errgroup.Group + + for _, host := range backupHosts { + runHost := host + errgrp.Go(func() error { + return pki.ExtractBackupBundleOnHost(ctx, runHost, c.SystemImages.Alpine, services.EtcdSnapshotPath, c.PrivateRegistriesMap) + }) + } + return errgrp.Wait() +} + +func (c *Cluster) getBackupHosts() []*hosts.Host { + var backupHosts []*hosts.Host + if len(c.Services.Etcd.ExternalURLs) > 0 { + backupHosts = c.ControlPlaneHosts + } else { + // Save certificates on etcd and controlplane hosts + backupHosts = hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, nil) + } + return backupHosts +} diff --git a/cmd/etcd.go b/cmd/etcd.go index efd750c5..56f22691 100644 --- a/cmd/etcd.go +++ b/cmd/etcd.go @@ -69,6 +69,10 @@ func SnapshotSaveEtcdHosts( return err } + if err := kubeCluster.SaveBackupCertificateBundle(ctx); err != nil { + return err + } + log.Infof(ctx, "Finished saving snapshot [%s] on all etcd hosts", snapshotName) return nil } @@ -91,7 +95,9 @@ func RestoreEtcdSnapshot( if err := kubeCluster.RestoreEtcdSnapshot(ctx, snapshotName); err != nil { return err } - + if err := kubeCluster.ExtractBackupCertificateBundle(ctx); err != nil { + return err + } log.Infof(ctx, "Finished restoring snapshot [%s] on all etcd hosts", snapshotName) return nil } diff --git a/pki/constants.go b/pki/constants.go index 3e9bcec7..f6b5bdba 100644 --- a/pki/constants.go +++ b/pki/constants.go @@ -8,6 +8,7 @@ const ( CertificatesSecretName = "k8s-certs" TempCertPath = "/etc/kubernetes/.tmp/" ClusterConfig = "cluster.yml" + BundleCertPath = "/backup/pki.bundle.tar.gz" CACertName = "kube-ca" KubeAPICertName = "kube-apiserver" diff --git a/pki/pki.go b/pki/pki.go index 489c1ddd..3535cf9e 100644 --- a/pki/pki.go +++ b/pki/pki.go @@ -6,7 +6,11 @@ import ( "crypto/x509" "fmt" "net" + "path" + "strings" + "github.com/docker/docker/api/types/container" + "github.com/rancher/rke/docker" "github.com/rancher/rke/hosts" "github.com/rancher/rke/log" "github.com/rancher/types/apis/management.cattle.io/v3" @@ -29,9 +33,10 @@ type CertificatePKI struct { } const ( - etcdRole = "etcd" - controlRole = "controlplane" - workerRole = "worker" + etcdRole = "etcd" + controlRole = "controlplane" + workerRole = "worker" + BundleCertContainer = "rke-bundle-cert" ) func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) (map[string]CertificatePKI, error) { @@ -207,3 +212,64 @@ func RegenerateEtcdCertificate( log.Infof(ctx, "[certificates] Successfully generated new etcd-%s certificate and key", etcdHost.InternalAddress) return crtMap, nil } + +func SaveBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSystemImage, etcdSnapshotPath string, prsMap map[string]v3.PrivateRegistry) error { + imageCfg := &container.Config{ + Cmd: []string{ + "sh", + "-c", + fmt.Sprintf("tar czvf %s %s", BundleCertPath, path.Join(host.PrefixPath, TempCertPath)), + }, + Image: alpineSystemImage, + } + hostCfg := &container.HostConfig{ + + Binds: []string{ + fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")), + fmt.Sprintf("%s:/backup:z", etcdSnapshotPath), + }, + Privileged: true, + } + if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, BundleCertContainer, host.Address, "certificates", prsMap); err != nil { + return err + } + status, err := docker.WaitForContainer(ctx, host.DClient, host.Address, BundleCertContainer) + if err != nil { + return err + } + if status != 0 { + return fmt.Errorf("Failed to run certificate bundle compress, exit status is: %d", status) + } + return docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer) +} + +func ExtractBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSystemImage, etcdSnapshotPath string, prsMap map[string]v3.PrivateRegistry) error { + fullTempCertPath := path.Join(host.PrefixPath, TempCertPath) + imageCfg := &container.Config{ + Cmd: []string{ + "sh", + "-c", + fmt.Sprintf("if [ -f %s ];then tar xzvf %s -C %s --strip-components %d; fi", BundleCertPath, BundleCertPath, fullTempCertPath, len(strings.Split(fullTempCertPath, "/"))-1), + }, + Image: alpineSystemImage, + } + hostCfg := &container.HostConfig{ + + Binds: []string{ + fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")), + fmt.Sprintf("%s:/backup:z", etcdSnapshotPath), + }, + Privileged: true, + } + if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, BundleCertContainer, host.Address, "certificates", prsMap); err != nil { + return err + } + status, err := docker.WaitForContainer(ctx, host.DClient, host.Address, BundleCertContainer) + if err != nil { + return err + } + if status != 0 { + return fmt.Errorf("Failed to run certificate bundle extract, exit status is: %d", status) + } + return docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer) +} diff --git a/services/etcd.go b/services/etcd.go index 7ffc7306..da5e71e8 100644 --- a/services/etcd.go +++ b/services/etcd.go @@ -58,6 +58,9 @@ func RunEtcdPlane( if err := RunEtcdSnapshotSave(ctx, host, prsMap, alpineImage, etcdSnapshot.Creation, etcdSnapshot.Retention, EtcdSnapshotContainerName, false); err != nil { return err } + if err := pki.SaveBackupBundleOnHost(ctx, host, alpineImage, EtcdSnapshotPath, prsMap); err != nil { + return err + } } if err := createLogLink(ctx, host, EtcdContainerName, ETCDRole, alpineImage, prsMap); err != nil { return err