1
0
mirror of https://github.com/rancher/rke.git synced 2025-08-01 23:33:39 +00:00

Add certificate bundle backup with etcd snapshot-save

This commit is contained in:
galal-hussein 2018-05-29 17:21:24 +02:00 committed by Alena Prokharchyk
parent f6d391b664
commit 2da8682463
5 changed files with 118 additions and 4 deletions

View File

@ -11,6 +11,7 @@ import (
"github.com/rancher/rke/k8s"
"github.com/rancher/rke/log"
"github.com/rancher/rke/pki"
"github.com/rancher/rke/services"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"k8s.io/client-go/kubernetes"
@ -247,3 +248,40 @@ func fetchCertificatesFromEtcd(ctx context.Context, kubeCluster *Cluster) ([]byt
clientkey := cert.EncodePrivateKeyPEM(certificates[pki.KubeNodeCertName].Key)
return clientCert, clientkey, nil
}
func (c *Cluster) SaveBackupCertificateBundle(ctx context.Context) error {
backupHosts := c.getBackupHosts()
var errgrp errgroup.Group
for _, host := range backupHosts {
runHost := host
errgrp.Go(func() error {
return pki.SaveBackupBundleOnHost(ctx, runHost, c.SystemImages.Alpine, services.EtcdSnapshotPath, c.PrivateRegistriesMap)
})
}
return errgrp.Wait()
}
func (c *Cluster) ExtractBackupCertificateBundle(ctx context.Context) error {
backupHosts := c.getBackupHosts()
var errgrp errgroup.Group
for _, host := range backupHosts {
runHost := host
errgrp.Go(func() error {
return pki.ExtractBackupBundleOnHost(ctx, runHost, c.SystemImages.Alpine, services.EtcdSnapshotPath, c.PrivateRegistriesMap)
})
}
return errgrp.Wait()
}
func (c *Cluster) getBackupHosts() []*hosts.Host {
var backupHosts []*hosts.Host
if len(c.Services.Etcd.ExternalURLs) > 0 {
backupHosts = c.ControlPlaneHosts
} else {
// Save certificates on etcd and controlplane hosts
backupHosts = hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, nil)
}
return backupHosts
}

View File

@ -69,6 +69,10 @@ func SnapshotSaveEtcdHosts(
return err
}
if err := kubeCluster.SaveBackupCertificateBundle(ctx); err != nil {
return err
}
log.Infof(ctx, "Finished saving snapshot [%s] on all etcd hosts", snapshotName)
return nil
}
@ -91,7 +95,9 @@ func RestoreEtcdSnapshot(
if err := kubeCluster.RestoreEtcdSnapshot(ctx, snapshotName); err != nil {
return err
}
if err := kubeCluster.ExtractBackupCertificateBundle(ctx); err != nil {
return err
}
log.Infof(ctx, "Finished restoring snapshot [%s] on all etcd hosts", snapshotName)
return nil
}

View File

@ -8,6 +8,7 @@ const (
CertificatesSecretName = "k8s-certs"
TempCertPath = "/etc/kubernetes/.tmp/"
ClusterConfig = "cluster.yml"
BundleCertPath = "/backup/pki.bundle.tar.gz"
CACertName = "kube-ca"
KubeAPICertName = "kube-apiserver"

View File

@ -6,7 +6,11 @@ import (
"crypto/x509"
"fmt"
"net"
"path"
"strings"
"github.com/docker/docker/api/types/container"
"github.com/rancher/rke/docker"
"github.com/rancher/rke/hosts"
"github.com/rancher/rke/log"
"github.com/rancher/types/apis/management.cattle.io/v3"
@ -29,9 +33,10 @@ type CertificatePKI struct {
}
const (
etcdRole = "etcd"
controlRole = "controlplane"
workerRole = "worker"
etcdRole = "etcd"
controlRole = "controlplane"
workerRole = "worker"
BundleCertContainer = "rke-bundle-cert"
)
func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) (map[string]CertificatePKI, error) {
@ -207,3 +212,64 @@ func RegenerateEtcdCertificate(
log.Infof(ctx, "[certificates] Successfully generated new etcd-%s certificate and key", etcdHost.InternalAddress)
return crtMap, nil
}
func SaveBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSystemImage, etcdSnapshotPath string, prsMap map[string]v3.PrivateRegistry) error {
imageCfg := &container.Config{
Cmd: []string{
"sh",
"-c",
fmt.Sprintf("tar czvf %s %s", BundleCertPath, path.Join(host.PrefixPath, TempCertPath)),
},
Image: alpineSystemImage,
}
hostCfg := &container.HostConfig{
Binds: []string{
fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
fmt.Sprintf("%s:/backup:z", etcdSnapshotPath),
},
Privileged: true,
}
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, BundleCertContainer, host.Address, "certificates", prsMap); err != nil {
return err
}
status, err := docker.WaitForContainer(ctx, host.DClient, host.Address, BundleCertContainer)
if err != nil {
return err
}
if status != 0 {
return fmt.Errorf("Failed to run certificate bundle compress, exit status is: %d", status)
}
return docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer)
}
func ExtractBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSystemImage, etcdSnapshotPath string, prsMap map[string]v3.PrivateRegistry) error {
fullTempCertPath := path.Join(host.PrefixPath, TempCertPath)
imageCfg := &container.Config{
Cmd: []string{
"sh",
"-c",
fmt.Sprintf("if [ -f %s ];then tar xzvf %s -C %s --strip-components %d; fi", BundleCertPath, BundleCertPath, fullTempCertPath, len(strings.Split(fullTempCertPath, "/"))-1),
},
Image: alpineSystemImage,
}
hostCfg := &container.HostConfig{
Binds: []string{
fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
fmt.Sprintf("%s:/backup:z", etcdSnapshotPath),
},
Privileged: true,
}
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, BundleCertContainer, host.Address, "certificates", prsMap); err != nil {
return err
}
status, err := docker.WaitForContainer(ctx, host.DClient, host.Address, BundleCertContainer)
if err != nil {
return err
}
if status != 0 {
return fmt.Errorf("Failed to run certificate bundle extract, exit status is: %d", status)
}
return docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer)
}

View File

@ -58,6 +58,9 @@ func RunEtcdPlane(
if err := RunEtcdSnapshotSave(ctx, host, prsMap, alpineImage, etcdSnapshot.Creation, etcdSnapshot.Retention, EtcdSnapshotContainerName, false); err != nil {
return err
}
if err := pki.SaveBackupBundleOnHost(ctx, host, alpineImage, EtcdSnapshotPath, prsMap); err != nil {
return err
}
}
if err := createLogLink(ctx, host, EtcdContainerName, ETCDRole, alpineImage, prsMap); err != nil {
return err