mirror of
https://github.com/rancher/rke.git
synced 2025-08-01 23:33:39 +00:00
Add certificate bundle backup with etcd snapshot-save
This commit is contained in:
parent
f6d391b664
commit
2da8682463
@ -11,6 +11,7 @@ import (
|
||||
"github.com/rancher/rke/k8s"
|
||||
"github.com/rancher/rke/log"
|
||||
"github.com/rancher/rke/pki"
|
||||
"github.com/rancher/rke/services"
|
||||
"github.com/sirupsen/logrus"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@ -247,3 +248,40 @@ func fetchCertificatesFromEtcd(ctx context.Context, kubeCluster *Cluster) ([]byt
|
||||
clientkey := cert.EncodePrivateKeyPEM(certificates[pki.KubeNodeCertName].Key)
|
||||
return clientCert, clientkey, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) SaveBackupCertificateBundle(ctx context.Context) error {
|
||||
backupHosts := c.getBackupHosts()
|
||||
var errgrp errgroup.Group
|
||||
|
||||
for _, host := range backupHosts {
|
||||
runHost := host
|
||||
errgrp.Go(func() error {
|
||||
return pki.SaveBackupBundleOnHost(ctx, runHost, c.SystemImages.Alpine, services.EtcdSnapshotPath, c.PrivateRegistriesMap)
|
||||
})
|
||||
}
|
||||
return errgrp.Wait()
|
||||
}
|
||||
|
||||
func (c *Cluster) ExtractBackupCertificateBundle(ctx context.Context) error {
|
||||
backupHosts := c.getBackupHosts()
|
||||
var errgrp errgroup.Group
|
||||
|
||||
for _, host := range backupHosts {
|
||||
runHost := host
|
||||
errgrp.Go(func() error {
|
||||
return pki.ExtractBackupBundleOnHost(ctx, runHost, c.SystemImages.Alpine, services.EtcdSnapshotPath, c.PrivateRegistriesMap)
|
||||
})
|
||||
}
|
||||
return errgrp.Wait()
|
||||
}
|
||||
|
||||
func (c *Cluster) getBackupHosts() []*hosts.Host {
|
||||
var backupHosts []*hosts.Host
|
||||
if len(c.Services.Etcd.ExternalURLs) > 0 {
|
||||
backupHosts = c.ControlPlaneHosts
|
||||
} else {
|
||||
// Save certificates on etcd and controlplane hosts
|
||||
backupHosts = hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, nil)
|
||||
}
|
||||
return backupHosts
|
||||
}
|
||||
|
@ -69,6 +69,10 @@ func SnapshotSaveEtcdHosts(
|
||||
return err
|
||||
}
|
||||
|
||||
if err := kubeCluster.SaveBackupCertificateBundle(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Infof(ctx, "Finished saving snapshot [%s] on all etcd hosts", snapshotName)
|
||||
return nil
|
||||
}
|
||||
@ -91,7 +95,9 @@ func RestoreEtcdSnapshot(
|
||||
if err := kubeCluster.RestoreEtcdSnapshot(ctx, snapshotName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := kubeCluster.ExtractBackupCertificateBundle(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Infof(ctx, "Finished restoring snapshot [%s] on all etcd hosts", snapshotName)
|
||||
return nil
|
||||
}
|
||||
|
@ -8,6 +8,7 @@ const (
|
||||
CertificatesSecretName = "k8s-certs"
|
||||
TempCertPath = "/etc/kubernetes/.tmp/"
|
||||
ClusterConfig = "cluster.yml"
|
||||
BundleCertPath = "/backup/pki.bundle.tar.gz"
|
||||
|
||||
CACertName = "kube-ca"
|
||||
KubeAPICertName = "kube-apiserver"
|
||||
|
72
pki/pki.go
72
pki/pki.go
@ -6,7 +6,11 @@ import (
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"net"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/docker/docker/api/types/container"
|
||||
"github.com/rancher/rke/docker"
|
||||
"github.com/rancher/rke/hosts"
|
||||
"github.com/rancher/rke/log"
|
||||
"github.com/rancher/types/apis/management.cattle.io/v3"
|
||||
@ -29,9 +33,10 @@ type CertificatePKI struct {
|
||||
}
|
||||
|
||||
const (
|
||||
etcdRole = "etcd"
|
||||
controlRole = "controlplane"
|
||||
workerRole = "worker"
|
||||
etcdRole = "etcd"
|
||||
controlRole = "controlplane"
|
||||
workerRole = "worker"
|
||||
BundleCertContainer = "rke-bundle-cert"
|
||||
)
|
||||
|
||||
func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineConfig, configPath, configDir string) (map[string]CertificatePKI, error) {
|
||||
@ -207,3 +212,64 @@ func RegenerateEtcdCertificate(
|
||||
log.Infof(ctx, "[certificates] Successfully generated new etcd-%s certificate and key", etcdHost.InternalAddress)
|
||||
return crtMap, nil
|
||||
}
|
||||
|
||||
func SaveBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSystemImage, etcdSnapshotPath string, prsMap map[string]v3.PrivateRegistry) error {
|
||||
imageCfg := &container.Config{
|
||||
Cmd: []string{
|
||||
"sh",
|
||||
"-c",
|
||||
fmt.Sprintf("tar czvf %s %s", BundleCertPath, path.Join(host.PrefixPath, TempCertPath)),
|
||||
},
|
||||
Image: alpineSystemImage,
|
||||
}
|
||||
hostCfg := &container.HostConfig{
|
||||
|
||||
Binds: []string{
|
||||
fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
|
||||
fmt.Sprintf("%s:/backup:z", etcdSnapshotPath),
|
||||
},
|
||||
Privileged: true,
|
||||
}
|
||||
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, BundleCertContainer, host.Address, "certificates", prsMap); err != nil {
|
||||
return err
|
||||
}
|
||||
status, err := docker.WaitForContainer(ctx, host.DClient, host.Address, BundleCertContainer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if status != 0 {
|
||||
return fmt.Errorf("Failed to run certificate bundle compress, exit status is: %d", status)
|
||||
}
|
||||
return docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer)
|
||||
}
|
||||
|
||||
func ExtractBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSystemImage, etcdSnapshotPath string, prsMap map[string]v3.PrivateRegistry) error {
|
||||
fullTempCertPath := path.Join(host.PrefixPath, TempCertPath)
|
||||
imageCfg := &container.Config{
|
||||
Cmd: []string{
|
||||
"sh",
|
||||
"-c",
|
||||
fmt.Sprintf("if [ -f %s ];then tar xzvf %s -C %s --strip-components %d; fi", BundleCertPath, BundleCertPath, fullTempCertPath, len(strings.Split(fullTempCertPath, "/"))-1),
|
||||
},
|
||||
Image: alpineSystemImage,
|
||||
}
|
||||
hostCfg := &container.HostConfig{
|
||||
|
||||
Binds: []string{
|
||||
fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
|
||||
fmt.Sprintf("%s:/backup:z", etcdSnapshotPath),
|
||||
},
|
||||
Privileged: true,
|
||||
}
|
||||
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, BundleCertContainer, host.Address, "certificates", prsMap); err != nil {
|
||||
return err
|
||||
}
|
||||
status, err := docker.WaitForContainer(ctx, host.DClient, host.Address, BundleCertContainer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if status != 0 {
|
||||
return fmt.Errorf("Failed to run certificate bundle extract, exit status is: %d", status)
|
||||
}
|
||||
return docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer)
|
||||
}
|
||||
|
@ -58,6 +58,9 @@ func RunEtcdPlane(
|
||||
if err := RunEtcdSnapshotSave(ctx, host, prsMap, alpineImage, etcdSnapshot.Creation, etcdSnapshot.Retention, EtcdSnapshotContainerName, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := pki.SaveBackupBundleOnHost(ctx, host, alpineImage, EtcdSnapshotPath, prsMap); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := createLogLink(ctx, host, EtcdContainerName, ETCDRole, alpineImage, prsMap); err != nil {
|
||||
return err
|
||||
|
Loading…
Reference in New Issue
Block a user