Use supported rke-tools for backup and restore in legacy clusters

cluster/etcd.go

@@ -3,7 +3,6 @@ package cluster
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
@@ -12,13 +11,13 @@ import (
 	"github.com/rancher/rke/util"
 )
 
-const (
-	SupportedSyncToolsVersion = "0.1.22"
-)
-
 func (c *Cluster) SnapshotEtcd(ctx context.Context, snapshotName string) error {
+	backupImage := c.getBackupImage()
+	if !util.IsRancherBackupSupported(c.SystemImages.Alpine) {
+		log.Warnf(ctx, "Auto local backup sync is not supported in `%s`. Using `%s` instead.", c.SystemImages.Alpine, backupImage)
+	}
 	for _, host := range c.EtcdHosts {
-		if err := services.RunEtcdSnapshotSave(ctx, host, c.PrivateRegistriesMap, c.SystemImages.Alpine, snapshotName, true, c.Services.Etcd); err != nil {
+		if err := services.RunEtcdSnapshotSave(ctx, host, c.PrivateRegistriesMap, backupImage, snapshotName, true, c.Services.Etcd); err != nil {
 			return err
 		}
 	}
@@ -28,17 +27,19 @@ func (c *Cluster) SnapshotEtcd(ctx context.Context, snapshotName string) error {
 func (c *Cluster) PrepareBackup(ctx context.Context, snapshotPath string) error {
 	// local backup case
 	var backupServer *hosts.Host
-	// stop etcd on all etcd nodes, we need this because we start the backup server on the same port
+	backupImage := c.getBackupImage()
-	if !isAutoSyncSupported(c.SystemImages.Alpine) {
+	if !util.IsRancherBackupSupported(c.SystemImages.Alpine) {
-		log.Warnf(ctx, "Auto local backup sync is not supported. Use `rancher/rke-tools:%s` or up", SupportedSyncToolsVersion)
+		log.Warnf(ctx, "Auto local backup sync is not supported in `%s`. Using `%s` instead.", c.SystemImages.Alpine, backupImage)
-	} else if c.Services.Etcd.BackupConfig == nil || // legacy rke local backup
+	}
+	if c.Services.Etcd.BackupConfig == nil || // legacy rke local backup
 		(c.Services.Etcd.BackupConfig != nil && c.Services.Etcd.BackupConfig.S3BackupConfig == nil) { // rancher local backup, no s3
+		// stop etcd on all etcd nodes, we need this because we start the backup server on the same port
 		for _, host := range c.EtcdHosts {
 			if err := docker.StopContainer(ctx, host.DClient, host.Address, services.EtcdContainerName); err != nil {
 				log.Warnf(ctx, "failed to stop etcd container on host [%s]: %v", host.Address, err)
 			}
 			if backupServer == nil { // start the download server, only one node should have it!
-				if err := services.StartBackupServer(ctx, host, c.PrivateRegistriesMap, c.SystemImages.Alpine, snapshotPath); err != nil {
+				if err := services.StartBackupServer(ctx, host, c.PrivateRegistriesMap, backupImage, snapshotPath); err != nil {
 					log.Warnf(ctx, "failed to start backup server on host [%s]: %v", host.Address, err)
 					continue
 				}
@@ -50,7 +51,7 @@ func (c *Cluster) PrepareBackup(ctx context.Context, snapshotPath string) error
 			if backupServer != nil && host.Address == backupServer.Address { // we skip the backup server if it's there
 				continue
 			}
-			if err := services.DownloadEtcdSnapshotFromBackupServer(ctx, host, c.PrivateRegistriesMap, c.SystemImages.Alpine, snapshotPath, backupServer); err != nil {
+			if err := services.DownloadEtcdSnapshotFromBackupServer(ctx, host, c.PrivateRegistriesMap, backupImage, snapshotPath, backupServer); err != nil {
 				return err
 			}
 		}
@@ -63,7 +64,7 @@ func (c *Cluster) PrepareBackup(ctx context.Context, snapshotPath string) error
 	// s3 backup case
 	if c.Services.Etcd.BackupConfig != nil && c.Services.Etcd.BackupConfig.S3BackupConfig != nil {
 		for _, host := range c.EtcdHosts {
-			if err := services.DownloadEtcdSnapshotFromS3(ctx, host, c.PrivateRegistriesMap, c.SystemImages.Alpine, snapshotPath, c.Services.Etcd); err != nil {
+			if err := services.DownloadEtcdSnapshotFromS3(ctx, host, c.PrivateRegistriesMap, backupImage, snapshotPath, c.Services.Etcd); err != nil {
 				return err
 			}
 		}
@@ -75,6 +76,7 @@ func (c *Cluster) PrepareBackup(ctx context.Context, snapshotPath string) error
 	}
 	return nil
 }
+
 func (c *Cluster) RestoreEtcdSnapshot(ctx context.Context, snapshotPath string) error {
 	// Start restore process on all etcd hosts
 	initCluster := services.GetEtcdInitialCluster(c.EtcdHosts)
@@ -89,8 +91,10 @@ func (c *Cluster) RestoreEtcdSnapshot(ctx context.Context, snapshotPath string)
 func (c *Cluster) etcdSnapshotChecksum(ctx context.Context, snapshotPath string) bool {
 	log.Infof(ctx, "[etcd] Checking if all snapshots are identical")
 	etcdChecksums := []string{}
+	backupImage := c.getBackupImage()
+
 	for _, etcdHost := range c.EtcdHosts {
-		checksum, err := services.GetEtcdSnapshotChecksum(ctx, etcdHost, c.PrivateRegistriesMap, c.SystemImages.Alpine, snapshotPath)
+		checksum, err := services.GetEtcdSnapshotChecksum(ctx, etcdHost, c.PrivateRegistriesMap, backupImage, snapshotPath)
 		if err != nil {
 			return false
 		}
@@ -106,21 +110,9 @@ func (c *Cluster) etcdSnapshotChecksum(ctx context.Context, snapshotPath string)
 	return true
 }
 
-func isAutoSyncSupported(image string) bool {
+func (c *Cluster) getBackupImage() string {
-	v := strings.Split(image, ":")
+	if util.IsRancherBackupSupported(c.SystemImages.Alpine) {
-	last := v[len(v)-1]
+		return c.SystemImages.Alpine
-
-	sv, err := util.StrToSemVer(last)
-	if err != nil {
-		return false
 	}
-
+	return util.GetDefaultRKETools()
-	supported, err := util.StrToSemVer(SupportedSyncToolsVersion)
-	if err != nil {
-		return false
-	}
-	if sv.LessThan(*supported) {
-		return false
-	}
-	return true
 }

services/etcd.go

@@ -269,10 +269,14 @@ func IsEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.
 
 func RunEtcdSnapshotSave(ctx context.Context, etcdHost *hosts.Host, prsMap map[string]v3.PrivateRegistry, etcdSnapshotImage string, name string, once bool, es v3.ETCDService) error {
 	log.Infof(ctx, "[etcd] Saving snapshot [%s] on host [%s]", name, etcdHost.Address)
+	backupCmd := "etcd-backup"
+	if !util.IsRancherBackupSupported(etcdSnapshotImage) {
+		backupCmd = "rolling-backup"
+	}
 	imageCfg := &container.Config{
 		Cmd: []string{
 			"/opt/rke-tools/rke-etcd-backup",
-			"etcd-backup",
+			backupCmd,
 			"save",
 			"--cacert", pki.GetCertPath(pki.CACertName),
 			"--cert", pki.GetCertPath(pki.KubeNodeCertName),
@@ -330,7 +334,6 @@ func RunEtcdSnapshotSave(ctx context.Context, etcdHost *hosts.Host, prsMap map[s
 	if snapshotCont.State.Status == "exited" || snapshotCont.State.Restarting {
 		log.Warnf(ctx, "Etcd rolling snapshot container failed to start correctly")
 		return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotContainerName)
-
 	}
 	return nil
 }

util/util.go

@@ -12,6 +12,8 @@ import (
 
 const (
 	WorkerThreads = 50
+
+	SupportedSyncToolsVersion = "0.1.22"
 )
 
 func StrToSemVer(version string) (*semver.Version, error) {
@@ -108,3 +110,26 @@ func ValidateVersion(version string) error {
 
 	return nil
 }
+
+func GetDefaultRKETools() string {
+	return v3.AllK8sVersions[v3.DefaultK8s].Alpine
+}
+
+func IsRancherBackupSupported(image string) bool {
+	v := strings.Split(image, ":")
+	last := v[len(v)-1]
+
+	sv, err := StrToSemVer(last)
+	if err != nil {
+		return false
+	}
+
+	supported, err := StrToSemVer(SupportedSyncToolsVersion)
+	if err != nil {
+		return false
+	}
+	if sv.LessThan(*supported) {
+		return false
+	}
+	return true
+}