Add RemoveEtcdSnapshot

cluster/etcd.go

@@ -143,6 +143,19 @@ func (c *Cluster) RestoreEtcdSnapshot(ctx context.Context, snapshotPath string)
 	return nil
 }
 
+func (c *Cluster) RemoveEtcdSnapshot(ctx context.Context, snapshotName string) error {
+	backupImage := c.getBackupImage()
+	if !util.IsRancherBackupSupported(c.SystemImages.Alpine) {
+		log.Warnf(ctx, "Auto local backup sync is not supported in `%s`. Using `%s` instead.", c.SystemImages.Alpine, backupImage)
+	}
+	for _, host := range c.EtcdHosts {
+		if err := services.RunEtcdSnapshotRemove(ctx, host, c.PrivateRegistriesMap, backupImage, snapshotName, true, c.Services.Etcd); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (c *Cluster) etcdSnapshotChecksum(ctx context.Context, snapshotPath string) bool {
 	log.Infof(ctx, "[etcd] Checking if all snapshots are identical")
 	etcdChecksums := []string{}

cmd/etcd.go

@@ -220,3 +220,30 @@ func RestoreEtcdSnapshotFromCli(ctx *cli.Context) error {
 	_, _, _, _, _, err = RestoreEtcdSnapshot(context.Background(), rkeConfig, hosts.DialersOptions{}, flags, etcdSnapshotName)
 	return err
 }
+
+func SnapshotRemoveFromEtcdHosts(
+	ctx context.Context,
+	rkeConfig *v3.RancherKubernetesEngineConfig,
+	dialersOptions hosts.DialersOptions,
+	flags cluster.ExternalFlags, snapshotName string) error {
+
+	log.Infof(ctx, "Starting snapshot remove on etcd hosts")
+	kubeCluster, err := cluster.InitClusterObject(ctx, rkeConfig, flags)
+	if err != nil {
+		return err
+	}
+	if err := kubeCluster.SetupDialers(ctx, dialersOptions); err != nil {
+		return err
+	}
+
+	if err := kubeCluster.TunnelHosts(ctx, flags); err != nil {
+		return err
+	}
+
+	if err := kubeCluster.RemoveEtcdSnapshot(ctx, snapshotName); err != nil {
+		return err
+	}
+
+	log.Infof(ctx, "Finished removing snapshot [%s] from all etcd hosts", snapshotName)
+	return nil
+}

services/etcd.go

@@ -3,6 +3,7 @@ package services
 import (
 	"fmt"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -454,6 +455,53 @@ func RestoreEtcdSnapshot(ctx context.Context, etcdHost *hosts.Host, prsMap map[s
 	return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdRestoreContainerName)
 }
 
+func RunEtcdSnapshotRemove(ctx context.Context, etcdHost *hosts.Host, prsMap map[string]v3.PrivateRegistry, etcdSnapshotImage string, name string, once bool, es v3.ETCDService) error {
+	log.Infof(ctx, "[etcd] Removing snapshot [%s] from host [%s]", name, etcdHost.Address)
+	fullPath := fmt.Sprintf("/backup/%s", name)
+	// Make sure we have a safe path to remove
+	safePath, err := filepath.Match("/backup/*", fullPath)
+	if err != nil {
+		return fmt.Errorf("failed to validate snapshot name: %v", err)
+	}
+	if !safePath {
+		return fmt.Errorf("malformed snapshot path: %v", fullPath)
+	}
+
+	imageCfg := &container.Config{
+		Cmd: []string{
+			"rm",
+			"-f",
+			fullPath,
+		},
+		Image: etcdSnapshotImage,
+	}
+	hostCfg := &container.HostConfig{
+		Binds: []string{
+			fmt.Sprintf("%s:/backup", EtcdSnapshotPath),
+		},
+		RestartPolicy: container.RestartPolicy{Name: "no"},
+	}
+
+	if err := docker.DoRemoveContainer(ctx, etcdHost.DClient, EtcdSnapshotRemoveContainerName, etcdHost.Address); err != nil {
+		return err
+	}
+	if err := docker.DoRunContainer(ctx, etcdHost.DClient, imageCfg, hostCfg, EtcdSnapshotRemoveContainerName, etcdHost.Address, ETCDRole, prsMap); err != nil {
+		return err
+	}
+	status, _, stderr, err := docker.GetContainerOutput(ctx, etcdHost.DClient, EtcdSnapshotRemoveContainerName, etcdHost.Address)
+	if status != 0 || err != nil {
+		if removeErr := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotRemoveContainerName); removeErr != nil {
+			log.Warnf(ctx, "Failed to remove container [%s]: %v", removeErr)
+		}
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf("Failed to remove snapshot [%s] on host [%s], exit code [%d]: %v", EtcdSnapshotRemoveContainerName, etcdHost.Address, status, stderr)
+	}
+
+	return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotRemoveContainerName)
+}
+
 func GetEtcdSnapshotChecksum(ctx context.Context, etcdHost *hosts.Host, prsMap map[string]v3.PrivateRegistry, alpineImage, snapshotName string) (string, error) {
 	var checksum string
 	var err error

services/services.go

@@ -30,6 +30,7 @@ const (
 	EtcdContainerName               = "etcd"
 	EtcdSnapshotContainerName       = "etcd-rolling-snapshots"
 	EtcdSnapshotOnceContainerName   = "etcd-snapshot-once"
+	EtcdSnapshotRemoveContainerName = "etcd-remove-snapshot"
 	EtcdRestoreContainerName        = "etcd-restore"
 	EtcdDownloadBackupContainerName = "etcd-download-backup"
 	EtcdServeBackupContainerName    = "etcd-Serve-backup"