diff --git a/cluster/etcd.go b/cluster/etcd.go index 1744410e..3d9b9c3e 100644 --- a/cluster/etcd.go +++ b/cluster/etcd.go @@ -143,6 +143,19 @@ func (c *Cluster) RestoreEtcdSnapshot(ctx context.Context, snapshotPath string) return nil } +func (c *Cluster) RemoveEtcdSnapshot(ctx context.Context, snapshotName string) error { + backupImage := c.getBackupImage() + if !util.IsRancherBackupSupported(c.SystemImages.Alpine) { + log.Warnf(ctx, "Auto local backup sync is not supported in `%s`. Using `%s` instead.", c.SystemImages.Alpine, backupImage) + } + for _, host := range c.EtcdHosts { + if err := services.RunEtcdSnapshotRemove(ctx, host, c.PrivateRegistriesMap, backupImage, snapshotName, true, c.Services.Etcd); err != nil { + return err + } + } + return nil +} + func (c *Cluster) etcdSnapshotChecksum(ctx context.Context, snapshotPath string) bool { log.Infof(ctx, "[etcd] Checking if all snapshots are identical") etcdChecksums := []string{} diff --git a/cmd/etcd.go b/cmd/etcd.go index 79063dda..9fba28e2 100644 --- a/cmd/etcd.go +++ b/cmd/etcd.go @@ -220,3 +220,30 @@ func RestoreEtcdSnapshotFromCli(ctx *cli.Context) error { _, _, _, _, _, err = RestoreEtcdSnapshot(context.Background(), rkeConfig, hosts.DialersOptions{}, flags, etcdSnapshotName) return err } + +func SnapshotRemoveFromEtcdHosts( + ctx context.Context, + rkeConfig *v3.RancherKubernetesEngineConfig, + dialersOptions hosts.DialersOptions, + flags cluster.ExternalFlags, snapshotName string) error { + + log.Infof(ctx, "Starting snapshot remove on etcd hosts") + kubeCluster, err := cluster.InitClusterObject(ctx, rkeConfig, flags) + if err != nil { + return err + } + if err := kubeCluster.SetupDialers(ctx, dialersOptions); err != nil { + return err + } + + if err := kubeCluster.TunnelHosts(ctx, flags); err != nil { + return err + } + + if err := kubeCluster.RemoveEtcdSnapshot(ctx, snapshotName); err != nil { + return err + } + + log.Infof(ctx, "Finished removing snapshot [%s] from all etcd hosts", snapshotName) + return nil +} diff --git a/services/etcd.go b/services/etcd.go index 3d24ce5b..8a700404 100644 --- a/services/etcd.go +++ b/services/etcd.go @@ -3,6 +3,7 @@ package services import ( "fmt" "path" + "path/filepath" "strings" "time" @@ -454,6 +455,53 @@ func RestoreEtcdSnapshot(ctx context.Context, etcdHost *hosts.Host, prsMap map[s return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdRestoreContainerName) } +func RunEtcdSnapshotRemove(ctx context.Context, etcdHost *hosts.Host, prsMap map[string]v3.PrivateRegistry, etcdSnapshotImage string, name string, once bool, es v3.ETCDService) error { + log.Infof(ctx, "[etcd] Removing snapshot [%s] from host [%s]", name, etcdHost.Address) + fullPath := fmt.Sprintf("/backup/%s", name) + // Make sure we have a safe path to remove + safePath, err := filepath.Match("/backup/*", fullPath) + if err != nil { + return fmt.Errorf("failed to validate snapshot name: %v", err) + } + if !safePath { + return fmt.Errorf("malformed snapshot path: %v", fullPath) + } + + imageCfg := &container.Config{ + Cmd: []string{ + "rm", + "-f", + fullPath, + }, + Image: etcdSnapshotImage, + } + hostCfg := &container.HostConfig{ + Binds: []string{ + fmt.Sprintf("%s:/backup", EtcdSnapshotPath), + }, + RestartPolicy: container.RestartPolicy{Name: "no"}, + } + + if err := docker.DoRemoveContainer(ctx, etcdHost.DClient, EtcdSnapshotRemoveContainerName, etcdHost.Address); err != nil { + return err + } + if err := docker.DoRunContainer(ctx, etcdHost.DClient, imageCfg, hostCfg, EtcdSnapshotRemoveContainerName, etcdHost.Address, ETCDRole, prsMap); err != nil { + return err + } + status, _, stderr, err := docker.GetContainerOutput(ctx, etcdHost.DClient, EtcdSnapshotRemoveContainerName, etcdHost.Address) + if status != 0 || err != nil { + if removeErr := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotRemoveContainerName); removeErr != nil { + log.Warnf(ctx, "Failed to remove container [%s]: %v", removeErr) + } + if err != nil { + return err + } + return fmt.Errorf("Failed to remove snapshot [%s] on host [%s], exit code [%d]: %v", EtcdSnapshotRemoveContainerName, etcdHost.Address, status, stderr) + } + + return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotRemoveContainerName) +} + func GetEtcdSnapshotChecksum(ctx context.Context, etcdHost *hosts.Host, prsMap map[string]v3.PrivateRegistry, alpineImage, snapshotName string) (string, error) { var checksum string var err error diff --git a/services/services.go b/services/services.go index fa2ccb6e..ca7577c5 100644 --- a/services/services.go +++ b/services/services.go @@ -30,6 +30,7 @@ const ( EtcdContainerName = "etcd" EtcdSnapshotContainerName = "etcd-rolling-snapshots" EtcdSnapshotOnceContainerName = "etcd-snapshot-once" + EtcdSnapshotRemoveContainerName = "etcd-remove-snapshot" EtcdRestoreContainerName = "etcd-restore" EtcdDownloadBackupContainerName = "etcd-download-backup" EtcdServeBackupContainerName = "etcd-Serve-backup"