diff --git a/docker/docker.go b/docker/docker.go index 4a07080e..68c49769 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -485,3 +485,17 @@ func DoRestartContainer(ctx context.Context, dClient *client.Client, containerNa log.Infof(ctx, "[restart/%s] Successfully restarted container on host [%s]", containerName, hostname) return nil } + +func GetContainerOutput(ctx context.Context, dClient *client.Client, containerName, hostname string) (int64, string, string, error) { + status, err := WaitForContainer(ctx, dClient, hostname, containerName) + if err != nil { + return 1, "", "", err + } + + stderr, stdout, err := GetContainerLogsStdoutStderr(ctx, dClient, containerName, "1", false) + if err != nil { + return 1, "", "", err + } + + return status, stdout, stderr, nil +} diff --git a/services/etcd.go b/services/etcd.go index f9147a8f..d6773b88 100644 --- a/services/etcd.go +++ b/services/etcd.go @@ -290,16 +290,20 @@ func RunEtcdSnapshotSave(ctx context.Context, etcdHost *hosts.Host, prsMap map[s if err := docker.DoRunContainer(ctx, etcdHost.DClient, imageCfg, hostCfg, EtcdSnapshotOnceContainerName, etcdHost.Address, ETCDRole, prsMap); err != nil { return err } - status, err := docker.WaitForContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName) + status, _, stderr, err := docker.GetContainerOutput(ctx, etcdHost.DClient, EtcdSnapshotOnceContainerName, etcdHost.Address) if status != 0 || err != nil { - err := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName) - if err != nil { - return fmt.Errorf("Failed to take etcd snapshot exit code [%d], failed to exit container [%s]: %v ", status, EtcdDownloadBackupContainerName, err) + if removeErr := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName); removeErr != nil { + log.Warnf(ctx, "Failed to remove container [%s]: %v", removeErr) } - return fmt.Errorf("Failed to take etcd snapshot exit code [%d]: %v", status, err) + if err != nil { + return err + } + return fmt.Errorf("Failed to take one-time snapshot, exit code [%d]: %v", status, stderr) } + return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName) } + if err := docker.DoRunContainer(ctx, etcdHost.DClient, imageCfg, hostCfg, EtcdSnapshotContainerName, etcdHost.Address, ETCDRole, prsMap); err != nil { return err } @@ -349,12 +353,16 @@ func DownloadEtcdSnapshot(ctx context.Context, etcdHost *hosts.Host, prsMap map[ if err := docker.DoRunContainer(ctx, etcdHost.DClient, imageCfg, hostCfg, EtcdDownloadBackupContainerName, etcdHost.Address, ETCDRole, prsMap); err != nil { return err } - status, err := docker.WaitForContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdDownloadBackupContainerName) + + status, _, stderr, err := docker.GetContainerOutput(ctx, etcdHost.DClient, EtcdDownloadBackupContainerName, etcdHost.Address) if status != 0 || err != nil { - if err2 := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdDownloadBackupContainerName); err2 != nil { - log.Warnf(ctx, "Failed to delete backup download container [%s]: %v", EtcdDownloadBackupContainerName, err2) + if removeErr := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdDownloadBackupContainerName); removeErr != nil { + log.Warnf(ctx, "Failed to remove container [%s]: %v", removeErr) } - return fmt.Errorf("Failed to download etcd snapshot from s3, exit code [%d]: %v", status, err) + if err != nil { + return err + } + return fmt.Errorf("Failed to download etcd snapshot from s3, exit code [%d]: %v", status, stderr) } return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdDownloadBackupContainerName) }