rancher/rke
1
0
Fork 0
mirror of https://github.com/rancher/rke.git synced 2025-08-17 22:37:01 +00:00
Code Issues Releases Wiki Activity

Improve etcd snapshot logging

Browse Source
This commit is contained in:
Sebastiaan van Steenis 2019-08-21 23:08:30 +02:00 committed by Alena Prokharchyk
parent fef1c04eee
commit 65df510251
2 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/etcd.go
View File

@ -108,7 +108,7 @@ func SnapshotSaveEtcdHosts(
return err return err
} }
log.Infof(ctx, "Finished saving snapshot [%s] on all etcd hosts", snapshotName) log.Infof(ctx, "Finished saving/uploading snapshot [%s] on all etcd hosts", snapshotName)
return nil return nil
} }

28
services/etcd.go
View File

@ -305,16 +305,16 @@ func RunEtcdSnapshotSave(ctx context.Context, etcdHost *hosts.Host, prsMap map[s
Image: etcdSnapshotImage, Image: etcdSnapshotImage,
Env: es.ExtraEnv, Env: es.ExtraEnv,
} }
// Configure imageCfg for one time snapshot
if once { if once {
log.Infof(ctx, "[etcd] Running snapshot save once on host [%s]", etcdHost.Address)
imageCfg.Cmd = append(imageCfg.Cmd, "--once") imageCfg.Cmd = append(imageCfg.Cmd, "--once")
restartPolicy = "no" restartPolicy = "no"
// Configure imageCfg for rolling snapshots
} else if es.BackupConfig == nil { } else if es.BackupConfig == nil {
log.Infof(ctx, "[etcd] Running snapshot container [%s] on host [%s]", EtcdSnapshotOnceContainerName, etcdHost.Address)
imageCfg.Cmd = append(imageCfg.Cmd, "--retention="+es.Retention) imageCfg.Cmd = append(imageCfg.Cmd, "--retention="+es.Retention)
imageCfg.Cmd = append(imageCfg.Cmd, "--creation="+es.Creation) imageCfg.Cmd = append(imageCfg.Cmd, "--creation="+es.Creation)
} }
// Configure imageCfg for S3 backups
if es.BackupConfig != nil { if es.BackupConfig != nil {
imageCfg = configS3BackupImgCmd(ctx, imageCfg, es.BackupConfig) imageCfg = configS3BackupImgCmd(ctx, imageCfg, es.BackupConfig)
} }
@ -327,6 +327,8 @@ func RunEtcdSnapshotSave(ctx context.Context, etcdHost *hosts.Host, prsMap map[s
} }
if once { if once {
log.Infof(ctx, "[etcd] Running snapshot save once on host [%s]", etcdHost.Address)
logrus.Debugf("[etcd] Using command [%s] for snapshot save once container [%s] on host [%s]", getSanitizedSnapshotCmd(imageCfg, es.BackupConfig), EtcdSnapshotOnceContainerName, etcdHost.Address)
if err := docker.DoRemoveContainer(ctx, etcdHost.DClient, EtcdSnapshotOnceContainerName, etcdHost.Address); err != nil { if err := docker.DoRemoveContainer(ctx, etcdHost.DClient, EtcdSnapshotOnceContainerName, etcdHost.Address); err != nil {
return err return err
} }
@ -336,16 +338,18 @@ func RunEtcdSnapshotSave(ctx context.Context, etcdHost *hosts.Host, prsMap map[s
status, _, stderr, err := docker.GetContainerOutput(ctx, etcdHost.DClient, EtcdSnapshotOnceContainerName, etcdHost.Address) status, _, stderr, err := docker.GetContainerOutput(ctx, etcdHost.DClient, EtcdSnapshotOnceContainerName, etcdHost.Address)
if status != 0 || err != nil { if status != 0 || err != nil {
if removeErr := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName); removeErr != nil { if removeErr := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName); removeErr != nil {
log.Warnf(ctx, "Failed to remove container [%s]: %v", removeErr) log.Warnf(ctx, "[etcd] Failed to remove container [%s] on host [%s]: %v", removeErr, etcdHost.Address)
} }
if err != nil { if err != nil {
return err return err
} }
return fmt.Errorf("Failed to take one-time snapshot, exit code [%d]: %v", status, stderr) return fmt.Errorf("[etcd] Failed to take one-time snapshot on host [%s], exit code [%d]: %v", etcdHost.Address, status, stderr)
} }
return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName) return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName)
} }
log.Infof(ctx, "[etcd] Running snapshot container [%s] on host [%s]", EtcdSnapshotOnceContainerName, etcdHost.Address)
logrus.Debugf("[etcd] Running snapshot container [%s] with command [%s] on host [%s]", EtcdSnapshotContainerName, getSanitizedSnapshotCmd(imageCfg, es.BackupConfig), etcdHost.Address)
if err := docker.DoRemoveContainer(ctx, etcdHost.DClient, EtcdSnapshotContainerName, etcdHost.Address); err != nil { if err := docker.DoRemoveContainer(ctx, etcdHost.DClient, EtcdSnapshotContainerName, etcdHost.Address); err != nil {
return err return err
} }
@ -359,7 +363,7 @@ func RunEtcdSnapshotSave(ctx context.Context, etcdHost *hosts.Host, prsMap map[s
} }
time.Sleep(EtcdSnapshotWaitTime * time.Second) time.Sleep(EtcdSnapshotWaitTime * time.Second)
if snapshotCont.State.Status == "exited" || snapshotCont.State.Restarting { if snapshotCont.State.Status == "exited" || snapshotCont.State.Restarting {
log.Warnf(ctx, "Etcd rolling snapshot container failed to start correctly") log.Warnf(ctx, "[etcd] etcd rolling snapshot container failed to start correctly")
return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotContainerName) return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotContainerName)
} }
return nil return nil
@ -594,13 +598,17 @@ func configS3BackupImgCmd(ctx context.Context, imageCfg *container.Config, bc *v
"--s3-bucketName=" + bc.S3BackupConfig.BucketName, "--s3-bucketName=" + bc.S3BackupConfig.BucketName,
"--s3-region=" + bc.S3BackupConfig.Region, "--s3-region=" + bc.S3BackupConfig.Region,
}...) }...)
s3Logline := fmt.Sprintf("[etcd] Snapshot will be uploaded to S3 compatible backend at [%s] in region [%s] to bucket [%s] using accesskey [%s]", bc.S3BackupConfig.Endpoint, bc.S3BackupConfig.Region, bc.S3BackupConfig.BucketName, bc.S3BackupConfig.AccessKey)
if bc.S3BackupConfig.CustomCA != "" { if bc.S3BackupConfig.CustomCA != "" {
caStr := base64.StdEncoding.EncodeToString([]byte(bc.S3BackupConfig.CustomCA)) caStr := base64.StdEncoding.EncodeToString([]byte(bc.S3BackupConfig.CustomCA))
cmd = append(cmd, "--s3-endpoint-ca="+caStr) cmd = append(cmd, "--s3-endpoint-ca="+caStr)
s3Logline += fmt.Sprintf(" and using endpoint CA [%s]", caStr)
} }
if bc.S3BackupConfig.Folder != "" { if bc.S3BackupConfig.Folder != "" {
cmd = append(cmd, "--s3-folder="+bc.S3BackupConfig.Folder) cmd = append(cmd, "--s3-folder="+bc.S3BackupConfig.Folder)
s3Logline += fmt.Sprintf(" and using folder [%s]", bc.S3BackupConfig.Folder)
} }
log.Infof(ctx, s3Logline)
} }
imageCfg.Cmd = append(imageCfg.Cmd, cmd...) imageCfg.Cmd = append(imageCfg.Cmd, cmd...)
return imageCfg return imageCfg
@ -721,3 +729,11 @@ func setEtcdPermissions(ctx context.Context, etcdHost *hosts.Host, prsMap map[st
} }
return docker.DoRunOnetimeContainer(ctx, etcdHost.DClient, imageCfg, hostCfg, EtcdPermFixContainerName, etcdHost.Address, ETCDRole, prsMap) return docker.DoRunOnetimeContainer(ctx, etcdHost.DClient, imageCfg, hostCfg, EtcdPermFixContainerName, etcdHost.Address, ETCDRole, prsMap)
} }
func getSanitizedSnapshotCmd(imageCfg *container.Config, bc *v3.BackupConfig) string {
cmd := strings.Join(imageCfg.Cmd, " ")
if bc != nil && bc.S3BackupConfig != nil {
return strings.Replace(cmd, bc.S3BackupConfig.SecretKey, "***", -1)
}
return cmd
}