rancher/rke
1
0
Fork 0
mirror of https://github.com/rancher/rke.git synced 2025-04-27 11:21:08 +00:00
Code Issues Releases Wiki Activity

Check etcd cluster health after member delete

Browse Source
This commit is contained in:
Sebastiaan van Steenis 2020-09-29 13:53:45 +02:00
parent 33c69c0108
commit 422dfff0fd
3 changed files with 40 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cluster/reconcile.go
View File

@ -194,7 +194,7 @@ func reconcileEtcd(ctx context.Context, currentCluster, kubeCluster *Cluster, ku
}
}
// handle etcd member delete
if err := deleteEtcdMembers(ctx, currentCluster, kubeCluster, kubeClient, clientCert, clientKey, etcdToDelete); err != nil {
if err := deleteEtcdMembers(ctx, currentCluster, kubeCluster, kubeClient, svcOptionData, clientCert, clientKey, etcdToDelete); err != nil {
return err
}
// handle etcd member add
@ -238,11 +238,20 @@ func addEtcdMembers(ctx context.Context, currentCluster, kubeCluster *Cluster, k
return nil
}
func deleteEtcdMembers(ctx context.Context, currentCluster, kubeCluster *Cluster, kubeClient *kubernetes.Clientset, clientCert, clientKey []byte, etcdToDelete []*hosts.Host) error {
func deleteEtcdMembers(ctx context.Context, currentCluster, kubeCluster *Cluster, kubeClient *kubernetes.Clientset, svcOptionData map[string]*v3.KubernetesServicesOptions, clientCert, clientKey []byte, etcdToDelete []*hosts.Host) error {
log.Infof(ctx, "[reconcile] Check etcd hosts to be deleted")
etcdNodePlanMap := make(map[string]v3.RKEConfigNodePlan)
for _, etcdMapHost := range kubeCluster.EtcdHosts {
svcOptions, err := kubeCluster.GetKubernetesServicesOptions(etcdMapHost.DockerInfo.OSType, svcOptionData)
if err != nil {
return err
}
etcdNodePlanMap[etcdMapHost.Address] = BuildRKEConfigNodePlan(ctx, kubeCluster, etcdMapHost, svcOptions)
}
for _, etcdHost := range etcdToDelete {
etcdHost.IsEtcd = false
if err := services.RemoveEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory, clientCert, clientKey); err != nil {
if err := services.RemoveEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory, clientCert, clientKey, etcdNodePlanMap); err != nil {
log.Warnf(ctx, "[reconcile] %v", err)
continue
}

29
services/etcd.go
View File

@ -195,8 +195,8 @@ func AddEtcdMember(ctx context.Context, toAddEtcdHost *hosts.Host, etcdHosts []*
return nil
}
func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte) error {
log.Infof(ctx, "[remove/%s] Removing member [etcd-%s] from etcd cluster", ETCDRole, etcdHost.HostnameOverride)
func RemoveEtcdMember(ctx context.Context, toDeleteEtcdHost *hosts.Host, etcdHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, cert, key []byte, etcdNodePlanMap map[string]v3.RKEConfigNodePlan) error {
log.Infof(ctx, "[remove/%s] Removing member [etcd-%s] from etcd cluster", ETCDRole, toDeleteEtcdHost.HostnameOverride)
var mID string
removed := false
for _, host := range etcdHosts {
@ -212,7 +212,7 @@ func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*ho
continue
}
for _, member := range members {
if member.Name == fmt.Sprintf("etcd-%s", etcdHost.HostnameOverride) {
if member.Name == fmt.Sprintf("etcd-%s", toDeleteEtcdHost.HostnameOverride) {
mID = member.ID
break
}
@ -221,13 +221,28 @@ func RemoveEtcdMember(ctx context.Context, etcdHost *hosts.Host, etcdHosts []*ho
logrus.Debugf("Failed to list etcd members from host [%s]: %v", host.Address, err)
continue
}
removed = true
break
etcdMemberDeletedTime := time.Now()
// Need to health check after successful member remove (especially for leader re-election)
// We will check all hosts to see if the cluster becomes healthy
var healthError error
_, _, healthCheckURL := GetProcessConfig(etcdNodePlanMap[host.Address].Processes[EtcdContainerName], host)
logrus.Infof("[remove/%s] Checking etcd cluster health on [etcd-%s] after removing [etcd-%s]", ETCDRole, host.HostnameOverride, toDeleteEtcdHost.HostnameOverride)
logrus.Debugf("[remove/%s] healthCheckURL for checking etcd cluster health on [etcd-%s] after removing [%s]: [%s]", ETCDRole, host.HostnameOverride, toDeleteEtcdHost.HostnameOverride, healthCheckURL)
healthError = isEtcdHealthy(localConnDialerFactory, host, cert, key, healthCheckURL)
if healthError == nil {
logrus.Infof("[remove/%s] etcd cluster health is healthy on [etcd-%s] after removing [etcd-%s]", ETCDRole, host.HostnameOverride, toDeleteEtcdHost.HostnameOverride)
etcdHealthyTime := time.Now()
diffTime := etcdHealthyTime.Sub(etcdMemberDeletedTime)
logrus.Debugf("Total time between etcd member deleted and etcd cluster healthy is: [%s]", diffTime)
removed = true
break
}
logrus.Warn(healthError)
}
if !removed {
return fmt.Errorf("Failed to delete etcd member [etcd-%s] from etcd cluster", etcdHost.HostnameOverride)
return fmt.Errorf("Failed to delete etcd member [etcd-%s] from etcd cluster", toDeleteEtcdHost.HostnameOverride)
}
log.Infof(ctx, "[remove/%s] Successfully removed member [etcd-%s] from etcd cluster", ETCDRole, etcdHost.HostnameOverride)
log.Infof(ctx, "[remove/%s] Successfully removed member [etcd-%s] from etcd cluster", ETCDRole, toDeleteEtcdHost.HostnameOverride)
return nil
}

10
services/etcd_util.go
View File

@ -44,7 +44,9 @@ func isEtcdHealthy(localConnDialerFactory hosts.DialerFactory, host *hosts.Host,
logrus.Debugf("[etcd] check etcd cluster health on host [%s]", host.Address)
var finalErr error
var healthy string
for i := 0; i < 3; i++ {
// given a max election timeout of 50000ms (50s), max re-election of 77 seconds was seen
// this allows for 18 * 5 seconds = 90 seconds of re-election
for i := 0; i < 18; i++ {
dialer, err := getEtcdDialer(localConnDialerFactory, host)
if err != nil {
return err
@ -67,12 +69,12 @@ func isEtcdHealthy(localConnDialerFactory hosts.DialerFactory, host *hosts.Host,
time.Sleep(5 * time.Second)
continue
}
// log in debug here as we don't want to log in warn on every iteration
// the error will be logged in the caller stack
logrus.Debugf("[etcd] etcd host [%s] reported healthy=%s", host.Address, healthy)
// Changed this from Debug to Info to inform user on what is happening
logrus.Infof("[etcd] etcd host [%s] reported healthy=%s", host.Address, healthy)
if healthy == "true" {
return nil
}
time.Sleep(5 * time.Second)
}
if finalErr != nil {
return fmt.Errorf("[etcd] host [%s] failed to check etcd health: %v", host.Address, finalErr)