github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 19:31:44 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #79677 from neolit123/etcd-add-retry

Browse Source 
kubeadm: run MemberAdd/Remove for etcd clients with exp-backoff retry
This commit is contained in:
Kubernetes Prow Robot 2019-07-04 05:44:28 -07:00 committed by GitHub
commit 7340b6341a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 33 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kubeadm/app/util/etcd/BUILD
View File

@ -9,6 +9,7 @@ go_library(
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/config:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/github.com/coreos/etcd/clientv3:go_default_library",
"//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",

40
cmd/kubeadm/app/util/etcd/etcd.go
View File

@ -29,6 +29,7 @@ import (
"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/pkg/transport"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
@ -36,6 +37,15 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/util/config"
)
// Exponential backoff for MemberAdd/Remove (values exclude jitter):
// 0, 50, 150, 350, 750, 1550, 3150, 6350, 12750 ms
var addRemoveBackoff = wait.Backoff{
Steps: 8,
Duration: 50 * time.Millisecond,
Factor: 2.0,
Jitter: 0.1,
}
// ClusterInterrogator is an interface to get etcd cluster related information
type ClusterInterrogator interface {
ClusterAvailable() (bool, error)
@ -187,11 +197,18 @@ func (c *Client) RemoveMember(id uint64) ([]Member, error) {
defer cli.Close()
// Remove an existing member from the cluster
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
resp, err := cli.MemberRemove(ctx, id)
cancel()
var lastError error
var resp *clientv3.MemberRemoveResponse
err = wait.ExponentialBackoff(addRemoveBackoff, func() (bool, error) {
resp, err = cli.MemberRemove(context.Background(), id)
if err == nil {
return true, nil
}
lastError = err
return false, nil
})
if err != nil {
return nil, err
return nil, lastError
}
// Returns the updated list of etcd members
@ -224,11 +241,18 @@ func (c *Client) AddMember(name string, peerAddrs string) ([]Member, error) {
defer cli.Close()
// Adds a new member to the cluster
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
resp, err := cli.MemberAdd(ctx, []string{peerAddrs})
cancel()
var lastError error
var resp *clientv3.MemberAddResponse
err = wait.ExponentialBackoff(addRemoveBackoff, func() (bool, error) {
resp, err = cli.MemberAdd(context.Background(), []string{peerAddrs})
if err == nil {
return true, nil
}
lastError = err
return false, nil
})
if err != nil {
return nil, err
return nil, lastError
}
// Returns the updated list of etcd members