From 19b2758919daa563522aec93956e326e786ff79f Mon Sep 17 00:00:00 2001 From: Lorenz Brun Date: Mon, 14 Jan 2019 20:47:41 +0100 Subject: [PATCH 1/2] Fix etcd healthcheck for consensus failures --- .../apiserver/pkg/storage/storagebackend/factory/etcd3.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index 6bac7424383..0d1b4a1fcf5 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -72,10 +72,12 @@ func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) { client := clientValue.Load().(*clientv3.Client) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - if _, err := client.Cluster.MemberList(ctx); err != nil { - return fmt.Errorf("error listing etcd members: %v", err) + // See https://github.com/etcd-io/etcd/blob/master/etcdctl/ctlv3/command/ep_command.go#L118 + _, err := client.Get(ctx, "health") + if err == nil { + return nil } - return nil + return fmt.Errorf("error getting data from etcd: %v", err) }, nil } From df3033a24095551db25dbab385ac3e630c8bd280 Mon Sep 17 00:00:00 2001 From: Lorenz Brun Date: Tue, 15 Jan 2019 21:22:07 +0100 Subject: [PATCH 2/2] Consider prefix in health check --- .../apiserver/pkg/storage/storagebackend/factory/etcd3.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index 0d1b4a1fcf5..801f8d6c920 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -19,6 +19,7 @@ package factory import ( "context" "fmt" + "path" "sync" "sync/atomic" "time" @@ -73,7 +74,7 @@ func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() // See https://github.com/etcd-io/etcd/blob/master/etcdctl/ctlv3/command/ep_command.go#L118 - _, err := client.Get(ctx, "health") + _, err := client.Get(ctx, path.Join(c.Prefix, "health")) if err == nil { return nil }