From 3ad4fededeafc6aff605b89334cf75c7351c69a2 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 14 Aug 2019 17:26:39 -0700 Subject: [PATCH 1/6] cluster/.../etcd/migrate: block etcd client creation until connection is up The new etcd balancer (>3.3.14, 3.4.0) uses an asynchronous resolver for endpoints. Without "WithBlock", the client may return before the connection is up. Signed-off-by: Gyuho Lee --- cluster/images/etcd/migrate/migrate_client.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cluster/images/etcd/migrate/migrate_client.go b/cluster/images/etcd/migrate/migrate_client.go index cfb019f7cde..da106eb97cf 100644 --- a/cluster/images/etcd/migrate/migrate_client.go +++ b/cluster/images/etcd/migrate/migrate_client.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "context" "fmt" "os" "os/exec" @@ -25,10 +26,9 @@ import ( "strings" "time" - "context" - clientv2 "github.com/coreos/etcd/client" "github.com/coreos/etcd/clientv3" + "google.golang.org/grpc" "k8s.io/klog" ) @@ -113,7 +113,13 @@ func (e *CombinedEtcdClient) clientV2() (clientv2.KeysAPI, error) { } func (e *CombinedEtcdClient) clientV3() (*clientv3.Client, error) { - return clientv3.New(clientv3.Config{Endpoints: []string{e.endpoint()}}) + return clientv3.New(clientv3.Config{ + Endpoints: []string{e.endpoint()}, + DialTimeout: 20 * time.Second, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + }) } // Backup creates a backup of an etcd2 data directory at the given backupDir. From eb1509a1d3ad6bbd6e465f93f32869bad3557117 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 14 Aug 2019 17:28:12 -0700 Subject: [PATCH 2/6] kubeadm/app/util/etcd: : block etcd client creation until connection is up The new etcd balancer (>3.3.14, 3.4.0) uses an asynchronous resolver for endpoints. Without "WithBlock", the client may return before the connection is up. Signed-off-by: Gyuho Lee --- cmd/kubeadm/app/util/etcd/etcd.go | 41 +++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/cmd/kubeadm/app/util/etcd/etcd.go b/cmd/kubeadm/app/util/etcd/etcd.go index c8ac23a929f..f64720d9a28 100644 --- a/cmd/kubeadm/app/util/etcd/etcd.go +++ b/cmd/kubeadm/app/util/etcd/etcd.go @@ -29,6 +29,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/pkg/transport" "github.com/pkg/errors" + "google.golang.org/grpc" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog" @@ -126,12 +127,20 @@ func NewFromCluster(client clientset.Interface, certificatesDir string) (*Client return etcdClient, nil } +// dialTimeout is the timeout for failing to establish a connection. +// It is set to 20 seconds as times shorter than that will cause TLS connections to fail +// on heavily loaded arm64 CPUs (issue #64649) +const dialTimeout = 20 * time.Second + // Sync synchronizes client's endpoints with the known endpoints from the etcd membership. func (c *Client) Sync() error { cli, err := clientv3.New(clientv3.Config{ Endpoints: c.Endpoints, - DialTimeout: 20 * time.Second, - TLS: c.TLS, + DialTimeout: dialTimeout, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: c.TLS, }) if err != nil { return err @@ -161,8 +170,11 @@ type Member struct { func (c *Client) GetMemberID(peerURL string) (uint64, error) { cli, err := clientv3.New(clientv3.Config{ Endpoints: c.Endpoints, - DialTimeout: 30 * time.Second, - TLS: c.TLS, + DialTimeout: dialTimeout, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: c.TLS, }) if err != nil { return 0, err @@ -188,8 +200,11 @@ func (c *Client) GetMemberID(peerURL string) (uint64, error) { func (c *Client) RemoveMember(id uint64) ([]Member, error) { cli, err := clientv3.New(clientv3.Config{ Endpoints: c.Endpoints, - DialTimeout: 30 * time.Second, - TLS: c.TLS, + DialTimeout: dialTimeout, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: c.TLS, }) if err != nil { return nil, err @@ -232,8 +247,11 @@ func (c *Client) AddMember(name string, peerAddrs string) ([]Member, error) { cli, err := clientv3.New(clientv3.Config{ Endpoints: c.Endpoints, - DialTimeout: 20 * time.Second, - TLS: c.TLS, + DialTimeout: dialTimeout, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: c.TLS, }) if err != nil { return nil, err @@ -320,8 +338,11 @@ func (c *Client) ClusterAvailable() (bool, error) { func (c *Client) GetClusterStatus() (map[string]*clientv3.StatusResponse, error) { cli, err := clientv3.New(clientv3.Config{ Endpoints: c.Endpoints, - DialTimeout: 5 * time.Second, - TLS: c.TLS, + DialTimeout: dialTimeout, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: c.TLS, }) if err != nil { return nil, err From b61433ef0f5e9ba20cb2e385859f59c7fe93b7d5 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 14 Aug 2019 17:28:35 -0700 Subject: [PATCH 3/6] test/integration: block etcd client creation until connection is up The new etcd balancer (>3.3.14, 3.4.0) uses an asynchronous resolver for endpoints. Without "WithBlock", the client may return before the connection is up. Signed-off-by: Gyuho Lee --- test/integration/utils.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/integration/utils.go b/test/integration/utils.go index 14a54ab9e46..276333d5b7a 100644 --- a/test/integration/utils.go +++ b/test/integration/utils.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "google.golang.org/grpc" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -82,8 +83,12 @@ func GetEtcdClients(config storagebackend.TransportConfig) (*clientv3.Client, cl } cfg := clientv3.Config{ - Endpoints: config.ServerList, - TLS: tlsConfig, + Endpoints: config.ServerList, + DialTimeout: 20 * time.Second, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: tlsConfig, } c, err := clientv3.New(cfg) From a254d0e2a67645948c9631d4bf11ef60aa26f5ae Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 14 Aug 2019 17:28:54 -0700 Subject: [PATCH 4/6] k8s/apiextensions-apiserver/test/integration: block etcd client creation until connection is up The new etcd balancer (>3.3.14, 3.4.0) uses an asynchronous resolver for endpoints. Without "WithBlock", the client may return before the connection is up. Signed-off-by: Gyuho Lee --- .../test/integration/objectmeta_test.go | 10 ++++++++-- .../test/integration/pruning_test.go | 10 ++++++++-- .../test/integration/storage/objectreader.go | 9 +++++++-- .../pkg/storage/storagebackend/factory/etcd3.go | 1 + 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go index 1ef3bd788a6..a5e0f2a6729 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go @@ -21,9 +21,11 @@ import ( "reflect" "strings" "testing" + "time" "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/pkg/transport" + "google.golang.org/grpc" "sigs.k8s.io/yaml" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -147,8 +149,12 @@ func TestInvalidObjectMetaInStorage(t *testing.T) { t.Fatal(err) } etcdConfig := clientv3.Config{ - Endpoints: restOptions.StorageConfig.Transport.ServerList, - TLS: tlsConfig, + Endpoints: restOptions.StorageConfig.Transport.ServerList, + DialTimeout: 20 * time.Second, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: tlsConfig, } etcdclient, err := clientv3.New(etcdConfig) if err != nil { diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go index 5cfacd77758..a33e2b0a81a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go @@ -21,9 +21,11 @@ import ( "reflect" "strings" "testing" + "time" "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/pkg/transport" + "google.golang.org/grpc" "sigs.k8s.io/yaml" @@ -331,8 +333,12 @@ func TestPruningFromStorage(t *testing.T) { t.Fatal(err) } etcdConfig := clientv3.Config{ - Endpoints: restOptions.StorageConfig.Transport.ServerList, - TLS: tlsConfig, + Endpoints: restOptions.StorageConfig.Transport.ServerList, + DialTimeout: 20 * time.Second, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: tlsConfig, } etcdclient, err := clientv3.New(etcdConfig) if err != nil { diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/objectreader.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/objectreader.go index 6be1bb28b85..e8449c0d904 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/objectreader.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/objectreader.go @@ -25,6 +25,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/pkg/transport" + "google.golang.org/grpc" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apiserver/pkg/registry/generic" @@ -112,8 +113,12 @@ func GetEtcdClients(config storagebackend.TransportConfig) (*clientv3.Client, cl } cfg := clientv3.Config{ - Endpoints: config.ServerList, - TLS: tlsConfig, + Endpoints: config.ServerList, + DialTimeout: 20 * time.Second, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: tlsConfig, } c, err := clientv3.New(cfg) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index f2e7452f4d2..627e547d035 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -111,6 +111,7 @@ func newETCD3Client(c storagebackend.TransportConfig) (*clientv3.Client, error) DialKeepAliveTime: keepaliveTime, DialKeepAliveTimeout: keepaliveTimeout, DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor), grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor), }, From 93b9545f4841522aeef2a85f516f49124f2883f5 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 14 Aug 2019 18:06:18 -0700 Subject: [PATCH 5/6] vendor: update with "update-vendor.sh" script Signed-off-by: Gyuho Lee --- cluster/images/etcd/migrate/BUILD | 1 + cmd/kubeadm/app/util/etcd/BUILD | 1 + staging/src/k8s.io/apiextensions-apiserver/go.mod | 1 + .../src/k8s.io/apiextensions-apiserver/test/integration/BUILD | 1 + .../apiextensions-apiserver/test/integration/storage/BUILD | 1 + test/integration/BUILD | 1 + 6 files changed, 6 insertions(+) diff --git a/cluster/images/etcd/migrate/BUILD b/cluster/images/etcd/migrate/BUILD index 33ffffce7bf..93af2cc384d 100644 --- a/cluster/images/etcd/migrate/BUILD +++ b/cluster/images/etcd/migrate/BUILD @@ -28,6 +28,7 @@ go_library( "//vendor/github.com/coreos/etcd/client:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kubeadm/app/util/etcd/BUILD b/cmd/kubeadm/app/util/etcd/BUILD index 5492f26fd68..4724f7023db 100644 --- a/cmd/kubeadm/app/util/etcd/BUILD +++ b/cmd/kubeadm/app/util/etcd/BUILD @@ -14,6 +14,7 @@ go_library( "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/go.mod b/staging/src/k8s.io/apiextensions-apiserver/go.mod index 6fff8d45368..c33a1882c73 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/go.mod +++ b/staging/src/k8s.io/apiextensions-apiserver/go.mod @@ -20,6 +20,7 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.3 github.com/stretchr/testify v1.3.0 + google.golang.org/grpc v1.23.0 gopkg.in/yaml.v2 v2.2.2 k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD index d562ce9042c..bff34280ce3 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD @@ -61,6 +61,7 @@ go_test( "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", "//vendor/sigs.k8s.io/yaml:go_default_library", ], diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/BUILD index 193788f2acf..184bd89fb03 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/BUILD @@ -13,6 +13,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", ], ) diff --git a/test/integration/BUILD b/test/integration/BUILD index 1ef467a0b48..5110e15cfcb 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -21,6 +21,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", ], ) From 21f976bb78d01669af23f4be2ba3ac6856004d0c Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Fri, 16 Aug 2019 19:21:20 -0700 Subject: [PATCH 6/6] cmd/kubeadm: remove "rpc/status" from import-restrictions Signed-off-by: Gyuho Lee --- cmd/kubeadm/.import-restrictions | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cmd/kubeadm/.import-restrictions b/cmd/kubeadm/.import-restrictions index 25fb82f6409..47a50dfc7a1 100644 --- a/cmd/kubeadm/.import-restrictions +++ b/cmd/kubeadm/.import-restrictions @@ -174,13 +174,7 @@ "golang.org/x/text/unicode/bidi", "golang.org/x/text/unicode/norm", "golang.org/x/text/width", - "golang.org/x/time/rate" - ] - }, - { - "SelectorRegexp": "google[.]golang[.]org", - "AllowedPrefixes": [ - "google.golang.org/genproto/googleapis/rpc/status", + "golang.org/x/time/rate", "google.golang.org/grpc" ] },