From c98e72841b1886a858d10c76859a9adbf145cfe7 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 20 Jun 2023 21:32:45 -0400 Subject: [PATCH] Add a connection backoff to talk to CRI impls We can add backoff for connection like we do in cadvisor: https://github.com/google/cadvisor/blob/master/container/containerd/client.go#L76-L80 for now, don't tune it, just use the default: https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md Signed-off-by: Davanum Srinivas --- pkg/kubelet/cri/remote/remote_image.go | 7 ++++++- pkg/kubelet/cri/remote/remote_runtime.go | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/cri/remote/remote_image.go b/pkg/kubelet/cri/remote/remote_image.go index f053345a06c..b444a28173f 100644 --- a/pkg/kubelet/cri/remote/remote_image.go +++ b/pkg/kubelet/cri/remote/remote_image.go @@ -25,6 +25,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" + "google.golang.org/grpc/backoff" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" @@ -55,7 +56,11 @@ func NewRemoteImageService(endpoint string, connectionTimeout time.Duration, tp ctx, cancel := context.WithTimeout(context.Background(), connectionTimeout) defer cancel() - dialOpts := []grpc.DialOption{} + dialOpts := []grpc.DialOption{ + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.DefaultConfig, + }), + } dialOpts = append(dialOpts, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialer), diff --git a/pkg/kubelet/cri/remote/remote_runtime.go b/pkg/kubelet/cri/remote/remote_runtime.go index 18e6bf7275f..1df3affbbea 100644 --- a/pkg/kubelet/cri/remote/remote_runtime.go +++ b/pkg/kubelet/cri/remote/remote_runtime.go @@ -27,6 +27,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" + "google.golang.org/grpc/backoff" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" @@ -79,7 +80,11 @@ func NewRemoteRuntimeService(endpoint string, connectionTimeout time.Duration, t ctx, cancel := context.WithTimeout(context.Background(), connectionTimeout) defer cancel() - dialOpts := []grpc.DialOption{} + dialOpts := []grpc.DialOption{ + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.DefaultConfig, + }), + } dialOpts = append(dialOpts, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialer),