devicemanager: fix race in stub

There is a race when the server is coming up and the subsequent dial on
the socket. Fix the race with a PollImmediate retry.

pkg/kubelet/cm/devicemanager/BUILD

@@ -32,6 +32,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1:go_default_library",
         "//staging/src/k8s.io/kubelet/pkg/apis/pluginregistration/v1:go_default_library",

pkg/kubelet/cm/devicemanager/device_plugin_stub.go

@@ -26,6 +26,7 @@ import (
 
 	"google.golang.org/grpc"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 	watcherapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1"
@@ -123,11 +124,21 @@ func (m *Stub) Start() error {
 		defer m.wg.Done()
 		m.server.Serve(sock)
 	}()
-	_, conn, err := dial(m.socket)
+
-	if err != nil {
+	var lastDialErr error
-		return err
+	wait.PollImmediate(1*time.Second, 10*time.Second, func() (bool, error) {
+		var conn *grpc.ClientConn
+		_, conn, lastDialErr = dial(m.socket)
+		if lastDialErr != nil {
+			return false, nil
 		}
 		conn.Close()
+		return true, nil
+	})
+	if lastDialErr != nil {
+		return lastDialErr
+	}
+
 	klog.Infof("Starting to serve on %v", m.socket)
 
 	return nil