Merge pull request #27823 from nikhiljindal/waitForSecret

Automatic merge from submit-queue

federation e2e: Adding retries to fetching secret in controller manager

Ref: https://github.com/kubernetes/kubernetes/pull/27708

Trying to fix the following failure in fed controller manager:
```
error in fetching secret: Get https://10.0.0.1:443/api/v1/namespaces/federation/secrets/federation-apiserver-secret: dial tcp 10.0.0.1:443: i/o timeout
```

I am not sure why that error is happening in the first place. kube-proxy should have been configured before fed controller manager pod comes up. I didnt find anything wrong in kube-proxy logs.
The request never reaches fed apiserver.
Lets see if adding retries helps.

cc @kubernetes/sig-cluster-federation @mml @colhom

federation/pkg/federation-controller/util/cluster_util.go

@@ -18,21 +18,26 @@ package util
 
 import (
 	"fmt"
+	"net"
+	"os"
+	"time"
+
 	"github.com/golang/glog"
 	federation_v1alpha1 "k8s.io/kubernetes/federation/apis/federation/v1alpha1"
+	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/client/restclient"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
 	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
 	clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
 	utilnet "k8s.io/kubernetes/pkg/util/net"
-	"net"
+	"k8s.io/kubernetes/pkg/util/wait"
-	"os"
 )
 
 const (
 	KubeAPIQPS              = 20.0
 	KubeAPIBurst            = 30
 	KubeconfigSecretDataKey = "kubeconfig"
+	getSecretTimeout        = 1 * time.Minute
 )
 
 func BuildClusterConfig(c *federation_v1alpha1.Cluster) (*restclient.Config, error) {
@@ -101,9 +106,20 @@ var KubeconfigGetterForSecret = func(secretName string) clientcmd.KubeconfigGett
 				return nil, fmt.Errorf("error in creating in-cluster client: %s", err)
 			}
 			data = []byte{}
-			secret, err := client.Secrets(namespace).Get(secretName)
+			var secret *api.Secret
+			err = wait.PollImmediate(1*time.Second, getSecretTimeout, func() (bool, error) {
+				secret, err = client.Secrets(namespace).Get(secretName)
+				if err == nil {
+					return true, nil
+				}
+				glog.Warningf("error in fetching secret: %s", err)
+				return false, nil
+			})
 			if err != nil {
-				return nil, fmt.Errorf("error in fetching secret: %s", err)
+				return nil, fmt.Errorf("timed out waiting for secret: %s", err)
+			}
+			if secret == nil {
+				return nil, fmt.Errorf("unexpected: received null secret %s", secretName)
 			}
 			ok := false
 			data, ok = secret.Data[KubeconfigSecretDataKey]