Fix mislooping in ssh.go. Add retries to AddSSHKeys.

(cherry picked from commit 4d5d0457ef)

pkg/cloudprovider/gce/gce.go

@@ -32,6 +32,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait"
 
 	"code.google.com/p/gcfg"
 	compute "code.google.com/p/google-api-go-client/compute/v1"
@@ -483,37 +484,46 @@ func (gce *GCECloud) getInstanceByName(name string) (*compute.Instance, error) {
 }
 
 func (gce *GCECloud) AddSSHKeyToAllInstances(user string, keyData []byte) error {
-	project, err := gce.service.Projects.Get(gce.projectID).Do()
+	return wait.Poll(2*time.Second, 30*time.Second, func() (bool, error) {
-	if err != nil {
+		project, err := gce.service.Projects.Get(gce.projectID).Do()
-		return err
+		if err != nil {
-	}
+			glog.Errorf("Could not get project: %v", err)
-	hostname, err := os.Hostname()
+			return false, nil
-	if err != nil {
-		return err
-	}
-	keyString := fmt.Sprintf("%s:%s %s@%s", user, strings.TrimSpace(string(keyData)), user, hostname)
-	found := false
-	for _, item := range project.CommonInstanceMetadata.Items {
-		if item.Key == "sshKeys" {
-			item.Value = addKey(item.Value, keyString)
-			found = true
-			break
 		}
-	}
+		hostname, err := os.Hostname()
-	if !found {
+		if err != nil {
-		// This is super unlikely, so log.
+			glog.Errorf("Could not get hostname: %v", err)
-		glog.Infof("Failed to find sshKeys metadata, creating a new item")
+			return false, nil
-		project.CommonInstanceMetadata.Items = append(project.CommonInstanceMetadata.Items,
+		}
-			&compute.MetadataItems{
+		keyString := fmt.Sprintf("%s:%s %s@%s", user, strings.TrimSpace(string(keyData)), user, hostname)
-				Key:   "sshKeys",
+		found := false
-				Value: keyString,
+		for _, item := range project.CommonInstanceMetadata.Items {
-			})
+			if item.Key == "sshKeys" {
-	}
+				item.Value = addKey(item.Value, keyString)
-	op, err := gce.service.Projects.SetCommonInstanceMetadata(gce.projectID, project.CommonInstanceMetadata).Do()
+				found = true
-	if err != nil {
+				break
-		return err
+			}
-	}
+		}
-	return gce.waitForGlobalOp(op)
+		if !found {
+			// This is super unlikely, so log.
+			glog.Infof("Failed to find sshKeys metadata, creating a new item")
+			project.CommonInstanceMetadata.Items = append(project.CommonInstanceMetadata.Items,
+				&compute.MetadataItems{
+					Key:   "sshKeys",
+					Value: keyString,
+				})
+		}
+		op, err := gce.service.Projects.SetCommonInstanceMetadata(gce.projectID, project.CommonInstanceMetadata).Do()
+		if err != nil {
+			glog.Errorf("Could not Set Metadata: %v", err)
+			return false, nil
+		}
+		if err := gce.waitForGlobalOp(op); err != nil {
+			glog.Errorf("Could not Set Metadata: %v", err)
+			return false, nil
+		}
+		return true, nil
+	})
 }
 
 func addKey(metadataBefore, keyString string) string {

pkg/util/ssh.go

@@ -223,11 +223,12 @@ func MakeSSHTunnels(user, keyfile string, addresses []string) (SSHTunnelList, er
 }
 
 func (l SSHTunnelList) Open() error {
-	for ix := range l {
+	for ix := 0; ix < len(l); ix++ {
 		if err := l[ix].Tunnel.Open(); err != nil {
 			// Remove a failed Open from the list.
 			glog.Errorf("Failed to open tunnel %v: %v", l[ix], err)
 			l = append(l[:ix], l[ix+1:]...)
+			ix--
 		}
 	}
 	if len(l) == 0 {