diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index cacb3d89a73..971fd648e47 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -333,6 +333,30 @@ function wait-for-jobs { fi } +# Robustly try to create a static ip. +# $1: The name of the ip to create +# $2: The name of the region to create the ip in. +function create-static-ip { + detect-project + local attempt=0 + local REGION="$2" + while true; do + if ! gcloud compute addresses create "$1" \ + --project "${PROJECT}" \ + --region "${REGION}" -q > /dev/null; then + if (( attempt > 4 )); then + echo -e "${color_red}Failed to create static ip $1 ${color_norm}" >&2 + exit 2 + fi + attempt=$(($attempt+1)) + echo -e "${color_yellow}Attempt $attempt failed to create static ip $1. Retrying.${color_norm}" >&2 + sleep $(($attempt * 5)) + else + break + fi + done +} + # Robustly try to create a firewall rule. # $1: The name of firewall rule. # $2: IP ranges. @@ -347,12 +371,13 @@ function create-firewall-rule { --source-ranges "$2" \ --target-tags "$3" \ --allow tcp,udp,icmp,esp,ah,sctp; then - if (( attempt > 5 )); then - echo -e "${color_red}Failed to create firewall rule $1 ${color_norm}" >&2 - exit 2 - fi - echo -e "${color_yellow}Attempt $(($attempt+1)) failed to create firewall rule $1. Retrying.${color_norm}" >&2 - attempt=$(($attempt+1)) + if (( attempt > 4 )); then + echo -e "${color_red}Failed to create firewall rule $1 ${color_norm}" >&2 + exit 2 + fi + echo -e "${color_yellow}Attempt $(($attempt+1)) failed to create firewall rule $1. Retrying.${color_norm}" >&2 + attempt=$(($attempt+1)) + sleep $(($attempt * 5)) else break fi @@ -654,7 +679,8 @@ function kube-up { # so extract the region name, which is the same as the zone but with the final # dash and characters trailing the dash removed. local REGION=${ZONE%-*} - MASTER_RESERVED_IP=$(gcloud compute addresses create "${MASTER_NAME}-ip" \ + create-static-ip "${MASTER_NAME}-ip" "${REGION}" + MASTER_RESERVED_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \ --project "${PROJECT}" \ --region "${REGION}" -q --format yaml | awk '/^address:/ { print $2 }') diff --git a/test/e2e/google_compute.go b/test/e2e/google_compute.go index 54fb454a052..f33a78a5725 100644 --- a/test/e2e/google_compute.go +++ b/test/e2e/google_compute.go @@ -21,6 +21,7 @@ import ( "os/exec" "regexp" "strings" + "time" "github.com/golang/glog" ) @@ -32,15 +33,22 @@ func createGCEStaticIP(name string) (string, error) { // NAME REGION ADDRESS STATUS // test-static-ip us-central1 104.197.143.7 RESERVED - output, err := exec.Command("gcloud", "compute", "addresses", "create", - name, "--project", testContext.CloudConfig.ProjectID, - "--region", "us-central1", "-q").CombinedOutput() - if err != nil { + var output []byte + var err error + for attempts := 0; attempts < 4; attempts++ { + output, err = exec.Command("gcloud", "compute", "addresses", "create", + name, "--project", testContext.CloudConfig.ProjectID, + "--region", "us-central1", "-q").CombinedOutput() + if err == nil { + break + } glog.Errorf("Creating static IP with name:%s in project: %s", name, testContext.CloudConfig.ProjectID) glog.Errorf("output: %s", output) + time.Sleep(time.Duration(5*attempts) * time.Second) + } + if err != nil { return "", err } - glog.Errorf("Creating static IP with name:%s in project: %s", name, testContext.CloudConfig.ProjectID) text := string(output) if strings.Contains(text, "RESERVED") { r, _ := regexp.Compile("[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+")