Merge pull request #12177 from ixdy/retry-gsutil-upload

Retry uploading/publishing CI builds to work around gsutil bug

build/common.sh

@@ -43,6 +43,7 @@ readonly KUBE_GCS_RELEASE_PREFIX=${KUBE_GCS_RELEASE_PREFIX-devel}/
 readonly KUBE_GCS_DOCKER_REG_PREFIX=${KUBE_GCS_DOCKER_REG_PREFIX-docker-reg}/
 readonly KUBE_GCS_LATEST_FILE=${KUBE_GCS_LATEST_FILE:-}
 readonly KUBE_GCS_LATEST_CONTENTS=${KUBE_GCS_LATEST_CONTENTS:-}
+readonly KUBE_GCS_DELETE_EXISTING="${KUBE_GCS_DELETE_EXISTING:-n}"
 
 # Constants
 readonly KUBE_BUILD_IMAGE_REPO=kube-build
@@ -908,13 +909,15 @@ function kube::release::gcs::copy_release_artifacts() {
   # First delete all objects at the destination
   if gsutil ls "${gcs_destination}" >/dev/null 2>&1; then
     kube::log::error "${gcs_destination} not empty."
-    read -p "Delete everything under ${gcs_destination}? [y/n] " -r || {
-      echo "EOF on prompt.  Skipping upload"
-      return
-    }
-    [[ $REPLY =~ ^[yY]$ ]] || {
-      echo "Skipping upload"
-      return
+    [[ ${KUBE_GCS_DELETE_EXISTING} =~ ^[yY]$ ]] || {
+      read -p "Delete everything under ${gcs_destination}? [y/n] " -r || {
+        echo "EOF on prompt.  Skipping upload"
+        return
+      }
+      [[ $REPLY =~ ^[yY]$ ]] || {
+        echo "Skipping upload"
+        return
+      }
     }
     gsutil -q -m rm -f -R "${gcs_destination}"
   fi

build/push-ci-build.sh

@@ -24,6 +24,7 @@ LATEST=$(git describe)
 KUBE_GCS_NO_CACHING=n
 KUBE_GCS_MAKE_PUBLIC=y
 KUBE_GCS_UPLOAD_RELEASE=y
+KUBE_GCS_DELETE_EXISTING=y
 KUBE_GCS_RELEASE_BUCKET=kubernetes-release
 KUBE_GCS_RELEASE_PREFIX="ci/${LATEST}"
 KUBE_GCS_LATEST_FILE="ci/latest.txt"
@@ -32,5 +33,11 @@ KUBE_GCS_LATEST_CONTENTS=${LATEST}
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "$KUBE_ROOT/build/common.sh"
 
-kube::release::gcs::release
-kube::release::gcs::publish_latest
+MAX_ATTEMPTS=3
+attempt=0
+while [[ ${attempt} -lt ${MAX_ATTEMPTS} ]]; do
+  kube::release::gcs::release && kube::release::gcs::publish_latest && break || true
+  attempt=$((attempt + 1))
+  sleep 5
+done
+[[ ${attempt} -lt ${MAX_ATTEMPTS} ]] || exit 1