diff --git a/test/kubemark/common.sh b/test/kubemark/common.sh index 032dffdaa4e..93cdc7c27f1 100644 --- a/test/kubemark/common.sh +++ b/test/kubemark/common.sh @@ -16,6 +16,7 @@ source "${KUBE_ROOT}/cluster/kubemark/config-default.sh" source "${KUBE_ROOT}/cluster/kubemark/util.sh" +source "${KUBE_ROOT}/cluster/kube-env.sh" detect-project &> /dev/null export PROJECT @@ -23,3 +24,22 @@ export PROJECT MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master" MASTER_TAG="kubemark-master" EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store" + +RETRIES=3 + +# Runs gcloud compute command with the given parameters. Up to $RETRIES will be made +# to execute the command. +# arguments: +# $@: all stuff that goes after 'gcloud compute ' +function run-gcloud-compute-with-retries { + for attempt in $(seq 1 ${RETRIES}); do + if ! gcloud compute $@; then + echo -e "${color_yellow}Attempt ${attempt} failed to $@. Retrying.${color_norm}" >& 2 + sleep $(($attempt * 5)) + else + return + fi + done + echo -e "${color_red} Failed to $@.${color_norm}" >& 2 + exit 1 +} diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 6db0c5f378a..5433e846a0e 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -46,39 +46,39 @@ cd $CURR_DIR GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}" -gcloud compute disks create "${MASTER_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} \ - --type "${MASTER_DISK_TYPE}" \ - --size "${MASTER_DISK_SIZE}" +run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \ + ${GCLOUD_COMMON_ARGS} \ + --type "${MASTER_DISK_TYPE}" \ + --size "${MASTER_DISK_SIZE}" -gcloud compute instances create "${MASTER_NAME}" \ - ${GCLOUD_COMMON_ARGS} \ - --machine-type "${MASTER_SIZE}" \ - --image-project="${MASTER_IMAGE_PROJECT}" \ - --image "${MASTER_IMAGE}" \ - --tags "${MASTER_TAG}" \ - --network "${NETWORK}" \ - --scopes "storage-ro,compute-rw,logging-write" \ - --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" +run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \ + ${GCLOUD_COMMON_ARGS} \ + --machine-type "${MASTER_SIZE}" \ + --image-project="${MASTER_IMAGE_PROJECT}" \ + --image "${MASTER_IMAGE}" \ + --tags "${MASTER_TAG}" \ + --network "${NETWORK}" \ + --scopes "storage-ro,compute-rw,logging-write" \ + --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" -gcloud compute firewall-rules create "${INSTANCE_PREFIX}-kubemark-master-https" \ - --project "${PROJECT}" \ - --network "${NETWORK}" \ - --source-ranges "0.0.0.0/0" \ - --target-tags "${MASTER_TAG}" \ - --allow "tcp:443" +run-gcloud-compute-with-retries firewall-rules create "${INSTANCE_PREFIX}-kubemark-master-https" \ + --project "${PROJECT}" \ + --network "${NETWORK}" \ + --source-ranges "0.0.0.0/0" \ + --target-tags "${MASTER_TAG}" \ + --allow "tcp:443" MASTER_IP=$(gcloud compute instances describe ${MASTER_NAME} \ --zone="${ZONE}" --project="${PROJECT}" | grep natIP: | cut -f2 -d":" | sed "s/ //g") if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store" - gcloud compute disks create "${EVENT_STORE_NAME}-pd" \ + run-gcloud-compute-with-retries disks create "${EVENT_STORE_NAME}-pd" \ ${GCLOUD_COMMON_ARGS} \ --type "${MASTER_DISK_TYPE}" \ --size "${MASTER_DISK_SIZE}" - gcloud compute instances create "${EVENT_STORE_NAME}" \ + run-gcloud-compute-with-retries instances create "${EVENT_STORE_NAME}" \ ${GCLOUD_COMMON_ARGS} \ --machine-type "${MASTER_SIZE}" \ --image-project="${MASTER_IMAGE_PROJECT}" \ @@ -119,15 +119,15 @@ done gcloud compute ssh --zone=${ZONE} --project="${PROJECT}" ${MASTER_NAME} \ --command="sudo mkdir /srv/kubernetes -p && \ - sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 -d > /srv/kubernetes/server.cert\" && \ - sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 -d > /srv/kubernetes/server.key\" && \ - sudo bash -c \"echo ${CA_CERT_BASE64} | base64 -d > /srv/kubernetes/ca.crt\" && \ - sudo bash -c \"echo ${KUBECFG_CERT_BASE64} | base64 -d > /srv/kubernetes/kubecfg.crt\" && \ - sudo bash -c \"echo ${KUBECFG_KEY_BASE64} | base64 -d > /srv/kubernetes/kubecfg.key\" && \ - sudo bash -c \"echo \"${KUBE_BEARER_TOKEN},admin,admin\" > /srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo \"${KUBELET_TOKEN},kubelet,kubelet\" >> /srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy\" >> /srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo admin,admin,admin > /srv/kubernetes/basic_auth.csv\"" + sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 -d > /srv/kubernetes/server.cert\" && \ + sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 -d > /srv/kubernetes/server.key\" && \ + sudo bash -c \"echo ${CA_CERT_BASE64} | base64 -d > /srv/kubernetes/ca.crt\" && \ + sudo bash -c \"echo ${KUBECFG_CERT_BASE64} | base64 -d > /srv/kubernetes/kubecfg.crt\" && \ + sudo bash -c \"echo ${KUBECFG_KEY_BASE64} | base64 -d > /srv/kubernetes/kubecfg.key\" && \ + sudo bash -c \"echo \"${KUBE_BEARER_TOKEN},admin,admin\" > /srv/kubernetes/known_tokens.csv\" && \ + sudo bash -c \"echo \"${KUBELET_TOKEN},kubelet,kubelet\" >> /srv/kubernetes/known_tokens.csv\" && \ + sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy\" >> /srv/kubernetes/known_tokens.csv\" && \ + sudo bash -c \"echo admin,admin,admin > /srv/kubernetes/basic_auth.csv\"" if [ "${RUN_FROM_DISTRO}" == "false" ]; then gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \