From 52498e6050c2b56010373b1166d92aa9ed407406 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Fri, 26 Aug 2016 10:56:29 +0200 Subject: [PATCH 1/2] Retry etcd starting in kubemark --- test/kubemark/start-kubemark-master.sh | 14 +++++++++-- test/kubemark/start-kubemark.sh | 32 -------------------------- 2 files changed, 12 insertions(+), 34 deletions(-) diff --git a/test/kubemark/start-kubemark-master.sh b/test/kubemark/start-kubemark-master.sh index acd9edca7fb..06ae512dcf5 100644 --- a/test/kubemark/start-kubemark-master.sh +++ b/test/kubemark/start-kubemark-master.sh @@ -19,8 +19,17 @@ EVENT_STORE_IP=$1 EVENT_STORE_URL="http://${EVENT_STORE_IP}:4002" NUM_NODES=$2 + +retry() { + for i in {1..4}; do + "$@" && return 0 || sleep $i + done + "$@" +} + if [ "${EVENT_STORE_IP}" == "127.0.0.1" ]; then - sudo docker run --net=host -v /var/etcd/data-events:/var/etcd/data -d \ + # Retry starting etcd to avoid pulling image errors. + retry sudo docker run --net=host -v /var/etcd/data-events:/var/etcd/data -d \ gcr.io/google_containers/etcd:3.0.4 /usr/local/bin/etcd \ --listen-peer-urls http://127.0.0.1:2381 \ --advertise-client-urls=http://127.0.0.1:4002 \ @@ -28,7 +37,8 @@ if [ "${EVENT_STORE_IP}" == "127.0.0.1" ]; then --data-dir=/var/etcd/data fi -sudo docker run --net=host -v /var/etcd/data:/var/etcd/data -d \ +# Retry starting etcd to avoid pulling image errors. +retry sudo docker run --net=host -v /var/etcd/data:/var/etcd/data -d \ gcr.io/google_containers/etcd:3.0.4 /usr/local/bin/etcd \ --listen-peer-urls http://127.0.0.1:2380 \ --advertise-client-urls=http://127.0.0.1:2379 \ diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 505ab622d48..37ff19d0a3c 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -115,38 +115,6 @@ run-gcloud-compute-with-retries firewall-rules create "${INSTANCE_PREFIX}-kubema --target-tags "${MASTER_TAG}" \ --allow "tcp:443" -if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then - EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store" - run-gcloud-compute-with-retries disks create "${EVENT_STORE_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} \ - --type "${MASTER_DISK_TYPE}" \ - --size "${MASTER_DISK_SIZE}" - - run-gcloud-compute-with-retries instances create "${EVENT_STORE_NAME}" \ - ${GCLOUD_COMMON_ARGS} \ - --machine-type "${MASTER_SIZE}" \ - --image-project="${MASTER_IMAGE_PROJECT}" \ - --image "${MASTER_IMAGE}" \ - --tags "${EVENT_STORE_NAME}" \ - --network "${NETWORK}" \ - --scopes "storage-ro,compute-rw,logging-write" \ - --disk "name=${EVENT_STORE_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" - - EVENT_STORE_IP=$(gcloud compute instances describe ${EVENT_STORE_NAME} \ - --zone="${ZONE}" --project="${PROJECT}" | grep networkIP: | cut -f2 -d":" | sed "s/ //g") - - until gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${EVENT_STORE_NAME}" --command="ls" &> /dev/null; do - sleep 1 - done - - gcloud compute ssh "${EVENT_STORE_NAME}" --zone="${ZONE}" --project="${PROJECT}" \ - --command="sudo docker run --net=host -d gcr.io/google_containers/etcd:3.0.4 /usr/local/bin/etcd \ - --listen-peer-urls http://127.0.0.1:2380 \ - --advertise-client-urls=http://127.0.0.1:4002 \ - --listen-client-urls=http://0.0.0.0:4002 \ - --data-dir=/var/etcd/data" -fi - ensure-temp-dir gen-kube-bearertoken create-certs ${MASTER_IP} From 752f41490526d4c372702232046b4aced99440f1 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Fri, 26 Aug 2016 11:00:28 +0200 Subject: [PATCH 2/2] Store logs from etcd in kubemark --- test/kubemark/start-kubemark-master.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/kubemark/start-kubemark-master.sh b/test/kubemark/start-kubemark-master.sh index 06ae512dcf5..6dadbcfa43f 100644 --- a/test/kubemark/start-kubemark-master.sh +++ b/test/kubemark/start-kubemark-master.sh @@ -29,21 +29,23 @@ retry() { if [ "${EVENT_STORE_IP}" == "127.0.0.1" ]; then # Retry starting etcd to avoid pulling image errors. - retry sudo docker run --net=host -v /var/etcd/data-events:/var/etcd/data -d \ - gcr.io/google_containers/etcd:3.0.4 /usr/local/bin/etcd \ + retry sudo docker run --net=host \ + -v /var/etcd/data-events:/var/etcd/data -v /var/log:/var/log -d \ + gcr.io/google_containers/etcd:3.0.4 /bin/sh -c "/usr/local/bin/etcd \ --listen-peer-urls http://127.0.0.1:2381 \ --advertise-client-urls=http://127.0.0.1:4002 \ --listen-client-urls=http://0.0.0.0:4002 \ - --data-dir=/var/etcd/data + --data-dir=/var/etcd/data 1>> /var/log/etcd-events.log 2>&1" fi # Retry starting etcd to avoid pulling image errors. -retry sudo docker run --net=host -v /var/etcd/data:/var/etcd/data -d \ - gcr.io/google_containers/etcd:3.0.4 /usr/local/bin/etcd \ +retry sudo docker run --net=host \ + -v /var/etcd/data:/var/etcd/data -v /var/log:/var/log -d \ + gcr.io/google_containers/etcd:3.0.4 /bin/sh -c "/usr/local/bin/etcd \ --listen-peer-urls http://127.0.0.1:2380 \ --advertise-client-urls=http://127.0.0.1:2379 \ --listen-client-urls=http://0.0.0.0:2379 \ - --data-dir=/var/etcd/data + --data-dir=/var/etcd/data 1>> /var/log/etcd.log 2>&1" # Increase the allowed number of open file descriptors ulimit -n 65536