From 3fc177ff7c2e21097a6d0c2359bf0d15f5384d39 Mon Sep 17 00:00:00 2001
From: gmarek <gmarek@google.com>
Date: Fri, 18 Nov 2016 13:19:48 +0100
Subject: [PATCH] Allow storing events on a separate disk

---
 test/kubemark/start-kubemark-master.sh | 55 +++++++++++++++-----------
 test/kubemark/start-kubemark.sh        | 14 ++++++-
 test/kubemark/stop-kubemark.sh         |  3 ++
 3 files changed, 47 insertions(+), 25 deletions(-)

diff --git a/test/kubemark/start-kubemark-master.sh b/test/kubemark/start-kubemark-master.sh
index d16087a425a..70be246141f 100644
--- a/test/kubemark/start-kubemark-master.sh
+++ b/test/kubemark/start-kubemark-master.sh
@@ -19,7 +19,9 @@
 EVENT_STORE_IP=$1
 EVENT_STORE_URL="http://${EVENT_STORE_IP}:4002"
 NUM_NODES=$2
-KUBEMARK_ETCD_IMAGE=$3
+EVENT_PD=$3
+# KUBEMARK_ETCD_IMAGE may be empty so it has to be kept as a last argument
+KUBEMARK_ETCD_IMAGE=$4
 if [[ -z "${KUBEMARK_ETCD_IMAGE}" ]]; then
   # Default etcd version.
   KUBEMARK_ETCD_IMAGE="2.2.1"
@@ -33,17 +35,18 @@ function retry() {
 }
 
 function mount-master-pd() {
-	if [[ ! -e /dev/disk/by-id/google-master-pd ]]; then
-		echo "Can't find master-pd. Skipping mount."
+	local -r pd_name=$1
+	local -r mount_point=$2
+	if [[ ! -e "/dev/disk/by-id/${pd_name}" ]]; then
+		echo "Can't find ${pd_name}. Skipping mount."
 		return
 	fi
-	device_info=$(ls -l "/dev/disk/by-id/google-master-pd")
-	relative_path=${device_info##* }
+	device_info=$(ls -l "/dev/disk/by-id/${pd_name}")
+	local relative_path=${device_info##* }
 	pd_device="/dev/disk/by-id/${relative_path}"
 
 	echo "Mounting master-pd"
-	local -r pd_path="/dev/disk/by-id/google-master-pd"
-	local -r mount_point="/mnt/disks/master-pd"
+	local -r pd_path="/dev/disk/by-id/${pd_name}"
 	# Format and mount the disk, create directories on it for all of the master's
 	# persistent data, and link them to where they're used.
 	mkdir -p "${mount_point}"
@@ -57,24 +60,28 @@ function mount-master-pd() {
 	echo "Mounting '${pd_path}' at '${mount_point}'"
 	mount -o discard,defaults "${pd_path}" "${mount_point}"
 	echo "Mounted master-pd '${pd_path}' at '${mount_point}'"
-
-	# Contains all the data stored in etcd.
-	mkdir -m 700 -p "${mount_point}/var/etcd"
-	ln -s -f "${mount_point}/var/etcd" /var/etcd
-	mkdir -p /etc/srv
-	# Contains the dynamically generated apiserver auth certs and keys.
-	mkdir -p "${mount_point}/srv/kubernetes"
-	ln -s -f "${mount_point}/srv/kubernetes" /etc/srv/kubernetes
-	# Directory for kube-apiserver to store SSH key (if necessary).
-	mkdir -p "${mount_point}/srv/sshproxy"
-	ln -s -f "${mount_point}/srv/sshproxy" /etc/srv/sshproxy
-
-	if ! id etcd &>/dev/null; then
-		useradd -s /sbin/nologin -d /var/etcd etcd
-	fi
 }
 
-mount-master-pd
+main_etcd_mount_point="/mnt/disks/master-pd"
+mount-master-pd "google-master-pd" "${main_etcd_mount_point}"
+# Contains all the data stored in etcd.
+mkdir -m 700 -p "${main_etcd_mount_point}/var/etcd"
+ln -s -f "${main_etcd_mount_point}/var/etcd" /var/etcd
+mkdir -p /etc/srv
+# Contains the dynamically generated apiserver auth certs and keys.
+mkdir -p "${main_etcd_mount_point}/srv/kubernetes"
+ln -s -f "${main_etcd_mount_point}/srv/kubernetes" /etc/srv/kubernetes
+# Directory for kube-apiserver to store SSH key (if necessary).
+mkdir -p "${main_etcd_mount_point}/srv/sshproxy"
+ln -s -f "${main_etcd_mount_point}/srv/sshproxy" /etc/srv/sshproxy
+
+if [ "${EVENT_PD:-false}" == "true" ]; then
+	event_etcd_mount_point="/mnt/disks/master-event-pd"
+	mount-master-pd "google-master-event-pd" "${event_etcd_mount_point}"
+	# Contains all the data stored in event etcd.
+	mkdir -m 700 -p "${event_etcd_mount_point}/var/etcd/events"
+	ln -s -f "${event_etcd_mount_point}/var/etcd/events" /var/etcd/events
+fi
 
 ETCD_QUOTA_BYTES=""
 if [ "${KUBEMARK_ETCD_VERSION:0:2}" == "3." ]; then
@@ -87,7 +94,7 @@ fi
 if [ "${EVENT_STORE_IP}" == "127.0.0.1" ]; then
 	# Retry starting etcd to avoid pulling image errors.
 	retry sudo docker run --net=host \
-		-v /var/etcd/data-events:/var/etcd/data -v /var/log:/var/log -d \
+		-v /var/etcd/events/data:/var/etcd/data -v /var/log:/var/log -d \
 		gcr.io/google_containers/etcd:${KUBEMARK_ETCD_IMAGE} /bin/sh -c "/usr/local/bin/etcd \
 		--listen-peer-urls http://127.0.0.1:2381 \
 		--advertise-client-urls=http://127.0.0.1:4002 \
diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh
index 4275066e062..5a120d84aa8 100755
--- a/test/kubemark/start-kubemark.sh
+++ b/test/kubemark/start-kubemark.sh
@@ -90,6 +90,13 @@ run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \
   --type "${MASTER_DISK_TYPE}" \
   --size "${MASTER_DISK_SIZE}"
 
+if [ "${EVENT_PD:-false}" == "true" ]; then
+  run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \
+    ${GCLOUD_COMMON_ARGS} \
+    --type "${MASTER_DISK_TYPE}" \
+    --size "${MASTER_DISK_SIZE}"
+fi
+
 run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \
   --project "${PROJECT}" \
   --region "${REGION}" -q
@@ -109,6 +116,11 @@ run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \
   --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \
   --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no"
 
+if [ "${EVENT_PD:-false}" == "true" ]; then
+  echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}"
+  run-gcloud-compute-with-retries instances attach-disk "${MASTER_NAME}" --disk "${MASTER_NAME}-event-pd" --device-name="master-event-pd"
+fi
+
 run-gcloud-compute-with-retries firewall-rules create "${INSTANCE_PREFIX}-kubemark-master-https" \
   --project "${PROJECT}" \
   --network "${NETWORK}" \
@@ -157,7 +169,7 @@ gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \
 
 gcloud compute ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" \
   --command="chmod a+x configure-kubectl.sh && chmod a+x start-kubemark-master.sh && \
-             sudo ./start-kubemark-master.sh ${EVENT_STORE_IP:-127.0.0.1} ${NUM_NODES:-0} ${ETCD_IMAGE:-}"
+             sudo ./start-kubemark-master.sh ${EVENT_STORE_IP:-127.0.0.1} ${NUM_NODES:-0} ${EVENT_PD:-false} ${ETCD_IMAGE:-}"
 
 # create kubeconfig for Kubelet:
 KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
diff --git a/test/kubemark/stop-kubemark.sh b/test/kubemark/stop-kubemark.sh
index 2a787e0e92f..0d894250aaa 100755
--- a/test/kubemark/stop-kubemark.sh
+++ b/test/kubemark/stop-kubemark.sh
@@ -32,6 +32,9 @@ gcloud compute instances delete "${MASTER_NAME}" \
 gcloud compute disks delete "${MASTER_NAME}-pd" \
     ${GCLOUD_COMMON_ARGS} || true
 
+gcloud compute disks delete "${MASTER_NAME}-event-pd" \
+    ${GCLOUD_COMMON_ARGS} &> /dev/null || true
+
 gcloud compute addresses delete "${MASTER_NAME}-ip" \
     --project "${PROJECT}" \
     --region "${REGION}" \