From 69300c51b36e11c52a94ed1bc513ff74df86aef8 Mon Sep 17 00:00:00 2001
From: Justin Santa Barbara <justin@fathomdb.com>
Date: Fri, 12 Feb 2016 11:55:30 -0500
Subject: [PATCH 1/2] AWS kube-up: Reboot support with ephemeral storage

If the ephemeral volume is present and mounted, don't try to reinitialize
them.

Don't block the boot if the ephemeral volume is corrupt / missing -
this enables us to cope with a stop/start & presumably also corruption.
In this case, we'll reformat the ephemeral storage.

Fix #21157
---
 cluster/aws/templates/format-disks.sh | 28 ++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/cluster/aws/templates/format-disks.sh b/cluster/aws/templates/format-disks.sh
index 8d8d66a476c..b5dc0c45f54 100644
--- a/cluster/aws/templates/format-disks.sh
+++ b/cluster/aws/templates/format-disks.sh
@@ -18,6 +18,12 @@
 
 function ensure-local-disks() {
 
+# Skip if already mounted (a reboot)
+if ( grep "/mnt/ephemeral" /proc/mounts ); then
+  echo "Found /mnt/ephemeral in /proc/mounts; skipping local disk initialization"
+  return
+fi
+
 block_devices=()
 
 ephemeral_devices=$(curl --silent http://169.254.169.254/2014-11-05/meta-data/block-device-mapping/ | grep ephemeral)
@@ -61,10 +67,14 @@ else
   # Remove any existing mounts
   for block_device in ${block_devices}; do
     echo "Unmounting ${block_device}"
-    /bin/umount ${block_device}
+    /bin/umount ${block_device} || echo "Ignoring failure umounting ${block_device}"
     sed -i -e "\|^${block_device}|d" /etc/fstab
   done
 
+  # Remove any existing /mnt/ephemeral entry in /etc/fstab
+  sed -i -e "\|/mnt/ephemeral|d" /etc/fstab
+
+  # Mount the storage
   if [[ ${docker_storage} == "btrfs" ]]; then
     apt-get-install btrfs-tools
 
@@ -75,7 +85,7 @@ else
       echo "Found multiple ephemeral block devices, formatting with btrfs as RAID-0"
       mkfs.btrfs -f --data raid0 ${block_devices[@]}
     fi
-    echo "${block_devices[0]}  /mnt/ephemeral  btrfs  noatime  0 0" >> /etc/fstab
+    echo "${block_devices[0]}  /mnt/ephemeral  btrfs  noatime,nofail  0 0" >> /etc/fstab
     mkdir -p /mnt/ephemeral
     mount /mnt/ephemeral
 
@@ -89,7 +99,7 @@ else
     fi
 
     mkfs -t ext4 ${block_devices[0]}
-    echo "${block_devices[0]}  /mnt/ephemeral  ext4     noatime  0 0" >> /etc/fstab
+    echo "${block_devices[0]}  /mnt/ephemeral  ext4     noatime,nofail  0 0" >> /etc/fstab
     mkdir -p /mnt/ephemeral
     mount /mnt/ephemeral
 
@@ -129,7 +139,7 @@ else
       lvcreate -l 100%FREE -n kubernetes vg-ephemeral
       mkfs -t ext4 /dev/vg-ephemeral/kubernetes
       mkdir -p /mnt/ephemeral/kubernetes
-      echo "/dev/vg-ephemeral/kubernetes  /mnt/ephemeral/kubernetes  ext4  noatime  0 0" >> /etc/fstab
+      echo "/dev/vg-ephemeral/kubernetes  /mnt/ephemeral/kubernetes  ext4  noatime,nofail  0 0" >> /etc/fstab
       mount /mnt/ephemeral/kubernetes
 
       move_kubelet="/mnt/ephemeral/kubernetes"
@@ -146,7 +156,7 @@ else
       lvcreate -l 100%VG -n ephemeral vg-ephemeral
       mkfs -t ext4 /dev/vg-ephemeral/ephemeral
       mkdir -p /mnt/ephemeral
-      echo "/dev/vg-ephemeral/ephemeral  /mnt/ephemeral  ext4  noatime  0 0" >> /etc/fstab
+      echo "/dev/vg-ephemeral/ephemeral  /mnt/ephemeral  ext4  noatime,nofail 0 0" >> /etc/fstab
       mount /mnt/ephemeral
 
       mkdir -p /mnt/ephemeral/kubernetes
@@ -183,7 +193,9 @@ if [[ -n "${move_docker}" ]]; then
     mv /var/lib/docker ${move_docker}/
   fi
   mkdir -p ${move_docker}/docker
-  ln -s ${move_docker}/docker /var/lib/docker
+  if [[ ! -e /var/lib/docker ]]; then
+    ln -s ${move_docker}/docker /var/lib/docker
+  fi
   DOCKER_ROOT="${move_docker}/docker"
   DOCKER_OPTS="${DOCKER_OPTS:-} -g ${DOCKER_ROOT}"
 fi
@@ -195,7 +207,9 @@ if [[ -n "${move_kubelet}" ]]; then
     mv /var/lib/kubelet ${move_kubelet}/
   fi
   mkdir -p ${move_kubelet}/kubelet
-  ln -s ${move_kubelet}/kubelet /var/lib/kubelet
+  if [[ ! -e /var/lib/kubelet ]]; then
+    ln -s ${move_kubelet}/kubelet /var/lib/kubelet
+  fi
   KUBELET_ROOT="${move_kubelet}/kubelet"
 fi
 

From c68c019150e0f0533df56954f31ed190cc590fc0 Mon Sep 17 00:00:00 2001
From: Justin Santa Barbara <justin@fathomdb.com>
Date: Wed, 17 Feb 2016 09:57:37 -0500
Subject: [PATCH 2/2] AWS: Add comments to explain the symlink logic

---
 cluster/aws/templates/format-disks.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cluster/aws/templates/format-disks.sh b/cluster/aws/templates/format-disks.sh
index b5dc0c45f54..9aa04112a38 100644
--- a/cluster/aws/templates/format-disks.sh
+++ b/cluster/aws/templates/format-disks.sh
@@ -189,10 +189,13 @@ fi
 
 if [[ -n "${move_docker}" ]]; then
   # Move docker to e.g. /mnt
+  # but only if it is a directory, not a symlink left over from a previous run
   if [[ -d /var/lib/docker ]]; then
     mv /var/lib/docker ${move_docker}/
   fi
   mkdir -p ${move_docker}/docker
+  # If /var/lib/docker doesn't exist (it will exist if it is already a symlink),
+  # then symlink it to the ephemeral docker area
   if [[ ! -e /var/lib/docker ]]; then
     ln -s ${move_docker}/docker /var/lib/docker
   fi
@@ -203,10 +206,12 @@ fi
 if [[ -n "${move_kubelet}" ]]; then
   # Move /var/lib/kubelet to e.g. /mnt
   # (the backing for empty-dir volumes can use a lot of space!)
+  # (As with /var/lib/docker, only if it is a directory; skip if symlink)
   if [[ -d /var/lib/kubelet ]]; then
     mv /var/lib/kubelet ${move_kubelet}/
   fi
   mkdir -p ${move_kubelet}/kubelet
+  # Create symlink for /var/lib/kubelet, unless it is already a symlink
   if [[ ! -e /var/lib/kubelet ]]; then
     ln -s ${move_kubelet}/kubelet /var/lib/kubelet
   fi