Format ephemeral disks with btrfs, put Docker storage on there

Docker's default storage has severe bugs with older (stable) kernels.

Use btrfs to bypass those issues.

cluster/aws/templates/format-disks.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Discover all the ephemeral disks
+
+block_devices=()
+
+ephemeral_devices=$(curl --silent http://169.254.169.254/2014-11-05/meta-data/block-device-mapping/ | grep ephemeral)
+for ephemeral_device in $ephemeral_devices; do
+  echo "Checking ephemeral device: ${ephemeral_device}"
+  aws_device=$(curl --silent http://169.254.169.254/2014-11-05/meta-data/block-device-mapping/${ephemeral_device})
+
+  device_path=""
+  if [ -b /dev/$aws_device ]; then
+    device_path="/dev/$aws_device"
+  else
+    # Check for the xvd-style name
+    xvd_style=$(echo $aws_device | sed "s/sd/xvd/")
+    if [ -b /dev/$xvd_style ]; then
+      device_path="/dev/$xvd_style"
+    fi
+  fi
+
+  if [[ -z ${device_path} ]]; then
+    echo "  Could not find disk: ${ephemeral_device}@${aws_device}"
+  else
+    echo "  Detected ephemeral disk: ${ephemeral_device}@${device_path}"
+    block_devices+=(${device_path})
+  fi
+done
+
+# Format the ephemeral disks
+if [[ ${#block_devices[@]} == 0 ]]; then
+  echo "No ephemeral block devices found"
+else
+  echo "Block devices: ${block_devices}"
+
+  apt-get install --yes btrfs-tools
+
+  if [[ ${#block_devices[@]} == 1 ]]; then
+    echo "One ephemeral block device found; formatting with btrfs"
+    mkfs.btrfs -f ${block_devices[0]}
+  else
+    echo "Found multiple ephemeral block devices, formatting with btrfs as RAID-0"
+    mkfs.btrfs -f --data raid0 ${block_devices[@]}
+  fi
+  mount -t btrfs ${block_devices[0]} /mnt
+
+  # Move docker to /mnt if we have it
+  mkdir -p /mnt/docker
+  DOCKER_OPTS="${DOCKER_OPTS} -g /mnt/docker"
+fi
+

cluster/aws/templates/salt-minion.sh

@@ -30,6 +30,13 @@ grains:
   cloud: aws
 EOF
 
+
+if [[ -n "{DOCKER_OPTS}" ]]; then
+  cat <<EOF >>/etc/salt/minion.d/grains.conf
+  docker_opts: '$(echo "$DOCKER_OPTS" | sed -e "s/'/''/g")'
+EOF
+fi
+
 # Install Salt
 #
 # We specify -X to avoid a race condition that can cause minion failure to

cluster/aws/util.sh

@@ -421,6 +421,7 @@ function kube-up {
     echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'"
     echo "readonly ADMISSION_CONTROL='${ADMISSION_CONTROL:-}'"    
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/create-dynamic-salt-files.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/download-release.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/salt-master.sh"
@@ -498,6 +499,8 @@ function kube-up {
       echo "#! /bin/bash"
       echo "SALT_MASTER='${MASTER_INTERNAL_IP}'"
       echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'"
+      echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS:-}'"
+      grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"
       grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/salt-minion.sh"
     ) > "${KUBE_TEMP}/minion-start-${i}.sh"
     minion_id=$($AWS_CMD run-instances \