From 7afa78a2efb049e66d57bf07a9eee95347f7c1b6 Mon Sep 17 00:00:00 2001
From: Karl Isenberg <karl@mesosphere.io>
Date: Mon, 10 Aug 2015 15:45:20 -0700
Subject: [PATCH 1/6] [mesos/docker] Enhance kube-up to better support running
 in a container (for CI)

- Generate CA & API Server SSL key/cert in keygen docker image
  - Refactor SSL generation
  - Generate service account key & user files on local machine
- Enable kube-up to be run in a container (kubernetes-mesos-test)
- Add timeout env vars
- Pull docker images up front to avoid timeouts
- Remove docker image builds from test-setup
- Nuke logs dir before each kube-up
- Make run_in_docker work without KUBECONFIG defined
- Fix temp dir cleanup
- Add auth mount env var
  - Default to $HOME/tmp/kubernetes/auth
  - Outside of repo (which gets docker mounted when using kubernetes-mesos-test)
  - Inside $HOME (which gets vm mounted when using docker-machine or boot2docker)
- Add log dump dir env var
  - Default to $HOME/tmp/kubernetes/logs (for consistancy with auth dir)
- Enable errtrace
- Increase log level to aid CI debugging
---
 .../docker/common/bin/await-health-check      |   2 +-
 cluster/mesos/docker/common/bin/util-ssl.sh   | 169 ++++++++++++++++++
 .../docker/{ => common/bin}/util-temp-dir.sh  |  10 +-
 cluster/mesos/docker/config-default.sh        |  25 +++
 cluster/mesos/docker/deploy-addons.sh         |   3 +-
 cluster/mesos/docker/docker-compose.yml       |  52 ++++--
 cluster/mesos/docker/keygen/Dockerfile        |  18 ++
 cluster/mesos/docker/keygen/bin/kube-cagen.sh |  32 ++++
 .../mesos/docker/keygen/bin/kube-keygen.sh    |  45 +++++
 cluster/mesos/docker/keygen/build.sh          |  54 ++++++
 cluster/mesos/docker/util-ssl.sh              | 121 -------------
 cluster/mesos/docker/util.sh                  | 157 ++++++++++------
 12 files changed, 488 insertions(+), 200 deletions(-)
 create mode 100644 cluster/mesos/docker/common/bin/util-ssl.sh
 rename cluster/mesos/docker/{ => common/bin}/util-temp-dir.sh (91%)
 create mode 100644 cluster/mesos/docker/keygen/Dockerfile
 create mode 100755 cluster/mesos/docker/keygen/bin/kube-cagen.sh
 create mode 100755 cluster/mesos/docker/keygen/bin/kube-keygen.sh
 create mode 100755 cluster/mesos/docker/keygen/build.sh
 delete mode 100644 cluster/mesos/docker/util-ssl.sh

diff --git a/cluster/mesos/docker/common/bin/await-health-check b/cluster/mesos/docker/common/bin/await-health-check
index 577f5787ac0..7aa813ea1e3 100755
--- a/cluster/mesos/docker/common/bin/await-health-check
+++ b/cluster/mesos/docker/common/bin/await-health-check
@@ -33,7 +33,7 @@ fi
 address=${1:-}
 [ -z "${address}" ] && echo "No address supplied" && exit 1
 
-bin=$(cd $(dirname $0) && pwd -P)
+bin=$(cd "$(dirname ${BASH_SOURCE})" && pwd -P)
 
 echo "Waiting (up to ${duration}s) for ${address} to be healthy"
 if ! timeout "${duration}" bash -c "while ! ${bin}/health-check ${address}; do sleep 0.5; done"; then
diff --git a/cluster/mesos/docker/common/bin/util-ssl.sh b/cluster/mesos/docker/common/bin/util-ssl.sh
new file mode 100644
index 00000000000..5caa56c8c85
--- /dev/null
+++ b/cluster/mesos/docker/common/bin/util-ssl.sh
@@ -0,0 +1,169 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sourcable SSL functions
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+bin="$(cd "$(dirname "${BASH_SOURCE}")" && pwd -P)"
+source "${bin}/util-temp-dir.sh"
+
+function cluster::mesos::docker::find_openssl_config {
+  for candidate in "/etc/ssl/openssl.cnf" "/System/Library/OpenSSL/openssl.cnf"; do
+    if [ -f "${candidate}" ]; then
+      echo "${candidate}"
+      return 0
+    fi
+  done
+  echo "ERROR: cannot find openssl.cnf" 1>&2
+  return 1
+}
+
+function cluster::mesos::docker::create_root_certificate_authority {
+  local out_dir="$1"
+
+  # TODO(karlkfi): extract config
+  local subject="/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com"
+
+  echo "Creating private key" 1>&2
+  openssl genrsa -out "${out_dir}/root-ca.key" 2048
+
+  echo "Creating certificate sign request" 1>&2
+  openssl req -nodes -new -utf8 \
+    -key "${out_dir}/root-ca.key" \
+    -out "${out_dir}/root-ca.csr" \
+    -subj "${subject}"
+
+  echo "Signing new certificate with private key" 1>&2
+  openssl x509 -req -days 3650 \
+    -in "${out_dir}/root-ca.csr" \
+    -out "${out_dir}/root-ca.crt" \
+    -signkey "${out_dir}/root-ca.key"
+
+  echo "Key: ${out_dir}/root-ca.key" 1>&2
+  echo "Cert: ${out_dir}/root-ca.crt" 1>&2
+}
+
+# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names.
+# Uses the current dir for scratch work.
+function cluster::mesos::docker::create_apiserver_cert_inner {
+  local in_dir="$1"
+  local out_dir="$2"
+  local apiserver_ip="$3"
+  local service_ip="$4"
+  local workspace="$(pwd)"
+
+  if [ ! -f "${in_dir}/root-ca.key" ]; then
+    echo "Signing key not found: ${in_dir}/root-ca.key"
+    return 1
+  fi
+  if [ ! -f "${in_dir}/root-ca.crt" ]; then
+    echo "Root certificate not found: ${in_dir}/root-ca.key"
+    return 1
+  fi
+
+  mkdir -p "${out_dir}"
+
+  local openssl_cnf=$(cluster::mesos::docker::find_openssl_config)
+
+  # TODO(karlkfi): extract config
+  local subject="/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com"
+  local cluster_domain="cluster.local"
+  local service_name="kubernetes"
+  local service_namespace="default"
+  local subject_alt_name="IP:${apiserver_ip},IP:${service_ip},DNS:${service_name},DNS:${service_name}.${service_namespace},DNS:${service_name}.${service_namespace}.svc,DNS:${service_name}.${service_namespace}.svc.${cluster_domain}"
+
+  echo "Creating private key" 1>&2
+  openssl genrsa -out "${workspace}/apiserver.key" 2048
+
+  echo "Creating certificate sign request" 1>&2
+  openssl req -nodes -new -utf8 \
+    -key "${workspace}/apiserver.key" \
+    -out "${workspace}/apiserver.csr" \
+    -reqexts SAN \
+    -config <(cat "${openssl_cnf}"; echo -e "[SAN]\nsubjectAltName=${subject_alt_name}") \
+    -subj "${subject}"
+
+  echo "Validating certificate sign request" 1>&2
+  openssl req -text -noout -in "${workspace}/apiserver.csr" | grep -q "${service_name}.${service_namespace}.svc.${cluster_domain}"
+
+  echo "Signing new certificate with root certificate authority key" 1>&2
+  mkdir -p "${workspace}/demoCA/newcerts"
+  touch "${workspace}/demoCA/index.txt"
+  echo 1000 > "${workspace}/demoCA/serial"
+  openssl ca -batch \
+    -days 3650 \
+    -in "${workspace}/apiserver.csr" \
+    -cert "${in_dir}/root-ca.crt" \
+    -keyfile "${in_dir}/root-ca.key" \
+    -config <(sed 's/.*\(copy_extensions = copy\)/\1/' ${openssl_cnf})
+
+  echo "Validating signed certificate" 1>&2
+  openssl x509 -in "${workspace}/demoCA/newcerts/1000.pem" -text -noout | grep -q "${service_name}.${service_namespace}.svc.${cluster_domain}"
+
+  echo "Key: ${out_dir}/apiserver.key" 1>&2
+  cp "${workspace}/apiserver.key" "${out_dir}/apiserver.key"
+
+  echo "Cert: ${out_dir}/apiserver.crt" 1>&2
+  cp "${workspace}/demoCA/newcerts/1000.pem" "${out_dir}/apiserver.crt"
+}
+
+# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names.
+function cluster::mesos::docker::create_apiserver_cert {
+  local in_dir="$1" # must contain root-ca.crt & root-ca.key
+  local out_dir="$2"
+  local apiserver_ip="$3"
+  local service_ip="$4"
+
+  cluster::mesos::docker::run_in_temp_dir "k8sm-certs" \
+    "cluster::mesos::docker::create_apiserver_cert_inner" \
+      "${in_dir}" "${out_dir}" "${apiserver_ip}" "${service_ip}"
+}
+
+# Creates an rsa key (for signing service accounts).
+function cluster::mesos::docker::create_rsa_key {
+  local key_file_path="$1"
+
+  # buffer output until failure
+  local output=$((
+    openssl genrsa -out "${key_file_path}" 2048 || exit $?
+  ) 2>&1)
+  local exit_status="$?"
+  if [ "${exit_status}" != 0 ]; then
+    echo "${output}" 1>&2
+    return "${exit_status}"
+  fi
+
+  echo "Key: ${key_file_path}" 1>&2
+}
+
+# Creates a k8s token auth user file.
+# See /docs/admin/authentication.md
+function cluster::mesos::docker::create_token_user {
+  local user_name="$1"
+  echo "$(openssl rand -hex 32),${user_name},${user_name}"
+}
+
+# Creates a k8s basic auth user file.
+# See /docs/admin/authentication.md
+function cluster::mesos::docker::create_basic_user {
+  local user_name="$1"
+  local password="$2"
+  echo "${password},${user_name},${user_name}"
+}
diff --git a/cluster/mesos/docker/util-temp-dir.sh b/cluster/mesos/docker/common/bin/util-temp-dir.sh
similarity index 91%
rename from cluster/mesos/docker/util-temp-dir.sh
rename to cluster/mesos/docker/common/bin/util-temp-dir.sh
index 3e12a050a17..d44d4e2ae33 100644
--- a/cluster/mesos/docker/util-temp-dir.sh
+++ b/cluster/mesos/docker/common/bin/util-temp-dir.sh
@@ -19,6 +19,7 @@
 set -o errexit
 set -o nounset
 set -o pipefail
+set -o errtrace
 
 # Runs the supplied command string in a temporary workspace directory.
 function cluster::mesos::docker::run_in_temp_dir {
@@ -31,15 +32,16 @@ function cluster::mesos::docker::run_in_temp_dir {
   echo "Workspace created: ${workspace}" 1>&2
 
   cleanup() {
+    local -r workspace="$1"
     rm -rf "${workspace}"
     echo "Workspace deleted: ${workspace}" 1>&2
   }
-  trap 'cleanup' EXIT
+  trap "cleanup '${workspace}'" EXIT
 
   pushd "${workspace}" > /dev/null
-  (${cmd}) || return $?
+  ${cmd}
   popd > /dev/null
 
   trap - EXIT
-  cleanup
-}
\ No newline at end of file
+  cleanup "${workspace}"
+}
diff --git a/cluster/mesos/docker/config-default.sh b/cluster/mesos/docker/config-default.sh
index 1a1fd24c138..675ca691636 100755
--- a/cluster/mesos/docker/config-default.sh
+++ b/cluster/mesos/docker/config-default.sh
@@ -36,3 +36,28 @@ DNS_REPLICAS=1
 
 # Optional: Deploy cluster web interface.
 ENABLE_CLUSTER_UI=true
+
+# Timeout (in seconds) to wait for ssl certs to be generated
+KUBE_KEYGEN_TIMEOUT="${KUBE_KEYGEN_TIMEOUT:-60}"
+
+# Timeout (in seconds) to wait for Etcd to come up
+MESOS_DOCKER_ETCD_TIMEOUT="${MESOS_DOCKER_ETCD_TIMEOUT:-60}"
+
+# Timeout (in seconds) to wait for the Mesos Master to come up
+MESOS_DOCKER_MESOS_TIMEOUT="${MESOS_DOCKER_MESOS_TIMEOUT:-60}"
+
+# Timeout (in seconds) to wait for the API Server to come up
+MESOS_DOCKER_API_TIMEOUT="${MESOS_DOCKER_API_TIMEOUT:-180}"
+
+# Timeout (in seconds) to wait for each addon to come up
+MESOS_DOCKER_ADDON_TIMEOUT="${MESOS_DOCKER_ADDON_TIMEOUT:-180}"
+
+# Path to directory to dump logs to in case of kube-up failure.
+# If using docker-machine or boot2docker, should be under /Users (which is mounted from the host into the docker vm).
+# If running in a container, $HOME should be resolved outside of the container.
+MESOS_DOCKER_LOG_DIR="${MESOS_DOCKER_LOG_DIR:-${HOME}/tmp/kubernetes/log}"
+
+# Path to directory to store SSL certs/keys/tokens.
+# If using docker-machine or boot2docker, should be under /Users (which is mounted from the host into the docker vm).
+# If running in a container, $HOME should be resolved outside of the container.
+MESOS_DOCKER_AUTH_DIR="${MESOS_DOCKER_AUTH_DIR:-${HOME}/tmp/kubernetes/auth}"
diff --git a/cluster/mesos/docker/deploy-addons.sh b/cluster/mesos/docker/deploy-addons.sh
index 30e803c62b3..b18f25de56f 100755
--- a/cluster/mesos/docker/deploy-addons.sh
+++ b/cluster/mesos/docker/deploy-addons.sh
@@ -23,10 +23,11 @@
 set -o errexit
 set -o nounset
 set -o pipefail
+set -o errtrace
 
 KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd)
 source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/${KUBE_CONFIG_FILE-"config-default.sh"}"
-source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util-temp-dir.sh"
+source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/common/bin/util-temp-dir.sh"
 kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
 
 
diff --git a/cluster/mesos/docker/docker-compose.yml b/cluster/mesos/docker/docker-compose.yml
index 349b2a2a5a9..51cdd237809 100644
--- a/cluster/mesos/docker/docker-compose.yml
+++ b/cluster/mesos/docker/docker-compose.yml
@@ -75,13 +75,13 @@ apiserver:
   image: mesosphere/kubernetes-mesos
   entrypoint:
   - /bin/bash
-  - "-c"
+  - "-ceu"
   - >
     echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" &&
     (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo "  mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) &&
-    await-health-check http://etcd:4001/health &&
-    await-health-check http://mesosmaster1:5050/health &&
-    await-file -t=60 /var/run/kubernetes/auth/apiserver.crt &&
+    await-health-check "-t=${MESOS_DOCKER_ETCD_TIMEOUT}" http://etcd:4001/health &&
+    await-health-check "-t=${MESOS_DOCKER_MESOS_TIMEOUT}" http://mesosmaster1:5050/health &&
+    await-file "-t=${KUBE_KEYGEN_TIMEOUT}" /var/run/kubernetes/auth/apiserver.crt &&
     km apiserver
     --address=$(resolveip apiserver)
     --external-hostname=apiserver
@@ -98,10 +98,14 @@ apiserver:
     --cloud-config=/opt/mesos-cloud.conf
     --tls-cert-file=/var/run/kubernetes/auth/apiserver.crt
     --tls-private-key-file=/var/run/kubernetes/auth/apiserver.key
-    --v=2
+    --v=4
   ports: [ "8888:8888", "6443:6443" ]
+  environment:
+  - MESOS_DOCKER_ETCD_TIMEOUT
+  - MESOS_DOCKER_MESOS_TIMEOUT
+  - KUBE_KEYGEN_TIMEOUT
   volumes:
-  - ./certs/apiserver:/var/run/kubernetes/auth
+  - ${MESOS_DOCKER_AUTH_DIR}:/var/run/kubernetes/auth:ro
   links:
   - etcd
   - mesosmaster1
@@ -110,20 +114,23 @@ controller:
   image: mesosphere/kubernetes-mesos
   entrypoint:
   - /bin/bash
-  - "-c"
+  - "-ceu"
   - >
     echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" &&
     (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo "  mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) &&
-    await-health-check -t=60 http://mesosmaster1:5050/health &&
-    await-health-check -t=60 http://apiserver:8888/healthz &&
+    await-health-check "-t=${MESOS_DOCKER_MESOS_TIMEOUT}" http://mesosmaster1:5050/health &&
+    await-health-check "-t=${MESOS_DOCKER_API_TIMEOUT}" http://apiserver:8888/healthz &&
     km controller-manager
     --master=http://apiserver:8888
     --cloud-config=/opt/mesos-cloud.conf
     --service-account-private-key-file=/var/run/kubernetes/auth/service-accounts.key
     --root-ca-file=/var/run/kubernetes/auth/root-ca.crt
-    --v=2
+    --v=4
+  environment:
+  - MESOS_DOCKER_MESOS_TIMEOUT
+  - MESOS_DOCKER_API_TIMEOUT
   volumes:
-  - ./certs/controller:/var/run/kubernetes/auth
+  - ${MESOS_DOCKER_AUTH_DIR}:/var/run/kubernetes/auth:ro
   links:
   - mesosmaster1
   - apiserver
@@ -132,13 +139,13 @@ scheduler:
   image: mesosphere/kubernetes-mesos
   entrypoint:
   - /bin/bash
-  - "-c"
+  - "-ceu"
   - >
     echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" &&
     (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo "  mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) &&
-    await-health-check http://etcd:4001/health &&
-    await-health-check http://mesosmaster1:5050/health &&
-    await-health-check -t=60 http://apiserver:8888/healthz &&
+    await-health-check "-t=${MESOS_DOCKER_ETCD_TIMEOUT}" http://etcd:4001/health &&
+    await-health-check "-t=${MESOS_DOCKER_MESOS_TIMEOUT}" http://mesosmaster1:5050/health &&
+    await-health-check "-t=${MESOS_DOCKER_API_TIMEOUT}" http://apiserver:8888/healthz &&
     km scheduler
     --address=$(resolveip scheduler)
     --hostname-override=scheduler
@@ -148,10 +155,23 @@ scheduler:
     --mesos-master=mesosmaster1:5050
     --cluster-dns=10.10.10.10
     --cluster-domain=cluster.local
-    --v=2
+    --v=4
+  environment:
+  - MESOS_DOCKER_ETCD_TIMEOUT
+  - MESOS_DOCKER_MESOS_TIMEOUT
+  - MESOS_DOCKER_API_TIMEOUT
   links:
   - etcd
   - mesosmaster1
   - mesosslave1
   - mesosslave2
   - apiserver
+keygen:
+  image: mesosphere/kubernetes-mesos-keygen
+  command:
+  - apiserver
+  - /var/run/kubernetes/auth
+  volumes:
+  - ${MESOS_DOCKER_AUTH_DIR}:/var/run/kubernetes/auth
+  links:
+  - apiserver
diff --git a/cluster/mesos/docker/keygen/Dockerfile b/cluster/mesos/docker/keygen/Dockerfile
new file mode 100644
index 00000000000..59d92371d7a
--- /dev/null
+++ b/cluster/mesos/docker/keygen/Dockerfile
@@ -0,0 +1,18 @@
+FROM ubuntu:14.04.2
+MAINTAINER Mesosphere <support@mesosphere.io>
+
+RUN locale-gen en_US.UTF-8
+RUN dpkg-reconfigure locales
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+
+RUN apt-get update -qq && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \
+        curl \
+        openssl \
+        && \
+    apt-get clean
+
+COPY ./bin/* /usr/local/bin/
+
+ENTRYPOINT ["kube-keygen.sh"]
diff --git a/cluster/mesos/docker/keygen/bin/kube-cagen.sh b/cluster/mesos/docker/keygen/bin/kube-cagen.sh
new file mode 100755
index 00000000000..a68ad432761
--- /dev/null
+++ b/cluster/mesos/docker/keygen/bin/kube-cagen.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generates root certificate authority crt and key.
+# Writes to <out_dir> (use docker volume or docker export to retrieve files).
+# Params:
+#   out_dir  - dir to write crt and key to
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+source "util-ssl.sh"
+
+out_dir="${1:-}"
+[ -z "${out_dir}" ] && echo "No out_dir supplied (param 1)" && exit 1
+
+cluster::mesos::docker::create_root_certificate_authority "${out_dir}"
diff --git a/cluster/mesos/docker/keygen/bin/kube-keygen.sh b/cluster/mesos/docker/keygen/bin/kube-keygen.sh
new file mode 100755
index 00000000000..b990950bc18
--- /dev/null
+++ b/cluster/mesos/docker/keygen/bin/kube-keygen.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generates apiserver crt and key.
+# Requires provided hostname to be resolvable (use docker link).
+# Requires root certificate in <in_dir> (use docker volume).
+# Writes to <out_dir> (use docker volume or docker export to retrieve files).
+# Params:
+#   hostname - host name of the Kubernetes API Server to resolve into an IP
+#   in_dir   - dir to read root certificate from
+#   out_dir  - (Optional) dir to write crt and key to  (default=<in_dir>)
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+source "util-ssl.sh"
+
+hostname="${1:-}"
+[ -z "${hostname}" ] && echo "No hostname supplied (param 1)" && exit 1
+
+in_dir="${2:-}"
+[ -z "${in_dir}" ] && echo "No in_dir supplied (param 2)" && exit 1
+
+out_dir="${3:-${in_dir}}"
+
+# Certificate generation depends on IP being resolvable from the provided hostname.
+apiserver_ip="$(resolveip ${hostname})"
+apiservice_ip="10.10.10.1" #TODO(karlkfi): extract config
+
+cluster::mesos::docker::create_apiserver_cert "${in_dir}" "${out_dir}" "${apiserver_ip}" "${apiservice_ip}"
diff --git a/cluster/mesos/docker/keygen/build.sh b/cluster/mesos/docker/keygen/build.sh
new file mode 100755
index 00000000000..d753d539c60
--- /dev/null
+++ b/cluster/mesos/docker/keygen/build.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds a docker image that generates ssl certificates/keys/tokens required by kubernetes
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+IMAGE_REPO=${IMAGE_REPO:-mesosphere/kubernetes-mesos-keygen}
+IMAGE_TAG=${IMAGE_TAG:-latest}
+
+script_dir=$(cd $(dirname "${BASH_SOURCE}") && pwd -P)
+common_bin_path=$(cd ${script_dir}/../common/bin && pwd -P)
+KUBE_ROOT=$(cd ${script_dir}/../../../.. && pwd -P)
+
+source "${common_bin_path}/util-temp-dir.sh"
+
+cd "${KUBE_ROOT}"
+
+function build_image {
+  local -r workspace="$(pwd)"
+
+  echo "Copying files to workspace"
+
+  # binaries & scripts
+  mkdir -p "${workspace}/bin"
+  cp -a "${common_bin_path}/"* "${workspace}/bin/"
+  cp -a "${script_dir}/bin/"* "${workspace}/bin/"
+
+  # docker
+  cp -a "${script_dir}/Dockerfile" "${workspace}/"
+
+  echo "Building docker image ${IMAGE_REPO}:${IMAGE_TAG}"
+  set -o xtrace
+  docker build -t ${IMAGE_REPO}:${IMAGE_TAG} "$@" .
+  set +o xtrace
+  echo "Built docker image ${IMAGE_REPO}:${IMAGE_TAG}"
+}
+
+cluster::mesos::docker::run_in_temp_dir 'k8sm-keygen' 'build_image'
diff --git a/cluster/mesos/docker/util-ssl.sh b/cluster/mesos/docker/util-ssl.sh
deleted file mode 100644
index e03b084bea1..00000000000
--- a/cluster/mesos/docker/util-ssl.sh
+++ /dev/null
@@ -1,121 +0,0 @@
-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Sourcable SSL functions
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-script_dir=$(cd "$(dirname ${BASH_SOURCE})" && pwd -P)
-source "${script_dir}/util-temp-dir.sh"
-
-function cluster::mesos::docker::find_openssl_config {
-  for candidate in "/etc/ssl/openssl.cnf" "/System/Library/OpenSSL/openssl.cnf"; do
-    if [ -f "${candidate}" ]; then
-      echo "${candidate}"
-      return 0
-    fi
-  done
-  echo "ERROR: cannot find openssl.cnf" 1>&2
-  return 1
-}
-
-function cluster::mesos::docker::create_root_certificate_authority {
-  local certdir="$1"
-
-  openssl req -nodes -newkey rsa:2048 \
-    -keyout "${certdir}/root-ca.key" \
-    -out "${certdir}/root-ca.csr" \
-    -subj "/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com"
-
-  openssl x509 -req -days 3650 \
-    -in "${certdir}/root-ca.csr" \
-    -out "${certdir}/root-ca.crt" \
-    -signkey "${certdir}/root-ca.key"
-}
-
-# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names.
-# Uses the current dir for scratch work.
-function cluster::mesos::docker::create_apiserver_cert_inner {
-  local in_dir="$1" # must contain root-ca.crt & root-ca.key
-  local out_dir="$2"
-  local apiserver_ip="$3"
-  local service_ip="$4"
-  local workspace="$(pwd)"
-
-  mkdir -p "${out_dir}"
-
-  local OPENSSL_CNF=$(cluster::mesos::docker::find_openssl_config)
-
-  # create apiserver key and certificate sign request
-  local SANS="IP:${apiserver_ip},IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local"
-  openssl req -nodes -newkey rsa:2048 \
-    -keyout "${workspace}/apiserver.key" -out "${workspace}/apiserver.csr" \
-    -reqexts SAN -config <(cat "${OPENSSL_CNF}"; echo -e "[SAN]\nsubjectAltName=$SANS") \
-    -subj "/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com"
-
-  # sign with root-ca
-  mkdir -p ${workspace}/demoCA/newcerts
-  touch ${workspace}/demoCA/index.txt
-  echo 1000 > ${workspace}/demoCA/serial
-  openssl ca -cert "${in_dir}/root-ca.crt" -keyfile "${in_dir}/root-ca.key" \
-    -batch -days 3650 -in "${workspace}/apiserver.csr" \
-    -config <(sed 's/.*\(copy_extensions = copy\)/\1/' ${OPENSSL_CNF}) >/dev/null
-
-  # check certificate for subjectAltName extension
-  if ! openssl x509 -in "${workspace}/demoCA/newcerts/1000.pem" -text -noout | grep -q kubernetes.default.svc.cluster.local; then
-    echo "ERROR: openssl failed to add subjectAltName extension" 1>&2
-    return 1
-  fi
-
-  # write to out_dir
-  cp "${workspace}/demoCA/newcerts/1000.pem" "${out_dir}/apiserver.crt"
-  cp "${workspace}/apiserver.key" "${out_dir}/"
-}
-
-# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names.
-function cluster::mesos::docker::create_apiserver_cert {
-  local in_dir="$1" # must contain root-ca.crt & root-ca.key
-  local out_dir="$2"
-  local apiserver_ip="$3"
-  local service_ip="$4"
-
-  cluster::mesos::docker::run_in_temp_dir "k8sm-certs" \
-    "cluster::mesos::docker::create_apiserver_cert_inner" \
-      "${in_dir}" "${out_dir}" "${apiserver_ip}" "${service_ip}"
-}
-
-# Creates an rsa key (for signing service accounts).
-function cluster::mesos::docker::create_rsa_key {
-  local key_file_path="$1"
-  openssl genrsa -out "${key_file_path}" 2048
-}
-
-# Creates a k8s token auth user file.
-# See /docs/admin/authentication.md
-function cluster::mesos::docker::create_token_user {
-  local user_name="$1"
-  echo "$(openssl rand -hex 32),${user_name},${user_name}"
-}
-
-# Creates a k8s basic auth user file.
-# See /docs/admin/authentication.md
-function cluster::mesos::docker::create_basic_user {
-  local user_name="$1"
-  local password="$2"
-  echo "${password},${user_name},${user_name}"
-}
\ No newline at end of file
diff --git a/cluster/mesos/docker/util.sh b/cluster/mesos/docker/util.sh
index b202883c852..672675292fd 100644
--- a/cluster/mesos/docker/util.sh
+++ b/cluster/mesos/docker/util.sh
@@ -23,6 +23,7 @@
 set -o errexit
 set -o nounset
 set -o pipefail
+set -o errtrace
 
 KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd)
 provider_root="${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}"
@@ -30,13 +31,16 @@ compose_file="${provider_root}/docker-compose.yml"
 
 source "${provider_root}/${KUBE_CONFIG_FILE-"config-default.sh"}"
 source "${KUBE_ROOT}/cluster/common.sh"
-source "${provider_root}/util-ssl.sh"
+source "${provider_root}/common/bin/util-ssl.sh"
+
+log_dir="${MESOS_DOCKER_LOG_DIR}"
+auth_dir="${MESOS_DOCKER_AUTH_DIR}"
 
 
 # Run kubernetes scripts inside docker.
 # This bypasses the need to set up network routing when running docker in a VM (e.g. boot2docker).
 # Trap signals and kills the docker container for better signal handing
-function cluster::mesos::docker::run_in_docker {
+function cluster::mesos::docker::run_in_docker_test {
   entrypoint="$1"
   if [[ "${entrypoint}" = "./"* ]]; then
     # relative to project root
@@ -45,12 +49,18 @@ function cluster::mesos::docker::run_in_docker {
   shift
   args="$@"
 
+  # only mount KUBECONFIG if it exists, otherwise the directory will be created/owned by root
+  kube_config_mount=""
+  if [ -n "${KUBECONFIG:-}" ] && [ -e "${KUBECONFIG}" ]; then
+    kube_config_mount="-v \"$(dirname ${KUBECONFIG}):/root/.kube\""
+  fi
+
   container_id=$(
     docker run \
       -d \
       -e "KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER}" \
       -v "${KUBE_ROOT}:/go/src/github.com/GoogleCloudPlatform/kubernetes" \
-      -v "$(dirname ${KUBECONFIG}):/root/.kube" \
+      ${kube_config_mount} \
       -v "/var/run/docker.sock:/var/run/docker.sock" \
       --link docker_mesosmaster1_1:mesosmaster1 \
       --link docker_mesosslave1_1:mesosslave1 \
@@ -77,6 +87,36 @@ function cluster::mesos::docker::run_in_docker {
   return "${exit_status}"
 }
 
+# Run kube-cagen.sh inside docker.
+# Creating and signing in the same environment avoids a subject comparison string_mask issue.
+function cluster::mesos::docker::run_in_docker_cagen {
+  out_dir="$1"
+
+  container_id=$(
+    docker run \
+      -d \
+      -v "${out_dir}:/var/run/kubernetes/auth" \
+      --entrypoint="kube-cagen.sh" \
+      mesosphere/kubernetes-mesos-keygen \
+      "/var/run/kubernetes/auth"
+  )
+
+  docker logs -f "${container_id}" &
+
+  # trap and kill for better signal handing
+  trap 'echo "Killing container ${container_id}" 1>&2 && docker kill ${container_id}' INT TERM
+  exit_status=$(docker wait "${container_id}")
+  trap - INT TERM
+
+  if [ "$exit_status" != 0 ]; then
+    echo "Exited ${exit_status}" 1>&2
+  fi
+
+  docker rm -f "${container_id}" > /dev/null
+
+  return "${exit_status}"
+}
+
 # Generate kubeconfig data for the created cluster.
 function create-kubeconfig {
   local kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
@@ -89,8 +129,8 @@ function create-kubeconfig {
     touch "${KUBECONFIG}"
   fi
 
-  local token="$(cut -d, -f1 ${provider_root}/certs/apiserver/token-users)"
-  "${kubectl}" config set-cluster "${CONTEXT}" --server="${KUBE_SERVER}" --certificate-authority="${provider_root}/certs/root-ca.crt"
+  local token="$(cut -d, -f1 ${auth_dir}/token-users)"
+  "${kubectl}" config set-cluster "${CONTEXT}" --server="${KUBE_SERVER}" --certificate-authority="${auth_dir}/root-ca.crt"
   "${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="cluster-admin"
   "${kubectl}" config set-credentials cluster-admin --token="${token}"
   "${kubectl}" config use-context "${CONTEXT}" --cluster="${CONTEXT}"
@@ -150,70 +190,76 @@ function verify-prereqs {
   # mesosphere/kubernetes-mesos-test, etc.
 }
 
-# Instantiate a kubernetes cluster.
-function kube-up {
-  # TODO: version images (k8s version, git sha, and dirty state) to avoid re-building them every time.
-  if [ "${MESOS_DOCKER_SKIP_BUILD:-false}" != "true" ]; then
-    echo "Building Docker images" 1>&2
-    "${provider_root}/km/build.sh"
-    "${provider_root}/test/build.sh"
-  fi
+# Initialize
+function cluster::mesos::docker::init_auth {
+  local -r auth_dir="$1"
 
-  local certdir="${provider_root}/certs"
-
-  # create mount volume dirs
-  local apiserver_certs_dir="${certdir}/apiserver"
-  local controller_certs_dir="${certdir}/controller"
-  # clean certs directory from previous runs
-  if [ -d "${apiserver_certs_dir}" ]; then
-    rm -rf "${apiserver_certs_dir}"/*
-  fi
-  mkdir -p "${apiserver_certs_dir}" "${controller_certs_dir}"
+  #TODO(karlkfi): reuse existing credentials/certs/keys
+  mkdir -p "${auth_dir}"
+  rm -rf "${auth_dir}"/*
 
   echo "Creating root certificate authority" 1>&2
-  cluster::mesos::docker::create_root_certificate_authority "${certdir}"
-  cp "${certdir}/root-ca.crt" "${controller_certs_dir}/"
+  cluster::mesos::docker::run_in_docker_cagen "${auth_dir}"
 
   echo "Creating service-account rsa key" 1>&2
-  cluster::mesos::docker::create_rsa_key "${certdir}/service-accounts.key"
-  cp "${certdir}/service-accounts.key" "${apiserver_certs_dir}/"
-  cp "${certdir}/service-accounts.key" "${controller_certs_dir}/"
+  cluster::mesos::docker::create_rsa_key "${auth_dir}/service-accounts.key"
 
   echo "Creating cluster-admin token user" 1>&2
-  cluster::mesos::docker::create_token_user "cluster-admin" > "${apiserver_certs_dir}/token-users"
+  cluster::mesos::docker::create_token_user "cluster-admin" > "${auth_dir}/token-users"
 
   echo "Creating admin basic auth user" 1>&2
-  cluster::mesos::docker::create_basic_user "admin" "admin" > "${apiserver_certs_dir}/basic-users"
+  cluster::mesos::docker::create_basic_user "admin" "admin" > "${auth_dir}/basic-users"
+}
 
-  # log dump on failure
-  trap 'cluster::mesos::docker::dump_logs' ERR
+# Instantiate a kubernetes cluster.
+function kube-up {
+  # Nuke logs up front so that we know any existing logs came from the last kube-up
+  mkdir -p "${log_dir}"
+  rm -rf "${log_dir}"/*
+
+  if [ "${MESOS_DOCKER_SKIP_BUILD:-false}" != "true" ]; then
+    # Pull before `docker-compose up` to avoid timeouts between container runs.
+    # Pull before building images (but only if built) to avoid overwriting locally built images.
+    echo "Pulling docker images" 1>&2
+    docker-compose -f "${compose_file}" pull
+
+    echo "Building Docker images" 1>&2
+    # TODO: version images (k8s version, git sha, and dirty state) to avoid re-building them every time.
+    "${provider_root}/km/build.sh"
+    "${provider_root}/test/build.sh"
+    "${provider_root}/keygen/build.sh"
+  fi
+
+  cluster::mesos::docker::init_auth "${auth_dir}"
+
+  # Dump logs on premature exit (errexit triggers exit).
+  # Trap EXIT instead of ERR, because ERR can trigger multiple times with errtrace enabled.
+  trap "cluster::mesos::docker::dump_logs '${log_dir}'" EXIT
 
   echo "Starting ${KUBERNETES_PROVIDER} cluster" 1>&2
+  export MESOS_DOCKER_ETCD_TIMEOUT="${MESOS_DOCKER_ETCD_TIMEOUT}"
+  export MESOS_DOCKER_MESOS_TIMEOUT="${MESOS_DOCKER_MESOS_TIMEOUT}"
+  export MESOS_DOCKER_API_TIMEOUT="${MESOS_DOCKER_API_TIMEOUT}"
+  export KUBE_KEYGEN_TIMEOUT="${KUBE_KEYGEN_TIMEOUT}"
+  export MESOS_DOCKER_AUTH_DIR="${MESOS_DOCKER_AUTH_DIR}"
   docker-compose -f "${compose_file}" up -d
 
+  # await-health-check requires GNU timeout
+  # apiserver hostname resolved by docker
+  cluster::mesos::docker::run_in_docker_test await-health-check "-t=${MESOS_DOCKER_API_TIMEOUT}" http://apiserver:8888/healthz
+
   detect-master
   detect-minions
-
-  # The apiserver is waiting for its certificate, which depends on the IP docker chose.
-  echo "Creating apiserver certificate" 1>&2
-  local apiserer_ip="$(cut -f1 -d: <<<${KUBE_MASTER_IP})"
-  local apiservice_ip="10.10.10.1"
-  cluster::mesos::docker::create_apiserver_cert \
-    "${certdir}" "${apiserver_certs_dir}" "${apiserer_ip}" "${apiservice_ip}"
-
-  # KUBECONFIG needs to exist before run_in_docker mounts it, otherwise it will be owned by root
   create-kubeconfig
 
-  # await-health-check could be run locally, but it would require GNU timeout installed on mac...
-  # "${provider_root}/common/bin/await-health-check" -t=120 ${KUBE_SERVER}/healthz
-  cluster::mesos::docker::run_in_docker await-health-check -t=120 http://apiserver:8888/healthz
-
   echo "Deploying Addons" 1>&2
   KUBE_SERVER=${KUBE_SERVER} "${provider_root}/deploy-addons.sh"
 
   # Wait for addons to deploy
-  cluster::mesos::docker::await_ready "kube-dns"
-  cluster::mesos::docker::await_ready "kube-ui"
+  cluster::mesos::docker::await_ready "kube-dns" "${MESOS_DOCKER_ADDON_TIMEOUT}"
+  cluster::mesos::docker::await_ready "kube-ui" "${MESOS_DOCKER_ADDON_TIMEOUT}"
+
+  trap - EXIT
 }
 
 function validate-cluster {
@@ -236,10 +282,7 @@ function kube-down {
 }
 
 function test-setup {
-  echo "Building required docker images" 1>&2
-  "${KUBE_ROOT}/cluster/mesos/docker/km/build.sh"
-  "${KUBE_ROOT}/cluster/mesos/docker/test/build.sh"
-  "${KUBE_ROOT}/cluster/mesos/docker/mesos-slave/build.sh"
+  echo "TODO: test-setup" 1>&2
 }
 
 # Execute after running tests to perform any required clean-up
@@ -267,8 +310,8 @@ function restart-apiserver {
 
 # Waits for a kube-system pod (of the provided name) to have the phase/status "Running".
 function cluster::mesos::docker::await_ready {
-  local pod_name=$1
-  local max_attempts=60
+  local pod_name="$1"
+  local max_attempts="$2"
   local phase="Unknown"
   echo -n "${pod_name}: "
   local n=0
@@ -295,10 +338,10 @@ function cluster::mesos::docker::addon_status {
 }
 
 function cluster::mesos::docker::dump_logs {
-  local log_dir="${provider_root}/logs"
-  echo "Dumping logs to '${log_dir}'" 1>&2
-  mkdir -p "${log_dir}"
+  local out_dir="$1"
+  echo "Dumping logs to '${out_dir}'" 1>&2
+  mkdir -p "${out_dir}"
   while read name; do
-    docker logs "${name}" &> "${log_dir}/${name}.log"
+    docker logs "${name}" &> "${out_dir}/${name}.log"
   done < <(docker-compose -f "${compose_file}" ps -q | xargs docker inspect --format '{{.Name}}')
 }

From 1d16b21dd25f6df5c13aa139dc482a611a8b5638 Mon Sep 17 00:00:00 2001
From: Karl Isenberg <karl@mesosphere.io>
Date: Fri, 14 Aug 2015 20:17:03 -0700
Subject: [PATCH 2/6] [mesos/docker] Configure controller-manager to advertize
 its externally accessible IP

---
 cluster/mesos/docker/docker-compose.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cluster/mesos/docker/docker-compose.yml b/cluster/mesos/docker/docker-compose.yml
index 51cdd237809..03fb7e81912 100644
--- a/cluster/mesos/docker/docker-compose.yml
+++ b/cluster/mesos/docker/docker-compose.yml
@@ -121,6 +121,7 @@ controller:
     await-health-check "-t=${MESOS_DOCKER_MESOS_TIMEOUT}" http://mesosmaster1:5050/health &&
     await-health-check "-t=${MESOS_DOCKER_API_TIMEOUT}" http://apiserver:8888/healthz &&
     km controller-manager
+    --address=$(resolveip controller)
     --master=http://apiserver:8888
     --cloud-config=/opt/mesos-cloud.conf
     --service-account-private-key-file=/var/run/kubernetes/auth/service-accounts.key

From 2ea10ec5b7dfd7f89bae9a985efd5f6f601fd650 Mon Sep 17 00:00:00 2001
From: Karl Isenberg <karl@mesosphere.io>
Date: Mon, 17 Aug 2015 13:50:50 -0700
Subject: [PATCH 3/6] [mesos/docker] Replace auth/log dirs with
 MESOS_DOCKER_WORK_DIR

- Mount mesos slave workspaces in MESOS_DOCKER_WORK_DIR
- Clear mesos slave workspace in kube-up
- Add MESOS_DOCKER_IMAGE_DIR to configure docker-in-docker image storage
- Lazily eval auth/log dirs
- Improve mesos slave missing error in detect-minions
---
 cluster/mesos/docker/config-default.sh  | 16 +++++++-----
 cluster/mesos/docker/docker-compose.yml | 14 +++++++----
 cluster/mesos/docker/util.sh            | 33 ++++++++++++++++++-------
 3 files changed, 43 insertions(+), 20 deletions(-)

diff --git a/cluster/mesos/docker/config-default.sh b/cluster/mesos/docker/config-default.sh
index 675ca691636..9798210ae08 100755
--- a/cluster/mesos/docker/config-default.sh
+++ b/cluster/mesos/docker/config-default.sh
@@ -52,12 +52,16 @@ MESOS_DOCKER_API_TIMEOUT="${MESOS_DOCKER_API_TIMEOUT:-180}"
 # Timeout (in seconds) to wait for each addon to come up
 MESOS_DOCKER_ADDON_TIMEOUT="${MESOS_DOCKER_ADDON_TIMEOUT:-180}"
 
-# Path to directory to dump logs to in case of kube-up failure.
+# Path to directory on the host to use as the root for multiple docker volumes.
+# ${MESOS_DOCKER_WORK_DIR}/log - storage of component logs (written on deploy failure)
+# ${MESOS_DOCKER_WORK_DIR}/auth - storage of SSL certs/keys/tokens
+# ${MESOS_DOCKER_WORK_DIR}/<component>/mesos - storage of mesos slave work (e.g. task logs)
 # If using docker-machine or boot2docker, should be under /Users (which is mounted from the host into the docker vm).
 # If running in a container, $HOME should be resolved outside of the container.
-MESOS_DOCKER_LOG_DIR="${MESOS_DOCKER_LOG_DIR:-${HOME}/tmp/kubernetes/log}"
+MESOS_DOCKER_WORK_DIR="${MESOS_DOCKER_WORK_DIR:-${HOME}/tmp/kubernetes}"
 
-# Path to directory to store SSL certs/keys/tokens.
-# If using docker-machine or boot2docker, should be under /Users (which is mounted from the host into the docker vm).
-# If running in a container, $HOME should be resolved outside of the container.
-MESOS_DOCKER_AUTH_DIR="${MESOS_DOCKER_AUTH_DIR:-${HOME}/tmp/kubernetes/auth}"
+# Path to directory to store mesos slave docker-in-docker images & volumes.
+# Usage: ${MESOS_DOCKER_IMAGE_DIR}/<component>/docker
+# Must not be either an AUFS mount or an SMB/CIFS mount.
+# If using docker-machine or boot2docker, should NOT be under /Users (which is mounted from the host into the docker vm).
+MESOS_DOCKER_IMAGE_DIR="${MESOS_DOCKER_IMAGE_DIR:-/var/tmp/kubernetes}"
\ No newline at end of file
diff --git a/cluster/mesos/docker/docker-compose.yml b/cluster/mesos/docker/docker-compose.yml
index 03fb7e81912..6137a61aefc 100644
--- a/cluster/mesos/docker/docker-compose.yml
+++ b/cluster/mesos/docker/docker-compose.yml
@@ -40,6 +40,7 @@ mesosslave1:
   - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[21000-21099]
   - MESOS_SWITCH_USER=0
   - MESOS_CONTAINERIZERS=docker,mesos
+  - MESOS_WORK_DIR=/var/tmp/mesos
   - DOCKER_NETWORK_OFFSET=0.0.1.0
   - DOCKER_DAEMON_ARGS=--log-level=error
   links:
@@ -47,7 +48,8 @@ mesosslave1:
   - mesosmaster1
   - "ambassador:apiserver"
   volumes:
-  - /var/tmp/mesosslave1:/var/lib/docker
+  - ${MESOS_DOCKER_WORK_DIR}/mesosslave1/mesos:/var/tmp/mesos
+  - ${MESOS_DOCKER_IMAGE_DIR}/mesosslave1/docker:/var/lib/docker
 mesosslave2:
   hostname: mesosslave2
   privileged: true
@@ -62,6 +64,7 @@ mesosslave2:
   - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[21000-21099]
   - MESOS_SWITCH_USER=0
   - MESOS_CONTAINERIZERS=docker,mesos
+  - MESOS_WORK_DIR=/var/tmp/mesos
   - DOCKER_NETWORK_OFFSET=0.0.2.0
   - DOCKER_DAEMON_ARGS=--log-level=error
   links:
@@ -69,7 +72,8 @@ mesosslave2:
   - mesosmaster1
   - "ambassador:apiserver"
   volumes:
-  - /var/tmp/mesosslave2:/var/lib/docker
+  - ${MESOS_DOCKER_WORK_DIR}/mesosslave2/mesos:/var/tmp/mesos
+  - ${MESOS_DOCKER_IMAGE_DIR}/mesosslave2/docker:/var/lib/docker
 apiserver:
   hostname: apiserver
   image: mesosphere/kubernetes-mesos
@@ -105,7 +109,7 @@ apiserver:
   - MESOS_DOCKER_MESOS_TIMEOUT
   - KUBE_KEYGEN_TIMEOUT
   volumes:
-  - ${MESOS_DOCKER_AUTH_DIR}:/var/run/kubernetes/auth:ro
+  - ${MESOS_DOCKER_WORK_DIR}/auth:/var/run/kubernetes/auth:ro
   links:
   - etcd
   - mesosmaster1
@@ -131,7 +135,7 @@ controller:
   - MESOS_DOCKER_MESOS_TIMEOUT
   - MESOS_DOCKER_API_TIMEOUT
   volumes:
-  - ${MESOS_DOCKER_AUTH_DIR}:/var/run/kubernetes/auth:ro
+  - ${MESOS_DOCKER_WORK_DIR}/auth:/var/run/kubernetes/auth:ro
   links:
   - mesosmaster1
   - apiserver
@@ -173,6 +177,6 @@ keygen:
   - apiserver
   - /var/run/kubernetes/auth
   volumes:
-  - ${MESOS_DOCKER_AUTH_DIR}:/var/run/kubernetes/auth
+  - ${MESOS_DOCKER_WORK_DIR}/auth:/var/run/kubernetes/auth
   links:
   - apiserver
diff --git a/cluster/mesos/docker/util.sh b/cluster/mesos/docker/util.sh
index 672675292fd..2cc2053c448 100644
--- a/cluster/mesos/docker/util.sh
+++ b/cluster/mesos/docker/util.sh
@@ -33,9 +33,6 @@ source "${provider_root}/${KUBE_CONFIG_FILE-"config-default.sh"}"
 source "${KUBE_ROOT}/cluster/common.sh"
 source "${provider_root}/common/bin/util-ssl.sh"
 
-log_dir="${MESOS_DOCKER_LOG_DIR}"
-auth_dir="${MESOS_DOCKER_AUTH_DIR}"
-
 
 # Run kubernetes scripts inside docker.
 # This bypasses the need to set up network routing when running docker in a VM (e.g. boot2docker).
@@ -119,6 +116,7 @@ function cluster::mesos::docker::run_in_docker_cagen {
 
 # Generate kubeconfig data for the created cluster.
 function create-kubeconfig {
+  local -r auth_dir="${MESOS_DOCKER_WORK_DIR}/auth"
   local kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
 
   export CONTEXT="${KUBERNETES_PROVIDER}"
@@ -157,7 +155,7 @@ function detect-master {
 
   docker_id=$(docker ps --filter="name=docker_apiserver" --quiet)
   if [[ "${docker_id}" == *'\n'* ]]; then
-    echo "ERROR: Multiple API Servers running in docker" 1>&2
+    echo "ERROR: Multiple API Servers running" 1>&2
     return 1
   fi
 
@@ -175,6 +173,10 @@ function detect-master {
 # but might not have a Kublet running unless a kubernetes task has been scheduled on them.
 function detect-minions {
   docker_ids=$(docker ps --filter="name=docker_mesosslave" --quiet)
+  if [ -z "${docker_ids}" ]; then
+    echo "ERROR: Mesos slave(s) not running" 1>&2
+    return 1
+  fi
   while read -r docker_id; do
     minion_ip=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "${docker_id}")
     KUBE_MINION_IP_ADDRESSES+=("${minion_ip}")
@@ -192,9 +194,11 @@ function verify-prereqs {
 
 # Initialize
 function cluster::mesos::docker::init_auth {
-  local -r auth_dir="$1"
+  local -r auth_dir="${MESOS_DOCKER_WORK_DIR}/auth"
 
   #TODO(karlkfi): reuse existing credentials/certs/keys
+  # Nuke old auth
+  echo "Creating Auth Dir: ${auth_dir}" 1>&2
   mkdir -p "${auth_dir}"
   rm -rf "${auth_dir}"/*
 
@@ -213,7 +217,17 @@ function cluster::mesos::docker::init_auth {
 
 # Instantiate a kubernetes cluster.
 function kube-up {
-  # Nuke logs up front so that we know any existing logs came from the last kube-up
+  # Nuke old mesos-slave workspaces
+  for ((i=1; i <= NUM_MINIONS; i++)) do
+    local work_dir="${MESOS_DOCKER_WORK_DIR}/mesosslave${i}/mesos"
+    echo "Creating Mesos Work Dir: ${work_dir}" 1>&2
+    mkdir -p "${work_dir}"
+    rm -rf "${work_dir}"/*
+  done
+
+  local -r log_dir="${MESOS_DOCKER_WORK_DIR}/log"
+
+  # Nuke old logs
   mkdir -p "${log_dir}"
   rm -rf "${log_dir}"/*
 
@@ -230,7 +244,7 @@ function kube-up {
     "${provider_root}/keygen/build.sh"
   fi
 
-  cluster::mesos::docker::init_auth "${auth_dir}"
+  cluster::mesos::docker::init_auth
 
   # Dump logs on premature exit (errexit triggers exit).
   # Trap EXIT instead of ERR, because ERR can trigger multiple times with errtrace enabled.
@@ -241,7 +255,8 @@ function kube-up {
   export MESOS_DOCKER_MESOS_TIMEOUT="${MESOS_DOCKER_MESOS_TIMEOUT}"
   export MESOS_DOCKER_API_TIMEOUT="${MESOS_DOCKER_API_TIMEOUT}"
   export KUBE_KEYGEN_TIMEOUT="${KUBE_KEYGEN_TIMEOUT}"
-  export MESOS_DOCKER_AUTH_DIR="${MESOS_DOCKER_AUTH_DIR}"
+  export MESOS_DOCKER_WORK_DIR="${MESOS_DOCKER_WORK_DIR}"
+  export MESOS_DOCKER_IMAGE_DIR="${MESOS_DOCKER_IMAGE_DIR}"
   docker-compose -f "${compose_file}" up -d
 
   # await-health-check requires GNU timeout
@@ -330,7 +345,7 @@ function cluster::mesos::docker::await_ready {
 
 # Prints the status of the kube-system pod specified
 function cluster::mesos::docker::addon_status {
-  local pod_name=$1
+  local pod_name="$1"
   local kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
   local phase=$("${kubectl}" get pods --namespace=kube-system -l k8s-app=${pod_name} -o template --template="{{(index .items 0).status.phase}}" 2>/dev/null)
   phase="${phase:-Unknown}"

From 87da2ed6de4403f88cb1932ed6a1a0d581858af2 Mon Sep 17 00:00:00 2001
From: Karl Isenberg <karl@mesosphere.io>
Date: Mon, 17 Aug 2015 17:20:55 -0700
Subject: [PATCH 4/6] Update mesos-slave-dind to use Docker 1.8.1

---
 cluster/mesos/docker/docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cluster/mesos/docker/docker-compose.yml b/cluster/mesos/docker/docker-compose.yml
index 6137a61aefc..fca56d19113 100644
--- a/cluster/mesos/docker/docker-compose.yml
+++ b/cluster/mesos/docker/docker-compose.yml
@@ -29,7 +29,7 @@ mesosmaster1:
 mesosslave1:
   hostname: mesosslave1
   privileged: true
-  image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404
+  image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404.docker181
   entrypoint: [ "bash", "-c", "wrapdocker mesos-slave --hostname=$(getent hosts mesosslave1 | cut -d' ' -f1 | sort -u | tail -1)" ]
   command: ~
   environment:
@@ -53,7 +53,7 @@ mesosslave1:
 mesosslave2:
   hostname: mesosslave2
   privileged: true
-  image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404
+  image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404.docker181
   entrypoint: [ "bash", "-c", "wrapdocker mesos-slave --hostname=$(getent hosts mesosslave2 | cut -d' ' -f1 | sort -u | tail -1)" ]
   command: ~
   environment:

From 9b5d525e3de2db62d6168853cd94201bbe9ddd06 Mon Sep 17 00:00:00 2001
From: Karl Isenberg <karl@mesosphere.io>
Date: Mon, 17 Aug 2015 18:27:41 -0700
Subject: [PATCH 5/6] [mesos/docker] Upgrade mesos-docker-test image to use
 docker 1.8.1

---
 cluster/mesos/docker/test/Dockerfile | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/cluster/mesos/docker/test/Dockerfile b/cluster/mesos/docker/test/Dockerfile
index 19cd78a595a..93f5ced8a5d 100644
--- a/cluster/mesos/docker/test/Dockerfile
+++ b/cluster/mesos/docker/test/Dockerfile
@@ -26,12 +26,13 @@ RUN apt-get update -qq && \
 # Install latest Docker
 # RUN curl -sSL https://get.docker.com/ubuntu/ | sh
 
-# Install Docker 1.6.2 (docker 1.7 has regressions)
-RUN echo deb https://get.docker.com/ubuntu docker main > /etc/apt/sources.list.d/docker.list && \
-    apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 && \
+# Install Docker 1.8.1 explicitly
+RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D && \
+    mkdir -p /etc/apt/sources.list.d && \
+    echo deb https://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list && \
     apt-get update -qq && \
     DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \
-        lxc-docker-1.6.2 \
+        docker-engine=1.8.1-0~trusty \
         && \
     apt-get clean
 

From 053cb26dde51f97a631e13fb87cbd6a7ec79f80f Mon Sep 17 00:00:00 2001
From: Karl Isenberg <karl@mesosphere.io>
Date: Tue, 18 Aug 2015 16:50:13 -0700
Subject: [PATCH 6/6] [mesos/docker] Expose DOCKER_DAEMON_ARGS

---
 cluster/mesos/docker/config-default.sh  | 4 +++-
 cluster/mesos/docker/docker-compose.yml | 6 ++----
 cluster/mesos/docker/util.sh            | 1 +
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/cluster/mesos/docker/config-default.sh b/cluster/mesos/docker/config-default.sh
index 9798210ae08..e56446d1324 100755
--- a/cluster/mesos/docker/config-default.sh
+++ b/cluster/mesos/docker/config-default.sh
@@ -64,4 +64,6 @@ MESOS_DOCKER_WORK_DIR="${MESOS_DOCKER_WORK_DIR:-${HOME}/tmp/kubernetes}"
 # Usage: ${MESOS_DOCKER_IMAGE_DIR}/<component>/docker
 # Must not be either an AUFS mount or an SMB/CIFS mount.
 # If using docker-machine or boot2docker, should NOT be under /Users (which is mounted from the host into the docker vm).
-MESOS_DOCKER_IMAGE_DIR="${MESOS_DOCKER_IMAGE_DIR:-/var/tmp/kubernetes}"
\ No newline at end of file
+MESOS_DOCKER_IMAGE_DIR="${MESOS_DOCKER_IMAGE_DIR:-/var/tmp/kubernetes}"
+
+DOCKER_DAEMON_ARGS="${DOCKER_DAEMON_ARGS:---log-level=error}"
diff --git a/cluster/mesos/docker/docker-compose.yml b/cluster/mesos/docker/docker-compose.yml
index fca56d19113..42ed9521b92 100644
--- a/cluster/mesos/docker/docker-compose.yml
+++ b/cluster/mesos/docker/docker-compose.yml
@@ -42,14 +42,13 @@ mesosslave1:
   - MESOS_CONTAINERIZERS=docker,mesos
   - MESOS_WORK_DIR=/var/tmp/mesos
   - DOCKER_NETWORK_OFFSET=0.0.1.0
-  - DOCKER_DAEMON_ARGS=--log-level=error
+  - DOCKER_DAEMON_ARGS
   links:
   - etcd
   - mesosmaster1
   - "ambassador:apiserver"
   volumes:
   - ${MESOS_DOCKER_WORK_DIR}/mesosslave1/mesos:/var/tmp/mesos
-  - ${MESOS_DOCKER_IMAGE_DIR}/mesosslave1/docker:/var/lib/docker
 mesosslave2:
   hostname: mesosslave2
   privileged: true
@@ -66,14 +65,13 @@ mesosslave2:
   - MESOS_CONTAINERIZERS=docker,mesos
   - MESOS_WORK_DIR=/var/tmp/mesos
   - DOCKER_NETWORK_OFFSET=0.0.2.0
-  - DOCKER_DAEMON_ARGS=--log-level=error
+  - DOCKER_DAEMON_ARGS
   links:
   - etcd
   - mesosmaster1
   - "ambassador:apiserver"
   volumes:
   - ${MESOS_DOCKER_WORK_DIR}/mesosslave2/mesos:/var/tmp/mesos
-  - ${MESOS_DOCKER_IMAGE_DIR}/mesosslave2/docker:/var/lib/docker
 apiserver:
   hostname: apiserver
   image: mesosphere/kubernetes-mesos
diff --git a/cluster/mesos/docker/util.sh b/cluster/mesos/docker/util.sh
index 2cc2053c448..ff51e8eee69 100644
--- a/cluster/mesos/docker/util.sh
+++ b/cluster/mesos/docker/util.sh
@@ -257,6 +257,7 @@ function kube-up {
   export KUBE_KEYGEN_TIMEOUT="${KUBE_KEYGEN_TIMEOUT}"
   export MESOS_DOCKER_WORK_DIR="${MESOS_DOCKER_WORK_DIR}"
   export MESOS_DOCKER_IMAGE_DIR="${MESOS_DOCKER_IMAGE_DIR}"
+  export DOCKER_DAEMON_ARGS="${DOCKER_DAEMON_ARGS}"
   docker-compose -f "${compose_file}" up -d
 
   # await-health-check requires GNU timeout