Rackspace - Switch to CoreOS for standard cluster

The Rackspace setup for Kubernetes now leverages CoreOS instead of Ubuntu. We've dropped Salt completely for our cluster.

cluster/rackspace/cloud-config/master-cloud-config.yaml

@@ -0,0 +1,153 @@
+#cloud-config
+
+write_files:
+  - path: /opt/bin/regen-minion-list.sh
+    permissions: 0755
+    content: |
+      #!/bin/sh
+      m=$(echo $(etcdctl ls --recursive /corekube/minions | cut -d/ -f4 | sort) | tr ' ' ,)
+      echo "Found $m"
+      mkdir -p /run/apiserver
+      echo "MINIONS=$m" > /run/apiserver/minions.env
+  - path: /opt/bin/git-kubernetes-nginx.sh
+    permissions: 0755
+    content: |
+      #!/bin/bash
+      git clone https://github.com/doublerr/kubernetes_nginx /opt/kubernetes_nginx
+      /usr/bin/cp /opt/.kubernetes_auth /opt/kubernetes_nginx/.kubernetes_auth
+      docker build -t kubernetes_nginx:latest /opt/kubernetes_nginx
+  - path: /opt/bin/download-release.sh
+    permissions: 0755
+    content: |
+      #!/bin/bash
+      OBJECT_URL="CLOUD_FILES_URL"
+      echo "Downloading release ($OBJECT_URL)"
+      wget $OBJECT_URL -O /opt/kubernetes.tar.gz
+      echo "Unpacking release"
+      rm -rf /opt/kubernetes || false
+      tar xzf /opt/kubernetes.tar.gz -C /opt/
+  - path: /opt/.kubernetes_auth
+    permissions: 0600
+    content: |
+      KUBE_USER:KUBE_PASSWORD
+
+coreos:
+  etcd:
+    name: kubernetes-master
+    discovery: https://discovery.etcd.io/DISCOVERY_ID
+    addr: $private_ipv4:4001
+    peer-addr: $private_ipv4:7001
+    peer-bind-addr: $private_ipv4:7001
+
+  fleet:
+    public-ip: $private_ipv4
+    metadata: kubernetes_role=master
+
+  update:
+    reboot-strategy: etcd-lock
+
+  units:
+    #- name: nova-agent-watcher.service
+    #  command: try-restart
+    - name: etcd.service
+      command: start
+    - name: fleet.service
+      command: start
+    - name: download-release.service
+      command: start
+      content: |
+        [Unit]
+        Description=Downloads Kubernetes Release
+        After=network-online.target
+        Requires=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=/usr/bin/bash /opt/bin/download-release.sh
+    - name: master-apiserver.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes API Server
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        After=network-online.target
+        Requires=network-online.target
+        After=minion-finder.service
+        Requires=minion-finder.service
+        After=download-release.service
+        Requires=download-release.service
+        [Service]
+        EnvironmentFile=-/run/apiserver/minions.env
+        ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/platforms/linux/amd64/apiserver /opt/bin/apiserver
+        ExecStart=/opt/bin/apiserver --address=127.0.0.1 --port=8080 --machines=${MINIONS} --etcd_servers=http://127.0.0.1:4001 --logtostderr=true
+        Restart=always
+        RestartSec=2
+    - name: master-apiserver-sighup.path
+      command: start
+      content: |
+        [Path]
+        PathChanged=/run/apiserver/minions.env
+    - name: master-apiserver-sighup.service
+      command: start
+      content: |
+        [Service]
+        ExecStart=/usr/bin/pkill -SIGHUP -f apiserver
+    - name: minion-finder.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Minion finder
+        After=network-online.target
+        Requires=network-online.target
+        After=etcd.service
+        Requires=etcd.service
+        [Service]
+        ExecStartPre=/opt/bin/regen-minion-list.sh
+        ExecStart=/usr/bin/etcdctl exec-watch --recursive /corekube/minions -- /opt/bin/regen-minion-list.sh
+        Restart=always
+        RestartSec=30
+    - name: master-controller-manager.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Controller Manager
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        After=network-online.target
+        Requires=network-online.target
+        After=master-apiserver.service
+        Requires=master-apiserver.service
+        [Service]
+        ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/platforms/linux/amd64/controller-manager /opt/bin/controller-manager
+        ExecStart=/opt/bin/controller-manager --master=127.0.0.1:8080 --logtostderr=true
+        Restart=always
+        RestartSec=2
+    - name: master-scheduler.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Scheduler
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        After=network-online.target
+        Requires=network-online.target
+        After=master-apiserver.service
+        Requires=master-apiserver.service
+        [Service]
+        ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/platforms/linux/amd64/scheduler /opt/bin/scheduler
+        ExecStart=/opt/bin/scheduler --master=127.0.0.1:8080 --logtostderr=true
+        Restart=always
+        RestartSec=10
+    - name: kubernetes-nginx.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Nginx Service
+        After=network-online.target
+        Requires=network-online.target
+        After=docker.service
+        Requires=docker.service
+        [Service]
+        ExecStartPre=/opt/bin/git-kubernetes-nginx.sh
+        ExecStart=/usr/bin/docker run --rm --net="host" -p "443:443" -t --name "kubernetes_nginx" kubernetes_nginx
+        ExecStop=/usr/bin/docker stop kubernetes_nginx
+        Restart=always
+        RestartSec=15

cluster/rackspace/cloud-config/minion-cloud-config.yaml

@@ -0,0 +1,224 @@
+#cloud-config
+
+write_files:
+  - path: /opt/bin/kube-net-update.sh
+    permissions: 0755
+    content: |
+      #!/bin/sh
+      set -x -e
+      nh=${ETCD_WATCH_KEY##*/}
+      net=$ETCD_WATCH_VALUE
+      case $ETCD_WATCH_ACTION in
+      set) ip route replace $net via $nh dev eth2 metric 900 ;;
+      expire) ip route del $net via $nh metric 900 ;;
+      esac
+  - path: /opt/bin/download-release.sh
+    permissions: 0755
+    content: |
+      #!/bin/bash
+      OBJECT_URL="http://storage.googleapis.com/kubernetes-releases-56726/devel/kubernetes.tar.gz"
+      echo "Downloading release ($OBJECT_URL)"
+      wget $OBJECT_URL -O /opt/kubernetes.tar.gz
+      echo "Unpacking release"
+      rm -rf /opt/kubernetes || false
+      tar xzf /opt/kubernetes.tar.gz -C /opt/
+  - path: /opt/kubernetes-manifests/cadvisor.manifest
+    permissions: 0755
+    content: |
+      version: v1beta2
+      id: cadvisor-agent
+      containers:
+        - name: cadvisor
+          image: google/cadvisor:latest
+          ports:
+            - name: http
+              containerPort: 8080
+              hostPort: 4194
+          volumeMounts:
+            - name: varrun
+              mountPath: /var/run
+              readOnly: false
+            - name: varlibdocker
+              mountPath: /var/lib/docker
+              readOnly: true
+            - name: cgroups
+              mountPath: /sys/fs/cgroup
+              readOnly: true
+      volumes:
+        - name: varrun
+          source:
+            hostDir:
+              path: /var/run
+        - name: varlibdocker
+          source:
+            hostDir:
+              path: /var/lib/docker
+        - name: cgroups
+          source:
+            hostDir:
+              path: /sys/fs/cgroup
+
+coreos:
+  etcd:
+    name: kubernetes-minion-INDEX
+    discovery: https://discovery.etcd.io/DISCOVERY_ID
+    addr: $private_ipv4:4001
+    peer-addr: $private_ipv4:7001
+    peer-bind-addr: $private_ipv4:7001
+
+  fleet:
+    public-ip: $private_ipv4
+    metadata: kubernetes_role=minion
+
+  update:
+    reboot-strategy: etcd-lock
+
+  units:
+    - name: etcd.service
+      command: start
+    - name: fleet.service
+      command: start
+    - name: download-release.service
+      command: start
+      content: |
+        [Unit]
+        Description=Downloads Kubernetes Release
+        After=network-online.target
+        Requires=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=/usr/bin/bash /opt/bin/download-release.sh
+    - name: minion-kubelet.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Kubelet
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        After=network-online.target
+        Requires=network-online.target
+        After=docker.service
+        Requires=docker.service
+        After=download-release.service
+        Requires=download-release.service
+        [Service]
+        ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/platforms/linux/amd64/kubelet /opt/bin/kubelet
+        ExecStart=/opt/bin/kubelet --address=$private_ipv4 --hostname_override=$private_ipv4 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true --config=/opt/kubernetes-manifests
+        Restart=always
+        RestartSec=2
+    - name: minion-proxy.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Proxy
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        After=network-online.target
+        Requires=network-online.target
+        After=docker.service
+        Requires=docker.service
+        After=download-release.service
+        Requires=download-release.service
+        [Service]
+        ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/platforms/linux/amd64/proxy /opt/bin/proxy
+        ExecStart=/opt/bin/proxy --bind_address=$private_ipv4 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true
+        Restart=always
+        RestartSec=2
+    - name: minion-advertiser.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Minion Advertiser
+        After=etcd.service
+        Requires=etcd.service
+        After=minion-kubelet.service
+        [Service]
+        ExecStart=/bin/sh -c 'while :; do etcdctl set /corekube/minions/$private_ipv4 $private_ipv4 --ttl 300; sleep 120; done'
+        Restart=always
+        RestartSec=120
+    - name: net-advertiser.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Network Advertiser
+        After=etcd.service
+        Requires=etcd.service
+        After=minion-kubelet.service
+        [Service]
+        ExecStart=/bin/sh -c 'eth2_ip=$$(ip -o -f inet a show dev eth2 | sed "s/.* inet \([0-9.]\+\).*/\1/"); while :; do etcdctl set /corekube/net/$$eth2_ip 10.240.INDEX.0/24 --ttl 300; sleep 120; done'
+        Restart=always
+        RestartSec=120
+    - name: net-router.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Network Router
+        After=etcd.service
+        Requires=etcd.service
+        After=minion-kubelet.service
+        [Service]
+        ExecStart=/usr/bin/etcdctl exec-watch --recursive /corekube/net -- /opt/bin/kube-net-update.sh
+        Restart=always
+        RestartSec=120
+    - name: cbr0.netdev
+      command: start
+      content: |
+        [NetDev]
+        Kind=bridge
+        Name=cbr0
+    - name: cbr0.network
+      command: start
+      content: |
+        [Match]
+        Name=cbr0
+
+        [Network]
+        Address=10.240.INDEX.1/24
+    - name: nat.service
+      command: start
+      content: |
+        [Unit]
+        Description=NAT container->outside traffic
+
+        [Service]
+        ExecStart=/usr/sbin/iptables -t nat -A POSTROUTING -o eth0 -s 10.240.INDEX.0/24 -j MASQUERADE
+        ExecStart=/usr/sbin/iptables -t nat -A POSTROUTING -o eth1 -s 10.240.INDEX.0/24 -j MASQUERADE
+        RemainAfterExit=yes
+        Type=oneshot
+    - name: docker.service
+      command: start
+      content: |
+        [Unit]
+        After=network.target
+        Description=Docker Application Container Engine
+        Documentation=http://docs.docker.io
+
+        [Service]
+        ExecStartPre=/bin/mount --make-rprivate /
+        ExecStart=/usr/bin/docker -d -s=btrfs -H fd:// -b cbr0 --iptables=false
+        Restart=always
+        RestartSec=30
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: format-data.service
+      command: start
+      content: |
+        [Unit]
+        Description=Formats data drive
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=/usr/sbin/wipefs -f /dev/xvde1
+        ExecStart=/usr/sbin/mkfs.btrfs -f /dev/xvde1
+    - name: var-lib-docker-volumes.mount
+      command: start
+      content: |
+        [Unit]
+        Description=Mount data drive to /var/lib/docker/volumes
+        Requires=format-data.service
+        After=format-data.service
+        Before=docker.service
+        [Mount]
+        What=/dev/xvde1
+        Where=/var/lib/docker/volumes
+        Type=btrfs

cluster/rackspace/config-default.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sane defaults for dev environments. The following variables can be easily overriden
+# by setting each as a ENV variable ahead of time:
+# KUBE_IMAGE, KUBE_MASTER_FLAVOR, KUBE_MINION_FLAVOR, NUM_MINIONS, NOVA_NETWORK and SSH_KEY_NAME
+
+# Shared
+KUBE_IMAGE="${KUBE_IMAGE-b63e1435-a46f-4726-b984-e3f15ae92753}" # CoreOS(Beta) 
+SSH_KEY_NAME="${SSH_KEY_NAME-id_kubernetes}"
+NOVA_NETWORK_LABEL="kubernetes-pool-net"
+NOVA_NETWORK_CIDR="${NOVA_NETWORK-192.168.0.0/24}"
+INSTANCE_PREFIX="kubernetes"
+
+# Master
+KUBE_MASTER_FLAVOR="${KUBE_MASTER_FLAVOR-performance1-1}"
+MASTER_NAME="${INSTANCE_PREFIX}-master"
+MASTER_TAG="tags=${INSTANCE_PREFIX}-master"
+
+# Minion
+KUBE_MINION_FLAVOR="${KUBE_MINION_FLAVOR-performance1-2}"
+RAX_NUM_MINIONS="${RAX_NUM_MINIONS-4}"
+MINION_TAG="tags=${INSTANCE_PREFIX}-minion"
+MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${RAX_NUM_MINIONS}}))
+KUBE_NETWORK=($(eval echo "10.240.{1..${RAX_NUM_MINIONS}}.0/24"))
+PORTAL_NET="10.0.0.0/16"

cluster/rackspace/kube-up.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Bring up a Kubernetes cluster.
+#
+# If the full release name (gs://<bucket>/<release>) is passed in then we take
+# that directly.  If not then we assume we are doing development stuff and take
+# the defaults in the release config.
+
+# exit on any error
+set -e
+
+source $(dirname $0)/../kube-env.sh
+source $(dirname $0)/../$KUBERNETES_PROVIDER/util.sh
+
+echo "Starting cluster using provider: $KUBERNETES_PROVIDER"
+
+verify-prereqs
+kube-up
+
+# skipping validation for now until since machines show up as private IPs
+# source $(dirname $0)/validate-cluster.sh
+
+echo "Done"

cluster/rackspace/util.sh

@@ -0,0 +1,272 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A library of helper functions for deploying on Rackspace
+
+# Use the config file specified in $KUBE_CONFIG_FILE, or default to
+# config-default.sh.
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+source $(dirname ${BASH_SOURCE})/${KUBE_CONFIG_FILE-"config-default.sh"}
+
+verify-prereqs() {
+  # Make sure that prerequisites are installed.
+  for x in nova; do
+    if [ "$(which $x)" == "" ]; then
+      echo "cluster/rackspace/util.sh:  Can't find $x in PATH, please fix and retry."
+      exit 1
+    fi
+  done
+}
+
+# Ensure that we have a password created for validating to the master.  Will
+# read from $HOME/.kubernetres_auth if available.
+#
+# Vars set:
+#   KUBE_USER
+#   KUBE_PASSWORD
+get-password() {
+  local file="$HOME/.kubernetes_auth"
+  if [[ -r "$file" ]]; then
+    KUBE_USER=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["User"]')
+    KUBE_PASSWORD=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["Password"]')
+    return
+  fi
+  KUBE_USER=admin
+  KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))')
+
+  # Store password for reuse.
+  cat << EOF > "$file"
+{
+  "User": "$KUBE_USER",
+  "Password": "$KUBE_PASSWORD"
+}
+EOF
+  chmod 0600 "$file"
+}
+
+rax-ssh-key() {
+  if [ ! -f $HOME/.ssh/${SSH_KEY_NAME} ]; then
+    echo "cluster/rackspace/util.sh: Generating SSH KEY ${HOME}/.ssh/${SSH_KEY_NAME}"
+    ssh-keygen -f ${HOME}/.ssh/${SSH_KEY_NAME} -N '' > /dev/null
+  fi
+
+  if ! $(nova keypair-list | grep $SSH_KEY_NAME > /dev/null 2>&1); then
+    echo "cluster/rackspace/util.sh: Uploading key to Rackspace:"
+    echo -e "\tnova keypair-add ${SSH_KEY_NAME} --pub-key ${HOME}/.ssh/${SSH_KEY_NAME}.pub"
+    nova keypair-add ${SSH_KEY_NAME} --pub-key ${HOME}/.ssh/${SSH_KEY_NAME}.pub > /dev/null 2>&1
+  else
+    echo "cluster/rackspace/util.sh: SSH key ${SSH_KEY_NAME}.pub already uploaded"
+  fi
+}
+
+find-release-tars() {
+  SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz"
+  if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
+    SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz"
+  fi
+  if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
+    echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz"
+    exit 1
+  fi
+}
+
+find-object-url() {
+
+  RELEASE=kubernetes-releases-${OS_USERNAME}/devel/kubernetes-server-linux-amd64.tar.gz
+
+  TEMP_URL=$(swiftly -A ${OS_AUTH_URL} -U ${OS_USERNAME} -K ${OS_PASSWORD} tempurl GET $RELEASE)
+  echo "cluster/rackspace/util.sh: Object temp URL:"
+  echo -e "\t${TEMP_URL}"
+
+}
+
+rax-boot-master() {
+
+  DISCOVERY_URL=$(curl https://discovery.etcd.io/new)
+  DISCOVERY_ID=$(echo "${DISCOVERY_URL}" | cut -f 4 -d /)
+  echo "cluster/rackspace/util.sh: etcd discovery URL: ${DISCOVERY_URL}"
+
+  get-password
+  find-object-url
+
+# Copy cloud-config to KUBE_TEMP and work some sed magic
+  sed -e "s|DISCOVERY_ID|${DISCOVERY_ID}|" \
+      -e "s|CLOUD_FILES_URL|${TEMP_URL}|" \
+      -e "s|KUBE_USER|${KUBE_USER}|" \
+      -e "s|KUBE_PASSWORD|${KUBE_PASSWORD}|" \
+      -e "s|PORTAL_NET|${PORTAL_NET}|" \
+      $(dirname $0)/rackspace/cloud-config/master-cloud-config.yaml > $KUBE_TEMP/master-cloud-config.yaml
+
+
+  MASTER_BOOT_CMD="nova boot \
+--key-name ${SSH_KEY_NAME} \
+--flavor ${KUBE_MASTER_FLAVOR} \
+--image ${KUBE_IMAGE} \
+--meta ${MASTER_TAG} \
+--meta ETCD=${DISCOVERY_ID} \
+--user-data ${KUBE_TEMP}/master-cloud-config.yaml \
+--config-drive true \
+--nic net-id=${NETWORK_UUID} \
+${MASTER_NAME}"
+
+  echo "cluster/rackspace/util.sh: Booting ${MASTER_NAME} with following command:"
+  echo -e "\t$MASTER_BOOT_CMD"
+  $MASTER_BOOT_CMD
+}
+
+rax-boot-minions() {
+
+  cp $(dirname $0)/rackspace/cloud-config/minion-cloud-config.yaml \
+  ${KUBE_TEMP}/minion-cloud-config.yaml
+
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+
+    sed -e "s|DISCOVERY_ID|${DISCOVERY_ID}|" \
+        -e "s|INDEX|$((i + 1))|g" \
+        -e "s|CLOUD_FILES_URL|${TEMP_URL}|" \
+    $(dirname $0)/rackspace/cloud-config/minion-cloud-config.yaml > $KUBE_TEMP/minion-cloud-config-$(($i + 1)).yaml
+
+
+    MINION_BOOT_CMD="nova boot \
+--key-name ${SSH_KEY_NAME} \
+--flavor ${KUBE_MINION_FLAVOR} \
+--image ${KUBE_IMAGE} \
+--meta ${MINION_TAG} \
+--user-data ${KUBE_TEMP}/minion-cloud-config-$(( i +1 )).yaml \
+--config-drive true \
+--nic net-id=${NETWORK_UUID} \
+${MINION_NAMES[$i]}"
+
+    echo "cluster/rackspace/util.sh: Booting ${MINION_NAMES[$i]} with following command:"
+    echo -e "\t$MINION_BOOT_CMD"
+    $MINION_BOOT_CMD
+  done
+}
+
+rax-nova-network() {
+  if ! $(nova network-list | grep $NOVA_NETWORK_LABEL > /dev/null 2>&1); then
+    SAFE_CIDR=$(echo $NOVA_NETWORK_CIDR | tr -d '\\')
+    NETWORK_CREATE_CMD="nova network-create $NOVA_NETWORK_LABEL $SAFE_CIDR"
+
+    echo "cluster/rackspace/util.sh: Creating cloud network with following command:"
+    echo -e "\t${NETWORK_CREATE_CMD}"
+
+    $NETWORK_CREATE_CMD
+  else
+    echo "cluster/rackspace/util.sh: Using existing cloud network $NOVA_NETWORK_LABEL"
+  fi
+}
+
+detect-minions() {
+  KUBE_MINION_IP_ADDRESSES=()
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+    local minion_ip=$(nova show --minimal ${MINION_NAMES[$i]} \
+      | grep accessIPv4 | awk '{print $4}')
+    echo "cluster/rackspace/util.sh: Found ${MINION_NAMES[$i]} at ${minion_ip}"
+    KUBE_MINION_IP_ADDRESSES+=("${minion_ip}")
+  done
+  if [ -z "$KUBE_MINION_IP_ADDRESSES" ]; then
+    echo "cluster/rackspace/util.sh: Could not detect Kubernetes minion nodes.  Make sure you've launched a cluster with 'kube-up.sh'"
+    exit 1
+  fi
+
+}
+
+detect-master() {
+  KUBE_MASTER=${MASTER_NAME}
+
+  KUBE_MASTER_IP=$(nova show $KUBE_MASTER --minimal | grep accessIPv4 | awk '{print $4}')
+}
+
+# $1 should be the network you would like to get an IP address for
+detect-master-nova-net() {
+  KUBE_MASTER=${MASTER_NAME}
+
+  MASTER_IP=$(nova show $KUBE_MASTER --minimal | grep $1 | awk '{print $5}')
+}
+
+kube-up() {
+
+  SCRIPT_DIR=$(CDPATH="" cd $(dirname $0); pwd)
+  # Find the release to use.  Generally it will be passed when doing a 'prod'
+  # install and will default to the release/config.sh version when doing a
+  # developer up.
+  #find-object-url $CONTAINER output/release/$TAR_FILE
+
+  # Create a temp directory to hold scripts that will be uploaded to master/minions
+  KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
+  trap "rm -rf ${KUBE_TEMP}" EXIT
+
+  get-password
+  python $(dirname $0)/../third_party/htpasswd/htpasswd.py -b -c ${KUBE_TEMP}/htpasswd $KUBE_USER $KUBE_PASSWORD
+  HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd)
+
+  rax-nova-network
+  NETWORK_UUID=$(nova network-list | grep -i ${NOVA_NETWORK_LABEL} | awk '{print $2}')
+
+  # create and upload ssh key if necessary
+  rax-ssh-key
+
+  echo "cluster/rackspace/util.sh: Starting Cloud Servers"
+  rax-boot-master
+
+  rax-boot-minions
+
+  FAIL=0
+  for job in `jobs -p`
+  do
+      wait $job || let "FAIL+=1"
+  done
+  if (( $FAIL != 0 )); then
+    echo "${FAIL} commands failed.  Exiting."
+    exit 2
+  fi
+
+  detect-master
+
+  echo "Waiting for cluster initialization."
+  echo
+  echo "  This will continually check to see if the API for kubernetes is reachable."
+  echo "  This might loop forever if there was some uncaught error during start"
+  echo "  up."
+  echo
+
+  #This will fail until apiserver salt is updated
+  until $(curl --insecure --user ${KUBE_USER}:${KUBE_PASSWORD} --max-time 5 \
+          --fail --output /dev/null --silent https://${KUBE_MASTER_IP}/api/v1beta1/pods); do
+      printf "."
+      sleep 2
+  done
+
+  echo "Kubernetes cluster created."
+
+  # Don't bail on errors, we want to be able to print some info.
+  set +e
+
+  detect-minions
+
+  echo "All minions may not be online yet, this is okay."
+  echo
+  echo "Kubernetes cluster is running.  The master is running at:"
+  echo
+  echo "  https://${KUBE_MASTER_IP}"
+  echo
+  echo "The user name and password to use is located in ~/.kubernetes_auth."
+  echo
+  echo "Security note: The server above uses a self signed certificate.  This is"
+  echo "    subject to \"Man in the middle\" type attacks."
+  echo
+}