deploy update

cluster/azure/config-default.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+INSTANCE_PREFIX=kubenertes
+AZ_LOCATION='West US'
+TAG=testing
+AZ_CS_PREFIX=kube
+AZ_VNET=shchTest
+AZ_SUBNET=Subnet-1
+AZ_IMAGE=b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04-LTS-amd64-server-20140618.1-en-us-30GB
+AZ_CS="" # is set in azure/util.sh verify-prereqs 
+
+AZ_SSH_KEY=$HOME/.ssh/azure_rsa
+AZ_SSH_CERT=$HOME/.ssh/azure.pem
+
+NUM_MINIONS=2
+
+MASTER_NAME="${INSTANCE_PREFIX}-master"
+MASTER_TAG="${INSTANCE_PREFIX}-master"
+MINION_TAG="${INSTANCE_PREFIX}-minion"
+MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
+MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
+MINION_SCOPES=""
+

cluster/azure/templates/common.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Retry a download until we get it.
+#
+# $1 is the URL to download
+download-or-bust() {
+  local -r url="$1"
+  local -r file="${url##*/}"
+  rm -f "$file"
+  until [[ -e "${file}" ]]; do
+    curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "$url"
+    md5sum "$file"
+  done
+}
+
+# Install salt from GCS.  See README.md for instructions on how to update these
+# debs.
+#
+# $1 If set to --master, also install the master
+install-salt() {
+  apt-get update
+
+  mkdir -p /var/cache/salt-install
+  cd /var/cache/salt-install
+
+  TARS=(
+    libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
+    python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
+    salt-common_2014.1.13+ds-1~bpo70+1_all.deb
+    salt-minion_2014.1.13+ds-1~bpo70+1_all.deb
+  )
+  if [[ ${1-} == '--master' ]]; then
+    TARS+=(salt-master_2014.1.13+ds-1~bpo70+1_all.deb)
+  fi
+  URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
+
+  for tar in "${TARS[@]}"; do
+    download-or-bust "${URL_BASE}/${tar}"
+    dpkg -i "${tar}"
+  done
+
+# This will install any of the unmet dependencies from above.
+  apt-get install -f -y
+
+}

cluster/azure/templates/create-dynamic-salt-files.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Create the overlay files for the salt tree.  We create these in a separate
+# place so that we can blow away the rest of the salt configs on a kube-push and
+# re-apply these.
+
+mkdir -p /srv/salt-overlay/pillar
+cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
+node_instance_prefix: $NODE_INSTANCE_PREFIX
+EOF
+
+mkdir -p /srv/salt-overlay/salt/nginx
+echo $MASTER_HTPASSWD > /srv/salt-overlay/salt/nginx/htpasswd

cluster/azure/templates/download-release.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Download and install release
+
+# This script assumes that the environment variable MASTER_RELEASE_TAR contains
+# the release tar to download and unpack.  It is meant to be pushed to the
+# master and run.
+
+
+echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
+download-or-bust "$SERVER_BINARY_TAR_URL"
+
+echo "Downloading binary release tar ($SALT_TAR_URL)"
+download-or-bust "$SALT_TAR_URL"
+
+echo "Unpacking Salt tree"
+rm -rf kubernetes
+tar xzf "${SALT_TAR_URL##*/}"
+
+echo "Running release install script"
+sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}"

cluster/azure/templates/salt-master.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Prepopulate the name of the Master
+mkdir -p /etc/salt/minion.d
+echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
+
+cat <<EOF >/etc/salt/minion.d/grains.conf
+grains:
+  roles:
+    - kubernetes-master
+  cloud: azure
+EOF
+
+# Auto accept all keys from minions that try to join
+mkdir -p /etc/salt/master.d
+cat <<EOF >/etc/salt/master.d/auto-accept.conf
+auto_accept: True
+EOF
+
+cat <<EOF >/etc/salt/master.d/reactor.conf
+# React to new minions starting by running highstate on them.
+reactor:
+  - 'salt/minion/*/start':
+    - /srv/reactor/highstate-new.sls
+EOF
+
+mkdir -p /srv/salt/nginx
+echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd
+
+mkdir -p /etc/openvpn
+umask=$(umask)
+umask 0066
+echo "$CA_CRT" > /etc/openvpn/ca.crt
+echo "$SERVER_CRT" > /etc/openvpn/server.crt
+echo "$SERVER_KEY" > /etc/openvpn/server.key
+umask $umask
+
+cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
+log_level: debug
+log_level_logfile: debug
+EOF
+
+cat <<EOF >/etc/salt/master.d/log-level-debug.d
+log_level: debug
+log_level_logfile: debug
+EOF
+
+install-salt --master
+
+# Wait a few minutes and trigger another Salt run to better recover from
+# any transient errors.
+echo "Sleeping 180"
+sleep 180
+salt-call state.highstate || true
+

cluster/azure/templates/salt-minion.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mkdir -p /etc/openvpn
+umask=$(umask)
+umask 0066
+echo "$CA_CRT" > /etc/openvpn/ca.crt
+echo "$CLIENT_CRT" > /etc/openvpn/client.crt
+echo "$CLIENT_KEY" > /etc/openvpn/client.key
+umask $umask
+
+# Prepopulate the name of the Master
+mkdir -p /etc/salt/minion.d
+echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
+
+cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
+log_level: debug
+log_level_logfile: debug
+EOF
+
+# Our minions will have a pool role to distinguish them from the master.
+#cat <<EOF >/etc/salt/minion.d/grains.conf
+#grains:
+#  roles:
+#    - kubernetes-pool
+#  cbr-cidr: $MINION_IP_RANGE
+#  cloud: gce
+#EOF
+
+hostnamef=$(hostname -f)
+sudo apt-get install ipcalc
+netmask=$(ipcalc $MINION_IP_RANGE | grep Netmask | awk '{ print $2 }')
+network=$(ipcalc $MINION_IP_RANGE | grep Address | awk '{ print $2 }')
+cbrstring="$network $netmask"
+
+# Our minions will have a pool role to distinguish them from the master.
+cat <<EOF >/etc/salt/minion.d/grains.conf
+grains:
+  roles:
+    - kubernetes-pool
+  cbr-cidr: $MINION_IP_RANGE
+  cloud: azure
+  hostnamef: $hostnamef
+  cbr-string: $cbrstring
+EOF
+
+install-salt
+
+# Wait a few minutes and trigger another Salt run to better recover from
+# any transient errors.
+echo "Sleeping 180"
+sleep 180
+salt-call state.highstate || true

cluster/azure/util.sh

@@ -0,0 +1,566 @@
+#!/bin/bash
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A library of helper functions and constant for the local config.
+
+# Use the config file specified in $KUBE_CONFIG_FILE, or default to
+# config-default.sh.
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+source "${KUBE_ROOT}/cluster/azure/${KUBE_CONFIG_FILE-"config-default.sh"}"
+
+function json_val () {
+    python -c 'import json,sys;obj=json.load(sys.stdin);print obj'$1'';
+}
+
+# Verify prereqs
+function verify-prereqs {
+	if [ -z "$(which azure)" ]; then
+		echo "Couldn't find azure in PATH"
+		echo "  please install with 'npm install azure-cli'"
+		exit 1
+	fi
+
+	if [ -z "$(azure account list | grep true)" ]; then
+		echo "Default azure account not set"
+		echo "  please set with 'azure account set'"
+		exit 1
+	fi
+	
+  account=$(azure account list | grep true | awk '{ print $2 }')
+  if which md5 > /dev/null 2>&1; then
+    AZ_HSH=$(md5 -q -s "$AZ_SUBNET@$account")
+  else
+    AZ_HSH=$(echo -n "$AZ_SUBNET@$account" | md5sum)
+  fi
+  
+  AZ_HSH=${AZ_HSH:0:7}
+  AZ_STG=kube$AZ_HSH
+  echo "==> AZ_STG: $AZ_STG"
+  
+  AZ_CS="$AZ_CS_PREFIX-$AZ_HSH"
+  echo "==> AZ_CS: $AZ_CS"
+  
+  CONTAINER=kube-$TAG
+  echo "==> CONTAINER: $CONTAINER"
+}
+
+# Create a temp dir that'll be deleted at the end of this bash session.
+#
+# Vars set:
+#   KUBE_TEMP
+function ensure-temp-dir {
+  if [[ -z ${KUBE_TEMP-} ]]; then
+    KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
+    trap 'rm -rf "${KUBE_TEMP}"' EXIT
+  fi
+}
+
+# Verify and find the various tar files that we are going to use on the server.
+#
+# Vars set:
+#   SERVER_BINARY_TAR
+#   SALT_TAR
+function find-release-tars {
+  SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz"
+  if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
+    SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz"
+  fi
+  if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
+    echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz"
+    exit 1
+  fi
+
+  SALT_TAR="${KUBE_ROOT}/server/kubernetes-salt.tar.gz"
+  if [[ ! -f "$SALT_TAR" ]]; then
+    SALT_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-salt.tar.gz"
+  fi
+  if [[ ! -f "$SALT_TAR" ]]; then
+    echo "!!! Cannot find kubernetes-salt.tar.gz"
+    exit 1
+  fi
+}
+
+
+# Take the local tar files and upload them to Azure Storage.  They will then be
+# downloaded by the master as part of the start up script for the master.
+#
+# Assumed vars:
+#   SERVER_BINARY_TAR
+#   SALT_TAR
+# Vars set:
+#   SERVER_BINARY_TAR_URL
+#   SALT_TAR_URL
+function upload-server-tars() {
+  SERVER_BINARY_TAR_URL=
+  SALT_TAR_URL=
+  
+  echo "==> SERVER_BINARY_TAR: $SERVER_BINARY_TAR"
+  echo "==> SALT_TAR: $SALT_TAR"
+  
+  echo "+++ Staging server tars to Azure Storage: $AZ_STG"
+  local server_binary_url="${SERVER_BINARY_TAR##*/}"
+  local salt_url="${SALT_TAR##*/}"
+  
+  SERVER_BINARY_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$server_binary_url"
+  SALT_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$salt_url"
+
+  echo "==> SERVER_BINARY_TAR_URL: $SERVER_BINARY_TAR_URL"
+  echo "==> SALT_TAR_URL: $SALT_TAR_URL"
+  
+  
+  echo "--> Checking storage exsists..."
+  if [ -z "$(azure storage account show $AZ_STG 2>/dev/null | \
+    grep data)" ]; then
+    echo "--> Creating storage..."
+    azure storage account create -l "$AZ_LOCATION" $AZ_STG
+  fi
+
+  echo "--> Getting storage key..."
+  stg_key=$(azure storage account keys list $AZ_STG --json | \
+      json_val '["primaryKey"]')
+
+  echo "--> Checking storage container exsists..."
+  if [ -z "$(azure storage container show -a $AZ_STG -k "$stg_key" \
+      $CONTAINER 2>/dev/null | grep data)" ]; then
+      echo "--> Creating storage container..."
+      azure storage container create \
+          -a $AZ_STG \
+          -k "$stg_key" \
+          -p Blob \
+          $CONTAINER
+  fi
+
+  echo "--> Checking server binary exists in the container..."
+  if [ -n "$(azure storage blob show -a $AZ_STG -k "$stg_key" \
+      $CONTAINER $server_binary_url 2>/dev/null | grep data)" ]; then
+      echo "--> Deleting server binary in the container..."
+      azure storage blob delete \
+          -a $AZ_STG \
+          -k "$stg_key" \
+          $CONTAINER \
+          $server_binary_url
+  fi
+  
+  echo "--> Uploading server binary to the container..."
+  azure storage blob upload \
+      -a $AZ_STG \
+      -k "$stg_key" \
+      $SERVER_BINARY_TAR \
+      $CONTAINER \
+      $server_binary_url
+      
+  echo "--> Checking salt data exists in the container..."
+  if [ -n "$(azure storage blob show -a $AZ_STG -k "$stg_key" \
+      $CONTAINER $salt_url 2>/dev/null | grep data)" ]; then
+      echo "--> Deleting salt data in the container..."
+      azure storage blob delete \
+          -a $AZ_STG \
+          -k "$stg_key" \
+          $CONTAINER \
+          $salt_url
+  fi
+
+  echo "--> Uploading salt data to the container..."
+  azure storage blob upload \
+      -a $AZ_STG \
+      -k "$stg_key" \
+      $SALT_TAR \
+      $CONTAINER \
+      $salt_url
+}
+
+# Detect the information about the minions
+#
+# Assumed vars:
+#   MINION_NAMES
+#   ZONE
+# Vars set:
+#   
+function detect-minions () {
+  if [ -z "$AZ_CS" ]; then
+   verify-prereqs
+  fi
+  ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}"))
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+      MINION_NAMES[$i]=$(ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net hostname -f)
+  done
+
+}
+
+# Detect the IP for the master
+#
+# Assumed vars:
+#   MASTER_NAME
+#   ZONE
+# Vars set:
+#   KUBE_MASTER
+#   KUBE_MASTER_IP
+function detect-master () {
+  if [ -z "$AZ_CS" ]; then
+   verify-prereqs
+  fi
+  
+  KUBE_MASTER=${MASTER_NAME}
+  KUBE_MASTER_IP="${AZ_CS}.cloudapp.net"
+  echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)"
+
+}
+
+# Ensure that we have a password created for validating to the master.  Will
+# read from $HOME/.kubernetres_auth if available.
+#
+# Vars set:
+#   KUBE_USER
+#   KUBE_PASSWORD
+function get-password {
+  local file="$HOME/.kubernetes_auth"
+  if [[ -r "$file" ]]; then
+    KUBE_USER=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["User"]')
+    KUBE_PASSWORD=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["Password"]')
+    return
+  fi
+  KUBE_USER=admin
+  KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))')
+
+  # Remove this code, since in all use cases I can see, we are overwriting this
+  # at cluster creation time.
+  cat << EOF > "$file"
+{
+  "User": "$KUBE_USER",
+  "Password": "$KUBE_PASSWORD"
+}
+EOF
+  chmod 0600 "$file"
+}
+
+# Generate authentication token for admin user. Will
+# read from $HOME/.kubernetes_auth if available.
+#
+# Vars set:
+#   KUBE_ADMIN_TOKEN
+function get-admin-token {
+  local file="$HOME/.kubernetes_auth"
+  if [[ -r "$file" ]]; then
+    KUBE_ADMIN_TOKEN=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["BearerToken"]')
+    return
+  fi
+  KUBE_ADMIN_TOKEN=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(32))')
+}
+
+# Instantiate a kubernetes cluster
+#
+# Assumed vars
+#   KUBE_ROOT
+#   <Various vars set in config file>
+function kube-up {
+  # Make sure we have the tar files staged on Azure Storage
+  find-release-tars
+  upload-server-tars
+
+  ensure-temp-dir
+
+  get-password
+  python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
+    -b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
+  local htpasswd
+  htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
+  
+  
+    # Generate openvpn certs
+  echo "--> Generating openvpn certs" 
+  echo 01 > ${KUBE_TEMP}/ca.srl
+  openssl genrsa -out ${KUBE_TEMP}/ca.key
+  openssl req -new -x509 -days 1095 \
+      -key ${KUBE_TEMP}/ca.key \
+      -out ${KUBE_TEMP}/ca.crt \
+      -subj "/CN=openvpn-ca"
+  openssl genrsa -out ${KUBE_TEMP}/server.key
+  openssl req -new \
+      -key ${KUBE_TEMP}/server.key \
+      -out ${KUBE_TEMP}/server.csr \
+      -subj "/CN=server"
+  openssl x509 -req -days 1095 \
+      -in ${KUBE_TEMP}/server.csr \
+      -CA ${KUBE_TEMP}/ca.crt \
+      -CAkey ${KUBE_TEMP}/ca.key \
+      -CAserial ${KUBE_TEMP}/ca.srl \
+      -out ${KUBE_TEMP}/server.crt
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+      openssl genrsa -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.key
+      openssl req -new \
+          -key ${KUBE_TEMP}/${MINION_NAMES[$i]}.key \
+          -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \
+          -subj "/CN=${MINION_NAMES[$i]}"
+      openssl x509 -req -days 1095 \
+          -in ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \
+          -CA ${KUBE_TEMP}/ca.crt \
+          -CAkey ${KUBE_TEMP}/ca.key \
+          -CAserial ${KUBE_TEMP}/ca.srl \
+          -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt
+  done
+
+  # Build up start up script for master
+  echo "--> Building up start up script for master"
+  (
+    echo "#!/bin/bash"
+    echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\""
+    echo "SERVER_CRT=\"$(cat ${KUBE_TEMP}/server.crt)\""
+    echo "SERVER_KEY=\"$(cat ${KUBE_TEMP}/server.key)\""
+    echo "mkdir -p /var/cache/kubernetes-install"
+    echo "cd /var/cache/kubernetes-install"
+    echo "readonly MASTER_NAME='${MASTER_NAME}'"
+    echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-minion'"
+    echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
+    echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
+    echo "readonly MASTER_HTPASSWD='${htpasswd}'"
+    grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/common.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/create-dynamic-salt-files.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/download-release.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/salt-master.sh"
+  ) > "${KUBE_TEMP}/master-start.sh"
+  
+  cat "${KUBE_TEMP}/master-start.sh" > ~/kubernetes/start-files/master-start.sh
+ 
+
+
+  if [ ! -f $AZ_SSH_KEY ]; then
+      ssh-keygen -f $AZ_SSH_KEY -N ''
+  fi
+
+  if [ ! -f $AZ_SSH_CERT ]; then
+      openssl req -new -x509 -days 1095 -key $AZ_SSH_KEY -out $AZ_SSH_CERT \
+          -subj "/CN=azure-ssh-key"
+  fi
+
+  if [ -z "$(azure network vnet show $AZ_VNET 2>/dev/null | grep data)" ]; then
+      #azure network vnet create with $AZ_SUBNET
+      #FIXME not working
+      echo error create vnet $AZ_VNET with subnet $AZ_SUBNET
+      exit 1
+  fi
+  
+  echo "--> Starting VM"
+  azure vm create \
+      -w $AZ_VNET \
+      -n $MASTER_NAME \
+      -l "$AZ_LOCATION" \
+      -t $AZ_SSH_CERT \
+      -e 22000 -P \
+      -d ${KUBE_TEMP}/master-start.sh \
+      -b $AZ_SUBNET \
+      $AZ_CS $AZ_IMAGE $USER
+
+  ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}"))
+
+
+  #Build up start up script for minions
+  echo "--> Building up start up script for minions"
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+    (
+      echo "#!/bin/bash"
+      echo "MASTER_NAME='${MASTER_NAME}'"
+      echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\""
+      echo "CLIENT_CRT=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt)\""
+      echo "CLIENT_KEY=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.key)\""
+      echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'"
+      grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/common.sh"
+      grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/salt-minion.sh"
+    ) > "${KUBE_TEMP}/minion-start-${i}.sh"
+
+  cat "${KUBE_TEMP}/minion-start-${i}.sh" > ~/kubernetes/start-files/minion-start-${i}.sh
+
+  echo "--> Starting VM"
+  azure vm create \
+      -c -w $AZ_VNET \
+      -n ${MINION_NAMES[$i]} \
+      -l "$AZ_LOCATION" \
+      -t $AZ_SSH_CERT \
+      -e ${ssh_ports[$i]} -P \
+      -d ${KUBE_TEMP}/minion-start-${i}.sh \
+      -b $AZ_SUBNET \
+      $AZ_CS $AZ_IMAGE $USER
+  done
+
+  echo "--> Createing endpoint"
+  azure vm endpoint create $MASTER_NAME 443
+
+  local fail=0
+  local job
+  for job in $(jobs -p); do
+      wait "${job}" || fail=$((fail + 1))
+  done
+  if (( $fail != 0 )); then
+    echo "${fail} commands failed.  Exiting." >&2
+    exit 2
+  fi
+
+
+  detect-master > /dev/null
+  
+  echo "==> KUBE_USER:KUBE_PASSWORD: ${KUBE_USER}:${KUBE_PASSWORD}"
+  echo "==> KUBE_MASTER_IP: ${KUBE_MASTER_IP}"
+
+  echo "Waiting for cluster initialization."
+  echo
+  echo "  This will continually check to see if the API for kubernetes is reachable."
+  echo "  This might loop forever if there was some uncaught error during start"
+  echo "  up."
+  echo
+
+
+  until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \
+          --fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
+      printf "."
+      sleep 2
+  done
+
+  printf "\n"
+  echo "Kubernetes cluster created."
+  echo "Sanity checking cluster..."
+
+  sleep 5
+
+  # Basic sanity checking
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+      # Make sure docker is installed
+      echo "--> Making sure docker is installed on ${MINION_NAMES[$i]}."
+      ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net which docker > /dev/null || {
+        echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely" >&2
+        echo "to work correctly. Please run ./cluster/kube-down.sh and re-create the" >&2
+        echo "cluster. (sorry!)" >&2
+        exit 1
+      }
+  done
+
+  echo
+  echo "Kubernetes cluster is running.  The master is running at:"
+  echo
+  echo "  https://${KUBE_MASTER_IP}"
+  echo
+  echo "The user name and password to use is located in ~/.kubernetes_auth."
+  echo
+  
+  echo "--> nginx auth"
+
+  local kube_cert=".kubecfg.crt"
+  local kube_key=".kubecfg.key"
+  local ca_cert=".kubernetes.ca.crt"
+
+  # TODO: generate ADMIN (and KUBELET) tokens and put those in the master's
+  # config file.  Distribute the same way the htpasswd is done.
+  (umask 077
+   ssh -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net sudo cat /srv/kubernetes/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null
+   ssh -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net sudo cat /srv/kubernetes/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null
+   ssh -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net sudo cat /srv/kubernetes/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null
+
+  cat << EOF > ~/.kubernetes_auth
+{
+  "User": "$KUBE_USER",
+  "Password": "$KUBE_PASSWORD",
+  "CAFile": "$HOME/$ca_cert",
+  "CertFile": "$HOME/$kube_cert",
+  "KeyFile": "$HOME/$kube_key"
+}
+EOF
+
+   chmod 0600 ~/.kubernetes_auth "${HOME}/${kube_cert}" \
+     "${HOME}/${kube_key}" "${HOME}/${ca_cert}"
+  )
+}
+
+# Delete a kubernetes cluster
+function kube-down {
+  echo "Bringing down cluster"
+  
+  set +e
+  azure vm delete $MASTER_NAME -b -q
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+    azure vm delete ${MINION_NAMES[$i]} -b -q
+  done
+
+  wait
+
+}
+
+# Update a kubernetes cluster with latest source
+#function kube-push {
+#  detect-project
+#  detect-master
+
+  # Make sure we have the tar files staged on Azure Storage
+#  find-release-tars
+#  upload-server-tars
+
+#  (
+#    echo "#! /bin/bash"
+#    echo "mkdir -p /var/cache/kubernetes-install"
+#    echo "cd /var/cache/kubernetes-install"
+#    echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
+#    echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
+#    grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/common.sh"
+#    grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/download-release.sh"
+#    echo "echo Executing configuration"
+#    echo "sudo salt '*' mine.update"
+#    echo "sudo salt --force-color '*' state.highstate"
+#   ) | gcutil ssh --project "$PROJECT" --zone "$ZONE" "$KUBE_MASTER" sudo bash
+
+#  get-password
+
+#  echo
+#  echo "Kubernetes cluster is running.  The master is running at:"
+#  echo
+#  echo "  https://${KUBE_MASTER_IP}"
+# echo
+#  echo "The user name and password to use is located in ~/.kubernetes_auth."
+#  echo
+
+#}
+
+# -----------------------------------------------------------------------------
+# Cluster specific test helpers used from hack/e2e-test.sh
+
+# Execute prior to running tests to build a release if required for env.
+#
+# Assumed Vars:
+#   KUBE_ROOT
+function test-build-release {
+  # Make a release
+  "${KUBE_ROOT}/build/release.sh"
+}
+
+# SSH to a node by name ($1) and run a command ($2).
+function ssh-to-node {
+  local node="$1"
+  local cmd="$2"
+  ssh --ssh_arg "-o LogLevel=quiet" "${node}" "${cmd}"
+}
+
+# Restart the kube-proxy on a node ($1)
+function restart-kube-proxy {
+  ssh-to-node "$1" "sudo /etc/init.d/kube-proxy restart"
+}
+
+# Setup monitoring using heapster and InfluxDB
+function setup-monitoring {
+  echo "not implemented"  >/dev/null
+}
+
+function teardown-monitoring {
+  echo "not implemented"  >/dev/null
+}
+
+
+