Merge pull request #1008 from MSOpenTech/azure-fix

Refactor Azure deploy scripts
This commit is contained in:
Daniel Smith 2014-08-25 11:45:33 -07:00
commit 12af8a2161
7 changed files with 286 additions and 175 deletions

View File

@ -14,13 +14,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Tear down a Kubernetes cluster.
SCRIPT_DIR=$(CDPATH="" cd $(dirname $0); pwd)
source $SCRIPT_DIR/../../release/azure/config.sh
source $SCRIPT_DIR/../util.sh
source $SCRIPT_DIR/../release/azure/config.sh
echo "Bringing down cluster"
azure vm delete $MASTER_NAME -b -q
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
azure vm delete ${MINION_NAMES[$i]} -b -q
done
AZ_SSH_KEY=$HOME/.ssh/azure_rsa
AZ_SSH_CERT=$HOME/.ssh/azure.pem
AZ_IMAGE=b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04-LTS-amd64-server-20140618.1-en-us-30GB
AZ_SUBNET=Subnet-1
AZ_VNET=kube-$AZ_HSH
AZ_CS=kube-$AZ_HSH
NUM_MINIONS=4
MASTER_NAME="${INSTANCE_PREFIX}-master"
MASTER_TAG="${INSTANCE_PREFIX}-master"
MINION_TAG="${INSTANCE_PREFIX}-minion"
MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
MINION_SCOPES=""

View File

@ -1,138 +0,0 @@
#!/bin/bash
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Bring up a Kubernetes cluster.
# exit on any error
set -eu
set -o pipefail
SCRIPT_DIR=$(CDPATH="" cd $(dirname $0); pwd)
source $SCRIPT_DIR/../../release/azure/config.sh
source $SCRIPT_DIR/../util.sh
KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
trap "rm -rf ${KUBE_TEMP}" EXIT
get-password
echo "Using password: $user:$passwd"
python $SCRIPT_DIR/../../third_party/htpasswd/htpasswd.py -b -c \
${KUBE_TEMP}/htpasswd $user $passwd
HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd)
# Build up start up script for master
(
echo "#!/bin/bash"
echo "MASTER_NAME=${MASTER_NAME}"
echo "MASTER_RELEASE_TAR=${FULL_URL}"
echo "MASTER_HTPASSWD='${HTPASSWD}'"
grep -v "^#" $SCRIPT_DIR/templates/download-release.sh
grep -v "^#" $SCRIPT_DIR/templates/salt-master.sh
) > ${KUBE_TEMP}/master-start.sh
echo "Starting VMs"
if [ ! -f $AZ_SSH_KEY ]; then
ssh-keygen -f $AZ_SSH_KEY -N ''
fi
if [ ! -f $AZ_SSH_CERT ]; then
openssl req -new -key $AZ_SSH_KEY -out ${KUBE_TEMP}/temp.csr \
-subj "/C=US/ST=WA/L=Redmond/O=Azure-CLI/CN=Azure"
openssl req -x509 -key $AZ_SSH_KEY -in ${KUBE_TEMP}/temp.csr \
-out $AZ_SSH_CERT -days 1095
rm ${KUBE_TEMP}/temp.csr
fi
if [ -z "$(azure network vnet show $AZ_VNET 2>/dev/null | grep data)" ]; then
#azure network vnet create with $AZ_SUBNET
#FIXME not working
echo error create vnet $AZ_VNET with subnet $AZ_SUBNET
exit 1
fi
azure vm create \
-w $AZ_VNET \
-n $MASTER_NAME \
-l "$AZ_LOCATION" \
-t $AZ_SSH_CERT \
-e 22000 -P \
-d ${KUBE_TEMP}/master-start.sh \
-b $AZ_SUBNET \
$AZ_CS $AZ_IMAGE $USER
ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}"))
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
(
echo "#!/bin/bash"
echo "MASTER_NAME=${MASTER_NAME}"
echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}"
grep -v "^#" $SCRIPT_DIR/templates/salt-minion.sh
) > ${KUBE_TEMP}/minion-start-${i}.sh
azure vm create \
-c -w $AZ_VNET \
-n ${MINION_NAMES[$i]} \
-l "$AZ_LOCATION" \
-t $AZ_SSH_CERT \
-e ${ssh_ports[$i]} -P \
-d ${KUBE_TEMP}/minion-start-${i}.sh \
-b $AZ_SUBNET \
$AZ_CS $AZ_IMAGE $USER
done
azure vm endpoint create $MASTER_NAME 443
echo "Waiting for cluster initialization."
echo
echo " This will continually check to see if the API for kubernetes is reachable."
echo " This might loop forever if there was some uncaught error during start"
echo " up."
echo
until $(curl --insecure --user ${user}:${passwd} --max-time 5 \
--fail --output /dev/null --silent https://$AZ_CS.cloudapp.net/api/v1beta1/pods); do
printf "."
sleep 2
done
# Basic sanity checking
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
# Make sure docker is installed
ssh -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net which docker > /dev/null
if [ "$?" != "0" ]; then
echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)"
exit 1
fi
# Make sure the kubelet is running
ssh -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net /etc/init.d/kubelet status
if [ "$?" != "0" ]; then
echo "Kubelet failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)"
exit 1
fi
done
echo
echo "Kubernetes cluster is running. Access the master at:"
echo
echo " https://${user}:${passwd}@$AZ_CS.cloudapp.net"
echo
echo "Security note: The server above uses a self signed certificate. This is"
echo " subject to \"Man in the middle\" type attacks."

253
cluster/azure/util.sh Normal file
View File

@ -0,0 +1,253 @@
#!/bin/bash
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
SCRIPT_DIR=$(CDPATH="" cd $(dirname $0); pwd)
# Use the config file specified in $KUBE_CONFIG_FILE, or default to
# config-default.sh.
source ${SCRIPT_DIR}/azure/${KUBE_CONFIG_FILE-"config-default.sh"}
function detect-minions () {
ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}"))
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
MINION_NAMES[$i]=$(ssh -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net hostname -f)
done
}
function detect-master () {
KUBE_MASTER_IP=${AZ_CS}.cloudapp.net
echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)"
}
function get-password {
file=${HOME}/.kubernetes_auth
if [ -e ${file} ]; then
user=$(cat $file | python -c 'import json,sys;print json.load(sys.stdin)["User"]')
passwd=$(cat $file | python -c 'import json,sys;print json.load(sys.stdin)["Password"]')
return
fi
user=admin
passwd=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))')
# Store password for reuse.
cat << EOF > ~/.kubernetes_auth
{
"User": "$user",
"Password": "$passwd"
}
EOF
chmod 0600 ~/.kubernetes_auth
}
# Verify prereqs
function verify-prereqs {
echo "OK"
# Already done in sourcing config-default, which sources
# release/azure/config.sh
}
# Instantiate a kubernetes cluster
function kube-up {
KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
trap "rm -rf ${KUBE_TEMP}" EXIT
get-password
echo "Using password: $user:$passwd"
python $SCRIPT_DIR/../third_party/htpasswd/htpasswd.py -b -c \
${KUBE_TEMP}/htpasswd $user $passwd
HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd)
# Build up start up script for master
(
echo "#!/bin/bash"
echo "MASTER_NAME=${MASTER_NAME}"
echo "MASTER_RELEASE_TAR=${FULL_URL}"
echo "MASTER_HTPASSWD='${HTPASSWD}'"
grep -v "^#" $SCRIPT_DIR/azure/templates/download-release.sh
grep -v "^#" $SCRIPT_DIR/azure/templates/salt-master.sh
) > ${KUBE_TEMP}/master-start.sh
echo "Starting VMs"
if [ ! -f $AZ_SSH_KEY ]; then
ssh-keygen -f $AZ_SSH_KEY -N ''
fi
if [ ! -f $AZ_SSH_CERT ]; then
openssl req -new -key $AZ_SSH_KEY -out ${KUBE_TEMP}/temp.csr \
-subj "/C=US/ST=WA/L=Redmond/O=Azure-CLI/CN=Azure"
openssl req -x509 -key $AZ_SSH_KEY -in ${KUBE_TEMP}/temp.csr \
-out $AZ_SSH_CERT -days 1095
rm ${KUBE_TEMP}/temp.csr
fi
if [ -z "$(azure network vnet show $AZ_VNET 2>/dev/null | grep data)" ]; then
#azure network vnet create with $AZ_SUBNET
#FIXME not working
echo error create vnet $AZ_VNET with subnet $AZ_SUBNET
exit 1
fi
azure vm create \
-w $AZ_VNET \
-n $MASTER_NAME \
-l "$AZ_LOCATION" \
-t $AZ_SSH_CERT \
-e 22000 -P \
-d ${KUBE_TEMP}/master-start.sh \
-b $AZ_SUBNET \
$AZ_CS $AZ_IMAGE $USER
ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}"))
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
(
echo "#!/bin/bash"
echo "MASTER_NAME=${MASTER_NAME}"
echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}"
grep -v "^#" $SCRIPT_DIR/azure/templates/salt-minion.sh
) > ${KUBE_TEMP}/minion-start-${i}.sh
azure vm create \
-c -w $AZ_VNET \
-n ${MINION_NAMES[$i]} \
-l "$AZ_LOCATION" \
-t $AZ_SSH_CERT \
-e ${ssh_ports[$i]} -P \
-d ${KUBE_TEMP}/minion-start-${i}.sh \
-b $AZ_SUBNET \
$AZ_CS $AZ_IMAGE $USER
done
azure vm endpoint create $MASTER_NAME 443
echo "Waiting for cluster initialization."
echo
echo " This will continually check to see if the API for kubernetes is reachable."
echo " This might loop forever if there was some uncaught error during start"
echo " up."
echo
until $(curl --insecure --user ${user}:${passwd} --max-time 5 \
--fail --output /dev/null --silent https://$AZ_CS.cloudapp.net/api/v1beta1/pods); do
printf "."
sleep 2
done
# Basic sanity checking
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
# Make sure docker is installed
ssh -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net which docker > /dev/null
if [ "$?" != "0" ]; then
echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)"
exit 1
fi
# Make sure the kubelet is running
ssh -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net /etc/init.d/kubelet status
if [ "$?" != "0" ]; then
echo "Kubelet failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)"
exit 1
fi
done
echo
echo "Kubernetes cluster is running. Access the master at:"
echo
echo " https://${user}:${passwd}@$AZ_CS.cloudapp.net"
echo
echo "Security note: The server above uses a self signed certificate. This is"
echo " subject to \"Man in the middle\" type attacks."
}
# Delete a kubernetes cluster
function kube-down {
echo "Bringing down cluster"
set +e
azure vm delete $MASTER_NAME -b -q
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
azure vm delete ${MINION_NAMES[$i]} -b -q
done
}
# # Update a kubernetes cluster with latest source
# function kube-push {
# # Find the release to use. Generally it will be passed when doing a 'prod'
# # install and will default to the release/config.sh version when doing a
# # developer up.
# find-release $1
# # Detect the project into $PROJECT
# detect-master
# (
# echo MASTER_RELEASE_TAR=$RELEASE_NORMALIZED/master-release.tgz
# grep -v "^#" $(dirname $0)/templates/download-release.sh
# echo "echo Executing configuration"
# echo "sudo salt '*' mine.update"
# echo "sudo salt --force-color '*' state.highstate"
# ) | gcutil ssh --project ${PROJECT} --zone ${ZONE} $KUBE_MASTER bash
# get-password
# echo "Kubernetes cluster is updated. Access the master at:"
# echo
# echo " https://${user}:${passwd}@${KUBE_MASTER_IP}"
# echo
# }
# # Execute prior to running tests to build a release if required for env
# function test-build-release {
# # Build source
# ${KUBE_REPO_ROOT}/hack/build-go.sh
# # Make a release
# $(dirname $0)/../release/release.sh
# }
# # Execute prior to running tests to initialize required structure
# function test-setup {
# # Detect the project into $PROJECT if it isn't set
# # gce specific
# detect-project
# if [[ ${ALREADY_UP} -ne 1 ]]; then
# # Open up port 80 & 8080 so common containers on minions can be reached
# gcutil addfirewall \
# --norespect_terminal_width \
# --project ${PROJECT} \
# --target_tags ${MINION_TAG} \
# --allowed tcp:80,tcp:8080 \
# --network ${NETWORK} \
# ${MINION_TAG}-${INSTANCE_PREFIX}-http-alt
# fi
# }
# # Execute after running tests to perform any required clean-up
# function test-teardown {
# echo "Shutting down test cluster in background."
# gcutil deletefirewall \
# --project ${PROJECT} \
# --norespect_terminal_width \
# --force \
# ${MINION_TAG}-${INSTANCE_PREFIX}-http-alt || true > /dev/null
# $(dirname $0)/../cluster/kube-down.sh > /dev/null
# }

View File

@ -37,7 +37,7 @@ for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
# Grep returns an exit status of 1 when line is not found, so we need the : to always return a 0 exit status
count=$(grep -c ${MINION_NAMES[i]} ${MINIONS_FILE}) || :
if [ "$count" == "0" ]; then
echo "Failed to find ${MINION_NAMES[i]}, cluster is probably broken."
echo "Failed to find ${MINION_NAMES[i]}, cluster is probably broken."
exit 1
fi
@ -48,7 +48,6 @@ for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
exit 1
else
echo "Kubelet is successfully installed on ${MINION_NAMES[$i]}"
fi
fi
done
echo "Cluster validation succeeded"

View File

@ -20,7 +20,7 @@ set -o errexit
set -o nounset
set -o pipefail
hackdir=$(dirname "$0")
hackdir=$(CDPATH="" cd $(dirname $0); pwd)
# Set the environment variables required by the build.
. "${hackdir}/config-go.sh"

View File

@ -12,11 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
INSTANCE_PREFIX=kubenertes
AZ_LOCATION='West US'
AZ_SSH_KEY=$HOME/.ssh/azure
AZ_SSH_CERT=$HOME/.ssh/azure.pem
AZ_IMAGE=b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04-LTS-amd64-server-20140618.1-en-us-30GB
AZ_SUBNET=Subnet-1
TAG=testing
if [ -z "$(which azure)" ]; then
@ -32,19 +29,12 @@ if [ -z "$(azure account list | grep true)" ]; then
fi
account=$(azure account list | grep true | awk '{ print $2 }')
if which md5 > /dev/null 2>&1; then
hsh=$(md5 -q -s $account)
AZ_HSH=$(md5 -q -s $account)
else
hsh=$(echo -n "$account" | md5sum)
AZ_HSH=$(echo -n "$account" | md5sum)
fi
hsh=${hsh:0:7}
STG_ACCOUNT=kube$hsh
AZ_VNET=kube-$hsh
AZ_CS=kube-$hsh
AZ_HSH=${AZ_HSH:0:7}
AZ_STG=kube$AZ_HSH
CONTAINER=kube-$TAG
FULL_URL="https://${STG_ACCOUNT}.blob.core.windows.net/$CONTAINER/master-release.tgz"
FULL_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/master-release.tgz"

View File

@ -26,38 +26,37 @@ function json_val () {
}
source $SCRIPT_DIR/config.sh
source $SCRIPT_DIR/../../cluster/${KUBE_CONFIG_FILE-"config-default.sh"}
$SCRIPT_DIR/../build-release.sh $INSTANCE_PREFIX
if [ -z "$(azure storage account show $STG_ACCOUNT 2>/dev/null | \
if [ -z "$(azure storage account show $AZ_STG 2>/dev/null | \
grep data)" ]; then
azure storage account create -l "$AZ_LOCATION" $STG_ACCOUNT
azure storage account create -l "$AZ_LOCATION" $AZ_STG
fi
stg_key=$(azure storage account keys list $STG_ACCOUNT --json | \
stg_key=$(azure storage account keys list $AZ_STG --json | \
json_val '["primaryKey"]')
if [ -z "$(azure storage container show -a $STG_ACCOUNT -k "$stg_key" \
if [ -z "$(azure storage container show -a $AZ_STG -k "$stg_key" \
$CONTAINER 2>/dev/null | grep data)" ]; then
azure storage container create \
-a $STG_ACCOUNT \
-a $AZ_STG \
-k "$stg_key" \
-p Blob \
$CONTAINER
fi
if [ -n "$(azure storage blob show -a $STG_ACCOUNT -k "$stg_key" \
if [ -n "$(azure storage blob show -a $AZ_STG -k "$stg_key" \
$CONTAINER master-release.tgz 2>/dev/null | grep data)" ]; then
azure storage blob delete \
-a $STG_ACCOUNT \
-a $AZ_STG \
-k "$stg_key" \
$CONTAINER \
master-release.tgz
fi
azure storage blob upload \
-a $STG_ACCOUNT \
-a $AZ_STG \
-k "$stg_key" \
$SCRIPT_DIR/../../output/release/master-release.tgz \
$CONTAINER \