github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 12:15:52 +00:00
Code Issues Projects Releases Wiki Activity

Updates and formatting to azure scripts.

Browse Source
This commit is contained in:
Jeff Mendoza 2014-11-18 14:10:18 -08:00
parent 6ea7191592
commit 1ee5ab5e5e
3 changed files with 365 additions and 365 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cluster/azure/config-default.sh
View File

@ -14,19 +14,19 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
INSTANCE_PREFIX=kubenertes INSTANCE_PREFIX=kubernetes
AZ_LOCATION='West US' AZ_LOCATION='West US'
TAG=testing TAG=testing
AZ_CS_PREFIX=kube AZ_CS_PREFIX=kube
AZ_VNET=shchTest AZ_VNET=MyVnet
AZ_SUBNET=Subnet-1 AZ_SUBNET=Subnet-1
AZ_IMAGE=b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04-LTS-amd64-server-20140618.1-en-us-30GB AZ_IMAGE=b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04_1-LTS-amd64-server-20140927-en-us-30GB
AZ_CS="" # is set in azure/util.sh verify-prereqs AZ_CS="" # is set in azure/util.sh verify-prereqs
AZ_SSH_KEY=$HOME/.ssh/azure_rsa AZ_SSH_KEY=$HOME/.ssh/azure_rsa
AZ_SSH_CERT=$HOME/.ssh/azure.pem AZ_SSH_CERT=$HOME/.ssh/azure.pem
NUM_MINIONS=2 NUM_MINIONS=4
MASTER_NAME="${INSTANCE_PREFIX}-master" MASTER_NAME="${INSTANCE_PREFIX}-master"
MASTER_TAG="${INSTANCE_PREFIX}-master" MASTER_TAG="${INSTANCE_PREFIX}-master"
@ -35,3 +35,4 @@ MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24")) MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
MINION_SCOPES="" MINION_SCOPES=""
PORTAL_NET="10.250.0.0/16"

1
cluster/azure/templates/create-dynamic-salt-files.sh
View File

@ -21,6 +21,7 @@
mkdir -p /srv/salt-overlay/pillar mkdir -p /srv/salt-overlay/pillar
cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
node_instance_prefix: $NODE_INSTANCE_PREFIX node_instance_prefix: $NODE_INSTANCE_PREFIX
portal_net: $PORTAL_NET
EOF EOF
mkdir -p /srv/salt-overlay/salt/nginx mkdir -p /srv/salt-overlay/salt/nginx

718
cluster/azure/util.sh
View File

@ -21,40 +21,60 @@
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/azure/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/azure/${KUBE_CONFIG_FILE-"config-default.sh"}"
function azure_call {
params=""
for param in "$@"
do
params="${params} \"${param}\""
done
rv=1
stderr="getaddrinfo ENOTFOUND"
while [ $rv -ne 0 -a -n "$(echo $stderr | grep "getaddrinfo ENOTFOUND")" ]; do
set +e
{ stderr=$(bash -c "azure $params" 2>&1 1>&3-) ;} 3>&1
rv=$?
set -e
done
if [ $rv -ne 0 ]; then
echo $stderr 1>&2
exit
fi
}
function json_val () { function json_val () {
python -c 'import json,sys;obj=json.load(sys.stdin);print obj'$1''; python -c 'import json,sys;obj=json.load(sys.stdin);print obj'$1'';
} }
# Verify prereqs # Verify prereqs
function verify-prereqs { function verify-prereqs {
if [ -z "$(which azure)" ]; then if [ -z "$(which azure)" ]; then
echo "Couldn't find azure in PATH" echo "Couldn't find azure in PATH"
echo " please install with 'npm install azure-cli'" echo " please install with 'npm install azure-cli'"
exit 1 exit 1
fi fi
if [ -z "$(azure account list | grep true)" ]; then if [ -z "$(azure_call account list | grep true)" ]; then
echo "Default azure account not set" echo "Default azure account not set"
echo " please set with 'azure account set'" echo " please set with 'azure account set'"
exit 1 exit 1
fi fi
account=$(azure account list | grep true | awk '{ print $2 }') account=$(azure_call account list | grep true)
if which md5 > /dev/null 2>&1; then if which md5 > /dev/null 2>&1; then
AZ_HSH=$(md5 -q -s "$AZ_SUBNET@$account") AZ_HSH=$(md5 -q -s "$account")
else else
AZ_HSH=$(echo -n "$AZ_SUBNET@$account" | md5sum) AZ_HSH=$(echo -n "$account" | md5sum)
fi fi
AZ_HSH=${AZ_HSH:0:7} AZ_HSH=${AZ_HSH:0:7}
AZ_STG=kube$AZ_HSH AZ_STG=kube$AZ_HSH
echo "==> AZ_STG: $AZ_STG" echo "==> AZ_STG: $AZ_STG"
AZ_CS="$AZ_CS_PREFIX-$AZ_HSH" AZ_CS="$AZ_CS_PREFIX-$AZ_HSH"
echo "==> AZ_CS: $AZ_CS" echo "==> AZ_CS: $AZ_CS"
CONTAINER=kube-$TAG CONTAINER=kube-$TAG
echo "==> CONTAINER: $CONTAINER" echo "==> CONTAINER: $CONTAINER"
} }
# Create a temp dir that'll be deleted at the end of this bash session. # Create a temp dir that'll be deleted at the end of this bash session.
@ -62,10 +82,10 @@ function verify-prereqs {
# Vars set: # Vars set:
# KUBE_TEMP # KUBE_TEMP
function ensure-temp-dir { function ensure-temp-dir {
if [[ -z ${KUBE_TEMP-} ]]; then if [[ -z ${KUBE_TEMP-} ]]; then
KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX) KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
trap 'rm -rf "${KUBE_TEMP}"' EXIT trap 'rm -rf "${KUBE_TEMP}"' EXIT
fi fi
} }
# Verify and find the various tar files that we are going to use on the server. # Verify and find the various tar files that we are going to use on the server.
@ -74,23 +94,23 @@ function ensure-temp-dir {
# SERVER_BINARY_TAR # SERVER_BINARY_TAR
# SALT_TAR # SALT_TAR
function find-release-tars { function find-release-tars {
SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz" SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz"
if [[ ! -f "$SERVER_BINARY_TAR" ]]; then if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz" SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz"
fi fi
if [[ ! -f "$SERVER_BINARY_TAR" ]]; then if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz" echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz"
exit 1 exit 1
fi fi
SALT_TAR="${KUBE_ROOT}/server/kubernetes-salt.tar.gz" SALT_TAR="${KUBE_ROOT}/server/kubernetes-salt.tar.gz"
if [[ ! -f "$SALT_TAR" ]]; then if [[ ! -f "$SALT_TAR" ]]; then
SALT_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-salt.tar.gz" SALT_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-salt.tar.gz"
fi fi
if [[ ! -f "$SALT_TAR" ]]; then if [[ ! -f "$SALT_TAR" ]]; then
echo "!!! Cannot find kubernetes-salt.tar.gz" echo "!!! Cannot find kubernetes-salt.tar.gz"
exit 1 exit 1
fi fi
} }
@ -104,82 +124,81 @@ function find-release-tars {
# SERVER_BINARY_TAR_URL # SERVER_BINARY_TAR_URL
# SALT_TAR_URL # SALT_TAR_URL
function upload-server-tars() { function upload-server-tars() {
SERVER_BINARY_TAR_URL= SERVER_BINARY_TAR_URL=
SALT_TAR_URL= SALT_TAR_URL=
echo "==> SERVER_BINARY_TAR: $SERVER_BINARY_TAR"
echo "==> SALT_TAR: $SALT_TAR"
echo "+++ Staging server tars to Azure Storage: $AZ_STG"
local server_binary_url="${SERVER_BINARY_TAR##*/}"
local salt_url="${SALT_TAR##*/}"
SERVER_BINARY_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$server_binary_url"
SALT_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$salt_url"
echo "==> SERVER_BINARY_TAR_URL: $SERVER_BINARY_TAR_URL" echo "==> SERVER_BINARY_TAR: $SERVER_BINARY_TAR"
echo "==> SALT_TAR_URL: $SALT_TAR_URL" echo "==> SALT_TAR: $SALT_TAR"
echo "+++ Staging server tars to Azure Storage: $AZ_STG"
echo "--> Checking storage exsists..." local server_binary_url="${SERVER_BINARY_TAR##*/}"
if [ -z "$(azure storage account show $AZ_STG 2>/dev/null | \ local salt_url="${SALT_TAR##*/}"
SERVER_BINARY_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$server_binary_url"
SALT_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$salt_url"
echo "==> SERVER_BINARY_TAR_URL: $SERVER_BINARY_TAR_URL"
echo "==> SALT_TAR_URL: $SALT_TAR_URL"
echo "--> Checking storage exsists..."
if [ -z "$(azure_call storage account show $AZ_STG 2>/dev/null | \
grep data)" ]; then grep data)" ]; then
echo "--> Creating storage..." echo "--> Creating storage..."
azure storage account create -l "$AZ_LOCATION" $AZ_STG azure_call storage account create -l "$AZ_LOCATION" $AZ_STG
fi fi
echo "--> Getting storage key..." echo "--> Getting storage key..."
stg_key=$(azure storage account keys list $AZ_STG --json | \ stg_key=$(azure_call storage account keys list $AZ_STG --json | \
json_val '["primaryKey"]') json_val '["primaryKey"]')
echo "--> Checking storage container exsists..." echo "--> Checking storage container exsists..."
if [ -z "$(azure storage container show -a $AZ_STG -k "$stg_key" \ if [ -z "$(azure_call storage container show -a $AZ_STG -k "$stg_key" \
$CONTAINER 2>/dev/null | grep data)" ]; then $CONTAINER 2>/dev/null | grep data)" ]; then
echo "--> Creating storage container..." echo "--> Creating storage container..."
azure storage container create \ azure_call storage container create \
-a $AZ_STG \ -a $AZ_STG \
-k "$stg_key" \ -k "$stg_key" \
-p Blob \ -p Blob \
$CONTAINER $CONTAINER
fi fi
echo "--> Checking server binary exists in the container..." echo "--> Checking server binary exists in the container..."
if [ -n "$(azure storage blob show -a $AZ_STG -k "$stg_key" \ if [ -n "$(azure_call storage blob show -a $AZ_STG -k "$stg_key" \
$CONTAINER $server_binary_url 2>/dev/null | grep data)" ]; then $CONTAINER $server_binary_url 2>/dev/null | grep data)" ]; then
echo "--> Deleting server binary in the container..." echo "--> Deleting server binary in the container..."
azure storage blob delete \ azure_call storage blob delete \
-a $AZ_STG \ -a $AZ_STG \
-k "$stg_key" \ -k "$stg_key" \
$CONTAINER \ $CONTAINER \
$server_binary_url $server_binary_url
fi fi
echo "--> Uploading server binary to the container..."
azure storage blob upload \
-a $AZ_STG \
-k "$stg_key" \
$SERVER_BINARY_TAR \
$CONTAINER \
$server_binary_url
echo "--> Checking salt data exists in the container..."
if [ -n "$(azure storage blob show -a $AZ_STG -k "$stg_key" \
$CONTAINER $salt_url 2>/dev/null | grep data)" ]; then
echo "--> Deleting salt data in the container..."
azure storage blob delete \
-a $AZ_STG \
-k "$stg_key" \
$CONTAINER \
$salt_url
fi
echo "--> Uploading salt data to the container..." echo "--> Uploading server binary to the container..."
azure storage blob upload \ azure_call storage blob upload \
-a $AZ_STG \ -a $AZ_STG \
-k "$stg_key" \ -k "$stg_key" \
$SALT_TAR \ $SERVER_BINARY_TAR \
$CONTAINER \ $CONTAINER \
$salt_url $server_binary_url
echo "--> Checking salt data exists in the container..."
if [ -n "$(azure_call storage blob show -a $AZ_STG -k "$stg_key" \
$CONTAINER $salt_url 2>/dev/null | grep data)" ]; then
echo "--> Deleting salt data in the container..."
azure_call storage blob delete \
-a $AZ_STG \
-k "$stg_key" \
$CONTAINER \
$salt_url
fi
echo "--> Uploading salt data to the container..."
azure_call storage blob upload \
-a $AZ_STG \
-k "$stg_key" \
$SALT_TAR \
$CONTAINER \
$salt_url
} }
# Detect the information about the minions # Detect the information about the minions
@ -188,16 +207,15 @@ function upload-server-tars() {
# MINION_NAMES # MINION_NAMES
# ZONE # ZONE
# Vars set: # Vars set:
# #
function detect-minions () { function detect-minions () {
if [ -z "$AZ_CS" ]; then if [ -z "$AZ_CS" ]; then
verify-prereqs verify-prereqs
fi fi
ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}")) ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}"))
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
MINION_NAMES[$i]=$(ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net hostname -f) MINION_NAMES[$i]=$(ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net hostname -f)
done done
} }
# Detect the IP for the master # Detect the IP for the master
@ -209,14 +227,13 @@ function detect-minions () {
# KUBE_MASTER # KUBE_MASTER
# KUBE_MASTER_IP # KUBE_MASTER_IP
function detect-master () { function detect-master () {
if [ -z "$AZ_CS" ]; then if [ -z "$AZ_CS" ]; then
verify-prereqs verify-prereqs
fi fi
KUBE_MASTER=${MASTER_NAME}
KUBE_MASTER_IP="${AZ_CS}.cloudapp.net"
echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)"
KUBE_MASTER=${MASTER_NAME}
KUBE_MASTER_IP="${AZ_CS}.cloudapp.net"
echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)"
} }
# Ensure that we have a password created for validating to the master. Will # Ensure that we have a password created for validating to the master. Will
@ -226,24 +243,24 @@ function detect-master () {
# KUBE_USER # KUBE_USER
# KUBE_PASSWORD # KUBE_PASSWORD
function get-password { function get-password {
local file="$HOME/.kubernetes_auth" local file="$HOME/.kubernetes_auth"
if [[ -r "$file" ]]; then if [[ -r "$file" ]]; then
KUBE_USER=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["User"]') KUBE_USER=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["User"]')
KUBE_PASSWORD=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["Password"]') KUBE_PASSWORD=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["Password"]')
return return
fi fi
KUBE_USER=admin KUBE_USER=admin
KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))') KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))')
# Remove this code, since in all use cases I can see, we are overwriting this # Remove this code, since in all use cases I can see, we are overwriting this
# at cluster creation time. # at cluster creation time.
cat << EOF > "$file" cat << EOF > "$file"
{ {
"User": "$KUBE_USER", "User": "$KUBE_USER",
"Password": "$KUBE_PASSWORD" "Password": "$KUBE_PASSWORD"
} }
EOF EOF
chmod 0600 "$file" chmod 0600 "$file"
} }
# Generate authentication token for admin user. Will # Generate authentication token for admin user. Will
@ -252,12 +269,12 @@ EOF
# Vars set: # Vars set:
# KUBE_ADMIN_TOKEN # KUBE_ADMIN_TOKEN
function get-admin-token { function get-admin-token {
local file="$HOME/.kubernetes_auth" local file="$HOME/.kubernetes_auth"
if [[ -r "$file" ]]; then if [[ -r "$file" ]]; then
KUBE_ADMIN_TOKEN=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["BearerToken"]') KUBE_ADMIN_TOKEN=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["BearerToken"]')
return return
fi fi
KUBE_ADMIN_TOKEN=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(32))') KUBE_ADMIN_TOKEN=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(32))')
} }
# Instantiate a kubernetes cluster # Instantiate a kubernetes cluster
@ -266,207 +283,192 @@ function get-admin-token {
# KUBE_ROOT # KUBE_ROOT
# <Various vars set in config file> # <Various vars set in config file>
function kube-up { function kube-up {
# Make sure we have the tar files staged on Azure Storage # Make sure we have the tar files staged on Azure Storage
find-release-tars find-release-tars
upload-server-tars upload-server-tars
ensure-temp-dir ensure-temp-dir
get-password
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
-b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
local htpasswd
htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
get-password
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
-b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
local htpasswd
htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
# Generate openvpn certs # Generate openvpn certs
echo "--> Generating openvpn certs" echo "--> Generating openvpn certs"
echo 01 > ${KUBE_TEMP}/ca.srl echo 01 > ${KUBE_TEMP}/ca.srl
openssl genrsa -out ${KUBE_TEMP}/ca.key openssl genrsa -out ${KUBE_TEMP}/ca.key
openssl req -new -x509 -days 1095 \ openssl req -new -x509 -days 1095 \
-key ${KUBE_TEMP}/ca.key \ -key ${KUBE_TEMP}/ca.key \
-out ${KUBE_TEMP}/ca.crt \ -out ${KUBE_TEMP}/ca.crt \
-subj "/CN=openvpn-ca" -subj "/CN=openvpn-ca"
openssl genrsa -out ${KUBE_TEMP}/server.key openssl genrsa -out ${KUBE_TEMP}/server.key
openssl req -new \ openssl req -new \
-key ${KUBE_TEMP}/server.key \ -key ${KUBE_TEMP}/server.key \
-out ${KUBE_TEMP}/server.csr \ -out ${KUBE_TEMP}/server.csr \
-subj "/CN=server" -subj "/CN=server"
openssl x509 -req -days 1095 \ openssl x509 -req -days 1095 \
-in ${KUBE_TEMP}/server.csr \ -in ${KUBE_TEMP}/server.csr \
-CA ${KUBE_TEMP}/ca.crt \ -CA ${KUBE_TEMP}/ca.crt \
-CAkey ${KUBE_TEMP}/ca.key \ -CAkey ${KUBE_TEMP}/ca.key \
-CAserial ${KUBE_TEMP}/ca.srl \ -CAserial ${KUBE_TEMP}/ca.srl \
-out ${KUBE_TEMP}/server.crt -out ${KUBE_TEMP}/server.crt
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
openssl genrsa -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.key openssl genrsa -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.key
openssl req -new \ openssl req -new \
-key ${KUBE_TEMP}/${MINION_NAMES[$i]}.key \ -key ${KUBE_TEMP}/${MINION_NAMES[$i]}.key \
-out ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \ -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \
-subj "/CN=${MINION_NAMES[$i]}" -subj "/CN=${MINION_NAMES[$i]}"
openssl x509 -req -days 1095 \ openssl x509 -req -days 1095 \
-in ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \ -in ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \
-CA ${KUBE_TEMP}/ca.crt \ -CA ${KUBE_TEMP}/ca.crt \
-CAkey ${KUBE_TEMP}/ca.key \ -CAkey ${KUBE_TEMP}/ca.key \
-CAserial ${KUBE_TEMP}/ca.srl \ -CAserial ${KUBE_TEMP}/ca.srl \
-out ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt
done done
# Build up start up script for master # Build up start up script for master
echo "--> Building up start up script for master" echo "--> Building up start up script for master"
(
echo "#!/bin/bash"
echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\""
echo "SERVER_CRT=\"$(cat ${KUBE_TEMP}/server.crt)\""
echo "SERVER_KEY=\"$(cat ${KUBE_TEMP}/server.key)\""
echo "mkdir -p /var/cache/kubernetes-install"
echo "cd /var/cache/kubernetes-install"
echo "readonly MASTER_NAME='${MASTER_NAME}'"
echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-minion'"
echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
echo "readonly MASTER_HTPASSWD='${htpasswd}'"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/common.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/create-dynamic-salt-files.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/download-release.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/salt-master.sh"
) > "${KUBE_TEMP}/master-start.sh"
cat "${KUBE_TEMP}/master-start.sh" > ~/kubernetes/start-files/master-start.sh
if [ ! -f $AZ_SSH_KEY ]; then
ssh-keygen -f $AZ_SSH_KEY -N ''
fi
if [ ! -f $AZ_SSH_CERT ]; then
openssl req -new -x509 -days 1095 -key $AZ_SSH_KEY -out $AZ_SSH_CERT \
-subj "/CN=azure-ssh-key"
fi
if [ -z "$(azure network vnet show $AZ_VNET 2>/dev/null | grep data)" ]; then
#azure network vnet create with $AZ_SUBNET
#FIXME not working
echo error create vnet $AZ_VNET with subnet $AZ_SUBNET
exit 1
fi
echo "--> Starting VM"
azure vm create \
-w $AZ_VNET \
-n $MASTER_NAME \
-l "$AZ_LOCATION" \
-t $AZ_SSH_CERT \
-e 22000 -P \
-d ${KUBE_TEMP}/master-start.sh \
-b $AZ_SUBNET \
$AZ_CS $AZ_IMAGE $USER
ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}"))
#Build up start up script for minions
echo "--> Building up start up script for minions"
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
( (
echo "#!/bin/bash" echo "#!/bin/bash"
echo "MASTER_NAME='${MASTER_NAME}'" echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\""
echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\"" echo "SERVER_CRT=\"$(cat ${KUBE_TEMP}/server.crt)\""
echo "CLIENT_CRT=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt)\"" echo "SERVER_KEY=\"$(cat ${KUBE_TEMP}/server.key)\""
echo "CLIENT_KEY=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.key)\"" echo "mkdir -p /var/cache/kubernetes-install"
echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'" echo "cd /var/cache/kubernetes-install"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/common.sh" echo "readonly MASTER_NAME='${MASTER_NAME}'"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/salt-minion.sh" echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-minion'"
) > "${KUBE_TEMP}/minion-start-${i}.sh" echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
echo "readonly MASTER_HTPASSWD='${htpasswd}'"
echo "readonly PORTAL_NET='${PORTAL_NET}'"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/common.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/create-dynamic-salt-files.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/download-release.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/salt-master.sh"
) > "${KUBE_TEMP}/master-start.sh"
cat "${KUBE_TEMP}/minion-start-${i}.sh" > ~/kubernetes/start-files/minion-start-${i}.sh if [ ! -f $AZ_SSH_KEY ]; then
ssh-keygen -f $AZ_SSH_KEY -N ''
fi
echo "--> Starting VM" if [ ! -f $AZ_SSH_CERT ]; then
azure vm create \ openssl req -new -x509 -days 1095 -key $AZ_SSH_KEY -out $AZ_SSH_CERT \
-c -w $AZ_VNET \ -subj "/CN=azure-ssh-key"
-n ${MINION_NAMES[$i]} \ fi
-l "$AZ_LOCATION" \
-t $AZ_SSH_CERT \
-e ${ssh_ports[$i]} -P \
-d ${KUBE_TEMP}/minion-start-${i}.sh \
-b $AZ_SUBNET \
$AZ_CS $AZ_IMAGE $USER
done
echo "--> Createing endpoint" if [ -z "$(azure_call network vnet show $AZ_VNET 2>/dev/null | grep data)" ]; then
azure vm endpoint create $MASTER_NAME 443 #azure network vnet create with $AZ_SUBNET
#FIXME not working
local fail=0 echo error create vnet $AZ_VNET with subnet $AZ_SUBNET
local job
for job in $(jobs -p); do
wait "${job}" || fail=$((fail + 1))
done
if (( $fail != 0 )); then
echo "${fail} commands failed. Exiting." >&2
exit 2
fi
detect-master > /dev/null
echo "==> KUBE_USER:KUBE_PASSWORD: ${KUBE_USER}:${KUBE_PASSWORD}"
echo "==> KUBE_MASTER_IP: ${KUBE_MASTER_IP}"
echo "Waiting for cluster initialization."
echo
echo " This will continually check to see if the API for kubernetes is reachable."
echo " This might loop forever if there was some uncaught error during start"
echo " up."
echo
until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \
--fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
printf "."
sleep 2
done
printf "\n"
echo "Kubernetes cluster created."
echo "Sanity checking cluster..."
sleep 5
# Basic sanity checking
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
# Make sure docker is installed
echo "--> Making sure docker is installed on ${MINION_NAMES[$i]}."
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net which docker > /dev/null || {
echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely" >&2
echo "to work correctly. Please run ./cluster/kube-down.sh and re-create the" >&2
echo "cluster. (sorry!)" >&2
exit 1 exit 1
} fi
done
echo echo "--> Starting VM"
echo "Kubernetes cluster is running. The master is running at:" azure_call vm create \
echo -w $AZ_VNET \
echo " https://${KUBE_MASTER_IP}" -n $MASTER_NAME \
echo -l "$AZ_LOCATION" \
echo "The user name and password to use is located in ~/.kubernetes_auth." -t $AZ_SSH_CERT \
echo -e 22000 -P \
-d ${KUBE_TEMP}/master-start.sh \
echo "--> nginx auth" -b $AZ_SUBNET \
$AZ_CS $AZ_IMAGE $USER
local kube_cert=".kubecfg.crt" ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}"))
local kube_key=".kubecfg.key"
local ca_cert=".kubernetes.ca.crt"
# TODO: generate ADMIN (and KUBELET) tokens and put those in the master's #Build up start up script for minions
# config file. Distribute the same way the htpasswd is done. echo "--> Building up start up script for minions"
(umask 077 for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
ssh -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net sudo cat /srv/kubernetes/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null (
ssh -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net sudo cat /srv/kubernetes/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null echo "#!/bin/bash"
ssh -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net sudo cat /srv/kubernetes/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null echo "MASTER_NAME='${MASTER_NAME}'"
echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\""
echo "CLIENT_CRT=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt)\""
echo "CLIENT_KEY=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.key)\""
echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/common.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/salt-minion.sh"
) > "${KUBE_TEMP}/minion-start-${i}.sh"
cat << EOF > ~/.kubernetes_auth echo "--> Starting VM"
azure_call vm create \
-c -w $AZ_VNET \
-n ${MINION_NAMES[$i]} \
-l "$AZ_LOCATION" \
-t $AZ_SSH_CERT \
-e ${ssh_ports[$i]} -P \
-d ${KUBE_TEMP}/minion-start-${i}.sh \
-b $AZ_SUBNET \
$AZ_CS $AZ_IMAGE $USER
done
echo "--> Createing endpoint"
azure_call vm endpoint create $MASTER_NAME 443
detect-master > /dev/null
echo "==> KUBE_MASTER_IP: ${KUBE_MASTER_IP}"
echo "Waiting for cluster initialization."
echo
echo " This will continually check to see if the API for kubernetes is reachable."
echo " This might loop forever if there was some uncaught error during start"
echo " up."
echo
until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \
--fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
printf "."
sleep 2
done
printf "\n"
echo "Kubernetes cluster created."
echo "Sanity checking cluster..."
# Wait for salt on the minions
sleep 30
# Basic sanity checking
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
# Make sure docker is installed
echo "--> Making sure docker is installed on ${MINION_NAMES[$i]}."
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} \
$AZ_CS.cloudapp.net which docker > /dev/null || {
echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely" >&2
echo "to work correctly. Please run ./cluster/kube-down.sh and re-create the" >&2
echo "cluster. (sorry!)" >&2
exit 1
}
done
echo
echo "Kubernetes cluster is running. The master is running at:"
echo
echo " https://${KUBE_MASTER_IP}"
echo
echo "The user name and password to use is located in ~/.kubernetes_auth."
echo
echo "--> nginx auth"
local kube_cert=".kubecfg.crt"
local kube_key=".kubecfg.key"
local ca_cert=".kubernetes.ca.crt"
# TODO: generate ADMIN (and KUBELET) tokens and put those in the master's
# config file. Distribute the same way the htpasswd is done.
(umask 077
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net \
sudo cat /srv/kubernetes/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net \
sudo cat /srv/kubernetes/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net \
sudo cat /srv/kubernetes/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null
cat << EOF > ~/.kubernetes_auth
{ {
"User": "$KUBE_USER", "User": "$KUBE_USER",
"Password": "$KUBE_PASSWORD", "Password": "$KUBE_PASSWORD",
@ -476,23 +478,22 @@ function kube-up {
} }
EOF EOF
chmod 0600 ~/.kubernetes_auth "${HOME}/${kube_cert}" \ chmod 0600 ~/.kubernetes_auth "${HOME}/${kube_cert}" \
"${HOME}/${kube_key}" "${HOME}/${ca_cert}" "${HOME}/${kube_key}" "${HOME}/${ca_cert}"
) )
} }
# Delete a kubernetes cluster # Delete a kubernetes cluster
function kube-down { function kube-down {
echo "Bringing down cluster" echo "Bringing down cluster"
set +e
azure vm delete $MASTER_NAME -b -q
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
azure vm delete ${MINION_NAMES[$i]} -b -q
done
wait set +e
azure_call vm delete $MASTER_NAME -b -q
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
azure_call vm delete ${MINION_NAMES[$i]} -b -q
done
wait
} }
# Update a kubernetes cluster with latest source # Update a kubernetes cluster with latest source
@ -500,7 +501,7 @@ function kube-down {
# detect-project # detect-project
# detect-master # detect-master
# Make sure we have the tar files staged on Azure Storage # Make sure we have the tar files staged on Azure Storage
# find-release-tars # find-release-tars
# upload-server-tars # upload-server-tars
@ -537,30 +538,27 @@ function kube-down {
# Assumed Vars: # Assumed Vars:
# KUBE_ROOT # KUBE_ROOT
function test-build-release { function test-build-release {
# Make a release # Make a release
"${KUBE_ROOT}/build/release.sh" "${KUBE_ROOT}/build/release.sh"
} }
# SSH to a node by name ($1) and run a command ($2). # SSH to a node by name ($1) and run a command ($2).
function ssh-to-node { function ssh-to-node {
local node="$1" local node="$1"
local cmd="$2" local cmd="$2"
ssh --ssh_arg "-o LogLevel=quiet" "${node}" "${cmd}" ssh --ssh_arg "-o LogLevel=quiet" "${node}" "${cmd}"
} }
# Restart the kube-proxy on a node ($1) # Restart the kube-proxy on a node ($1)
function restart-kube-proxy { function restart-kube-proxy {
ssh-to-node "$1" "sudo /etc/init.d/kube-proxy restart" ssh-to-node "$1" "sudo /etc/init.d/kube-proxy restart"
} }
# Setup monitoring using heapster and InfluxDB # Setup monitoring using heapster and InfluxDB
function setup-monitoring { function setup-monitoring {
echo "not implemented" >/dev/null echo "not implemented" >/dev/null
} }
function teardown-monitoring { function teardown-monitoring {
echo "not implemented" >/dev/null echo "not implemented" >/dev/null
} }