diff --git a/cluster/azure/kube-down.sh b/cluster/azure/config-default.sh old mode 100755 new mode 100644 similarity index 54% rename from cluster/azure/kube-down.sh rename to cluster/azure/config-default.sh index 804ef6d2190..cfc3e830ec5 --- a/cluster/azure/kube-down.sh +++ b/cluster/azure/config-default.sh @@ -14,13 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Tear down a Kubernetes cluster. SCRIPT_DIR=$(CDPATH="" cd $(dirname $0); pwd) -source $SCRIPT_DIR/../../release/azure/config.sh -source $SCRIPT_DIR/../util.sh +source $SCRIPT_DIR/../release/azure/config.sh -echo "Bringing down cluster" -azure vm delete $MASTER_NAME -b -q -for (( i=0; i<${#MINION_NAMES[@]}; i++)); do - azure vm delete ${MINION_NAMES[$i]} -b -q -done +AZ_SSH_KEY=$HOME/.ssh/azure_rsa +AZ_SSH_CERT=$HOME/.ssh/azure.pem +AZ_IMAGE=b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04-LTS-amd64-server-20140618.1-en-us-30GB +AZ_SUBNET=Subnet-1 +AZ_VNET=kube-$AZ_HSH +AZ_CS=kube-$AZ_HSH + +NUM_MINIONS=4 + +MASTER_NAME="${INSTANCE_PREFIX}-master" +MASTER_TAG="${INSTANCE_PREFIX}-master" +MINION_TAG="${INSTANCE_PREFIX}-minion" +MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) +MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24")) +MINION_SCOPES="" diff --git a/cluster/azure/kube-up.sh b/cluster/azure/kube-up.sh deleted file mode 100755 index e7d0e0bc021..00000000000 --- a/cluster/azure/kube-up.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Bring up a Kubernetes cluster. - -# exit on any error -set -eu -set -o pipefail -SCRIPT_DIR=$(CDPATH="" cd $(dirname $0); pwd) - -source $SCRIPT_DIR/../../release/azure/config.sh -source $SCRIPT_DIR/../util.sh - -KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX) -trap "rm -rf ${KUBE_TEMP}" EXIT - -get-password -echo "Using password: $user:$passwd" -python $SCRIPT_DIR/../../third_party/htpasswd/htpasswd.py -b -c \ - ${KUBE_TEMP}/htpasswd $user $passwd -HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd) - -# Build up start up script for master -( - echo "#!/bin/bash" - echo "MASTER_NAME=${MASTER_NAME}" - echo "MASTER_RELEASE_TAR=${FULL_URL}" - echo "MASTER_HTPASSWD='${HTPASSWD}'" - grep -v "^#" $SCRIPT_DIR/templates/download-release.sh - grep -v "^#" $SCRIPT_DIR/templates/salt-master.sh -) > ${KUBE_TEMP}/master-start.sh - -echo "Starting VMs" - -if [ ! -f $AZ_SSH_KEY ]; then - ssh-keygen -f $AZ_SSH_KEY -N '' -fi - -if [ ! -f $AZ_SSH_CERT ]; then - openssl req -new -key $AZ_SSH_KEY -out ${KUBE_TEMP}/temp.csr \ - -subj "/C=US/ST=WA/L=Redmond/O=Azure-CLI/CN=Azure" - openssl req -x509 -key $AZ_SSH_KEY -in ${KUBE_TEMP}/temp.csr \ - -out $AZ_SSH_CERT -days 1095 - rm ${KUBE_TEMP}/temp.csr -fi - -if [ -z "$(azure network vnet show $AZ_VNET 2>/dev/null | grep data)" ]; then - #azure network vnet create with $AZ_SUBNET - #FIXME not working - echo error create vnet $AZ_VNET with subnet $AZ_SUBNET - exit 1 -fi - -azure vm create \ - -w $AZ_VNET \ - -n $MASTER_NAME \ - -l "$AZ_LOCATION" \ - -t $AZ_SSH_CERT \ - -e 22000 -P \ - -d ${KUBE_TEMP}/master-start.sh \ - -b $AZ_SUBNET \ - $AZ_CS $AZ_IMAGE $USER - -ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}")) - -for (( i=0; i<${#MINION_NAMES[@]}; i++)); do - ( - echo "#!/bin/bash" - echo "MASTER_NAME=${MASTER_NAME}" - echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}" - grep -v "^#" $SCRIPT_DIR/templates/salt-minion.sh - ) > ${KUBE_TEMP}/minion-start-${i}.sh - - azure vm create \ - -c -w $AZ_VNET \ - -n ${MINION_NAMES[$i]} \ - -l "$AZ_LOCATION" \ - -t $AZ_SSH_CERT \ - -e ${ssh_ports[$i]} -P \ - -d ${KUBE_TEMP}/minion-start-${i}.sh \ - -b $AZ_SUBNET \ - $AZ_CS $AZ_IMAGE $USER -done - -azure vm endpoint create $MASTER_NAME 443 - -echo "Waiting for cluster initialization." -echo -echo " This will continually check to see if the API for kubernetes is reachable." -echo " This might loop forever if there was some uncaught error during start" -echo " up." -echo - -until $(curl --insecure --user ${user}:${passwd} --max-time 5 \ - --fail --output /dev/null --silent https://$AZ_CS.cloudapp.net/api/v1beta1/pods); do - printf "." - sleep 2 -done - -# Basic sanity checking -for (( i=0; i<${#MINION_NAMES[@]}; i++)); do - # Make sure docker is installed - ssh -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net which docker > /dev/null - if [ "$?" != "0" ]; then - echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly." - echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)" - exit 1 - fi - - # Make sure the kubelet is running - ssh -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net /etc/init.d/kubelet status - if [ "$?" != "0" ]; then - echo "Kubelet failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly." - echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)" - exit 1 - fi -done - -echo -echo "Kubernetes cluster is running. Access the master at:" -echo -echo " https://${user}:${passwd}@$AZ_CS.cloudapp.net" -echo -echo "Security note: The server above uses a self signed certificate. This is" -echo " subject to \"Man in the middle\" type attacks." diff --git a/cluster/azure/util.sh b/cluster/azure/util.sh new file mode 100644 index 00000000000..5302200fb41 --- /dev/null +++ b/cluster/azure/util.sh @@ -0,0 +1,253 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR=$(CDPATH="" cd $(dirname $0); pwd) + +# Use the config file specified in $KUBE_CONFIG_FILE, or default to +# config-default.sh. +source ${SCRIPT_DIR}/azure/${KUBE_CONFIG_FILE-"config-default.sh"} + +function detect-minions () { + ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}")) + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + MINION_NAMES[$i]=$(ssh -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net hostname -f) + done +} + +function detect-master () { + KUBE_MASTER_IP=${AZ_CS}.cloudapp.net + echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)" +} + +function get-password { + file=${HOME}/.kubernetes_auth + if [ -e ${file} ]; then + user=$(cat $file | python -c 'import json,sys;print json.load(sys.stdin)["User"]') + passwd=$(cat $file | python -c 'import json,sys;print json.load(sys.stdin)["Password"]') + return + fi + user=admin + passwd=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))') + + # Store password for reuse. + cat << EOF > ~/.kubernetes_auth +{ + "User": "$user", + "Password": "$passwd" +} +EOF + chmod 0600 ~/.kubernetes_auth +} + +# Verify prereqs +function verify-prereqs { + echo "OK" + # Already done in sourcing config-default, which sources + # release/azure/config.sh +} + +# Instantiate a kubernetes cluster +function kube-up { + KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX) + trap "rm -rf ${KUBE_TEMP}" EXIT + + get-password + echo "Using password: $user:$passwd" + python $SCRIPT_DIR/../third_party/htpasswd/htpasswd.py -b -c \ + ${KUBE_TEMP}/htpasswd $user $passwd + HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd) + + # Build up start up script for master + ( + echo "#!/bin/bash" + echo "MASTER_NAME=${MASTER_NAME}" + echo "MASTER_RELEASE_TAR=${FULL_URL}" + echo "MASTER_HTPASSWD='${HTPASSWD}'" + grep -v "^#" $SCRIPT_DIR/azure/templates/download-release.sh + grep -v "^#" $SCRIPT_DIR/azure/templates/salt-master.sh + ) > ${KUBE_TEMP}/master-start.sh + + echo "Starting VMs" + + if [ ! -f $AZ_SSH_KEY ]; then + ssh-keygen -f $AZ_SSH_KEY -N '' + fi + + if [ ! -f $AZ_SSH_CERT ]; then + openssl req -new -key $AZ_SSH_KEY -out ${KUBE_TEMP}/temp.csr \ + -subj "/C=US/ST=WA/L=Redmond/O=Azure-CLI/CN=Azure" + openssl req -x509 -key $AZ_SSH_KEY -in ${KUBE_TEMP}/temp.csr \ + -out $AZ_SSH_CERT -days 1095 + rm ${KUBE_TEMP}/temp.csr + fi + + if [ -z "$(azure network vnet show $AZ_VNET 2>/dev/null | grep data)" ]; then + #azure network vnet create with $AZ_SUBNET + #FIXME not working + echo error create vnet $AZ_VNET with subnet $AZ_SUBNET + exit 1 + fi + + azure vm create \ + -w $AZ_VNET \ + -n $MASTER_NAME \ + -l "$AZ_LOCATION" \ + -t $AZ_SSH_CERT \ + -e 22000 -P \ + -d ${KUBE_TEMP}/master-start.sh \ + -b $AZ_SUBNET \ + $AZ_CS $AZ_IMAGE $USER + + ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}")) + + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + ( + echo "#!/bin/bash" + echo "MASTER_NAME=${MASTER_NAME}" + echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}" + grep -v "^#" $SCRIPT_DIR/azure/templates/salt-minion.sh + ) > ${KUBE_TEMP}/minion-start-${i}.sh + + azure vm create \ + -c -w $AZ_VNET \ + -n ${MINION_NAMES[$i]} \ + -l "$AZ_LOCATION" \ + -t $AZ_SSH_CERT \ + -e ${ssh_ports[$i]} -P \ + -d ${KUBE_TEMP}/minion-start-${i}.sh \ + -b $AZ_SUBNET \ + $AZ_CS $AZ_IMAGE $USER + done + + azure vm endpoint create $MASTER_NAME 443 + + echo "Waiting for cluster initialization." + echo + echo " This will continually check to see if the API for kubernetes is reachable." + echo " This might loop forever if there was some uncaught error during start" + echo " up." + echo + + until $(curl --insecure --user ${user}:${passwd} --max-time 5 \ + --fail --output /dev/null --silent https://$AZ_CS.cloudapp.net/api/v1beta1/pods); do + printf "." + sleep 2 + done + + # Basic sanity checking + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + # Make sure docker is installed + ssh -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net which docker > /dev/null + if [ "$?" != "0" ]; then + echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly." + echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)" + exit 1 + fi + + # Make sure the kubelet is running + ssh -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net /etc/init.d/kubelet status + if [ "$?" != "0" ]; then + echo "Kubelet failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly." + echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)" + exit 1 + fi + done + + echo + echo "Kubernetes cluster is running. Access the master at:" + echo + echo " https://${user}:${passwd}@$AZ_CS.cloudapp.net" + echo + echo "Security note: The server above uses a self signed certificate. This is" + echo " subject to \"Man in the middle\" type attacks." +} + +# Delete a kubernetes cluster +function kube-down { + echo "Bringing down cluster" + set +e + azure vm delete $MASTER_NAME -b -q + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + azure vm delete ${MINION_NAMES[$i]} -b -q + done +} + +# # Update a kubernetes cluster with latest source +# function kube-push { + +# # Find the release to use. Generally it will be passed when doing a 'prod' +# # install and will default to the release/config.sh version when doing a +# # developer up. +# find-release $1 + +# # Detect the project into $PROJECT +# detect-master + +# ( +# echo MASTER_RELEASE_TAR=$RELEASE_NORMALIZED/master-release.tgz +# grep -v "^#" $(dirname $0)/templates/download-release.sh +# echo "echo Executing configuration" +# echo "sudo salt '*' mine.update" +# echo "sudo salt --force-color '*' state.highstate" +# ) | gcutil ssh --project ${PROJECT} --zone ${ZONE} $KUBE_MASTER bash + +# get-password + +# echo "Kubernetes cluster is updated. Access the master at:" +# echo +# echo " https://${user}:${passwd}@${KUBE_MASTER_IP}" +# echo + +# } + +# # Execute prior to running tests to build a release if required for env +# function test-build-release { +# # Build source +# ${KUBE_REPO_ROOT}/hack/build-go.sh +# # Make a release +# $(dirname $0)/../release/release.sh +# } + +# # Execute prior to running tests to initialize required structure +# function test-setup { + +# # Detect the project into $PROJECT if it isn't set +# # gce specific +# detect-project + +# if [[ ${ALREADY_UP} -ne 1 ]]; then +# # Open up port 80 & 8080 so common containers on minions can be reached +# gcutil addfirewall \ +# --norespect_terminal_width \ +# --project ${PROJECT} \ +# --target_tags ${MINION_TAG} \ +# --allowed tcp:80,tcp:8080 \ +# --network ${NETWORK} \ +# ${MINION_TAG}-${INSTANCE_PREFIX}-http-alt +# fi + +# } + +# # Execute after running tests to perform any required clean-up +# function test-teardown { +# echo "Shutting down test cluster in background." +# gcutil deletefirewall \ +# --project ${PROJECT} \ +# --norespect_terminal_width \ +# --force \ +# ${MINION_TAG}-${INSTANCE_PREFIX}-http-alt || true > /dev/null +# $(dirname $0)/../cluster/kube-down.sh > /dev/null +# } diff --git a/cluster/validate-cluster.sh b/cluster/validate-cluster.sh index 0ce4770f960..1987d6926e7 100755 --- a/cluster/validate-cluster.sh +++ b/cluster/validate-cluster.sh @@ -37,7 +37,7 @@ for (( i=0; i<${#MINION_NAMES[@]}; i++)); do # Grep returns an exit status of 1 when line is not found, so we need the : to always return a 0 exit status count=$(grep -c ${MINION_NAMES[i]} ${MINIONS_FILE}) || : if [ "$count" == "0" ]; then - echo "Failed to find ${MINION_NAMES[i]}, cluster is probably broken." + echo "Failed to find ${MINION_NAMES[i]}, cluster is probably broken." exit 1 fi @@ -48,7 +48,6 @@ for (( i=0; i<${#MINION_NAMES[@]}; i++)); do exit 1 else echo "Kubelet is successfully installed on ${MINION_NAMES[$i]}" - fi + fi done echo "Cluster validation succeeded" - diff --git a/hack/build-go.sh b/hack/build-go.sh index b2044ae288f..eef27c72e22 100755 --- a/hack/build-go.sh +++ b/hack/build-go.sh @@ -20,7 +20,7 @@ set -o errexit set -o nounset set -o pipefail -hackdir=$(dirname "$0") +hackdir=$(CDPATH="" cd $(dirname $0); pwd) # Set the environment variables required by the build. . "${hackdir}/config-go.sh" diff --git a/release/azure/config.sh b/release/azure/config.sh index d6ba204e8e5..19738bd82fd 100644 --- a/release/azure/config.sh +++ b/release/azure/config.sh @@ -12,11 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +INSTANCE_PREFIX=kubenertes AZ_LOCATION='West US' -AZ_SSH_KEY=$HOME/.ssh/azure -AZ_SSH_CERT=$HOME/.ssh/azure.pem -AZ_IMAGE=b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04-LTS-amd64-server-20140618.1-en-us-30GB -AZ_SUBNET=Subnet-1 TAG=testing if [ -z "$(which azure)" ]; then @@ -32,19 +29,12 @@ if [ -z "$(azure account list | grep true)" ]; then fi account=$(azure account list | grep true | awk '{ print $2 }') - if which md5 > /dev/null 2>&1; then - hsh=$(md5 -q -s $account) + AZ_HSH=$(md5 -q -s $account) else - hsh=$(echo -n "$account" | md5sum) + AZ_HSH=$(echo -n "$account" | md5sum) fi -hsh=${hsh:0:7} - -STG_ACCOUNT=kube$hsh - -AZ_VNET=kube-$hsh -AZ_CS=kube-$hsh - +AZ_HSH=${AZ_HSH:0:7} +AZ_STG=kube$AZ_HSH CONTAINER=kube-$TAG - -FULL_URL="https://${STG_ACCOUNT}.blob.core.windows.net/$CONTAINER/master-release.tgz" +FULL_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/master-release.tgz" diff --git a/release/azure/release.sh b/release/azure/release.sh index ccf5ddf0b6c..323ceffac40 100755 --- a/release/azure/release.sh +++ b/release/azure/release.sh @@ -26,38 +26,37 @@ function json_val () { } source $SCRIPT_DIR/config.sh -source $SCRIPT_DIR/../../cluster/${KUBE_CONFIG_FILE-"config-default.sh"} $SCRIPT_DIR/../build-release.sh $INSTANCE_PREFIX -if [ -z "$(azure storage account show $STG_ACCOUNT 2>/dev/null | \ +if [ -z "$(azure storage account show $AZ_STG 2>/dev/null | \ grep data)" ]; then - azure storage account create -l "$AZ_LOCATION" $STG_ACCOUNT + azure storage account create -l "$AZ_LOCATION" $AZ_STG fi -stg_key=$(azure storage account keys list $STG_ACCOUNT --json | \ +stg_key=$(azure storage account keys list $AZ_STG --json | \ json_val '["primaryKey"]') -if [ -z "$(azure storage container show -a $STG_ACCOUNT -k "$stg_key" \ +if [ -z "$(azure storage container show -a $AZ_STG -k "$stg_key" \ $CONTAINER 2>/dev/null | grep data)" ]; then azure storage container create \ - -a $STG_ACCOUNT \ + -a $AZ_STG \ -k "$stg_key" \ -p Blob \ $CONTAINER fi -if [ -n "$(azure storage blob show -a $STG_ACCOUNT -k "$stg_key" \ +if [ -n "$(azure storage blob show -a $AZ_STG -k "$stg_key" \ $CONTAINER master-release.tgz 2>/dev/null | grep data)" ]; then azure storage blob delete \ - -a $STG_ACCOUNT \ + -a $AZ_STG \ -k "$stg_key" \ $CONTAINER \ master-release.tgz fi azure storage blob upload \ - -a $STG_ACCOUNT \ + -a $AZ_STG \ -k "$stg_key" \ $SCRIPT_DIR/../../output/release/master-release.tgz \ $CONTAINER \