Make vSphere scripts work for binary deploys

Fixes #1732.
This commit is contained in:
Pieter Noordhuis
2014-10-10 18:52:02 -07:00
parent 74998eead4
commit e90f98a52c
13 changed files with 467 additions and 334 deletions

View File

@@ -1,46 +0,0 @@
#!/bin/bash
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
function public-key {
local dir=${HOME}/.ssh
for f in $HOME/.ssh/{id_{rsa,dsa},*}.pub; do
if [ -r $f ]; then
echo $f
return
fi
done
echo "Can't find public key file..." 1>&2
exit 1
}
DISK=./kube/kube.vmdk
GUEST_ID=debian7_64Guest
PUBLIC_KEY_FILE=${PUBLIC_KEY_FILE-$(public-key)}
SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null"
# These need to be set
#export GOVC_URL=
#export GOVC_DATACENTER=
#export GOVC_DATASTORE=
#export GOVC_RESOURCE_POOL=
#export GOVC_NETWORK=
#export GOVC_GUEST_LOGIN='kube:kube'
# Set GOVC_INSECURE if the host in GOVC_URL is using a certificate that cannot
# be verified (i.e. a self-signed certificate), but IS trusted.
#export GOVC_INSECURE=1

View File

@@ -1,29 +0,0 @@
#!/bin/bash
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
source $(dirname ${BASH_SOURCE})/config-common.sh
NUM_MINIONS=4
INSTANCE_PREFIX=kubernetes
MASTER_NAME="${INSTANCE_PREFIX}-master"
MASTER_MEMORY_MB=1024
MASTER_CPU=1
MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
MINION_MEMORY_MB=2048
MINION_CPU=1

View File

@@ -1,29 +0,0 @@
#!/bin/bash
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
source $(dirname ${BASH_SOURCE})/config-common.sh
NUM_MINIONS=2
INSTANCE_PREFIX="e2e-test-${USER}"
MASTER_NAME="${INSTANCE_PREFIX}-master"
MASTER_MEMORY_MB=1024
MASTER_CPU=1
MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
MINION_MEMORY_MB=1024
MINION_CPU=1

View File

@@ -1,22 +0,0 @@
#!/bin/bash
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Remove kube.vm from /etc/hosts
sed -i -e 's/\b\w\+.vm\b//' /etc/hosts
# Update hostname in /etc/hosts and /etc/hostname
sed -i -e "s/\\bkube\\b/${MY_NAME}/g" /etc/host{s,name}
hostname ${MY_NAME}

View File

@@ -1,24 +0,0 @@
#!/bin/bash
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Install release
echo "Unpacking release"
rm -rf master-release || false
tar xzf master-release.tgz
echo "Running release install script"
sudo master-release/src/scripts/master-release-install.sh

View File

@@ -1,60 +0,0 @@
#!/bin/bash
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Use other Debian mirror
sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
# Prepopulate the name of the Master
mkdir -p /etc/salt/minion.d
echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
cat <<EOF >/etc/salt/minion.d/grains.conf
grains:
roles:
- kubernetes-master
cloud: vsphere
EOF
# Auto accept all keys from minions that try to join
mkdir -p /etc/salt/master.d
cat <<EOF >/etc/salt/master.d/auto-accept.conf
auto_accept: True
EOF
cat <<EOF >/etc/salt/master.d/reactor.conf
# React to new minions starting by running highstate on them.
reactor:
- 'salt/minion/*/start':
- /srv/reactor/start.sls
EOF
mkdir -p /srv/salt/nginx
echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd
# Install Salt
#
# We specify -X to avoid a race condition that can cause minion failure to
# install. See https://github.com/saltstack/salt-bootstrap/issues/270
#
# -M installs the master
if [ ! -x /etc/init.d/salt-master ]; then
wget -q -O - https://bootstrap.saltstack.com | sh -s -- -M -X
else
/etc/init.d/salt-master restart
/etc/init.d/salt-minion restart
fi
echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd

View File

@@ -1,55 +0,0 @@
#!/bin/bash
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Use other Debian mirror
sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
# Resolve hostname of master
if ! grep -q $MASTER_NAME /etc/hosts; then
echo "Adding host entry for $MASTER_NAME"
echo "$MASTER_IP $MASTER_NAME" >> /etc/hosts
fi
# Prepopulate the name of the Master
mkdir -p /etc/salt/minion.d
echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
# Turn on debugging for salt-minion
# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion
# Our minions will have a pool role to distinguish them from the master.
#
# Setting the "minion_ip" here causes the kubelet to use its IP for
# identification instead of its hostname.
#
cat <<EOF >/etc/salt/minion.d/grains.conf
grains:
minion_ip: $(ip route get 1.1.1.1 | awk '{print $7}')
roles:
- kubernetes-pool
- kubernetes-pool-vsphere
cbr-cidr: $MINION_IP_RANGE
EOF
# Install Salt
#
# We specify -X to avoid a race condition that can cause minion failure to
# install. See https://github.com/saltstack/salt-bootstrap/issues/270
if [ ! -x /etc/init.d/salt-minion ]; then
wget -q -O - https://bootstrap.saltstack.com | sh -s -- -X
else
/etc/init.d/salt-minion restart
fi

View File

@@ -1,306 +0,0 @@
#!/bin/bash
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A library of helper functions and constants for the local config.
# Use the config file specified in $KUBE_CONFIG_FILE, or default to
# config-default.sh.
source $(dirname ${BASH_SOURCE})/${KUBE_CONFIG_FILE-"config-default.sh"}
function detect-master {
KUBE_MASTER=${MASTER_NAME}
if [ -z "$KUBE_MASTER_IP" ]; then
KUBE_MASTER_IP=$(govc vm.ip ${MASTER_NAME})
fi
if [ -z "$KUBE_MASTER_IP" ]; then
echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'"
exit 1
fi
echo "Found ${KUBE_MASTER} at ${KUBE_MASTER_IP}"
}
function detect-minions {
KUBE_MINION_IP_ADDRESSES=()
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
local minion_ip=$(govc vm.ip ${MINION_NAMES[$i]})
echo "Found ${MINION_NAMES[$i]} at ${minion_ip}"
KUBE_MINION_IP_ADDRESSES+=("${minion_ip}")
done
if [ -z "$KUBE_MINION_IP_ADDRESSES" ]; then
echo "Could not detect Kubernetes minion nodes. Make sure you've launched a cluster with 'kube-up.sh'"
exit 1
fi
}
# Verify prereqs on host machine
function verify-prereqs {
if [ "$(which govc)" == "" ]; then
echo "Can't find govc in PATH, please install and retry."
echo ""
echo " go install github.com/vmware/govmomi/govc"
echo ""
exit 1
fi
}
# Run command over ssh
function kube-ssh {
local host=$1
shift
ssh ${SSH_OPTS} kube@${host} "$*" 2> /dev/null
}
# Instantiate a generic kubernetes virtual machine (master or minion)
function kube-up-vm {
local vm_name=$1
local vm_memory=$2
local vm_cpu=$3
local vm_ip=
govc vm.create \
-debug \
-m ${vm_memory} \
-c ${vm_cpu} \
-disk ${DISK} \
-g ${GUEST_ID} \
-link=true \
${vm_name}
# Retrieve IP first, to confirm the guest operations agent is running.
vm_ip=$(govc vm.ip ${vm_name})
govc guest.mkdir \
-vm ${vm_name} \
-p \
/home/kube/.ssh
govc guest.upload \
-vm ${vm_name} \
-f \
${PUBLIC_KEY_FILE} \
/home/kube/.ssh/authorized_keys
}
# Instantiate a kubernetes cluster
function kube-up {
# Build up start up script for master
KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
trap "rm -rf ${KUBE_TEMP}" EXIT
get-password
python $(dirname $0)/../third_party/htpasswd/htpasswd.py -b -c ${KUBE_TEMP}/htpasswd $user $passwd
HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd)
echo "Starting master VM (this can take a minute)..."
kube-up-vm ${MASTER_NAME} ${MASTER_MEMORY_MB-1024} ${MASTER_CPU-1}
# Prints master IP, so user can log in for debugging.
detect-master
echo
echo "Starting minion VMs (this can take a minute)..."
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
(
echo "#! /bin/bash"
echo "MY_NAME=${MINION_NAMES[$i]}"
grep -v "^#" $(dirname $0)/vsphere/templates/hostname.sh
echo "MASTER_NAME=${MASTER_NAME}"
echo "MASTER_IP=${KUBE_MASTER_IP}"
echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}"
grep -v "^#" $(dirname $0)/vsphere/templates/salt-minion.sh
) > ${KUBE_TEMP}/minion-start-${i}.sh
(
kube-up-vm ${MINION_NAMES[$i]} ${MINION_MEMORY_MB-1024} ${MINION_CPU-1}
MINION_IP=$(govc vm.ip ${MINION_NAMES[$i]})
govc guest.upload \
-vm ${MINION_NAMES[$i]} \
-perm 0700 \
-f \
${KUBE_TEMP}/minion-start-${i}.sh \
/home/kube/minion-start.sh
# Kickstart start script
kube-ssh ${MINION_IP} "nohup sudo ~/minion-start.sh < /dev/null 1> minion-start.out 2> minion-start.err &"
) &
done
FAIL=0
for job in `jobs -p`
do
wait $job || let "FAIL+=1"
done
if (( $FAIL != 0 )); then
echo "${FAIL} commands failed. Exiting."
exit 2
fi
# Print minion IPs, so user can log in for debugging.
detect-minions
echo
# Continue provisioning the master.
(
echo "#! /bin/bash"
echo "MY_NAME=${MASTER_NAME}"
grep -v "^#" $(dirname $0)/vsphere/templates/hostname.sh
echo "MASTER_NAME=${MASTER_NAME}"
echo "MASTER_HTPASSWD='${HTPASSWD}'"
grep -v "^#" $(dirname $0)/vsphere/templates/install-release.sh
grep -v "^#" $(dirname $0)/vsphere/templates/salt-master.sh
) > ${KUBE_TEMP}/master-start.sh
govc guest.upload \
-vm ${MASTER_NAME} \
-perm 0700 \
-f \
${KUBE_TEMP}/master-start.sh \
/home/kube/master-start.sh
govc guest.upload \
-vm ${MASTER_NAME} \
-f \
./_output/release/master-release.tgz \
/home/kube/master-release.tgz
# Kickstart start script
kube-ssh ${KUBE_MASTER_IP} "nohup sudo ~/master-start.sh < /dev/null 1> master-start.out 2> master-start.err &"
echo "Waiting for cluster initialization."
echo
echo " This will continually check to see if the API for kubernetes is reachable."
echo " This might loop forever if there was some uncaught error during start up."
echo
until $(curl --insecure --user ${user}:${passwd} --max-time 5 \
--fail --output /dev/null --silent https://${KUBE_MASTER_IP}/api/v1beta1/pods); do
printf "."
sleep 2
done
echo "Kubernetes cluster created."
echo
echo "Sanity checking cluster..."
sleep 5
# Don't bail on errors, we want to be able to print some info.
set +e
# Basic sanity checking
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
# Make sure docker is installed
kube-ssh ${KUBE_MINION_IP_ADDRESSES[$i]} which docker > /dev/null
if [ "$?" != "0" ]; then
echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)"
exit 1
fi
done
echo
echo "Kubernetes cluster is running. The master is running at:"
echo
echo " https://${KUBE_MASTER_IP}"
echo
echo "The user name and password to use is located in ~/.kubernetes_auth."
echo
echo "Security note: The server above uses a self signed certificate."
echo "This is subject to \"Man in the middle\" type attacks."
echo
}
# Delete a kubernetes cluster
function kube-down {
govc vm.destroy ${MASTER_NAME} &
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
govc vm.destroy ${MINION_NAMES[i]} &
done
wait
}
# Update a kubernetes cluster with latest source
function kube-push {
detect-master
govc guest.upload \
-vm ${MASTER_NAME} \
-f \
./_output/release/master-release.tgz \
/home/kube/master-release.tgz
(
grep -v "^#" $(dirname $0)/vsphere/templates/install-release.sh
echo "echo Executing configuration"
echo "sudo salt '*' mine.update"
echo "sudo salt --force-color '*' state.highstate"
) | kube-ssh ${KUBE_MASTER_IP} bash
get-password
echo
echo "Kubernetes cluster is updated. The master is running at:"
echo
echo " https://${KUBE_MASTER_IP}"
echo
echo "The user name and password to use is located in ~/.kubernetes_auth."
echo
}
# Execute prior to running tests to build a release if required for env
function test-build-release {
echo "TODO"
}
# Execute prior to running tests to initialize required structure
function test-setup {
echo "TODO"
}
# Execute after running tests to perform any required clean-up
function test-teardown {
echo "TODO"
}
# Set the {user} and {password} environment values required to interact with provider
function get-password {
file=${HOME}/.kubernetes_auth
if [ -e ${file} ]; then
user=$(cat $file | python -c 'import json,sys;print(json.load(sys.stdin)["User"])')
passwd=$(cat $file | python -c 'import json,sys;print(json.load(sys.stdin)["Password"])')
return
fi
user=admin
passwd=$(python -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))')
# Store password for reuse.
cat << EOF > ~/.kubernetes_auth
{
"User": "$user",
"Password": "$passwd"
}
EOF
chmod 0600 ~/.kubernetes_auth
}