diff --git a/cluster/kube-env.sh b/cluster/kube-env.sh index 9339f290a21..8fd202b35e5 100644 --- a/cluster/kube-env.sh +++ b/cluster/kube-env.sh @@ -18,6 +18,6 @@ # You can override the default provider by exporting the KUBERNETES_PROVIDER # variable in your bashrc # -# The valid values: 'gce', 'azure', 'vagrant', 'local' +# The valid values: 'gce', 'azure', 'vagrant', 'local', 'vsphere' KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER:-gce} diff --git a/cluster/saltbase/salt/apiserver/default b/cluster/saltbase/salt/apiserver/default index 8b4951d220e..858dc430525 100644 --- a/cluster/saltbase/salt/apiserver/default +++ b/cluster/saltbase/salt/apiserver/default @@ -26,6 +26,15 @@ {% elif grains.cloud is defined and grains.cloud == 'azure' %} MACHINES="{{ salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').values()|join(',', attribute='hostnamef') }}" {% set machines = "-machines $MACHINES" %} +{% else %} + # No cloud defined, collect IPs of minions as machines list. + # Use a bash array to build the value we need. It doesn't appear to be + # possible call functions map or zip, or use lambda's from Jinja. + MACHINE_IPS=() + {% for addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', expr_form='grain').values() %} + MACHINE_IPS+=( {{ addrs[0] }} ) + {% endfor %} + {% set machines = "-machines=$(echo ${MACHINE_IPS[@]} | xargs -n1 echo | paste -sd,)" %} {% endif %} DAEMON_ARGS="{{daemon_args}} {{address}} {{machines}} {{etcd_servers}} {{ minion_regexp }} {{ cloud_provider }}" diff --git a/cluster/saltbase/salt/static-routes/if-down b/cluster/saltbase/salt/static-routes/if-down new file mode 100644 index 00000000000..4c477cda149 --- /dev/null +++ b/cluster/saltbase/salt/static-routes/if-down @@ -0,0 +1,10 @@ +#!/bin/bash + +[ "$IFACE" == "eth0" ] || exit 0 + +{% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %} +{% if ip_addrs[0] != salt['network.ip_addrs']('eth0')[0] %} +{% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %} +route del -net {{ cidr }} +{% endif %} +{% endfor %} diff --git a/cluster/saltbase/salt/static-routes/if-up b/cluster/saltbase/salt/static-routes/if-up new file mode 100644 index 00000000000..d4318292f77 --- /dev/null +++ b/cluster/saltbase/salt/static-routes/if-up @@ -0,0 +1,10 @@ +#!/bin/bash + +[ "$IFACE" == "eth0" ] || exit 0 + +{% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %} +{% if ip_addrs[0] != salt['network.ip_addrs']('eth0')[0] %} +{% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %} +route add -net {{ cidr }} gw {{ ip_addrs[0] }} +{% endif %} +{% endfor %} diff --git a/cluster/saltbase/salt/static-routes/init.sls b/cluster/saltbase/salt/static-routes/init.sls new file mode 100644 index 00000000000..4416dbe5da0 --- /dev/null +++ b/cluster/saltbase/salt/static-routes/init.sls @@ -0,0 +1,30 @@ +# Add static routes to every minion to enable pods in the 10.244.x.x range to +# reach each other. This is suboptimal, but necessary to let every pod have +# its IP and have pods between minions be able to talk with each other. +# This will be obsolete when we figure out the right way to make this work. + +/etc/network/if-up.d/static-routes: + file.managed: + - source: salt://static-routes/if-up + - template: jinja + - user: root + - group: root + - mode: 755 + +/etc/network/if-down.d/static-routes: + file.managed: + - source: salt://static-routes/if-down + - template: jinja + - user: root + - group: root + - mode: 755 + +refresh routes: + cmd.wait_script: + - source: salt://static-routes/refresh + - cwd: /etc/network/ + - user: root + - group: root + - watch: + - file: /etc/network/if-up.d/static-routes + - file: /etc/network/if-down.d/static-routes diff --git a/cluster/saltbase/salt/static-routes/refresh b/cluster/saltbase/salt/static-routes/refresh new file mode 100644 index 00000000000..e988269073d --- /dev/null +++ b/cluster/saltbase/salt/static-routes/refresh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Fake an ifup/ifdown event +export IFACE=eth0 + +if-down.d/static-routes || true +if-up.d/static-routes || true diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 2c93627410a..1a7e038713e 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -19,3 +19,7 @@ base: - controller-manager - scheduler - nginx + + 'roles:kubernetes-pool-vsphere': + - match: grain + - static-routes diff --git a/cluster/validate-cluster.sh b/cluster/validate-cluster.sh index 0ce4770f960..99089292b97 100755 --- a/cluster/validate-cluster.sh +++ b/cluster/validate-cluster.sh @@ -33,6 +33,13 @@ detect-minions > /dev/null MINIONS_FILE=/tmp/minions $(dirname $0)/kubecfg.sh -template '{{range.Items}}{{.ID}}:{{end}}' list minions > ${MINIONS_FILE} +# On vSphere, use minion IPs as their names +if [ "$KUBERNETES_PROVIDER" == "vsphere" ]; then + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + MINION_NAMES[i]=${KUBE_MINION_IP_ADDRESSES[i]} + done +fi + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do # Grep returns an exit status of 1 when line is not found, so we need the : to always return a 0 exit status count=$(grep -c ${MINION_NAMES[i]} ${MINIONS_FILE}) || : diff --git a/cluster/vsphere/config-common.sh b/cluster/vsphere/config-common.sh new file mode 100644 index 00000000000..2bebd8e5f74 --- /dev/null +++ b/cluster/vsphere/config-common.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function public-key { + local dir=${HOME}/.ssh + + for f in $HOME/.ssh/{id_{rsa,dsa},*}.pub; do + if [ -r $f ]; then + echo $f + return + fi + done + + echo "Can't find public key file..." + exit 1 +} + +DISK=kube.vmdk +GUEST_ID=debian7_64Guest +PUBLIC_KEY_FILE=${PUBLIC_KEY_FILE-$(public-key)} +SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null" + +# These need to be set +#export GOVC_URL= +#export GOVC_DATACENTER= +#export GOVC_DATASTORE= +#export GOVC_RESOURCE_POOL= +#export GOVC_NETWORK= +#export GOVC_GUEST_LOGIN='kube:kube' diff --git a/cluster/vsphere/config-default.sh b/cluster/vsphere/config-default.sh new file mode 100755 index 00000000000..fb5e6e321c2 --- /dev/null +++ b/cluster/vsphere/config-default.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname ${BASH_SOURCE})/config-common.sh + +NUM_MINIONS=4 +INSTANCE_PREFIX=kubernetes + +MASTER_NAME="${INSTANCE_PREFIX}-master" +MASTER_MEMORY_MB=1024 +MASTER_CPU=1 + +MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) +MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24")) +MINION_MEMORY_MB=2048 +MINION_CPU=1 diff --git a/cluster/vsphere/config-test.sh b/cluster/vsphere/config-test.sh new file mode 100755 index 00000000000..3d36bc7cde2 --- /dev/null +++ b/cluster/vsphere/config-test.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname ${BASH_SOURCE})/config-common.sh + +NUM_MINIONS=2 +INSTANCE_PREFIX="e2e-test-${USER}" + +MASTER_NAME="${INSTANCE_PREFIX}-master" +MASTER_MEMORY_MB=1024 +MASTER_CPU=1 + +MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) +MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24")) +MINION_MEMORY_MB=1024 +MINION_CPU=1 diff --git a/cluster/vsphere/templates/hostname.sh b/cluster/vsphere/templates/hostname.sh new file mode 100755 index 00000000000..13b58a6d318 --- /dev/null +++ b/cluster/vsphere/templates/hostname.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Remove kube.vm from /etc/hosts +sed -i -e 's/\b\w\+.vm\b//' /etc/hosts + +# Update hostname in /etc/hosts and /etc/hostname +sed -i -e "s/\\bkube\\b/${MY_NAME}/g" /etc/host{s,name} +hostname ${MY_NAME} diff --git a/cluster/vsphere/templates/install-release.sh b/cluster/vsphere/templates/install-release.sh new file mode 100755 index 00000000000..877f150f13a --- /dev/null +++ b/cluster/vsphere/templates/install-release.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install release + +echo "Unpacking release" +rm -rf master-release || false +tar xzf master-release.tgz + +echo "Running release install script" +sudo master-release/src/scripts/master-release-install.sh diff --git a/cluster/vsphere/templates/salt-master.sh b/cluster/vsphere/templates/salt-master.sh new file mode 100755 index 00000000000..f2a16a52a67 --- /dev/null +++ b/cluster/vsphere/templates/salt-master.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use other Debian mirror +sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list + +# Prepopulate the name of the Master +mkdir -p /etc/salt/minion.d +echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf + +cat </etc/salt/minion.d/grains.conf +grains: + roles: + - kubernetes-master +EOF + +# Auto accept all keys from minions that try to join +mkdir -p /etc/salt/master.d +cat </etc/salt/master.d/auto-accept.conf +auto_accept: True +EOF + +cat </etc/salt/master.d/reactor.conf +# React to new minions starting by running highstate on them. +reactor: + - 'salt/minion/*/start': + - /srv/reactor/start.sls +EOF + +mkdir -p /srv/salt/nginx +echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd + +# Install Salt +# +# We specify -X to avoid a race condition that can cause minion failure to +# install. See https://github.com/saltstack/salt-bootstrap/issues/270 +# +# -M installs the master +if [ ! -x /etc/init.d/salt-master ]; then + wget -q -O - https://bootstrap.saltstack.com | sh -s -- -M -X +else + /etc/init.d/salt-master restart + /etc/init.d/salt-minion restart +fi + +echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd diff --git a/cluster/vsphere/templates/salt-minion.sh b/cluster/vsphere/templates/salt-minion.sh new file mode 100755 index 00000000000..75a6aab2631 --- /dev/null +++ b/cluster/vsphere/templates/salt-minion.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use other Debian mirror +sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list + +# Resolve hostname of master +if ! grep -q $MASTER_NAME /etc/hosts; then + echo "Adding host entry for $MASTER_NAME" + echo "$MASTER_IP $MASTER_NAME" >> /etc/hosts +fi + +# Prepopulate the name of the Master +mkdir -p /etc/salt/minion.d +echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf + +# Turn on debugging for salt-minion +# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion + +# Our minions will have a pool role to distinguish them from the master. +# +# Setting the "minion_ip" here causes the kubelet to use its IP for +# identification instead of its hostname. +# +cat </etc/salt/minion.d/grains.conf +grains: + minion_ip: $(ip route get 1.1.1.1 | awk '{print $7}') + roles: + - kubernetes-pool + - kubernetes-pool-vsphere + cbr-cidr: $MINION_IP_RANGE +EOF + +# Install Salt +# +# We specify -X to avoid a race condition that can cause minion failure to +# install. See https://github.com/saltstack/salt-bootstrap/issues/270 +if [ ! -x /etc/init.d/salt-minion ]; then + wget -q -O - https://bootstrap.saltstack.com | sh -s -- -X +else + /etc/init.d/salt-minion restart +fi diff --git a/cluster/vsphere/util.sh b/cluster/vsphere/util.sh new file mode 100644 index 00000000000..79b52f8e796 --- /dev/null +++ b/cluster/vsphere/util.sh @@ -0,0 +1,285 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions and constants for the local config. + +# Use the config file specified in $KUBE_CONFIG_FILE, or default to +# config-default.sh. +source $(dirname ${BASH_SOURCE})/${KUBE_CONFIG_FILE-"config-default.sh"} + +function detect-master { + KUBE_MASTER=${MASTER_NAME} + if [ -z "$KUBE_MASTER_IP" ]; then + KUBE_MASTER_IP=$(govc vm.ip ${MASTER_NAME}) + fi + if [ -z "$KUBE_MASTER_IP" ]; then + echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" + exit 1 + fi + echo "Found ${KUBE_MASTER} at ${KUBE_MASTER_IP}" +} + +function detect-minions { + KUBE_MINION_IP_ADDRESSES=() + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + local minion_ip=$(govc vm.ip ${MINION_NAMES[$i]}) + echo "Found ${MINION_NAMES[$i]} at ${minion_ip}" + KUBE_MINION_IP_ADDRESSES+=("${minion_ip}") + done + if [ -z "$KUBE_MINION_IP_ADDRESSES" ]; then + echo "Could not detect Kubernetes minion nodes. Make sure you've launched a cluster with 'kube-up.sh'" + exit 1 + fi +} + +# Verify prereqs on host machine +function verify-prereqs { + if [ "$(which govc)" == "" ]; then + echo "Can't find govc in PATH, please install and retry." + echo "" + echo " go install github.com/vmware/govmomi/govc" + echo "" + exit 1 + fi +} + +# Run command over ssh +function kube-ssh { + local host=$1 + shift + ssh ${SSH_OPTS} kube@${host} "$*" 2> /dev/null +} + +# Instantiate a generic kubernetes virtual machine (master or minion) +function kube-up-vm { + local vm_name=$1 + local vm_memory=$2 + local vm_cpu=$3 + local vm_ip= + + govc vm.create \ + -debug \ + -m ${vm_memory} \ + -c ${vm_cpu} \ + -disk ${DISK} \ + -g ${GUEST_ID} \ + -link=true \ + ${vm_name} + + # Retrieve IP first, to confirm the guest operations agent is running. + vm_ip=$(govc vm.ip ${vm_name}) + + govc guest.mkdir \ + -vm ${vm_name} \ + -p \ + /home/kube/.ssh + + govc guest.upload \ + -vm ${vm_name} \ + -f \ + ${PUBLIC_KEY_FILE} \ + /home/kube/.ssh/authorized_keys +} + +# Instantiate a kubernetes cluster +function kube-up { + # Build up start up script for master + KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX) + trap "rm -rf ${KUBE_TEMP}" EXIT + + get-password + echo "Using password: $user:$passwd" + echo + python $(dirname $0)/../third_party/htpasswd/htpasswd.py -b -c ${KUBE_TEMP}/htpasswd $user $passwd + HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd) + + echo "Starting master VM (this can take a minute)..." + + kube-up-vm ${MASTER_NAME} ${MASTER_MEMORY_MB-1024} ${MASTER_CPU-1} + + # Prints master IP, so user can log in for debugging. + detect-master + echo + + echo "Starting minion VMs (this can take a minute)..." + + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + ( + echo "#! /bin/bash" + echo "MY_NAME=${MINION_NAMES[$i]}" + grep -v "^#" $(dirname $0)/vsphere/templates/hostname.sh + echo "MASTER_NAME=${MASTER_NAME}" + echo "MASTER_IP=${KUBE_MASTER_IP}" + echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}" + grep -v "^#" $(dirname $0)/vsphere/templates/salt-minion.sh + ) > ${KUBE_TEMP}/minion-start-${i}.sh + + ( + kube-up-vm ${MINION_NAMES[$i]} ${MINION_MEMORY_MB-1024} ${MINION_CPU-1} + + MINION_IP=$(govc vm.ip ${MINION_NAMES[$i]}) + + govc guest.upload \ + -vm ${MINION_NAMES[$i]} \ + -perm 0700 \ + -f \ + ${KUBE_TEMP}/minion-start-${i}.sh \ + /home/kube/minion-start.sh + + # Kickstart start script + kube-ssh ${MINION_IP} "nohup sudo ~/minion-start.sh < /dev/null 1> minion-start.out 2> minion-start.err &" + ) & + done + + FAIL=0 + for job in `jobs -p` + do + wait $job || let "FAIL+=1" + done + if (( $FAIL != 0 )); then + echo "${FAIL} commands failed. Exiting." + exit 2 + fi + + # Print minion IPs, so user can log in for debugging. + detect-minions + echo + + # Continue provisioning the master. + + ( + echo "#! /bin/bash" + echo "MY_NAME=${MASTER_NAME}" + grep -v "^#" $(dirname $0)/vsphere/templates/hostname.sh + echo "MASTER_NAME=${MASTER_NAME}" + echo "MASTER_HTPASSWD='${HTPASSWD}'" + echo "MINION_NAMES=( ${MINION_NAMES[@]} )" + echo "KUBE_MINION_IP_ADDRESSES=( ${KUBE_MINION_IP_ADDRESSES[@]} )" + grep -v "^#" $(dirname $0)/vsphere/templates/install-release.sh + grep -v "^#" $(dirname $0)/vsphere/templates/salt-master.sh + ) > ${KUBE_TEMP}/master-start.sh + + govc guest.upload \ + -vm ${MASTER_NAME} \ + -perm 0700 \ + -f \ + ${KUBE_TEMP}/master-start.sh \ + /home/kube/master-start.sh + + govc guest.upload \ + -vm ${MASTER_NAME} \ + -f \ + ./output/release/master-release.tgz \ + /home/kube/master-release.tgz + + # Kickstart start script + kube-ssh ${KUBE_MASTER_IP} "nohup sudo ~/master-start.sh < /dev/null 1> master-start.out 2> master-start.err &" + + echo "Waiting for cluster initialization." + echo + echo " This will continually check to see if the API for kubernetes is reachable." + echo " This might loop forever if there was some uncaught error during start up." + echo + + until $(curl --insecure --user ${user}:${passwd} --max-time 5 \ + --fail --output /dev/null --silent https://${KUBE_MASTER_IP}/api/v1beta1/pods); do + printf "." + sleep 2 + done + + echo "Kubernetes cluster created." + echo + + echo "Sanity checking cluster..." + + sleep 5 + + # Don't bail on errors, we want to be able to print some info. + set +e + + # Basic sanity checking + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + # Make sure docker is installed + kube-ssh ${KUBE_MINION_IP_ADDRESSES[$i]} which docker > /dev/null + if [ "$?" != "0" ]; then + echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly." + echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)" + exit 1 + fi + done + + echo + echo "Kubernetes cluster is running. Access the master at:" + echo + echo " https://${user}:${passwd}@${KUBE_MASTER_IP}" + echo + echo "Security note: The server above uses a self signed certificate." + echo "This is subject to \"Man in the middle\" type attacks." + echo +} + +# Delete a kubernetes cluster +function kube-down { + govc vm.destroy ${MASTER_NAME} & + + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + govc vm.destroy ${MINION_NAMES[i]} & + done + + wait + +} + +# Update a kubernetes cluster with latest source +function kube-push { + echo "TODO" +} + +# Execute prior to running tests to build a release if required for env +function test-build-release { + echo "TODO" +} + +# Execute prior to running tests to initialize required structure +function test-setup { + echo "TODO" +} + +# Execute after running tests to perform any required clean-up +function test-teardown { + echo "TODO" +} + +# Set the {user} and {password} environment values required to interact with provider +function get-password { + file=${HOME}/.kubernetes_auth + if [ -e ${file} ]; then + user=$(cat $file | python -c 'import json,sys;print(json.load(sys.stdin)["User"])') + passwd=$(cat $file | python -c 'import json,sys;print(json.load(sys.stdin)["Password"])') + return + fi + user=admin + passwd=$(python -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))') + + # Store password for reuse. + cat << EOF > ~/.kubernetes_auth +{ + "User": "$user", + "Password": "$passwd" +} +EOF + chmod 0600 ~/.kubernetes_auth +} diff --git a/docs/getting-started-guides/vsphere.md b/docs/getting-started-guides/vsphere.md new file mode 100644 index 00000000000..ce2f27c64ca --- /dev/null +++ b/docs/getting-started-guides/vsphere.md @@ -0,0 +1,71 @@ +## Getting started with vSphere + +### Prerequisites + +1. You need administrator credentials to an ESXi machine or vCenter instance. +2. You must have Go (version 1.2 or later) installed: [www.golang.org](http://www.golang.org). +3. Install the govc tool to interact with ESXi/vCenter: + + ```sh + go get github.com/vmware/govmomi/govc + ``` +4. Install godep: + + ```sh + export GOBIN=/usr/local/go/bin + go get github.com/tools/godep + ``` + +5. Get the Kubernetes source: + + ```sh + git clone https://github.com/GoogleCloudPlatform/kubernetes.git + ``` + +### Setup + +Download a prebuilt Debian VMDK to be used as base image: + +```sh +wget http://storage.googleapis.com/govmomi/vmdk/kube.vmdk.gz{,.md5} +md5sum -c kube.vmdk.gz.md5 +gzip -d kube.vmdk.gz +``` + +Upload this VMDK to your vSphere instance: + +```sh +export GOVC_URL='https://user:pass@hostname/sdk' +export GOVC_DATASTORE='target datastore' +export GOVC_RESOURCE_POOL='resource pool with access to datastore' + +govc datastore.import kube.vmdk +``` + +Verify that the VMDK was correctly uploaded and expanded to 10GiB: + +```sh +govc datastore.ls +``` + +Take a look at the file `cluster/vsphere/config-common.sh` fill in the required +parameters. The guest login for the image that you imported is `kube:kube`. + +Now, let's continue with deploying Kubernetes: + +``` +cd kubernetes + +# Build a release +release/build-release.sh + +# Deploy Kubernetes (takes ~5 minutes, provided everything works out) +export KUBERNETES_PROVIDER=vsphere +cluster/kube-up.sh +``` + +Refer to the top level README and the getting started guide for Google Compute +Engine. Once you have successfully reached this point, your vSphere Kubernetes +deployment works just as any other one! + +**Enjoy!**