mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 19:31:44 +00:00
Deploy k8s to vSphere
This commit is contained in:
parent
bd2cbdc312
commit
ad7f131a5b
@ -18,6 +18,6 @@
|
|||||||
# You can override the default provider by exporting the KUBERNETES_PROVIDER
|
# You can override the default provider by exporting the KUBERNETES_PROVIDER
|
||||||
# variable in your bashrc
|
# variable in your bashrc
|
||||||
#
|
#
|
||||||
# The valid values: 'gce', 'azure', 'vagrant', 'local'
|
# The valid values: 'gce', 'azure', 'vagrant', 'local', 'vsphere'
|
||||||
|
|
||||||
KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER:-gce}
|
KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER:-gce}
|
||||||
|
@ -26,6 +26,15 @@
|
|||||||
{% elif grains.cloud is defined and grains.cloud == 'azure' %}
|
{% elif grains.cloud is defined and grains.cloud == 'azure' %}
|
||||||
MACHINES="{{ salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').values()|join(',', attribute='hostnamef') }}"
|
MACHINES="{{ salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').values()|join(',', attribute='hostnamef') }}"
|
||||||
{% set machines = "-machines $MACHINES" %}
|
{% set machines = "-machines $MACHINES" %}
|
||||||
|
{% else %}
|
||||||
|
# No cloud defined, collect IPs of minions as machines list.
|
||||||
|
# Use a bash array to build the value we need. It doesn't appear to be
|
||||||
|
# possible call functions map or zip, or use lambda's from Jinja.
|
||||||
|
MACHINE_IPS=()
|
||||||
|
{% for addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', expr_form='grain').values() %}
|
||||||
|
MACHINE_IPS+=( {{ addrs[0] }} )
|
||||||
|
{% endfor %}
|
||||||
|
{% set machines = "-machines=$(echo ${MACHINE_IPS[@]} | xargs -n1 echo | paste -sd,)" %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
DAEMON_ARGS="{{daemon_args}} {{address}} {{machines}} {{etcd_servers}} {{ minion_regexp }} {{ cloud_provider }}"
|
DAEMON_ARGS="{{daemon_args}} {{address}} {{machines}} {{etcd_servers}} {{ minion_regexp }} {{ cloud_provider }}"
|
||||||
|
10
cluster/saltbase/salt/static-routes/if-down
Normal file
10
cluster/saltbase/salt/static-routes/if-down
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
[ "$IFACE" == "eth0" ] || exit 0
|
||||||
|
|
||||||
|
{% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %}
|
||||||
|
{% if ip_addrs[0] != salt['network.ip_addrs']('eth0')[0] %}
|
||||||
|
{% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %}
|
||||||
|
route del -net {{ cidr }}
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
10
cluster/saltbase/salt/static-routes/if-up
Normal file
10
cluster/saltbase/salt/static-routes/if-up
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
[ "$IFACE" == "eth0" ] || exit 0
|
||||||
|
|
||||||
|
{% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %}
|
||||||
|
{% if ip_addrs[0] != salt['network.ip_addrs']('eth0')[0] %}
|
||||||
|
{% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %}
|
||||||
|
route add -net {{ cidr }} gw {{ ip_addrs[0] }}
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
30
cluster/saltbase/salt/static-routes/init.sls
Normal file
30
cluster/saltbase/salt/static-routes/init.sls
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
# Add static routes to every minion to enable pods in the 10.244.x.x range to
|
||||||
|
# reach each other. This is suboptimal, but necessary to let every pod have
|
||||||
|
# its IP and have pods between minions be able to talk with each other.
|
||||||
|
# This will be obsolete when we figure out the right way to make this work.
|
||||||
|
|
||||||
|
/etc/network/if-up.d/static-routes:
|
||||||
|
file.managed:
|
||||||
|
- source: salt://static-routes/if-up
|
||||||
|
- template: jinja
|
||||||
|
- user: root
|
||||||
|
- group: root
|
||||||
|
- mode: 755
|
||||||
|
|
||||||
|
/etc/network/if-down.d/static-routes:
|
||||||
|
file.managed:
|
||||||
|
- source: salt://static-routes/if-down
|
||||||
|
- template: jinja
|
||||||
|
- user: root
|
||||||
|
- group: root
|
||||||
|
- mode: 755
|
||||||
|
|
||||||
|
refresh routes:
|
||||||
|
cmd.wait_script:
|
||||||
|
- source: salt://static-routes/refresh
|
||||||
|
- cwd: /etc/network/
|
||||||
|
- user: root
|
||||||
|
- group: root
|
||||||
|
- watch:
|
||||||
|
- file: /etc/network/if-up.d/static-routes
|
||||||
|
- file: /etc/network/if-down.d/static-routes
|
7
cluster/saltbase/salt/static-routes/refresh
Normal file
7
cluster/saltbase/salt/static-routes/refresh
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Fake an ifup/ifdown event
|
||||||
|
export IFACE=eth0
|
||||||
|
|
||||||
|
if-down.d/static-routes || true
|
||||||
|
if-up.d/static-routes || true
|
@ -19,3 +19,7 @@ base:
|
|||||||
- controller-manager
|
- controller-manager
|
||||||
- scheduler
|
- scheduler
|
||||||
- nginx
|
- nginx
|
||||||
|
|
||||||
|
'roles:kubernetes-pool-vsphere':
|
||||||
|
- match: grain
|
||||||
|
- static-routes
|
||||||
|
@ -33,6 +33,13 @@ detect-minions > /dev/null
|
|||||||
MINIONS_FILE=/tmp/minions
|
MINIONS_FILE=/tmp/minions
|
||||||
$(dirname $0)/kubecfg.sh -template '{{range.Items}}{{.ID}}:{{end}}' list minions > ${MINIONS_FILE}
|
$(dirname $0)/kubecfg.sh -template '{{range.Items}}{{.ID}}:{{end}}' list minions > ${MINIONS_FILE}
|
||||||
|
|
||||||
|
# On vSphere, use minion IPs as their names
|
||||||
|
if [ "$KUBERNETES_PROVIDER" == "vsphere" ]; then
|
||||||
|
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||||
|
MINION_NAMES[i]=${KUBE_MINION_IP_ADDRESSES[i]}
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||||
# Grep returns an exit status of 1 when line is not found, so we need the : to always return a 0 exit status
|
# Grep returns an exit status of 1 when line is not found, so we need the : to always return a 0 exit status
|
||||||
count=$(grep -c ${MINION_NAMES[i]} ${MINIONS_FILE}) || :
|
count=$(grep -c ${MINION_NAMES[i]} ${MINIONS_FILE}) || :
|
||||||
|
42
cluster/vsphere/config-common.sh
Normal file
42
cluster/vsphere/config-common.sh
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2014 Google Inc. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
function public-key {
|
||||||
|
local dir=${HOME}/.ssh
|
||||||
|
|
||||||
|
for f in $HOME/.ssh/{id_{rsa,dsa},*}.pub; do
|
||||||
|
if [ -r $f ]; then
|
||||||
|
echo $f
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Can't find public key file..."
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
DISK=kube.vmdk
|
||||||
|
GUEST_ID=debian7_64Guest
|
||||||
|
PUBLIC_KEY_FILE=${PUBLIC_KEY_FILE-$(public-key)}
|
||||||
|
SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null"
|
||||||
|
|
||||||
|
# These need to be set
|
||||||
|
#export GOVC_URL=
|
||||||
|
#export GOVC_DATACENTER=
|
||||||
|
#export GOVC_DATASTORE=
|
||||||
|
#export GOVC_RESOURCE_POOL=
|
||||||
|
#export GOVC_NETWORK=
|
||||||
|
#export GOVC_GUEST_LOGIN='kube:kube'
|
29
cluster/vsphere/config-default.sh
Executable file
29
cluster/vsphere/config-default.sh
Executable file
@ -0,0 +1,29 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2014 Google Inc. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
source $(dirname ${BASH_SOURCE})/config-common.sh
|
||||||
|
|
||||||
|
NUM_MINIONS=4
|
||||||
|
INSTANCE_PREFIX=kubernetes
|
||||||
|
|
||||||
|
MASTER_NAME="${INSTANCE_PREFIX}-master"
|
||||||
|
MASTER_MEMORY_MB=1024
|
||||||
|
MASTER_CPU=1
|
||||||
|
|
||||||
|
MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
|
||||||
|
MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
|
||||||
|
MINION_MEMORY_MB=2048
|
||||||
|
MINION_CPU=1
|
29
cluster/vsphere/config-test.sh
Executable file
29
cluster/vsphere/config-test.sh
Executable file
@ -0,0 +1,29 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2014 Google Inc. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
source $(dirname ${BASH_SOURCE})/config-common.sh
|
||||||
|
|
||||||
|
NUM_MINIONS=2
|
||||||
|
INSTANCE_PREFIX="e2e-test-${USER}"
|
||||||
|
|
||||||
|
MASTER_NAME="${INSTANCE_PREFIX}-master"
|
||||||
|
MASTER_MEMORY_MB=1024
|
||||||
|
MASTER_CPU=1
|
||||||
|
|
||||||
|
MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
|
||||||
|
MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
|
||||||
|
MINION_MEMORY_MB=1024
|
||||||
|
MINION_CPU=1
|
22
cluster/vsphere/templates/hostname.sh
Executable file
22
cluster/vsphere/templates/hostname.sh
Executable file
@ -0,0 +1,22 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2014 Google Inc. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# Remove kube.vm from /etc/hosts
|
||||||
|
sed -i -e 's/\b\w\+.vm\b//' /etc/hosts
|
||||||
|
|
||||||
|
# Update hostname in /etc/hosts and /etc/hostname
|
||||||
|
sed -i -e "s/\\bkube\\b/${MY_NAME}/g" /etc/host{s,name}
|
||||||
|
hostname ${MY_NAME}
|
24
cluster/vsphere/templates/install-release.sh
Executable file
24
cluster/vsphere/templates/install-release.sh
Executable file
@ -0,0 +1,24 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2014 Google Inc. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# Install release
|
||||||
|
|
||||||
|
echo "Unpacking release"
|
||||||
|
rm -rf master-release || false
|
||||||
|
tar xzf master-release.tgz
|
||||||
|
|
||||||
|
echo "Running release install script"
|
||||||
|
sudo master-release/src/scripts/master-release-install.sh
|
59
cluster/vsphere/templates/salt-master.sh
Executable file
59
cluster/vsphere/templates/salt-master.sh
Executable file
@ -0,0 +1,59 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2014 Google Inc. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# Use other Debian mirror
|
||||||
|
sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
|
||||||
|
|
||||||
|
# Prepopulate the name of the Master
|
||||||
|
mkdir -p /etc/salt/minion.d
|
||||||
|
echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
|
||||||
|
|
||||||
|
cat <<EOF >/etc/salt/minion.d/grains.conf
|
||||||
|
grains:
|
||||||
|
roles:
|
||||||
|
- kubernetes-master
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Auto accept all keys from minions that try to join
|
||||||
|
mkdir -p /etc/salt/master.d
|
||||||
|
cat <<EOF >/etc/salt/master.d/auto-accept.conf
|
||||||
|
auto_accept: True
|
||||||
|
EOF
|
||||||
|
|
||||||
|
cat <<EOF >/etc/salt/master.d/reactor.conf
|
||||||
|
# React to new minions starting by running highstate on them.
|
||||||
|
reactor:
|
||||||
|
- 'salt/minion/*/start':
|
||||||
|
- /srv/reactor/start.sls
|
||||||
|
EOF
|
||||||
|
|
||||||
|
mkdir -p /srv/salt/nginx
|
||||||
|
echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd
|
||||||
|
|
||||||
|
# Install Salt
|
||||||
|
#
|
||||||
|
# We specify -X to avoid a race condition that can cause minion failure to
|
||||||
|
# install. See https://github.com/saltstack/salt-bootstrap/issues/270
|
||||||
|
#
|
||||||
|
# -M installs the master
|
||||||
|
if [ ! -x /etc/init.d/salt-master ]; then
|
||||||
|
wget -q -O - https://bootstrap.saltstack.com | sh -s -- -M -X
|
||||||
|
else
|
||||||
|
/etc/init.d/salt-master restart
|
||||||
|
/etc/init.d/salt-minion restart
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd
|
55
cluster/vsphere/templates/salt-minion.sh
Executable file
55
cluster/vsphere/templates/salt-minion.sh
Executable file
@ -0,0 +1,55 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2014 Google Inc. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# Use other Debian mirror
|
||||||
|
sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
|
||||||
|
|
||||||
|
# Resolve hostname of master
|
||||||
|
if ! grep -q $MASTER_NAME /etc/hosts; then
|
||||||
|
echo "Adding host entry for $MASTER_NAME"
|
||||||
|
echo "$MASTER_IP $MASTER_NAME" >> /etc/hosts
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Prepopulate the name of the Master
|
||||||
|
mkdir -p /etc/salt/minion.d
|
||||||
|
echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
|
||||||
|
|
||||||
|
# Turn on debugging for salt-minion
|
||||||
|
# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion
|
||||||
|
|
||||||
|
# Our minions will have a pool role to distinguish them from the master.
|
||||||
|
#
|
||||||
|
# Setting the "minion_ip" here causes the kubelet to use its IP for
|
||||||
|
# identification instead of its hostname.
|
||||||
|
#
|
||||||
|
cat <<EOF >/etc/salt/minion.d/grains.conf
|
||||||
|
grains:
|
||||||
|
minion_ip: $(ip route get 1.1.1.1 | awk '{print $7}')
|
||||||
|
roles:
|
||||||
|
- kubernetes-pool
|
||||||
|
- kubernetes-pool-vsphere
|
||||||
|
cbr-cidr: $MINION_IP_RANGE
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Install Salt
|
||||||
|
#
|
||||||
|
# We specify -X to avoid a race condition that can cause minion failure to
|
||||||
|
# install. See https://github.com/saltstack/salt-bootstrap/issues/270
|
||||||
|
if [ ! -x /etc/init.d/salt-minion ]; then
|
||||||
|
wget -q -O - https://bootstrap.saltstack.com | sh -s -- -X
|
||||||
|
else
|
||||||
|
/etc/init.d/salt-minion restart
|
||||||
|
fi
|
285
cluster/vsphere/util.sh
Normal file
285
cluster/vsphere/util.sh
Normal file
@ -0,0 +1,285 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2014 Google Inc. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# A library of helper functions and constants for the local config.
|
||||||
|
|
||||||
|
# Use the config file specified in $KUBE_CONFIG_FILE, or default to
|
||||||
|
# config-default.sh.
|
||||||
|
source $(dirname ${BASH_SOURCE})/${KUBE_CONFIG_FILE-"config-default.sh"}
|
||||||
|
|
||||||
|
function detect-master {
|
||||||
|
KUBE_MASTER=${MASTER_NAME}
|
||||||
|
if [ -z "$KUBE_MASTER_IP" ]; then
|
||||||
|
KUBE_MASTER_IP=$(govc vm.ip ${MASTER_NAME})
|
||||||
|
fi
|
||||||
|
if [ -z "$KUBE_MASTER_IP" ]; then
|
||||||
|
echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Found ${KUBE_MASTER} at ${KUBE_MASTER_IP}"
|
||||||
|
}
|
||||||
|
|
||||||
|
function detect-minions {
|
||||||
|
KUBE_MINION_IP_ADDRESSES=()
|
||||||
|
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||||
|
local minion_ip=$(govc vm.ip ${MINION_NAMES[$i]})
|
||||||
|
echo "Found ${MINION_NAMES[$i]} at ${minion_ip}"
|
||||||
|
KUBE_MINION_IP_ADDRESSES+=("${minion_ip}")
|
||||||
|
done
|
||||||
|
if [ -z "$KUBE_MINION_IP_ADDRESSES" ]; then
|
||||||
|
echo "Could not detect Kubernetes minion nodes. Make sure you've launched a cluster with 'kube-up.sh'"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Verify prereqs on host machine
|
||||||
|
function verify-prereqs {
|
||||||
|
if [ "$(which govc)" == "" ]; then
|
||||||
|
echo "Can't find govc in PATH, please install and retry."
|
||||||
|
echo ""
|
||||||
|
echo " go install github.com/vmware/govmomi/govc"
|
||||||
|
echo ""
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Run command over ssh
|
||||||
|
function kube-ssh {
|
||||||
|
local host=$1
|
||||||
|
shift
|
||||||
|
ssh ${SSH_OPTS} kube@${host} "$*" 2> /dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
# Instantiate a generic kubernetes virtual machine (master or minion)
|
||||||
|
function kube-up-vm {
|
||||||
|
local vm_name=$1
|
||||||
|
local vm_memory=$2
|
||||||
|
local vm_cpu=$3
|
||||||
|
local vm_ip=
|
||||||
|
|
||||||
|
govc vm.create \
|
||||||
|
-debug \
|
||||||
|
-m ${vm_memory} \
|
||||||
|
-c ${vm_cpu} \
|
||||||
|
-disk ${DISK} \
|
||||||
|
-g ${GUEST_ID} \
|
||||||
|
-link=true \
|
||||||
|
${vm_name}
|
||||||
|
|
||||||
|
# Retrieve IP first, to confirm the guest operations agent is running.
|
||||||
|
vm_ip=$(govc vm.ip ${vm_name})
|
||||||
|
|
||||||
|
govc guest.mkdir \
|
||||||
|
-vm ${vm_name} \
|
||||||
|
-p \
|
||||||
|
/home/kube/.ssh
|
||||||
|
|
||||||
|
govc guest.upload \
|
||||||
|
-vm ${vm_name} \
|
||||||
|
-f \
|
||||||
|
${PUBLIC_KEY_FILE} \
|
||||||
|
/home/kube/.ssh/authorized_keys
|
||||||
|
}
|
||||||
|
|
||||||
|
# Instantiate a kubernetes cluster
|
||||||
|
function kube-up {
|
||||||
|
# Build up start up script for master
|
||||||
|
KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
|
||||||
|
trap "rm -rf ${KUBE_TEMP}" EXIT
|
||||||
|
|
||||||
|
get-password
|
||||||
|
echo "Using password: $user:$passwd"
|
||||||
|
echo
|
||||||
|
python $(dirname $0)/../third_party/htpasswd/htpasswd.py -b -c ${KUBE_TEMP}/htpasswd $user $passwd
|
||||||
|
HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd)
|
||||||
|
|
||||||
|
echo "Starting master VM (this can take a minute)..."
|
||||||
|
|
||||||
|
kube-up-vm ${MASTER_NAME} ${MASTER_MEMORY_MB-1024} ${MASTER_CPU-1}
|
||||||
|
|
||||||
|
# Prints master IP, so user can log in for debugging.
|
||||||
|
detect-master
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "Starting minion VMs (this can take a minute)..."
|
||||||
|
|
||||||
|
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||||
|
(
|
||||||
|
echo "#! /bin/bash"
|
||||||
|
echo "MY_NAME=${MINION_NAMES[$i]}"
|
||||||
|
grep -v "^#" $(dirname $0)/vsphere/templates/hostname.sh
|
||||||
|
echo "MASTER_NAME=${MASTER_NAME}"
|
||||||
|
echo "MASTER_IP=${KUBE_MASTER_IP}"
|
||||||
|
echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}"
|
||||||
|
grep -v "^#" $(dirname $0)/vsphere/templates/salt-minion.sh
|
||||||
|
) > ${KUBE_TEMP}/minion-start-${i}.sh
|
||||||
|
|
||||||
|
(
|
||||||
|
kube-up-vm ${MINION_NAMES[$i]} ${MINION_MEMORY_MB-1024} ${MINION_CPU-1}
|
||||||
|
|
||||||
|
MINION_IP=$(govc vm.ip ${MINION_NAMES[$i]})
|
||||||
|
|
||||||
|
govc guest.upload \
|
||||||
|
-vm ${MINION_NAMES[$i]} \
|
||||||
|
-perm 0700 \
|
||||||
|
-f \
|
||||||
|
${KUBE_TEMP}/minion-start-${i}.sh \
|
||||||
|
/home/kube/minion-start.sh
|
||||||
|
|
||||||
|
# Kickstart start script
|
||||||
|
kube-ssh ${MINION_IP} "nohup sudo ~/minion-start.sh < /dev/null 1> minion-start.out 2> minion-start.err &"
|
||||||
|
) &
|
||||||
|
done
|
||||||
|
|
||||||
|
FAIL=0
|
||||||
|
for job in `jobs -p`
|
||||||
|
do
|
||||||
|
wait $job || let "FAIL+=1"
|
||||||
|
done
|
||||||
|
if (( $FAIL != 0 )); then
|
||||||
|
echo "${FAIL} commands failed. Exiting."
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Print minion IPs, so user can log in for debugging.
|
||||||
|
detect-minions
|
||||||
|
echo
|
||||||
|
|
||||||
|
# Continue provisioning the master.
|
||||||
|
|
||||||
|
(
|
||||||
|
echo "#! /bin/bash"
|
||||||
|
echo "MY_NAME=${MASTER_NAME}"
|
||||||
|
grep -v "^#" $(dirname $0)/vsphere/templates/hostname.sh
|
||||||
|
echo "MASTER_NAME=${MASTER_NAME}"
|
||||||
|
echo "MASTER_HTPASSWD='${HTPASSWD}'"
|
||||||
|
echo "MINION_NAMES=( ${MINION_NAMES[@]} )"
|
||||||
|
echo "KUBE_MINION_IP_ADDRESSES=( ${KUBE_MINION_IP_ADDRESSES[@]} )"
|
||||||
|
grep -v "^#" $(dirname $0)/vsphere/templates/install-release.sh
|
||||||
|
grep -v "^#" $(dirname $0)/vsphere/templates/salt-master.sh
|
||||||
|
) > ${KUBE_TEMP}/master-start.sh
|
||||||
|
|
||||||
|
govc guest.upload \
|
||||||
|
-vm ${MASTER_NAME} \
|
||||||
|
-perm 0700 \
|
||||||
|
-f \
|
||||||
|
${KUBE_TEMP}/master-start.sh \
|
||||||
|
/home/kube/master-start.sh
|
||||||
|
|
||||||
|
govc guest.upload \
|
||||||
|
-vm ${MASTER_NAME} \
|
||||||
|
-f \
|
||||||
|
./output/release/master-release.tgz \
|
||||||
|
/home/kube/master-release.tgz
|
||||||
|
|
||||||
|
# Kickstart start script
|
||||||
|
kube-ssh ${KUBE_MASTER_IP} "nohup sudo ~/master-start.sh < /dev/null 1> master-start.out 2> master-start.err &"
|
||||||
|
|
||||||
|
echo "Waiting for cluster initialization."
|
||||||
|
echo
|
||||||
|
echo " This will continually check to see if the API for kubernetes is reachable."
|
||||||
|
echo " This might loop forever if there was some uncaught error during start up."
|
||||||
|
echo
|
||||||
|
|
||||||
|
until $(curl --insecure --user ${user}:${passwd} --max-time 5 \
|
||||||
|
--fail --output /dev/null --silent https://${KUBE_MASTER_IP}/api/v1beta1/pods); do
|
||||||
|
printf "."
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Kubernetes cluster created."
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "Sanity checking cluster..."
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
# Don't bail on errors, we want to be able to print some info.
|
||||||
|
set +e
|
||||||
|
|
||||||
|
# Basic sanity checking
|
||||||
|
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||||
|
# Make sure docker is installed
|
||||||
|
kube-ssh ${KUBE_MINION_IP_ADDRESSES[$i]} which docker > /dev/null
|
||||||
|
if [ "$?" != "0" ]; then
|
||||||
|
echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
|
||||||
|
echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "Kubernetes cluster is running. Access the master at:"
|
||||||
|
echo
|
||||||
|
echo " https://${user}:${passwd}@${KUBE_MASTER_IP}"
|
||||||
|
echo
|
||||||
|
echo "Security note: The server above uses a self signed certificate."
|
||||||
|
echo "This is subject to \"Man in the middle\" type attacks."
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
# Delete a kubernetes cluster
|
||||||
|
function kube-down {
|
||||||
|
govc vm.destroy ${MASTER_NAME} &
|
||||||
|
|
||||||
|
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||||
|
govc vm.destroy ${MINION_NAMES[i]} &
|
||||||
|
done
|
||||||
|
|
||||||
|
wait
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
# Update a kubernetes cluster with latest source
|
||||||
|
function kube-push {
|
||||||
|
echo "TODO"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Execute prior to running tests to build a release if required for env
|
||||||
|
function test-build-release {
|
||||||
|
echo "TODO"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Execute prior to running tests to initialize required structure
|
||||||
|
function test-setup {
|
||||||
|
echo "TODO"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Execute after running tests to perform any required clean-up
|
||||||
|
function test-teardown {
|
||||||
|
echo "TODO"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Set the {user} and {password} environment values required to interact with provider
|
||||||
|
function get-password {
|
||||||
|
file=${HOME}/.kubernetes_auth
|
||||||
|
if [ -e ${file} ]; then
|
||||||
|
user=$(cat $file | python -c 'import json,sys;print(json.load(sys.stdin)["User"])')
|
||||||
|
passwd=$(cat $file | python -c 'import json,sys;print(json.load(sys.stdin)["Password"])')
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
user=admin
|
||||||
|
passwd=$(python -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))')
|
||||||
|
|
||||||
|
# Store password for reuse.
|
||||||
|
cat << EOF > ~/.kubernetes_auth
|
||||||
|
{
|
||||||
|
"User": "$user",
|
||||||
|
"Password": "$passwd"
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
chmod 0600 ~/.kubernetes_auth
|
||||||
|
}
|
71
docs/getting-started-guides/vsphere.md
Normal file
71
docs/getting-started-guides/vsphere.md
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
## Getting started with vSphere
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
|
||||||
|
1. You need administrator credentials to an ESXi machine or vCenter instance.
|
||||||
|
2. You must have Go (version 1.2 or later) installed: [www.golang.org](http://www.golang.org).
|
||||||
|
3. Install the govc tool to interact with ESXi/vCenter:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get github.com/vmware/govmomi/govc
|
||||||
|
```
|
||||||
|
4. Install godep:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export GOBIN=/usr/local/go/bin
|
||||||
|
go get github.com/tools/godep
|
||||||
|
```
|
||||||
|
|
||||||
|
5. Get the Kubernetes source:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git clone https://github.com/GoogleCloudPlatform/kubernetes.git
|
||||||
|
```
|
||||||
|
|
||||||
|
### Setup
|
||||||
|
|
||||||
|
Download a prebuilt Debian VMDK to be used as base image:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
wget http://storage.googleapis.com/govmomi/vmdk/kube.vmdk.gz{,.md5}
|
||||||
|
md5sum -c kube.vmdk.gz.md5
|
||||||
|
gzip -d kube.vmdk.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
Upload this VMDK to your vSphere instance:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export GOVC_URL='https://user:pass@hostname/sdk'
|
||||||
|
export GOVC_DATASTORE='target datastore'
|
||||||
|
export GOVC_RESOURCE_POOL='resource pool with access to datastore'
|
||||||
|
|
||||||
|
govc datastore.import kube.vmdk
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify that the VMDK was correctly uploaded and expanded to 10GiB:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
govc datastore.ls
|
||||||
|
```
|
||||||
|
|
||||||
|
Take a look at the file `cluster/vsphere/config-common.sh` fill in the required
|
||||||
|
parameters. The guest login for the image that you imported is `kube:kube`.
|
||||||
|
|
||||||
|
Now, let's continue with deploying Kubernetes:
|
||||||
|
|
||||||
|
```
|
||||||
|
cd kubernetes
|
||||||
|
|
||||||
|
# Build a release
|
||||||
|
release/build-release.sh
|
||||||
|
|
||||||
|
# Deploy Kubernetes (takes ~5 minutes, provided everything works out)
|
||||||
|
export KUBERNETES_PROVIDER=vsphere
|
||||||
|
cluster/kube-up.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Refer to the top level README and the getting started guide for Google Compute
|
||||||
|
Engine. Once you have successfully reached this point, your vSphere Kubernetes
|
||||||
|
deployment works just as any other one!
|
||||||
|
|
||||||
|
**Enjoy!**
|
Loading…
Reference in New Issue
Block a user