From 2dd57898d4fac3af8793148619bfe63e9ad492b4 Mon Sep 17 00:00:00 2001 From: Rajat Chopra Date: Thu, 7 Aug 2014 13:02:52 -0700 Subject: [PATCH 1/2] add ip per pod across vagrant minions --- .gitignore | 1 + Vagrantfile | 2 +- cluster/saltbase/salt/docker/init.sls | 6 +- cluster/saltbase/salt/sdn/init.sls | 16 ++++ cluster/saltbase/salt/top.sls | 2 + cluster/vagrant/pod-ip-test.sh | 44 ++++++++++ cluster/vagrant/provision-minion.sh | 3 + cluster/vagrant/provision-network.sh | 112 ++++++++++++++++++++++++++ 8 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 cluster/saltbase/salt/sdn/init.sls create mode 100644 cluster/vagrant/pod-ip-test.sh create mode 100755 cluster/vagrant/provision-network.sh diff --git a/.gitignore b/.gitignore index 5d28efcdf80..d8884a59f5c 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ Session.vim # Vagrant .vagrant +network_closure.sh # compiled binaries in third_party /third_party/pkg diff --git a/Vagrantfile b/Vagrantfile index 51eae9363e1..26f147a2429 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -45,7 +45,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| minion_ip = minion_ips[n] minion.vm.box = kube_box[kube_os]["name"] minion.vm.box_url = kube_box[kube_os]["box_url"] - minion.vm.provision "shell", inline: "/vagrant/cluster/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{minion_ip}" + minion.vm.provision "shell", inline: "/vagrant/cluster/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{minion_ip} #{minion_index}" minion.vm.network "private_network", ip: "#{minion_ip}" minion.vm.hostname = "kubernetes-minion-#{minion_index}" end diff --git a/cluster/saltbase/salt/docker/init.sls b/cluster/saltbase/salt/docker/init.sls index e154d64c6fd..aaefceafebe 100644 --- a/cluster/saltbase/salt/docker/init.sls +++ b/cluster/saltbase/salt/docker/init.sls @@ -4,6 +4,9 @@ {% set environment_file = '/etc/default/docker' %} {% endif %} +bridge-utils: + pkg.installed + {% if grains['os_family'] != 'RedHat' %} docker-repo: @@ -25,9 +28,6 @@ net.ipv4.ip_forward: sysctl.present: - value: 1 -bridge-utils: - pkg.installed - cbr0: container_bridge.ensure: - cidr: {{ grains['cbr-cidr'] }} diff --git a/cluster/saltbase/salt/sdn/init.sls b/cluster/saltbase/salt/sdn/init.sls new file mode 100644 index 00000000000..4b6a0e5274b --- /dev/null +++ b/cluster/saltbase/salt/sdn/init.sls @@ -0,0 +1,16 @@ +{% if grains['os_family'] == 'RedHat' %} + +openvswitch: + pkg: + - installed + service.running: + - enable: True + +sdn: + cmd.wait: + - name: /vagrant/network_closure.sh + - watch: + - pkg: docker-io + - pkg: openvswitch + +{% endif %} diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index eff1a260574..bb0b133b06f 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -12,6 +12,8 @@ base: - nsinit {% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} - openvpn-client +{% else %} + - sdn {% endif %} 'roles:kubernetes-master': diff --git a/cluster/vagrant/pod-ip-test.sh b/cluster/vagrant/pod-ip-test.sh new file mode 100644 index 00000000000..cf43bcba819 --- /dev/null +++ b/cluster/vagrant/pod-ip-test.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +cd $(dirname ${BASH_SOURCE})/../../ + +# start the cluster with 2 minions +export KUBERNETES_NUM_MINIONS=2 +export KUBERNETES_PROVIDER=vagrant +cluster/kube-up.sh + +echo "Pull an image that runs a web server" +vagrant ssh minion-1 -- sudo docker pull dockerfile/nginx +vagrant ssh minion-2 -- sudo docker pull dockerfile/nginx + +echo "Run the servers" +vagrant ssh minion-1 -- sudo docker run -d dockerfile/nginx +vagrant ssh minion-2 -- sudo docker run -d dockerfile/nginx + +echo "Run ping from minion-1 to docker bridges and to the containers on both minions" +vagrant ssh minion-1 -- 'ping -c 10 10.244.1.1 && ping -c 10 10.244.2.1 && ping -c 10 10.244.1.3 && ping -c 10 10.244.2.3' +echo "Same pinch from minion-2" +vagrant ssh minion-2 -- 'ping -c 10 10.244.1.1 && ping -c 10 10.244.2.1 && ping -c 10 10.244.1.3 && ping -c 10 10.244.2.3' + +echo "tcp check, curl to both the running webservers from both machines" +vagrant ssh minion-1 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80' +vagrant ssh minion-2 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80' + +echo "All good, destroy the cluster" +vagrant destroy -f diff --git a/cluster/vagrant/provision-minion.sh b/cluster/vagrant/provision-minion.sh index e03cd7104af..26f63215f98 100755 --- a/cluster/vagrant/provision-minion.sh +++ b/cluster/vagrant/provision-minion.sh @@ -52,3 +52,6 @@ if [ ! $(which salt-minion) ]; then systemctl enable salt-minion systemctl start salt-minion fi + +# run the networking setup +$(dirname $0)/provision-network.sh $@ diff --git a/cluster/vagrant/provision-network.sh b/cluster/vagrant/provision-network.sh new file mode 100755 index 00000000000..178f0b28ae0 --- /dev/null +++ b/cluster/vagrant/provision-network.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# exit on any error +set -e +source $(dirname $0)/provision-config.sh + +MINION_IP=$4 +MINION_ID=$5 +DOCKER_BRIDGE=kbr0 +OVS_SWITCH=obr0 +GRE_TUNNEL_BASE=gre +BRIDGE_BASE=10.244 +BRIDGE_ADDRESS=${BRIDGE_BASE}.${MINION_ID}.1 +BRIDGE_NETWORK=${BRIDGE_ADDRESS}/24 +BRIDGE_NETMASK=255.255.255.0 +NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/ +POST_NETWORK_SCRIPT=/vagrant/network_closure.sh + +# add docker bridge ifcfg file +cat < ${NETWORK_CONF_PATH}ifcfg-${DOCKER_BRIDGE} +# Generated by yours truly +DEVICE=${DOCKER_BRIDGE} +ONBOOT=yes +TYPE=Bridge +BOOTPROTO=static +IPADDR=${BRIDGE_ADDRESS} +NETMASK=${BRIDGE_NETMASK} +STP=yes +EOF + +# add the ovs bridge ifcfg file +cat < ${NETWORK_CONF_PATH}ifcfg-${OVS_SWITCH} +DEVICE=${OVS_SWITCH} +ONBOOT=yes +DEVICETYPE=ovs +TYPE=OVSBridge +BOOTPROTO=static +HOTPLUG=no +BRIDGE=${DOCKER_BRIDGE} +EOF + +# now loop through all other minions and create persistent gre tunnels +MINION_IPS=$3 +MINION_IP_ARRAY=(`echo ${MINION_IPS} | tr "," "\n"`) +GRE_NUM=0 +for remote_ip in "${MINION_IP_ARRAY[@]}" +do + if [ "${remote_ip}" == "${MINION_IP}" ]; then + continue + fi + ((GRE_NUM++)) || echo + GRE_TUNNEL=${GRE_TUNNEL_BASE}${GRE_NUM} + # ovs-vsctl add-port ${OVS_SWITCH} ${GRE_TUNNEL} -- set interface ${GRE_TUNNEL} type=gre options:remote_ip=${remote_ip} + cat < ${NETWORK_CONF_PATH}ifcfg-${GRE_TUNNEL} +DEVICE=${GRE_TUNNEL} +ONBOOT=yes +DEVICETYPE=ovs +TYPE=OVSTunnel +OVS_BRIDGE=${OVS_SWITCH} +OVS_TUNNEL_TYPE=gre +OVS_TUNNEL_OPTIONS="options:remote_ip=${remote_ip}" +EOF +done + +# add ip route rules such that all pod traffic flows through docker bridge and consequently to the gre tunnels +cat < /${NETWORK_CONF_PATH}route-${DOCKER_BRIDGE} +${BRIDGE_BASE}.0.0/16 dev ${DOCKER_BRIDGE} scope link src ${BRIDGE_ADDRESS} +EOF + + +# generate the post-configure script to be called by salt as cmd.wait +cat < ${POST_NETWORK_SCRIPT} +#!/bin/bash + +set -e +# NAT interface fails to revive on network restart, so OR-gate to true +systemctl restart network.service || true + +# set docker bridge up, and set stp on the ovs bridge +ip link set dev ${DOCKER_BRIDGE} up +ovs-vsctl set Bridge ${OVS_SWITCH} stp_enable=true + +# modify the docker service file such that it uses the kube docker bridge and not its own +sed -ie "s/ExecStart=\/usr\/bin\/docker -d/ExecStart=\/usr\/bin\/docker -d -b=${DOCKER_BRIDGE} --iptables=false/g" /usr/lib/systemd/system/docker.service +systemctl daemon-reload +systemctl restart docker.service + +# setup iptables masquerade rules so the pods can reach the internet +iptables -t nat -A POSTROUTING -s ${BRIDGE_BASE}.0.0/16 ! -d ${BRIDGE_BASE}.0.0/16 -j MASQUERADE + +# persist please +iptables-save >& /etc/sysconfig/iptables + +# self-destruct after doing the job +#rm -f ${POST_NETWORK_SCRIPT} +EOF + +chmod +x ${POST_NETWORK_SCRIPT} From a0b88e2f2d3f01a2b421df22ba755c224d148c0f Mon Sep 17 00:00:00 2001 From: Rajat Chopra Date: Fri, 22 Aug 2014 16:53:16 -0700 Subject: [PATCH 2/2] add test to check minion to master reachability; logfiling and some cosmetification. --- cluster/vagrant/pod-ip-test.sh | 96 +++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 18 deletions(-) mode change 100644 => 100755 cluster/vagrant/pod-ip-test.sh diff --git a/cluster/vagrant/pod-ip-test.sh b/cluster/vagrant/pod-ip-test.sh old mode 100644 new mode 100755 index cf43bcba819..56aba721fd9 --- a/cluster/vagrant/pod-ip-test.sh +++ b/cluster/vagrant/pod-ip-test.sh @@ -14,31 +14,91 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e +echoOK() { + TC='\e[' + RegB="${TC}0m" + if [ "$1" -eq "0" ]; then + Green="${TC}32m" + echo -e "[${Green}OK${RegB}]" + else + Red="${TC}31m" + echo -e "[${Red}FAIL${RegB}]" + echo "Check log file." + exit 1 + fi +} + +usage() { + echo "Usage options: [--logfile ]" +} + +logfile=/dev/null +while [[ $# > 0 ]]; do + key="$1" + shift + case $key in + -l|--logfile) + logfile="$1" + if [ "$logfile" == "" ]; then + usage + exit 1 + fi + shift + ;; + *) + # unknown option + usage + exit 1 + ;; + esac +done cd $(dirname ${BASH_SOURCE})/../../ -# start the cluster with 2 minions +echo All verbose output will be redirected to $logfile, use --logfile option to change. + +printf "Start the cluster with 2 minions .. " export KUBERNETES_NUM_MINIONS=2 export KUBERNETES_PROVIDER=vagrant -cluster/kube-up.sh -echo "Pull an image that runs a web server" -vagrant ssh minion-1 -- sudo docker pull dockerfile/nginx -vagrant ssh minion-2 -- sudo docker pull dockerfile/nginx +(cluster/kube-up.sh &>> $logfile) || true +echoOK $? -echo "Run the servers" -vagrant ssh minion-1 -- sudo docker run -d dockerfile/nginx -vagrant ssh minion-2 -- sudo docker run -d dockerfile/nginx +printf "Check if minion-1 can reach kubernetes master .. " +vagrant ssh minion-1 -- ping -c 10 kubernetes-master &>> $logfile +echoOK $? +printf "Check if minion-2 can reach kubernetes master .. " +vagrant ssh minion-2 -- ping -c 10 kubernetes-master &>> $logfile +echoOK $? -echo "Run ping from minion-1 to docker bridges and to the containers on both minions" -vagrant ssh minion-1 -- 'ping -c 10 10.244.1.1 && ping -c 10 10.244.2.1 && ping -c 10 10.244.1.3 && ping -c 10 10.244.2.3' -echo "Same pinch from minion-2" -vagrant ssh minion-2 -- 'ping -c 10 10.244.1.1 && ping -c 10 10.244.2.1 && ping -c 10 10.244.1.3 && ping -c 10 10.244.2.3' +printf "Pull an image that runs a web server on minion-1 .. " +vagrant ssh minion-1 -- 'sudo docker pull dockerfile/nginx' &>> $logfile +echoOK $? +printf "Pull an image that runs a web server on minion-2 .. " +vagrant ssh minion-2 -- 'sudo docker pull dockerfile/nginx' &>> $logfile +echoOK $? -echo "tcp check, curl to both the running webservers from both machines" -vagrant ssh minion-1 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80' -vagrant ssh minion-2 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80' +printf "Run the server on minion-1 .. " +vagrant ssh minion-1 -- sudo docker run -d dockerfile/nginx &>> $logfile +echoOK $? +printf "Run the server on minion-2 .. " +vagrant ssh minion-2 -- sudo docker run -d dockerfile/nginx &>> $logfile +echoOK $? -echo "All good, destroy the cluster" -vagrant destroy -f +printf "Run ping from minion-1 to docker bridges and to the containers on both minions .. " +vagrant ssh minion-1 -- 'ping -c 20 10.244.1.1 && ping -c 20 10.244.2.1 && ping -c 20 10.244.1.3 && ping -c 20 10.244.2.3' &>> $logfile +echoOK $? +printf "Same pinch from minion-2 .. " +vagrant ssh minion-2 -- 'ping -c 20 10.244.1.1 && ping -c 20 10.244.2.1 && ping -c 20 10.244.1.3 && ping -c 20 10.244.2.3' &>> $logfile +echoOK $? + +printf "tcp check, curl to both the running webservers from minion-1 .. " +vagrant ssh minion-1 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80' &>> $logfile +echoOK $? +printf "tcp check, curl to both the running webservers from minion-2 .. " +vagrant ssh minion-2 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80' &>> $logfile +echoOK $? + +printf "All good, destroy the cluster .. " +vagrant destroy -f &>> $logfile +echoOK $?