mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 19:56:01 +00:00
add ip per pod across vagrant minions
This commit is contained in:
parent
c92b7d5bb7
commit
2dd57898d4
1
.gitignore
vendored
1
.gitignore
vendored
@ -29,6 +29,7 @@ Session.vim
|
|||||||
|
|
||||||
# Vagrant
|
# Vagrant
|
||||||
.vagrant
|
.vagrant
|
||||||
|
network_closure.sh
|
||||||
|
|
||||||
# compiled binaries in third_party
|
# compiled binaries in third_party
|
||||||
/third_party/pkg
|
/third_party/pkg
|
||||||
|
2
Vagrantfile
vendored
2
Vagrantfile
vendored
@ -45,7 +45,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
|
|||||||
minion_ip = minion_ips[n]
|
minion_ip = minion_ips[n]
|
||||||
minion.vm.box = kube_box[kube_os]["name"]
|
minion.vm.box = kube_box[kube_os]["name"]
|
||||||
minion.vm.box_url = kube_box[kube_os]["box_url"]
|
minion.vm.box_url = kube_box[kube_os]["box_url"]
|
||||||
minion.vm.provision "shell", inline: "/vagrant/cluster/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{minion_ip}"
|
minion.vm.provision "shell", inline: "/vagrant/cluster/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{minion_ip} #{minion_index}"
|
||||||
minion.vm.network "private_network", ip: "#{minion_ip}"
|
minion.vm.network "private_network", ip: "#{minion_ip}"
|
||||||
minion.vm.hostname = "kubernetes-minion-#{minion_index}"
|
minion.vm.hostname = "kubernetes-minion-#{minion_index}"
|
||||||
end
|
end
|
||||||
|
@ -4,6 +4,9 @@
|
|||||||
{% set environment_file = '/etc/default/docker' %}
|
{% set environment_file = '/etc/default/docker' %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
bridge-utils:
|
||||||
|
pkg.installed
|
||||||
|
|
||||||
{% if grains['os_family'] != 'RedHat' %}
|
{% if grains['os_family'] != 'RedHat' %}
|
||||||
|
|
||||||
docker-repo:
|
docker-repo:
|
||||||
@ -25,9 +28,6 @@ net.ipv4.ip_forward:
|
|||||||
sysctl.present:
|
sysctl.present:
|
||||||
- value: 1
|
- value: 1
|
||||||
|
|
||||||
bridge-utils:
|
|
||||||
pkg.installed
|
|
||||||
|
|
||||||
cbr0:
|
cbr0:
|
||||||
container_bridge.ensure:
|
container_bridge.ensure:
|
||||||
- cidr: {{ grains['cbr-cidr'] }}
|
- cidr: {{ grains['cbr-cidr'] }}
|
||||||
|
16
cluster/saltbase/salt/sdn/init.sls
Normal file
16
cluster/saltbase/salt/sdn/init.sls
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
{% if grains['os_family'] == 'RedHat' %}
|
||||||
|
|
||||||
|
openvswitch:
|
||||||
|
pkg:
|
||||||
|
- installed
|
||||||
|
service.running:
|
||||||
|
- enable: True
|
||||||
|
|
||||||
|
sdn:
|
||||||
|
cmd.wait:
|
||||||
|
- name: /vagrant/network_closure.sh
|
||||||
|
- watch:
|
||||||
|
- pkg: docker-io
|
||||||
|
- pkg: openvswitch
|
||||||
|
|
||||||
|
{% endif %}
|
@ -12,6 +12,8 @@ base:
|
|||||||
- nsinit
|
- nsinit
|
||||||
{% if grains['cloud'] is defined and grains['cloud'] == 'azure' %}
|
{% if grains['cloud'] is defined and grains['cloud'] == 'azure' %}
|
||||||
- openvpn-client
|
- openvpn-client
|
||||||
|
{% else %}
|
||||||
|
- sdn
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
'roles:kubernetes-master':
|
'roles:kubernetes-master':
|
||||||
|
44
cluster/vagrant/pod-ip-test.sh
Normal file
44
cluster/vagrant/pod-ip-test.sh
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2014 Google Inc. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
cd $(dirname ${BASH_SOURCE})/../../
|
||||||
|
|
||||||
|
# start the cluster with 2 minions
|
||||||
|
export KUBERNETES_NUM_MINIONS=2
|
||||||
|
export KUBERNETES_PROVIDER=vagrant
|
||||||
|
cluster/kube-up.sh
|
||||||
|
|
||||||
|
echo "Pull an image that runs a web server"
|
||||||
|
vagrant ssh minion-1 -- sudo docker pull dockerfile/nginx
|
||||||
|
vagrant ssh minion-2 -- sudo docker pull dockerfile/nginx
|
||||||
|
|
||||||
|
echo "Run the servers"
|
||||||
|
vagrant ssh minion-1 -- sudo docker run -d dockerfile/nginx
|
||||||
|
vagrant ssh minion-2 -- sudo docker run -d dockerfile/nginx
|
||||||
|
|
||||||
|
echo "Run ping from minion-1 to docker bridges and to the containers on both minions"
|
||||||
|
vagrant ssh minion-1 -- 'ping -c 10 10.244.1.1 && ping -c 10 10.244.2.1 && ping -c 10 10.244.1.3 && ping -c 10 10.244.2.3'
|
||||||
|
echo "Same pinch from minion-2"
|
||||||
|
vagrant ssh minion-2 -- 'ping -c 10 10.244.1.1 && ping -c 10 10.244.2.1 && ping -c 10 10.244.1.3 && ping -c 10 10.244.2.3'
|
||||||
|
|
||||||
|
echo "tcp check, curl to both the running webservers from both machines"
|
||||||
|
vagrant ssh minion-1 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80'
|
||||||
|
vagrant ssh minion-2 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80'
|
||||||
|
|
||||||
|
echo "All good, destroy the cluster"
|
||||||
|
vagrant destroy -f
|
@ -52,3 +52,6 @@ if [ ! $(which salt-minion) ]; then
|
|||||||
systemctl enable salt-minion
|
systemctl enable salt-minion
|
||||||
systemctl start salt-minion
|
systemctl start salt-minion
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# run the networking setup
|
||||||
|
$(dirname $0)/provision-network.sh $@
|
||||||
|
112
cluster/vagrant/provision-network.sh
Executable file
112
cluster/vagrant/provision-network.sh
Executable file
@ -0,0 +1,112 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2014 Google Inc. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# exit on any error
|
||||||
|
set -e
|
||||||
|
source $(dirname $0)/provision-config.sh
|
||||||
|
|
||||||
|
MINION_IP=$4
|
||||||
|
MINION_ID=$5
|
||||||
|
DOCKER_BRIDGE=kbr0
|
||||||
|
OVS_SWITCH=obr0
|
||||||
|
GRE_TUNNEL_BASE=gre
|
||||||
|
BRIDGE_BASE=10.244
|
||||||
|
BRIDGE_ADDRESS=${BRIDGE_BASE}.${MINION_ID}.1
|
||||||
|
BRIDGE_NETWORK=${BRIDGE_ADDRESS}/24
|
||||||
|
BRIDGE_NETMASK=255.255.255.0
|
||||||
|
NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/
|
||||||
|
POST_NETWORK_SCRIPT=/vagrant/network_closure.sh
|
||||||
|
|
||||||
|
# add docker bridge ifcfg file
|
||||||
|
cat <<EOF > ${NETWORK_CONF_PATH}ifcfg-${DOCKER_BRIDGE}
|
||||||
|
# Generated by yours truly
|
||||||
|
DEVICE=${DOCKER_BRIDGE}
|
||||||
|
ONBOOT=yes
|
||||||
|
TYPE=Bridge
|
||||||
|
BOOTPROTO=static
|
||||||
|
IPADDR=${BRIDGE_ADDRESS}
|
||||||
|
NETMASK=${BRIDGE_NETMASK}
|
||||||
|
STP=yes
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# add the ovs bridge ifcfg file
|
||||||
|
cat <<EOF > ${NETWORK_CONF_PATH}ifcfg-${OVS_SWITCH}
|
||||||
|
DEVICE=${OVS_SWITCH}
|
||||||
|
ONBOOT=yes
|
||||||
|
DEVICETYPE=ovs
|
||||||
|
TYPE=OVSBridge
|
||||||
|
BOOTPROTO=static
|
||||||
|
HOTPLUG=no
|
||||||
|
BRIDGE=${DOCKER_BRIDGE}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# now loop through all other minions and create persistent gre tunnels
|
||||||
|
MINION_IPS=$3
|
||||||
|
MINION_IP_ARRAY=(`echo ${MINION_IPS} | tr "," "\n"`)
|
||||||
|
GRE_NUM=0
|
||||||
|
for remote_ip in "${MINION_IP_ARRAY[@]}"
|
||||||
|
do
|
||||||
|
if [ "${remote_ip}" == "${MINION_IP}" ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
((GRE_NUM++)) || echo
|
||||||
|
GRE_TUNNEL=${GRE_TUNNEL_BASE}${GRE_NUM}
|
||||||
|
# ovs-vsctl add-port ${OVS_SWITCH} ${GRE_TUNNEL} -- set interface ${GRE_TUNNEL} type=gre options:remote_ip=${remote_ip}
|
||||||
|
cat <<EOF > ${NETWORK_CONF_PATH}ifcfg-${GRE_TUNNEL}
|
||||||
|
DEVICE=${GRE_TUNNEL}
|
||||||
|
ONBOOT=yes
|
||||||
|
DEVICETYPE=ovs
|
||||||
|
TYPE=OVSTunnel
|
||||||
|
OVS_BRIDGE=${OVS_SWITCH}
|
||||||
|
OVS_TUNNEL_TYPE=gre
|
||||||
|
OVS_TUNNEL_OPTIONS="options:remote_ip=${remote_ip}"
|
||||||
|
EOF
|
||||||
|
done
|
||||||
|
|
||||||
|
# add ip route rules such that all pod traffic flows through docker bridge and consequently to the gre tunnels
|
||||||
|
cat <<EOF > /${NETWORK_CONF_PATH}route-${DOCKER_BRIDGE}
|
||||||
|
${BRIDGE_BASE}.0.0/16 dev ${DOCKER_BRIDGE} scope link src ${BRIDGE_ADDRESS}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
|
||||||
|
# generate the post-configure script to be called by salt as cmd.wait
|
||||||
|
cat <<EOF > ${POST_NETWORK_SCRIPT}
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
# NAT interface fails to revive on network restart, so OR-gate to true
|
||||||
|
systemctl restart network.service || true
|
||||||
|
|
||||||
|
# set docker bridge up, and set stp on the ovs bridge
|
||||||
|
ip link set dev ${DOCKER_BRIDGE} up
|
||||||
|
ovs-vsctl set Bridge ${OVS_SWITCH} stp_enable=true
|
||||||
|
|
||||||
|
# modify the docker service file such that it uses the kube docker bridge and not its own
|
||||||
|
sed -ie "s/ExecStart=\/usr\/bin\/docker -d/ExecStart=\/usr\/bin\/docker -d -b=${DOCKER_BRIDGE} --iptables=false/g" /usr/lib/systemd/system/docker.service
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl restart docker.service
|
||||||
|
|
||||||
|
# setup iptables masquerade rules so the pods can reach the internet
|
||||||
|
iptables -t nat -A POSTROUTING -s ${BRIDGE_BASE}.0.0/16 ! -d ${BRIDGE_BASE}.0.0/16 -j MASQUERADE
|
||||||
|
|
||||||
|
# persist please
|
||||||
|
iptables-save >& /etc/sysconfig/iptables
|
||||||
|
|
||||||
|
# self-destruct after doing the job
|
||||||
|
#rm -f ${POST_NETWORK_SCRIPT}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
chmod +x ${POST_NETWORK_SCRIPT}
|
Loading…
Reference in New Issue
Block a user