diff --git a/cluster/saltbase/salt/kubelet/init.sls b/cluster/saltbase/salt/kubelet/init.sls index 3e85b63bdef..b53717307c0 100644 --- a/cluster/saltbase/salt/kubelet/init.sls +++ b/cluster/saltbase/salt/kubelet/init.sls @@ -70,4 +70,7 @@ kubelet: - file: /etc/init.d/kubelet {% endif %} - file: /var/lib/kubelet/kubernetes_auth +{% if grains.network_mode is defined and grains.network_mode == 'openvswitch' %} + - sls: sdn +{% endif %} diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 3a34d349072..48af5cd93d9 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -41,6 +41,10 @@ base: {% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} - openvpn {% endif %} +{% if grains['cloud'] is defined and grains['cloud'] == 'vagrant' %} + - docker + - sdn +{% endif %} 'roles:kubernetes-pool-vsphere': - match: grain diff --git a/cluster/vagrant/config-default.sh b/cluster/vagrant/config-default.sh index bde0db27b23..f831bf32c74 100755 --- a/cluster/vagrant/config-default.sh +++ b/cluster/vagrant/config-default.sh @@ -29,12 +29,15 @@ export MASTER_NAME="${INSTANCE_PREFIX}-master" # Map out the IPs, names and container subnets of each minion export MINION_IP_BASE="10.245.1." MINION_CONTAINER_SUBNET_BASE="10.246" +MASTER_CONTAINER_NETMASK="255.255.255.0" +MASTER_CONTAINER_ADDR="${MINION_CONTAINER_SUBNET_BASE}.0.1" +MASTER_CONTAINER_SUBNET="${MINION_CONTAINER_SUBNET_BASE}.0.1/24" CONTAINER_SUBNET="${MINION_CONTAINER_SUBNET_BASE}.0.0/16" for ((i=0; i < NUM_MINIONS; i++)) do MINION_IPS[$i]="${MINION_IP_BASE}$((i+3))" MINION_NAMES[$i]="${INSTANCE_PREFIX}-minion-$((i+1))" - MINION_CONTAINER_SUBNETS[$i]="${MINION_CONTAINER_SUBNET_BASE}.${i}.1/24" - MINION_CONTAINER_ADDRS[$i]="${MINION_CONTAINER_SUBNET_BASE}.${i}.1" + MINION_CONTAINER_SUBNETS[$i]="${MINION_CONTAINER_SUBNET_BASE}.$((i+1)).1/24" + MINION_CONTAINER_ADDRS[$i]="${MINION_CONTAINER_SUBNET_BASE}.$((i+1)).1" MINION_CONTAINER_NETMASKS[$i]="255.255.255.0" VAGRANT_MINION_NAMES[$i]="minion-$((i+1))" done @@ -69,4 +72,4 @@ DNS_REPLICAS=1 # Optional: Enable setting flags for kube-apiserver to turn on behavior in active-dev RUNTIME_CONFIG="" -#RUNTIME_CONFIG="api/v1beta3" \ No newline at end of file +#RUNTIME_CONFIG="api/v1beta3" diff --git a/cluster/vagrant/provision-network.sh b/cluster/vagrant/provision-network.sh index 6a24936cf16..d18eb0085db 100755 --- a/cluster/vagrant/provision-network.sh +++ b/cluster/vagrant/provision-network.sh @@ -16,7 +16,8 @@ DOCKER_BRIDGE=kbr0 OVS_SWITCH=obr0 -GRE_TUNNEL_BASE=gre +DOCKER_OVS_TUN=tun0 +TUNNEL_BASE=gre NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/ POST_NETWORK_SCRIPT_DIR=/kubernetes-vagrant POST_NETWORK_SCRIPT=${POST_NETWORK_SCRIPT_DIR}/network_closure.sh @@ -24,55 +25,6 @@ POST_NETWORK_SCRIPT=${POST_NETWORK_SCRIPT_DIR}/network_closure.sh # ensure location of POST_NETWORK_SCRIPT exists mkdir -p $POST_NETWORK_SCRIPT_DIR -# add docker bridge ifcfg file -cat < ${NETWORK_CONF_PATH}ifcfg-${DOCKER_BRIDGE} -# Generated by yours truly -DEVICE=${DOCKER_BRIDGE} -ONBOOT=yes -TYPE=Bridge -BOOTPROTO=static -IPADDR=${MINION_CONTAINER_ADDR} -NETMASK=${MINION_CONTAINER_NETMASK} -STP=yes -EOF - -# add the ovs bridge ifcfg file -cat < ${NETWORK_CONF_PATH}ifcfg-${OVS_SWITCH} -DEVICE=${OVS_SWITCH} -ONBOOT=yes -DEVICETYPE=ovs -TYPE=OVSBridge -BOOTPROTO=static -HOTPLUG=no -BRIDGE=${DOCKER_BRIDGE} -EOF - -# now loop through all other minions and create persistent gre tunnels -GRE_NUM=0 -for remote_ip in "${MINION_IPS[@]}" -do - if [ "${remote_ip}" == "${MINION_IP}" ]; then - continue - fi - ((GRE_NUM++)) || echo - GRE_TUNNEL=${GRE_TUNNEL_BASE}${GRE_NUM} - # ovs-vsctl add-port ${OVS_SWITCH} ${GRE_TUNNEL} -- set interface ${GRE_TUNNEL} type=gre options:remote_ip=${remote_ip} - cat < ${NETWORK_CONF_PATH}ifcfg-${GRE_TUNNEL} -DEVICE=${GRE_TUNNEL} -ONBOOT=yes -DEVICETYPE=ovs -TYPE=OVSTunnel -OVS_BRIDGE=${OVS_SWITCH} -OVS_TUNNEL_TYPE=gre -OVS_TUNNEL_OPTIONS="options:remote_ip=${remote_ip}" -EOF -done - -# add ip route rules such that all pod traffic flows through docker bridge and consequently to the gre tunnels -cat < ${NETWORK_CONF_PATH}route-${DOCKER_BRIDGE} -${CONTAINER_SUBNET} dev ${DOCKER_BRIDGE} scope link src ${MINION_CONTAINER_ADDR} -EOF - # generate the post-configure script to be called by salt as cmd.wait cat < ${POST_NETWORK_SCRIPT} #!/bin/bash @@ -81,27 +33,58 @@ set -e # Only do this operation once, otherwise, we get docker.service files output on disk, and the command line arguments get applied multiple times grep -q kbr0 /etc/sysconfig/docker || { + CONTAINER_SUBNETS=(${MASTER_CONTAINER_SUBNET} ${MINION_CONTAINER_SUBNETS[@]}) + CONTAINER_IPS=(${MASTER_IP} ${MINION_IPS[@]}) + # Stop docker before making these updates systemctl stop docker - # NAT interface fails to revive on network restart, so OR-gate to true - systemctl restart network.service || true - - # set docker bridge up, and set stp on the ovs bridge + # create new docker bridge + ip link set dev ${DOCKER_BRIDGE} down || true + brctl delbr ${DOCKER_BRIDGE} || true + brctl addbr ${DOCKER_BRIDGE} ip link set dev ${DOCKER_BRIDGE} up - ovs-vsctl set Bridge ${OVS_SWITCH} stp_enable=true + ifconfig ${DOCKER_BRIDGE} ${CONTAINER_ADDR} netmask ${CONTAINER_NETMASK} up + + # add ovs bridge + ovs-vsctl del-br ${OVS_SWITCH} || true + ovs-vsctl add-br ${OVS_SWITCH} -- set Bridge ${OVS_SWITCH} fail-mode=secure + ovs-vsctl set bridge ${OVS_SWITCH} protocols=OpenFlow13 + ovs-vsctl del-port ${OVS_SWITCH} ${TUNNEL_BASE}0 || true + ovs-vsctl add-port ${OVS_SWITCH} ${TUNNEL_BASE}0 -- set Interface ${TUNNEL_BASE}0 type=${TUNNEL_BASE} options:remote_ip="flow" options:key="flow" ofport_request=10 + + # add tun device + ovs-vsctl del-port ${OVS_SWITCH} ${DOCKER_OVS_TUN} || true + ovs-vsctl add-port ${OVS_SWITCH} ${DOCKER_OVS_TUN} -- set Interface ${DOCKER_OVS_TUN} type=internal ofport_request=9 + brctl addif ${DOCKER_BRIDGE} ${DOCKER_OVS_TUN} + ip link set ${DOCKER_OVS_TUN} up + + + # add oflow rules, because we do not want to use stp + ovs-ofctl -O OpenFlow13 del-flows ${OVS_SWITCH} + + # now loop through all other minions and create persistent gre tunnels + NODE_INDEX=0 + for remote_ip in "\${CONTAINER_IPS[@]}" + do + if [ "\${remote_ip}" == "${NODE_IP}" ]; then + ovs-ofctl -O OpenFlow13 add-flow ${OVS_SWITCH} "table=0,ip,in_port=10,nw_dst=\${CONTAINER_SUBNETS[\${NODE_INDEX}]},actions=output:9" + ovs-ofctl -O OpenFlow13 add-flow ${OVS_SWITCH} "table=0,arp,in_port=10,nw_dst=\${CONTAINER_SUBNETS[\${NODE_INDEX}]},actions=output:9" + else + ovs-ofctl -O OpenFlow13 add-flow ${OVS_SWITCH} "table=0,in_port=9,ip,nw_dst=\${CONTAINER_SUBNETS[\${NODE_INDEX}]},actions=set_field:\${remote_ip}->tun_dst,output:10" + ovs-ofctl -O OpenFlow13 add-flow ${OVS_SWITCH} "table=0,in_port=9,arp,nw_dst=\${CONTAINER_SUBNETS[\${NODE_INDEX}]},actions=set_field:\${remote_ip}->tun_dst,output:10" + fi + ((NODE_INDEX++)) || true + done + + # add ip route rules such that all pod traffic flows through docker bridge and consequently to the gre tunnels + ip route add ${CONTAINER_SUBNET} dev ${DOCKER_BRIDGE} scope link src ${CONTAINER_ADDR} + # modify the docker service file such that it uses the kube docker bridge and not its own - #echo "OPTIONS=-b=kbr0 --iptables=false --selinux-enabled" > /etc/sysconfig/docker - echo "OPTIONS='-b=kbr0 --iptables=false --selinux-enabled ${DOCKER_OPTS}'" >/etc/sysconfig/docker + echo "OPTIONS='-b=kbr0 --selinux-enabled ${DOCKER_OPTS}'" >/etc/sysconfig/docker systemctl daemon-reload - systemctl restart docker.service - - # setup iptables masquerade rules so the pods can reach the internet - iptables -t nat -A POSTROUTING -s ${CONTAINER_SUBNET} ! -d ${CONTAINER_SUBNET} -j MASQUERADE - - # persist please - iptables-save >& /etc/sysconfig/iptables + systemctl start docker } EOF diff --git a/cluster/vagrant/util.sh b/cluster/vagrant/util.sh index ac707ca9a78..33285e6d202 100644 --- a/cluster/vagrant/util.sh +++ b/cluster/vagrant/util.sh @@ -69,6 +69,13 @@ function create-provision-scripts { echo "MASTER_IP='${MASTER_IP}'" echo "MINION_NAMES=(${MINION_NAMES[@]})" echo "MINION_IPS=(${MINION_IPS[@]})" + echo "NODE_IP='${MASTER_IP}'" + echo "CONTAINER_SUBNET='${CONTAINER_SUBNET}'" + echo "CONTAINER_NETMASK='${MASTER_CONTAINER_NETMASK}'" + echo "MASTER_CONTAINER_SUBNET='${MASTER_CONTAINER_SUBNET}'" + echo "CONTAINER_ADDR='${MASTER_CONTAINER_ADDR}'" + echo "MINION_CONTAINER_NETMASKS='${MINION_CONTAINER_NETMASKS[@]}'" + echo "MINION_CONTAINER_SUBNETS=(${MINION_CONTAINER_SUBNETS[@]})" echo "PORTAL_NET='${PORTAL_NET}'" echo "MASTER_USER='${MASTER_USER}'" echo "MASTER_PASSWD='${MASTER_PASSWD}'" @@ -80,6 +87,7 @@ function create-provision-scripts { echo "DNS_DOMAIN='${DNS_DOMAIN:-}'" echo "RUNTIME_CONFIG='${RUNTIME_CONFIG:-}'" grep -v "^#" "${KUBE_ROOT}/cluster/vagrant/provision-master.sh" + grep -v "^#" "${KUBE_ROOT}/cluster/vagrant/provision-network.sh" ) > "${KUBE_TEMP}/master-start.sh" for (( i=0; i<${#MINION_NAMES[@]}; i++)); do @@ -91,8 +99,11 @@ function create-provision-scripts { echo "MINION_IPS=(${MINION_IPS[@]})" echo "MINION_IP='${MINION_IPS[$i]}'" echo "MINION_ID='$i'" - echo "MINION_CONTAINER_ADDR='${MINION_CONTAINER_ADDRS[$i]}'" - echo "MINION_CONTAINER_NETMASK='${MINION_CONTAINER_NETMASKS[$i]}'" + echo "NODE_IP='${MINION_IPS[$i]}'" + echo "MASTER_CONTAINER_SUBNET='${MASTER_CONTAINER_SUBNET}'" + echo "CONTAINER_ADDR='${MINION_CONTAINER_ADDRS[$i]}'" + echo "CONTAINER_NETMASK='${MINION_CONTAINER_NETMASKS[$i]}'" + echo "MINION_CONTAINER_SUBNETS=(${MINION_CONTAINER_SUBNETS[@]})" echo "CONTAINER_SUBNET='${CONTAINER_SUBNET}'" echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS-}'" grep -v "^#" "${KUBE_ROOT}/cluster/vagrant/provision-minion.sh" @@ -257,6 +268,10 @@ function find-vagrant-name-by-ip { # Find the vagrant machien name based on the host name of the minion function find-vagrant-name-by-minion-name { local ip="$1" + if [[ "$ip" == "${INSTANCE_PREFIX}-master" ]]; then + echo "master" + return $? + fi local ip_pattern="${INSTANCE_PREFIX}-minion-(.*)" [[ $ip =~ $ip_pattern ]] || { @@ -280,7 +295,7 @@ function ssh-to-node { return 1 } - vagrant ssh "${machine}" -c "${cmd}" | grep -v "Connection to.*closed" + vagrant ssh "${machine}" -c "${cmd}" } # Restart the kube-proxy on a node ($1) @@ -288,6 +303,11 @@ function restart-kube-proxy { ssh-to-node "$1" "sudo systemctl restart kube-proxy" } +# Restart the apiserver +function restart-apiserver { + ssh-to-node "${master}" "sudo systemctl restart kube-apiserver" +} + function setup-monitoring-firewall { echo "TODO" 1>&2 } diff --git a/hack/e2e-suite/services.sh b/hack/e2e-suite/services.sh index a8b4ec9d031..73e049582fc 100755 --- a/hack/e2e-suite/services.sh +++ b/hack/e2e-suite/services.sh @@ -33,11 +33,6 @@ source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" prepare-e2e -if [[ "$KUBERNETES_PROVIDER" == "vagrant" ]]; then - echo "WARNING: Skipping services.sh for ${KUBERNETES_PROVIDER}. See https://github.com/GoogleCloudPlatform/kubernetes/issues/3655" - exit 0 -fi - function error() { echo "$@" >&2 exit 1 @@ -266,7 +261,7 @@ function verify_from_container() { for i in $(seq -s' ' 1 $4); do ok=false for j in $(seq -s' ' 1 10); do - if wget -q -T 1 -O - http://$2:$3; then + if wget -q -T 5 -O - http://$2:$3; then echo ok=true break @@ -420,7 +415,11 @@ verify_from_container "${svc3_name}" "${svc3_ip}" "${svc3_port}" \ # echo "Test 6: Restart the master, make sure portals come back." echo "Restarting the master" -ssh-to-node "${master}" "sudo /etc/init.d/kube-apiserver restart" +if [[ "$KUBERNETES_PROVIDER" == "vagrant" ]]; then + restart-apiserver "${master}" +else + ssh-to-node "${master}" "sudo /etc/init.d/kube-apiserver restart" +fi sleep 5 echo "Verifying the portals from the host" wait_for_service_up "${svc3_name}" "${svc3_ip}" "${svc3_port}" \