Deploy k8s to vSphere

cluster/saltbase/salt/apiserver/default

@@ -26,6 +26,15 @@
 {% elif grains.cloud is defined and grains.cloud == 'azure' %}
   MACHINES="{{ salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').values()|join(',', attribute='hostnamef') }}" 
   {% set machines = "-machines $MACHINES" %}
+{% else %}
+  # No cloud defined, collect IPs of minions as machines list.
+  # Use a bash array to build the value we need. It doesn't appear to be
+  # possible call functions map or zip, or use lambda's from Jinja.
+  MACHINE_IPS=()
+  {% for addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', expr_form='grain').values() %}
+  MACHINE_IPS+=( {{ addrs[0] }} )
+  {% endfor %}
+  {% set machines = "-machines=$(echo ${MACHINE_IPS[@]} | xargs -n1 echo | paste -sd,)" %}
 {% endif %}
 
 DAEMON_ARGS="{{daemon_args}} {{address}} {{machines}} {{etcd_servers}} {{ minion_regexp }} {{ cloud_provider }}"

cluster/saltbase/salt/static-routes/if-down

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+[ "$IFACE" == "eth0" ] || exit 0
+
+{% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %}
+{% if ip_addrs[0] != salt['network.ip_addrs']('eth0')[0] %}
+{% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %}
+route del -net {{ cidr }}
+{% endif %}
+{% endfor %}

cluster/saltbase/salt/static-routes/if-up

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+[ "$IFACE" == "eth0" ] || exit 0
+
+{% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %}
+{% if ip_addrs[0] != salt['network.ip_addrs']('eth0')[0] %}
+{% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %}
+route add -net {{ cidr }} gw {{ ip_addrs[0] }}
+{% endif %}
+{% endfor %}

cluster/saltbase/salt/static-routes/init.sls

@@ -0,0 +1,30 @@
+# Add static routes to every minion to enable pods in the 10.244.x.x range to
+# reach each other. This is suboptimal, but necessary to let every pod have
+# its IP and have pods between minions be able to talk with each other.
+# This will be obsolete when we figure out the right way to make this work.
+
+/etc/network/if-up.d/static-routes:
+  file.managed:
+    - source: salt://static-routes/if-up
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 755
+
+/etc/network/if-down.d/static-routes:
+  file.managed:
+    - source: salt://static-routes/if-down
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 755
+
+refresh routes:
+  cmd.wait_script:
+    - source: salt://static-routes/refresh
+    - cwd: /etc/network/
+    - user: root
+    - group: root
+    - watch:
+      - file: /etc/network/if-up.d/static-routes
+      - file: /etc/network/if-down.d/static-routes

cluster/saltbase/salt/static-routes/refresh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Fake an ifup/ifdown event
+export IFACE=eth0
+
+if-down.d/static-routes || true
+if-up.d/static-routes || true

cluster/saltbase/salt/top.sls

@@ -19,3 +19,7 @@ base:
     - controller-manager
     - scheduler
     - nginx
+
+  'roles:kubernetes-pool-vsphere':
+    - match: grain
+    - static-routes