WIP: Update Rackspace so node talks to apiserver

Kubernetes project has decided that it is better if kubelet
and kube-proxy use the apiserver REST interface to get and
set resources instead of accessing resource keys in etcd directly.
This is necessary to support kubelet reporting of events,
and also encapsulates the apiserver store details.

This means that the kubelet and kube-proxy need to know the
apiserver host(s) via a flag.

Since the Rackspace config already used etcd to advertise the
minions to the controller-manager, I used the same pattern to advertise
the apiserver(s) to the minions.

Setting --public_address_override=$private_ipv4 is intended to ensure that
the master serves its http interface on the right ethernet device, since I think
there are two on a droplet.

The new apiserver-advertiser.service puts the IPs of any apiservers in etcd.

The kubelet and kube-proxy now take an environment file which contains
the list of apiserver IPs, and that env var goes into a flag.  The
etcd_servers argument is removed -- the point is for these binaries
to not access etcd.

The new apiserver-finder.service watches for changes in etcd and
restarts kubelet and kube proxy when there are new apiservers.

cluster/rackspace/cloud-config/master-cloud-config.yaml

@@ -80,6 +80,18 @@ coreos:
         ExecStart=/opt/bin/kube-apiserver --address=127.0.0.1 --port=8080 --etcd_servers=http://127.0.0.1:4001 --portal_net=PORTAL_NET --logtostderr=true --cloud_provider=rackspace --cloud_config=/etc/cloud.conf --v=2
         Restart=always
         RestartSec=2
+    - name: apiserver-advertiser.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Apiserver Advertiser
+        After=etcd.service
+        Requires=etcd.service
+        After=master-apiserver.service
+        [Service]
+        ExecStart=/bin/sh -c 'etcdctl set /corekube/apiservers/$private_ipv4 $private_ipv4'
+        Restart=always
+        RestartSec=120
     - name: master-controller-manager.service
       command: start
       content: |

cluster/rackspace/cloud-config/minion-cloud-config.yaml

@@ -12,6 +12,14 @@ write_files:
       set) ip route replace $net via $nh dev eth2 metric 900 ;;
       expire) ip route del $net via $nh metric 900 ;;
       esac
+  - path: /opt/bin/regen-apiserver-list.sh
+    permissions: 0755
+    content: |
+      #!/bin/sh
+      m=$(echo $(etcdctl ls --recursive /corekube/apiservers | cut -d/ -f4 | sort) | tr ' ' ,)
+      mkdir -p /run/kubelet
+      echo "APISERVER_IPS=$m" > /run/kubelet/apiservers.env
+      echo "FIRST_APISERVER_URL=http://${m%%\,*}:7080" >> /run/kubelet/apiservers.env
   - path: /opt/bin/download-release.sh
     permissions: 0755
     content: |
@@ -67,8 +75,9 @@ coreos:
         After=download-release.service
         Requires=download-release.service
         [Service]
+        EnvironmentFile=/run/kubelet/apiserver.env
         ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kubelet /opt/bin/kubelet
-        ExecStart=/opt/bin/kubelet --address=$private_ipv4 --hostname_override=$private_ipv4 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true --config=/opt/kubernetes-manifests --cluster_dns=DNS_SERVER_IP --cluster_domain=DNS_DOMAIN
+        ExecStart=/opt/bin/kubelet --address=$private_ipv4 --hostname_override=$private_ipv4 --api_servers=$FIRST_APISERVER_URL --logtostderr=true --config=/opt/kubernetes-manifests --cluster_dns=DNS_SERVER_IP --cluster_domain=DNS_DOMAIN
         Restart=always
         RestartSec=2
     - name: minion-proxy.service
@@ -84,10 +93,31 @@ coreos:
         After=download-release.service
         Requires=download-release.service
         [Service]
+        EnvironmentFile=/run/kubelet/apiserver.env
         ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kube-proxy /opt/bin/kube-proxy
-        ExecStart=/opt/bin/kube-proxy --bind_address=$private_ipv4 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true --v=2
+        ExecStart=/opt/bin/kube-proxy --bind_address=$private_ipv4 --master=$FIRST_APISERVER_URL --logtostderr=true
         Restart=always
         RestartSec=2
+    - name: kubelet-sighup.path
+      command: start
+      content: |
+        [Path]
+        PathChanged=/run/kubelet/apiservers.env
+    - name: kubelet-sighup.service
+      command: start
+      content: |
+        [Service]
+        ExecStart=/usr/bin/pkill -SIGHUP -f kubelet
+    - name: kube-proxy-sighup.path
+      command: start
+      content: |
+        [Path]
+        PathChanged=/run/kubelet/apiservers.env
+    - name: kube-proxy-sighup.service
+      command: start
+      content: |
+        [Service]
+        ExecStart=/usr/bin/pkill -SIGHUP -f kube-proxy
     - name: minion-advertiser.service
       command: start
       content: |
@@ -100,6 +130,20 @@ coreos:
         ExecStart=/bin/sh -c 'while :; do etcdctl set /corekube/minions/$private_ipv4 $private_ipv4 --ttl 300; sleep 120; done'
         Restart=always
         RestartSec=120
+    - name: apiserver-finder.service
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes Apiserver finder
+        After=network-online.target
+        Requires=network-online.target
+        After=etcd.service
+        Requires=etcd.service
+        [Service]
+        ExecStartPre=/opt/bin/regen-apiserver-list.sh
+        ExecStart=/usr/bin/etcdctl exec-watch --recursive /corekube/apiservers -- /opt/bin/regen-apiserver-list.sh
+        Restart=always
+        RestartSec=30
     - name: net-advertiser.service
       command: start
       content: |