From 936f9cc95d00031c6ed4f44c35ab1f7df4b6fac2 Mon Sep 17 00:00:00 2001 From: Eric Tune Date: Wed, 14 Jan 2015 09:17:57 -0800 Subject: [PATCH] WIP: Update Rackspace so node talks to apiserver Kubernetes project has decided that it is better if kubelet and kube-proxy use the apiserver REST interface to get and set resources instead of accessing resource keys in etcd directly. This is necessary to support kubelet reporting of events, and also encapsulates the apiserver store details. This means that the kubelet and kube-proxy need to know the apiserver host(s) via a flag. Since the Rackspace config already used etcd to advertise the minions to the controller-manager, I used the same pattern to advertise the apiserver(s) to the minions. Setting --public_address_override=$private_ipv4 is intended to ensure that the master serves its http interface on the right ethernet device, since I think there are two on a droplet. The new apiserver-advertiser.service puts the IPs of any apiservers in etcd. The kubelet and kube-proxy now take an environment file which contains the list of apiserver IPs, and that env var goes into a flag. The etcd_servers argument is removed -- the point is for these binaries to not access etcd. The new apiserver-finder.service watches for changes in etcd and restarts kubelet and kube proxy when there are new apiservers. --- .../cloud-config/master-cloud-config.yaml | 12 +++++ .../cloud-config/minion-cloud-config.yaml | 48 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/cluster/rackspace/cloud-config/master-cloud-config.yaml b/cluster/rackspace/cloud-config/master-cloud-config.yaml index f97199f7945..ae449178f61 100644 --- a/cluster/rackspace/cloud-config/master-cloud-config.yaml +++ b/cluster/rackspace/cloud-config/master-cloud-config.yaml @@ -80,6 +80,18 @@ coreos: ExecStart=/opt/bin/kube-apiserver --address=127.0.0.1 --port=8080 --etcd_servers=http://127.0.0.1:4001 --portal_net=PORTAL_NET --logtostderr=true --cloud_provider=rackspace --cloud_config=/etc/cloud.conf --v=2 Restart=always RestartSec=2 + - name: apiserver-advertiser.service + command: start + content: | + [Unit] + Description=Kubernetes Apiserver Advertiser + After=etcd.service + Requires=etcd.service + After=master-apiserver.service + [Service] + ExecStart=/bin/sh -c 'etcdctl set /corekube/apiservers/$private_ipv4 $private_ipv4' + Restart=always + RestartSec=120 - name: master-controller-manager.service command: start content: | diff --git a/cluster/rackspace/cloud-config/minion-cloud-config.yaml b/cluster/rackspace/cloud-config/minion-cloud-config.yaml index 11cbadfdd0a..c55a63a3ad7 100644 --- a/cluster/rackspace/cloud-config/minion-cloud-config.yaml +++ b/cluster/rackspace/cloud-config/minion-cloud-config.yaml @@ -12,6 +12,14 @@ write_files: set) ip route replace $net via $nh dev eth2 metric 900 ;; expire) ip route del $net via $nh metric 900 ;; esac + - path: /opt/bin/regen-apiserver-list.sh + permissions: 0755 + content: | + #!/bin/sh + m=$(echo $(etcdctl ls --recursive /corekube/apiservers | cut -d/ -f4 | sort) | tr ' ' ,) + mkdir -p /run/kubelet + echo "APISERVER_IPS=$m" > /run/kubelet/apiservers.env + echo "FIRST_APISERVER_URL=http://${m%%\,*}:7080" >> /run/kubelet/apiservers.env - path: /opt/bin/download-release.sh permissions: 0755 content: | @@ -67,8 +75,9 @@ coreos: After=download-release.service Requires=download-release.service [Service] + EnvironmentFile=/run/kubelet/apiserver.env ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kubelet /opt/bin/kubelet - ExecStart=/opt/bin/kubelet --address=$private_ipv4 --hostname_override=$private_ipv4 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true --config=/opt/kubernetes-manifests --cluster_dns=DNS_SERVER_IP --cluster_domain=DNS_DOMAIN + ExecStart=/opt/bin/kubelet --address=$private_ipv4 --hostname_override=$private_ipv4 --api_servers=$FIRST_APISERVER_URL --logtostderr=true --config=/opt/kubernetes-manifests --cluster_dns=DNS_SERVER_IP --cluster_domain=DNS_DOMAIN Restart=always RestartSec=2 - name: minion-proxy.service @@ -84,10 +93,31 @@ coreos: After=download-release.service Requires=download-release.service [Service] + EnvironmentFile=/run/kubelet/apiserver.env ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kube-proxy /opt/bin/kube-proxy - ExecStart=/opt/bin/kube-proxy --bind_address=$private_ipv4 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true --v=2 + ExecStart=/opt/bin/kube-proxy --bind_address=$private_ipv4 --master=$FIRST_APISERVER_URL --logtostderr=true Restart=always RestartSec=2 + - name: kubelet-sighup.path + command: start + content: | + [Path] + PathChanged=/run/kubelet/apiservers.env + - name: kubelet-sighup.service + command: start + content: | + [Service] + ExecStart=/usr/bin/pkill -SIGHUP -f kubelet + - name: kube-proxy-sighup.path + command: start + content: | + [Path] + PathChanged=/run/kubelet/apiservers.env + - name: kube-proxy-sighup.service + command: start + content: | + [Service] + ExecStart=/usr/bin/pkill -SIGHUP -f kube-proxy - name: minion-advertiser.service command: start content: | @@ -100,6 +130,20 @@ coreos: ExecStart=/bin/sh -c 'while :; do etcdctl set /corekube/minions/$private_ipv4 $private_ipv4 --ttl 300; sleep 120; done' Restart=always RestartSec=120 + - name: apiserver-finder.service + command: start + content: | + [Unit] + Description=Kubernetes Apiserver finder + After=network-online.target + Requires=network-online.target + After=etcd.service + Requires=etcd.service + [Service] + ExecStartPre=/opt/bin/regen-apiserver-list.sh + ExecStart=/usr/bin/etcdctl exec-watch --recursive /corekube/apiservers -- /opt/bin/regen-apiserver-list.sh + Restart=always + RestartSec=30 - name: net-advertiser.service command: start content: |