From 1d9d11c8369a636c1ac67a95d62e116585fa218c Mon Sep 17 00:00:00 2001 From: Mike Danese Date: Wed, 4 Nov 2015 10:59:16 -0800 Subject: [PATCH] run kube-proxy in a static pod --- build/common.sh | 1 + cluster/saltbase/install.sh | 7 +- cluster/saltbase/pillar/docker-images.sls | 1 + .../saltbase/salt/kube-node-unpacker/init.sls | 43 ++++++ .../saltbase/salt/kube-node-unpacker/initd | 95 +++++++++++++ .../kube-node-unpacker.service | 9 ++ .../kube-node-unpacker/kube-node-unpacker.sh | 46 +++++++ cluster/saltbase/salt/kube-proxy/default | 27 ---- cluster/saltbase/salt/kube-proxy/init.sls | 96 ++++--------- cluster/saltbase/salt/kube-proxy/initd | 130 ------------------ .../salt/kube-proxy/kube-proxy.manifest | 54 ++++++++ .../salt/kube-proxy/kube-proxy.service | 12 -- cluster/saltbase/salt/supervisor/init.sls | 24 ---- cluster/saltbase/salt/top.sls | 1 + hack/lib/golang.sh | 1 + 15 files changed, 281 insertions(+), 266 deletions(-) create mode 100644 cluster/saltbase/salt/kube-node-unpacker/init.sls create mode 100755 cluster/saltbase/salt/kube-node-unpacker/initd create mode 100644 cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service create mode 100755 cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh delete mode 100644 cluster/saltbase/salt/kube-proxy/default delete mode 100644 cluster/saltbase/salt/kube-proxy/initd create mode 100644 cluster/saltbase/salt/kube-proxy/kube-proxy.manifest delete mode 100644 cluster/saltbase/salt/kube-proxy/kube-proxy.service diff --git a/build/common.sh b/build/common.sh index 16a136e4e3f..f61ad86e709 100755 --- a/build/common.sh +++ b/build/common.sh @@ -99,6 +99,7 @@ readonly KUBE_DOCKER_WRAPPED_BINARIES=( kube-apiserver,busybox kube-controller-manager,busybox kube-scheduler,busybox + kube-proxy,gcr.io/google_containers/debian-iptables:v1 ) # The set of addons images that should be prepopulated diff --git a/cluster/saltbase/install.sh b/cluster/saltbase/install.sh index c7fe553f228..b58c42b6b38 100755 --- a/cluster/saltbase/install.sh +++ b/cluster/saltbase/install.sh @@ -25,9 +25,10 @@ SALT_ROOT=$(dirname "${BASH_SOURCE}") readonly SALT_ROOT readonly KUBE_DOCKER_WRAPPED_BINARIES=( - kube-apiserver - kube-controller-manager - kube-scheduler + kube-apiserver + kube-controller-manager + kube-scheduler + kube-proxy ) readonly SERVER_BIN_TAR=${1-} diff --git a/cluster/saltbase/pillar/docker-images.sls b/cluster/saltbase/pillar/docker-images.sls index bad6c0509d9..64256914cc9 100644 --- a/cluster/saltbase/pillar/docker-images.sls +++ b/cluster/saltbase/pillar/docker-images.sls @@ -2,3 +2,4 @@ kube-apiserver_docker_tag: #kube-apiserver_docker_tag_value# kube-controller-manager_docker_tag: #kube-controller-manager_docker_tag_value# kube-scheduler_docker_tag: #kube-scheduler_docker_tag_value# +kube-proxy_docker_tag: #kube-proxy_docker_tag_value# diff --git a/cluster/saltbase/salt/kube-node-unpacker/init.sls b/cluster/saltbase/salt/kube-node-unpacker/init.sls new file mode 100644 index 00000000000..eb711a14094 --- /dev/null +++ b/cluster/saltbase/salt/kube-node-unpacker/init.sls @@ -0,0 +1,43 @@ +/etc/kubernetes/kube-node-unpacker.sh: + file.managed: + - source: salt://kube-node-unpacker/kube-node-unpacker.sh + - user: root + - group: root + - mode: 755 + +node-docker-image-tags: + file.touch: + - name: /srv/pillar/docker-images.sls + +{% if pillar.get('is_systemd') %} + +{{ pillar.get('systemd_system_path') }}/kube-node-unpacker.service: + file.managed: + - source: salt://kube-node-unpacker/kube-node-unpacker.service + - user: root + - group: root + cmd.wait: + - name: /opt/kubernetes/helpers/services bounce kube-node-unpacker + - watch: + - file: node-docker-image-tags + - file: /etc/kubernetes/kube-node-unpacker.sh + - file: {{ pillar.get('systemd_system_path') }}/kube-node-unpacker.service + +{% else %} + +/etc/init.d/kube-node-unpacker: + file.managed: + - source: salt://kube-node-unpacker/initd + - user: root + - group: root + - mode: 755 + +kube-node-unpacker: + service.running: + - enable: True + - restart: True + - watch: + - file: node-docker-image-tags + - file: /etc/kubernetes/kube-node-unpacker.sh + +{% endif %} diff --git a/cluster/saltbase/salt/kube-node-unpacker/initd b/cluster/saltbase/salt/kube-node-unpacker/initd new file mode 100755 index 00000000000..13a61126414 --- /dev/null +++ b/cluster/saltbase/salt/kube-node-unpacker/initd @@ -0,0 +1,95 @@ +#!/bin/bash +# +### BEGIN INIT INFO +# Provides: kube-node-unpacker +# Required-Start: $local_fs $network $syslog docker +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Kubernetes Node Unpacker +# Description: +# Unpacks docker images on Kubernetes nodes +### END INIT INFO + + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Kubernetes Node Unpacker" +NAME=kube-node-unpacker +DAEMON_LOG_FILE=/var/log/$NAME.log +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +KUBE_MASTER_ADDONS_SH=/etc/kubernetes/kube-node-unpacker.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + + + + +# +# Function that starts the daemon/service +# +do_start() +{ + ${KUBE_MASTER_ADDONS_SH} >${DAEMON_LOG_FILE} 2>&1 & + echo $! > ${PIDFILE} + disown +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + kill $(cat ${PIDFILE}) + rm ${PIDFILE} + return +} + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) log_end_msg 0 ;; + 2) exit 1 ;; + esac + ;; + status) + status_of_proc -p $PIDFILE $KUBE_MASTER_ADDONS_SH $NAME + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service b/cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service new file mode 100644 index 00000000000..bdb4609afb0 --- /dev/null +++ b/cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service @@ -0,0 +1,9 @@ +[Unit] +Description=Kubernetes Node Unpacker +Documentation=https://github.com/GoogleCloudPlatform/kubernetes + +[Service] +ExecStart=/etc/kubernetes/kube-node-unpacker.sh + +[Install] +WantedBy=multi-user.target diff --git a/cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh b/cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh new file mode 100755 index 00000000000..38e2aeebd45 --- /dev/null +++ b/cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# loadedImageFlags is a bit-flag to track which docker images loaded successfully. +let loadedImageFlags=0 + +while true; do + restart_docker=false + + if which docker 1>/dev/null 2>&1; then + + timeout 30 docker load -i /srv/salt/kube-bins/kube-proxy.tar 1>/dev/null 2>&1 + rc=$? + if [[ "${rc}" == 0 ]]; then + let loadedImageFlags="${loadedImageFlags}|1" + elif [[ "${rc}" == 124 ]]; then + restart_docker=true + fi + fi + + # required docker images got installed. exit while loop. + if [[ "${loadedImageFlags}" == 1 ]]; then break; fi + + # Sometimes docker load hang, restart docker daemon resolve the issue + if [[ "${restart_docker}" ]]; then service docker restart; fi + + # sleep for 15 seconds before attempting to load docker images again + sleep 15 + +done + +# Now exit. After kube-push, salt will notice that the service is down and it +# will start it and new docker images will be loaded. diff --git a/cluster/saltbase/salt/kube-proxy/default b/cluster/saltbase/salt/kube-proxy/default deleted file mode 100644 index d782e74d8d7..00000000000 --- a/cluster/saltbase/salt/kube-proxy/default +++ /dev/null @@ -1,27 +0,0 @@ -{% set daemon_args = "$DAEMON_ARGS" -%} -{% if grains['os_family'] == 'RedHat' -%} - {% set daemon_args = "" -%} -{% endif -%} - -{% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%} -{% if grains.api_servers is defined -%} - {% set api_servers = "--master=https://" + grains.api_servers -%} -{% else -%} - {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%} - {% set api_servers = "--master=https://" + ips[0][0] -%} -{% endif -%} - -# TODO: remove nginx for other cloud providers. -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ] %} - {% set api_servers_with_port = api_servers -%} -{% else -%} - {% set api_servers_with_port = api_servers + ":6443" -%} -{% endif -%} - -{% set test_args = "" -%} -{% if pillar['kubeproxy_test_args'] is defined -%} - {% set test_args=pillar['kubeproxy_test_args'] %} -{% endif -%} - -# test_args has to be kept at the end, so they'll overwrite any prior configuration -DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{kubeconfig}} {{pillar['log_level']}} {{test_args}}" diff --git a/cluster/saltbase/salt/kube-proxy/init.sls b/cluster/saltbase/salt/kube-proxy/init.sls index 3761b5bbce5..868235a3cdc 100644 --- a/cluster/saltbase/salt/kube-proxy/init.sls +++ b/cluster/saltbase/salt/kube-proxy/init.sls @@ -1,73 +1,3 @@ -{% if pillar.get('is_systemd') %} -{% set environment_file = '/etc/sysconfig/kube-proxy' %} -{% else %} -{% set environment_file = '/etc/default/kube-proxy' %} -{% endif %} - -/usr/local/bin/kube-proxy: - file.managed: - - source: salt://kube-bins/kube-proxy - - user: root - - group: root - - mode: 755 - -{{ environment_file }}: - file.managed: - - source: salt://kube-proxy/default - - template: jinja - - user: root - - group: root - - mode: 644 - -kube-proxy: - group.present: - - system: True - user.present: - - system: True - - gid_from_name: True - - shell: /sbin/nologin - - home: /var/kube-proxy - - require: - - group: kube-proxy - -{% if pillar.get('is_systemd') %} - -{{ pillar.get('systemd_system_path') }}/kube-proxy.service: - file.managed: - - source: salt://kube-proxy/kube-proxy.service - - user: root - - group: root - cmd.wait: - - name: /opt/kubernetes/helpers/services bounce kube-proxy - - watch: - - file: {{ environment_file }} - - file: {{ pillar.get('systemd_system_path') }}/kube-proxy.service - - file: /var/lib/kube-proxy/kubeconfig - -{% else %} - -/etc/init.d/kube-proxy: - file.managed: - - source: salt://kube-proxy/initd - - user: root - - group: root - - mode: 755 - -{% endif %} - -kube-proxy-service: - service.running: - - name: kube-proxy - - enable: True - - watch: - - file: {{ environment_file }} -{% if pillar.get('is_systemd') %} - - file: {{ pillar.get('systemd_system_path') }}/kube-proxy.service -{% else %} - - file: /etc/init.d/kube-proxy -{% endif %} - - file: /var/lib/kube-proxy/kubeconfig - /var/lib/kube-proxy/kubeconfig: file.managed: - source: salt://kube-proxy/kubeconfig @@ -75,3 +5,29 @@ kube-proxy-service: - group: root - mode: 400 - makedirs: true + +# kube-proxy in a static pod +/etc/kubernetes/manifests/kube-proxy.manifest: + file.managed: + - source: salt://kube-proxy/kube-proxy.manifest + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 + - require: + - service: docker + - service: kubelet + +/var/log/kube-proxy.log: + file.managed: + - user: root + - group: root + - mode: 644 + +#stop legacy kube-proxy service +stop_kube-proxy: + service.dead: + - name: kube-proxy + - enable: None diff --git a/cluster/saltbase/salt/kube-proxy/initd b/cluster/saltbase/salt/kube-proxy/initd deleted file mode 100644 index 658689949e3..00000000000 --- a/cluster/saltbase/salt/kube-proxy/initd +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -# -### BEGIN INIT INFO -# Provides: kube-proxy -# Required-Start: $local_fs $network $syslog -# Required-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: The Kubernetes network proxy -# Description: -# The Kubernetes network proxy enables network redirection and -# loadbalancing for dynamically placed containers. -### END INIT INFO - - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="The Kubernetes network proxy" -NAME=kube-proxy -DAEMON=/usr/local/bin/kube-proxy -DAEMON_ARGS="" -DAEMON_LOG_FILE=/var/log/$NAME.log -PIDFILE=/var/run/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -DAEMON_USER=root - -# Exit if the package is not installed -[ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.2-14) to ensure that this file is present -# and status_of_proc is working. -. /lib/lsb/init-functions - -# -# Function that starts the daemon/service -# -do_start() -{ - # Avoid a potential race at boot time when both monit and init.d start - # the same service - PIDS=$(pidof $DAEMON) - for PID in ${PIDS}; do - kill -9 $PID - done - - # Raise the file descriptor limit - we expect to open a lot of sockets! - ulimit -n 65536 - - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --quiet --background --no-close \ - --make-pidfile --pidfile $PIDFILE \ - --exec $DAEMON -c $DAEMON_USER --test > /dev/null \ - || return 1 - start-stop-daemon --start --quiet --background --no-close \ - --make-pidfile --pidfile $PIDFILE \ - --exec $DAEMON -c $DAEMON_USER -- \ - $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \ - || return 2 -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - - -case "$1" in - start) - log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) log_end_msg 0 || exit 0 ;; - 2) log_end_msg 1 || exit 1 ;; - esac - ;; - stop) - log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) log_end_msg 0 ;; - 2) exit 1 ;; - esac - ;; - status) - status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $? - ;; - - restart|force-reload) - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - *) - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest new file mode 100644 index 00000000000..7b891a9f83b --- /dev/null +++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest @@ -0,0 +1,54 @@ +{% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%} +{% if grains.api_servers is defined -%} + {% set api_servers = "--master=https://" + grains.api_servers -%} +{% else -%} + {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%} + {% set api_servers = "--master=https://" + ips[0][0] -%} +{% endif -%} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ] %} + {% set api_servers_with_port = api_servers -%} +{% else -%} + {% set api_servers_with_port = api_servers + ":6443" -%} +{% endif -%} +{% set test_args = "" -%} +{% if pillar['kubeproxy_test_args'] is defined -%} + {% set test_args=pillar['kubeproxy_test_args'] %} +{% endif -%} + +# kube-proxy podspec +apiVersion: v1 +kind: Pod +metadata: + name: kube-proxy + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: kube-proxy + image: gcr.io/google_containers/kube-proxy:{{pillar['kube-proxy_docker_tag']}} + command: + - /bin/sh + - -c + - kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{pillar['log_level']}} {{test_args}} 1>>/var/log/kube-proxy.log 2>&1 + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + - mountPath: /var/log + name: varlog + readOnly: false + - mountPath: /var/lib/kube-proxy/kubeconfig + name: kubeconfig + readOnly: false + volumes: + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host + - hostPath: + path: /var/lib/kube-proxy/kubeconfig + name: kubeconfig + - hostPath: + path: /var/log + name: varlog diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.service b/cluster/saltbase/salt/kube-proxy/kube-proxy.service deleted file mode 100644 index 8b11a323737..00000000000 --- a/cluster/saltbase/salt/kube-proxy/kube-proxy.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Kubernetes Kube-Proxy Server -Documentation=https://github.com/GoogleCloudPlatform/kubernetes - -[Service] -EnvironmentFile=/etc/sysconfig/kube-proxy -ExecStart=/usr/local/bin/kube-proxy "$DAEMON_ARGS" -Restart=on-failure -LimitNOFILE=65536 - -[Install] -WantedBy=multi-user.target diff --git a/cluster/saltbase/salt/supervisor/init.sls b/cluster/saltbase/salt/supervisor/init.sls index 6d13ed6ebdb..98a3700920d 100644 --- a/cluster/saltbase/salt/supervisor/init.sls +++ b/cluster/saltbase/salt/supervisor/init.sls @@ -52,30 +52,6 @@ monit: - mode: 755 - makedirs: True -{% if "kubernetes-pool" in grains.get('roles', []) %} -/etc/supervisor/conf.d/kube-proxy.conf: - file: - - managed - - source: salt://supervisor/kube-proxy.conf - - user: root - - group: root - - mode: 644 - - makedirs: True - - require_in: - - pkg: supervisor - - require: - - file: /usr/sbin/kube-proxy-checker.sh - -/usr/sbin/kube-proxy-checker.sh: - file: - - managed - - source: salt://supervisor/kube-proxy-checker.sh - - user: root - - group: root - - mode: 755 - - makedirs: True -{% endif %} - {% if grains['roles'][0] == 'kubernetes-master' -%} /etc/supervisor/conf.d/kube-addons.conf: file: diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index d47d6cceaee..c80c527e372 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -16,6 +16,7 @@ base: - helpers - cadvisor - kube-client-tools + - kube-node-unpacker - kubelet {% if pillar.get('network_provider', '').lower() == 'opencontrail' %} - opencontrail-networking-minion diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index fe054a7c271..7335f850330 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -129,6 +129,7 @@ readonly KUBE_STATIC_LIBRARIES=( kube-apiserver kube-controller-manager kube-scheduler + kube-proxy ) kube::golang::is_statically_linked_library() {