Merge pull request #17121 from mikedanese/kube-proxy-static

Auto commit by PR queue bot

build/common.sh

@@ -99,6 +99,7 @@ readonly KUBE_DOCKER_WRAPPED_BINARIES=(
   kube-apiserver,busybox
   kube-controller-manager,busybox
   kube-scheduler,busybox
+  kube-proxy,gcr.io/google_containers/debian-iptables:v1
 )
 
 # The set of addons images that should be prepopulated

cluster/saltbase/install.sh

@@ -25,9 +25,10 @@ SALT_ROOT=$(dirname "${BASH_SOURCE}")
 readonly SALT_ROOT
 
 readonly KUBE_DOCKER_WRAPPED_BINARIES=(
-      kube-apiserver
+  kube-apiserver
-      kube-controller-manager
+  kube-controller-manager
-      kube-scheduler
+  kube-scheduler
+  kube-proxy
 )
     
 readonly SERVER_BIN_TAR=${1-}

cluster/saltbase/pillar/docker-images.sls

@@ -2,3 +2,4 @@
 kube-apiserver_docker_tag: #kube-apiserver_docker_tag_value#
 kube-controller-manager_docker_tag: #kube-controller-manager_docker_tag_value#
 kube-scheduler_docker_tag: #kube-scheduler_docker_tag_value#
+kube-proxy_docker_tag: #kube-proxy_docker_tag_value#

cluster/saltbase/salt/kube-node-unpacker/init.sls

@@ -0,0 +1,43 @@
+/etc/kubernetes/kube-node-unpacker.sh:
+  file.managed:
+    - source: salt://kube-node-unpacker/kube-node-unpacker.sh
+    - user: root
+    - group: root
+    - mode: 755
+
+node-docker-image-tags:
+  file.touch:
+    - name: /srv/pillar/docker-images.sls
+
+{% if pillar.get('is_systemd') %}
+
+{{ pillar.get('systemd_system_path') }}/kube-node-unpacker.service:
+  file.managed:
+    - source: salt://kube-node-unpacker/kube-node-unpacker.service
+    - user: root
+    - group: root
+  cmd.wait:
+    - name: /opt/kubernetes/helpers/services bounce kube-node-unpacker
+    - watch:
+      - file: node-docker-image-tags
+      - file: /etc/kubernetes/kube-node-unpacker.sh
+      - file: {{ pillar.get('systemd_system_path') }}/kube-node-unpacker.service
+
+{% else %}
+
+/etc/init.d/kube-node-unpacker:
+  file.managed:
+    - source: salt://kube-node-unpacker/initd
+    - user: root
+    - group: root
+    - mode: 755
+
+kube-node-unpacker:
+  service.running:
+    - enable: True
+    - restart: True
+    - watch:
+      - file: node-docker-image-tags
+      - file: /etc/kubernetes/kube-node-unpacker.sh
+
+{% endif %}

cluster/saltbase/salt/kube-node-unpacker/initd

@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:   kube-node-unpacker
+# Required-Start:    $local_fs $network $syslog docker
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Kubernetes Node Unpacker
+# Description:
+#   Unpacks docker images on Kubernetes nodes
+### END INIT INFO
+
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="Kubernetes Node Unpacker"
+NAME=kube-node-unpacker
+DAEMON_LOG_FILE=/var/log/$NAME.log
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+KUBE_MASTER_ADDONS_SH=/etc/kubernetes/kube-node-unpacker.sh
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
+# and status_of_proc is working.
+. /lib/lsb/init-functions
+
+
+
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+    ${KUBE_MASTER_ADDONS_SH} </dev/null >>${DAEMON_LOG_FILE} 2>&1 &
+    echo $! > ${PIDFILE}
+    disown
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+    kill $(cat ${PIDFILE})
+    rm ${PIDFILE}
+    return
+}
+
+case "$1" in
+  start)
+        log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+                0|1) log_end_msg 0 || exit 0 ;;
+                2) log_end_msg 1 || exit 1 ;;
+        esac
+        ;;
+  stop)
+        log_daemon_msg "Stopping $DESC" "$NAME"
+        do_stop
+        case "$?" in
+                0|1) log_end_msg 0 ;;
+                2) exit 1 ;;
+        esac
+        ;;
+  status)
+        status_of_proc -p $PIDFILE $KUBE_MASTER_ADDONS_SH $NAME
+        ;;
+
+  restart|force-reload)
+        log_daemon_msg "Restarting $DESC" "$NAME"
+        do_stop
+        case "$?" in
+          0|1)
+                do_start
+                case "$?" in
+                        0) log_end_msg 0 ;;
+                        1) log_end_msg 1 ;; # Old process is still running
+                        *) log_end_msg 1 ;; # Failed to start
+                esac
+                ;;
+          *)
+                # Failed to stop
+                log_end_msg 1
+                ;;
+        esac
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac

cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Kubernetes Node Unpacker
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+
+[Service]
+ExecStart=/etc/kubernetes/kube-node-unpacker.sh
+
+[Install]
+WantedBy=multi-user.target

cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# loadedImageFlags is a bit-flag to track which docker images loaded successfully.
+let loadedImageFlags=0
+
+while true; do
+  restart_docker=false
+
+  if which docker 1>/dev/null 2>&1; then
+
+    timeout 30 docker load -i /srv/salt/kube-bins/kube-proxy.tar 1>/dev/null 2>&1
+    rc=$?
+    if [[ "${rc}" == 0 ]]; then
+      let loadedImageFlags="${loadedImageFlags}|1"
+    elif [[ "${rc}" == 124 ]]; then
+      restart_docker=true
+    fi
+  fi
+
+  # required docker images got installed. exit while loop.
+  if [[ "${loadedImageFlags}" == 1 ]]; then break; fi
+
+  # Sometimes docker load hang, restart docker daemon resolve the issue
+  if [[ "${restart_docker}" ]]; then service docker restart; fi
+
+  # sleep for 15 seconds before attempting to load docker images again
+  sleep 15
+
+done
+
+# Now exit. After kube-push, salt will notice that the service is down and it
+# will start it and new docker images will be loaded.

cluster/saltbase/salt/kube-proxy/default

@@ -1,27 +0,0 @@
-{% set daemon_args = "$DAEMON_ARGS" -%}
-{% if grains['os_family'] == 'RedHat' -%}
-	{% set daemon_args = "" -%}
-{% endif -%}
-
-{% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%}
-{% if grains.api_servers is defined -%}
-  {% set api_servers = "--master=https://" + grains.api_servers -%}
-{% else -%}
-  {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
-  {% set api_servers = "--master=https://" + ips[0][0] -%}
-{% endif -%}
-
-# TODO: remove nginx for other cloud providers.
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ]  %}
-   {% set api_servers_with_port = api_servers -%}
-{% else -%}
-  {% set api_servers_with_port = api_servers + ":6443" -%}
-{% endif -%}
-
-{% set test_args = "" -%}
-{% if pillar['kubeproxy_test_args'] is defined -%}
-  {% set test_args=pillar['kubeproxy_test_args'] %}
-{% endif -%}
-
-# test_args has to be kept at the end, so they'll overwrite any prior configuration
-DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{kubeconfig}} {{pillar['log_level']}} {{test_args}}"

cluster/saltbase/salt/kube-proxy/init.sls

@@ -1,73 +1,3 @@
-{% if pillar.get('is_systemd') %}
-{% set environment_file = '/etc/sysconfig/kube-proxy' %}
-{% else %}
-{% set environment_file = '/etc/default/kube-proxy' %}
-{% endif %}
-
-/usr/local/bin/kube-proxy:
-  file.managed:
-    - source: salt://kube-bins/kube-proxy
-    - user: root
-    - group: root
-    - mode: 755
-
-{{ environment_file }}:
-  file.managed:
-    - source: salt://kube-proxy/default
-    - template: jinja
-    - user: root
-    - group: root
-    - mode: 644
-
-kube-proxy:
-  group.present:
-    - system: True
-  user.present:
-    - system: True
-    - gid_from_name: True
-    - shell: /sbin/nologin
-    - home: /var/kube-proxy
-    - require:
-      - group: kube-proxy
-
-{% if pillar.get('is_systemd') %}
-
-{{ pillar.get('systemd_system_path') }}/kube-proxy.service:
-  file.managed:
-    - source: salt://kube-proxy/kube-proxy.service
-    - user: root
-    - group: root
-  cmd.wait:
-    - name: /opt/kubernetes/helpers/services bounce kube-proxy
-    - watch:
-      - file: {{ environment_file }}
-      - file: {{ pillar.get('systemd_system_path') }}/kube-proxy.service
-      - file: /var/lib/kube-proxy/kubeconfig
-
-{% else %}
-
-/etc/init.d/kube-proxy:
-  file.managed:
-    - source: salt://kube-proxy/initd
-    - user: root
-    - group: root
-    - mode: 755
-
-{% endif %}
-
-kube-proxy-service:
-  service.running:
-    - name: kube-proxy
-    - enable: True
-    - watch:
-      - file: {{ environment_file }}
-{% if pillar.get('is_systemd') %}
-      - file: {{ pillar.get('systemd_system_path') }}/kube-proxy.service
-{% else %}
-      - file: /etc/init.d/kube-proxy
-{% endif %}
-      - file: /var/lib/kube-proxy/kubeconfig
-
 /var/lib/kube-proxy/kubeconfig:
   file.managed:
     - source: salt://kube-proxy/kubeconfig
@@ -75,3 +5,29 @@ kube-proxy-service:
     - group: root
     - mode: 400
     - makedirs: true
+
+# kube-proxy in a static pod
+/etc/kubernetes/manifests/kube-proxy.manifest:
+  file.managed:
+    - source: salt://kube-proxy/kube-proxy.manifest
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 644
+    - makedirs: true
+    - dir_mode: 755
+    - require:
+      - service: docker
+      - service: kubelet
+
+/var/log/kube-proxy.log:
+  file.managed:
+    - user: root
+    - group: root
+    - mode: 644
+
+#stop legacy kube-proxy service 
+stop_kube-proxy:
+  service.dead:
+    - name: kube-proxy
+    - enable: None

cluster/saltbase/salt/kube-proxy/initd

@@ -1,130 +0,0 @@
-#!/bin/bash
-#
-### BEGIN INIT INFO
-# Provides:    kube-proxy
-# Required-Start:    $local_fs $network $syslog
-# Required-Stop:
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: The Kubernetes network proxy
-# Description:
-#   The Kubernetes network proxy enables network redirection and
-#   loadbalancing for dynamically placed containers.
-### END INIT INFO
-
-
-# PATH should only include /usr/* if it runs after the mountnfs.sh script
-PATH=/sbin:/usr/sbin:/bin:/usr/bin
-DESC="The Kubernetes network proxy"
-NAME=kube-proxy
-DAEMON=/usr/local/bin/kube-proxy
-DAEMON_ARGS=""
-DAEMON_LOG_FILE=/var/log/$NAME.log
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-DAEMON_USER=root
-
-# Exit if the package is not installed
-[ -x "$DAEMON" ] || exit 0
-
-# Read configuration variable file if it is present
-[ -r /etc/default/$NAME ] && . /etc/default/$NAME
-
-# Define LSB log_* functions.
-# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
-# and status_of_proc is working.
-. /lib/lsb/init-functions
-
-#
-# Function that starts the daemon/service
-#
-do_start()
-{
-        # Avoid a potential race at boot time when both monit and init.d start
-        # the same service
-        PIDS=$(pidof $DAEMON)
-        for PID in ${PIDS}; do
-            kill -9 $PID
-	done
-
-        # Raise the file descriptor limit - we expect to open a lot of sockets!
-        ulimit -n 65536
-
-        # Return
-        #   0 if daemon has been started
-        #   1 if daemon was already running
-        #   2 if daemon could not be started
-        start-stop-daemon --start --quiet --background --no-close \
-                --make-pidfile --pidfile $PIDFILE \
-                --exec $DAEMON -c $DAEMON_USER --test > /dev/null \
-                || return 1
-        start-stop-daemon --start --quiet --background --no-close \
-                --make-pidfile --pidfile $PIDFILE \
-                --exec $DAEMON -c $DAEMON_USER -- \
-                $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \
-                || return 2
-}
-
-#
-# Function that stops the daemon/service
-#
-do_stop()
-{
-        # Return
-        #   0 if daemon has been stopped
-        #   1 if daemon was already stopped
-        #   2 if daemon could not be stopped
-        #   other if a failure occurred
-        start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME
-        RETVAL="$?"
-        [ "$RETVAL" = 2 ] && return 2
-        # Many daemons don't delete their pidfiles when they exit.
-        rm -f $PIDFILE
-        return "$RETVAL"
-}
-
-
-case "$1" in
-  start)
-        log_daemon_msg "Starting $DESC" "$NAME"
-        do_start
-        case "$?" in
-                0|1) log_end_msg 0 || exit 0 ;;
-                2) log_end_msg 1 || exit 1 ;;
-        esac
-        ;;
-  stop)
-        log_daemon_msg "Stopping $DESC" "$NAME"
-        do_stop
-        case "$?" in
-                0|1) log_end_msg 0 ;;
-                2) exit 1 ;;
-        esac
-        ;;
-  status)
-        status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $?
-        ;;
-
-  restart|force-reload)
-        log_daemon_msg "Restarting $DESC" "$NAME"
-        do_stop
-        case "$?" in
-          0|1)
-                do_start
-                case "$?" in
-                        0) log_end_msg 0 ;;
-                        1) log_end_msg 1 ;; # Old process is still running
-                        *) log_end_msg 1 ;; # Failed to start
-                esac
-                ;;
-          *)
-                # Failed to stop
-                log_end_msg 1
-                ;;
-        esac
-        ;;
-  *)
-        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
-        exit 3
-        ;;
-esac

cluster/saltbase/salt/kube-proxy/kube-proxy.manifest

@@ -0,0 +1,54 @@
+{% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%}
+{% if grains.api_servers is defined -%}
+  {% set api_servers = "--master=https://" + grains.api_servers -%}
+{% else -%}
+  {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
+  {% set api_servers = "--master=https://" + ips[0][0] -%}
+{% endif -%}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ]  %}
+  {% set api_servers_with_port = api_servers -%}
+{% else -%}
+  {% set api_servers_with_port = api_servers + ":6443" -%}
+{% endif -%}
+{% set test_args = "" -%}
+{% if pillar['kubeproxy_test_args'] is defined -%}
+  {% set test_args=pillar['kubeproxy_test_args'] %}
+{% endif -%}
+
+# kube-proxy podspec
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-proxy
+    image: gcr.io/google_containers/kube-proxy:{{pillar['kube-proxy_docker_tag']}}
+    command:
+    - /bin/sh
+    - -c
+    - kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{pillar['log_level']}} {{test_args}} 1>>/var/log/kube-proxy.log 2>&1
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: /var/log
+      name: varlog
+      readOnly: false
+    - mountPath: /var/lib/kube-proxy/kubeconfig
+      name: kubeconfig
+      readOnly: false
+  volumes:
+  - hostPath:
+      path: /usr/share/ca-certificates
+    name: ssl-certs-host
+  - hostPath:
+      path: /var/lib/kube-proxy/kubeconfig
+    name: kubeconfig
+  - hostPath:
+      path: /var/log
+    name: varlog

cluster/saltbase/salt/kube-proxy/kube-proxy.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=Kubernetes Kube-Proxy Server
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-
-[Service]
-EnvironmentFile=/etc/sysconfig/kube-proxy
-ExecStart=/usr/local/bin/kube-proxy "$DAEMON_ARGS"
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

cluster/saltbase/salt/supervisor/init.sls

@@ -52,30 +52,6 @@ monit:
     - mode: 755
     - makedirs: True
 
-{% if "kubernetes-pool" in grains.get('roles', []) %}
-/etc/supervisor/conf.d/kube-proxy.conf:
-  file:
-    - managed
-    - source: salt://supervisor/kube-proxy.conf
-    - user: root
-    - group: root
-    - mode: 644
-    - makedirs: True
-    - require_in: 
-      - pkg: supervisor
-    - require: 
-      - file: /usr/sbin/kube-proxy-checker.sh
-
-/usr/sbin/kube-proxy-checker.sh:
-  file:
-    - managed
-    - source: salt://supervisor/kube-proxy-checker.sh
-    - user: root
-    - group: root
-    - mode: 755
-    - makedirs: True
-{% endif %}
-
 {% if grains['roles'][0] == 'kubernetes-master' -%}
 /etc/supervisor/conf.d/kube-addons.conf:
   file:

cluster/saltbase/salt/top.sls

@@ -16,6 +16,7 @@ base:
     - helpers
     - cadvisor
     - kube-client-tools
+    - kube-node-unpacker
     - kubelet
 {% if pillar.get('network_provider', '').lower() == 'opencontrail' %}
     - opencontrail-networking-minion

hack/lib/golang.sh

@@ -129,6 +129,7 @@ readonly KUBE_STATIC_LIBRARIES=(
   kube-apiserver
   kube-controller-manager
   kube-scheduler
+  kube-proxy
 )
 
 kube::golang::is_statically_linked_library() {