diff --git a/build/common.sh b/build/common.sh index 7492dbed8ff..38996f73895 100755 --- a/build/common.sh +++ b/build/common.sh @@ -905,9 +905,7 @@ function kube::release::package_kube_manifests_tarball() { cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${dst_dir}" cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${dst_dir}" cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${dst_dir}" - cp "${salt_dir}/kube-addons/namespace.yaml" "${dst_dir}" - cp "${salt_dir}/kube-addons/kube-addons.sh" "${dst_dir}" - cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${dst_dir}" + cp "${salt_dir}/kube-addons/kube-addon-manager.yaml" "${dst_dir}" cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}" cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}" local objects diff --git a/cluster/addons/addon-manager/.gitignore b/cluster/addons/addon-manager/.gitignore new file mode 100644 index 00000000000..4eb4f5f7b24 --- /dev/null +++ b/cluster/addons/addon-manager/.gitignore @@ -0,0 +1 @@ +kubectl diff --git a/cluster/addons/addon-manager/Dockerfile b/cluster/addons/addon-manager/Dockerfile new file mode 100644 index 00000000000..09653ab99a8 --- /dev/null +++ b/cluster/addons/addon-manager/Dockerfile @@ -0,0 +1,24 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:2.7-slim + +RUN pip install pyyaml + +ADD kube-addons.sh /opt/ +ADD kube-addon-update.sh /opt/ +ADD namespace.yaml /opt/ +ADD kubectl /usr/local/bin/kubectl + +CMD /opt/kube-addons.sh diff --git a/cluster/addons/addon-manager/Makefile b/cluster/addons/addon-manager/Makefile new file mode 100644 index 00000000000..bdd5f2a8ec0 --- /dev/null +++ b/cluster/addons/addon-manager/Makefile @@ -0,0 +1,34 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +IMAGE=gcr.io/google-containers/kube-addon-manager +VERSION=v1 +KUBECTL_VERSION=v1.2.3 + +.PHONY: build push container + +build: kubectl + docker build -t "$(IMAGE):$(VERSION)" . + +kubectl: + curl "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ + -o kubectl + chmod +x kubectl + +push: build + gcloud docker push "$(IMAGE):$(VERSION)" + +clean: + rm kubectl + docker rmi -f "$(IMAGE):$(VERSION)" diff --git a/cluster/saltbase/salt/kube-addons/kube-addon-update.sh b/cluster/addons/addon-manager/kube-addon-update.sh similarity index 99% rename from cluster/saltbase/salt/kube-addons/kube-addon-update.sh rename to cluster/addons/addon-manager/kube-addon-update.sh index 5f20bffa7c1..dcaf3518c69 100755 --- a/cluster/saltbase/salt/kube-addons/kube-addon-update.sh +++ b/cluster/addons/addon-manager/kube-addon-update.sh @@ -98,7 +98,7 @@ function log() { function get-object-kind-from-file() { # prints to stdout, so log cannot be used #WARNING: only yaml is supported - cat $1 | ${PYTHON} -c ''' + cat $1 | python -c ''' try: import pipes,sys,yaml y = yaml.load(sys.stdin) @@ -120,7 +120,7 @@ function get-object-nsname-from-file() { # prints to stdout, so log cannot be used #WARNING: only yaml is supported #addons that do not specify a namespace are assumed to be in "default". - cat $1 | ${PYTHON} -c ''' + cat $1 | python -c ''' try: import pipes,sys,yaml y = yaml.load(sys.stdin) diff --git a/cluster/saltbase/salt/kube-addons/kube-addons.sh b/cluster/addons/addon-manager/kube-addons.sh old mode 100644 new mode 100755 similarity index 89% rename from cluster/saltbase/salt/kube-addons/kube-addons.sh rename to cluster/addons/addon-manager/kube-addons.sh index 9607aaf330e..c532a1d46b2 --- a/cluster/saltbase/salt/kube-addons/kube-addons.sh +++ b/cluster/addons/addon-manager/kube-addons.sh @@ -19,21 +19,11 @@ # managed result is of that. Start everything below that directory. KUBECTL=${KUBECTL_BIN:-/usr/local/bin/kubectl} -ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-600} +ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-10} SYSTEM_NAMESPACE=kube-system trusty_master=${TRUSTY_MASTER:-false} -function ensure_python() { - if ! python --version > /dev/null 2>&1; then - echo "No python on the machine, will use a python image" - local -r PYTHON_IMAGE=gcr.io/google_containers/python:v1 - export PYTHON="docker run --interactive --rm --net=none ${PYTHON_IMAGE} python" - else - export PYTHON=python - fi -} - # $1 filename of addon to start. # $2 count of tries to start the addon. # $3 delay in seconds between two consecutive tries @@ -74,10 +64,8 @@ function create-resource-from-string() { # managed result is of that. Start everything below that directory. echo "== Kubernetes addon manager started at $(date -Is) with ADDON_CHECK_INTERVAL_SEC=${ADDON_CHECK_INTERVAL_SEC} ==" -ensure_python - # Create the namespace that will be used to host the cluster-level add-ons. -start_addon /etc/kubernetes/addons/namespace.yaml 100 10 "" & +start_addon /opt/namespace.yaml 100 10 "" & # Wait for the default service account to be created in the kube-system namespace. token_found="" diff --git a/cluster/saltbase/salt/kube-addons/namespace.yaml b/cluster/addons/addon-manager/namespace.yaml similarity index 100% rename from cluster/saltbase/salt/kube-addons/namespace.yaml rename to cluster/addons/addon-manager/namespace.yaml diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index 229dcba8051..d3d0551fb6e 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -679,13 +679,6 @@ prepare_kube_addons() { setup_addon_manifests "admission-controls" "limit-range" fi - # Prepare the scripts for running addons. - addon_script_dir="/var/lib/cloud/scripts/kubernetes" - mkdir -p "${addon_script_dir}" - cp "${addon_src_dir}/kube-addons.sh" "${addon_script_dir}" - cp "${addon_src_dir}/kube-addon-update.sh" "${addon_script_dir}" - chmod 544 "${addon_script_dir}/"*.sh - # In case that some GCE customized trusty may have a read-only /root. - mount -t tmpfs tmpfs /root - mount --bind -o remount,rw,noexec /root + # Place addon manager pod manifest + cp "${addon_src_dir}/kube-addon-manager.yaml" /etc/kubernetes/manifests } diff --git a/cluster/gce/trusty/master.yaml b/cluster/gce/trusty/master.yaml index 9fe719289b2..5401951afca 100644 --- a/cluster/gce/trusty/master.yaml +++ b/cluster/gce/trusty/master.yaml @@ -180,38 +180,6 @@ script prepare_kube_addons end script ---==================================== -MIME-Version: 1.0 -Content-Type: text/upstart-job; charset="us-ascii" -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="kube-addons.conf" - -#upstart-job - -description "Run kubernetes addon pods" - -start on stopped kube-master-components - -respawn - -script - set -o errexit - set -o nounset - - . /etc/kube-env - export HOME="/root" - export KUBECTL_BIN="/usr/bin/kubectl" - export TRUSTY_MASTER="true" - if [ -n "${TEST_ADDON_CHECK_INTERVAL_SEC:-}" ]; then - export TEST_ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC} - fi - # Run the script to start and monitoring addon manifest changes. - exec /var/lib/cloud/scripts/kubernetes/kube-addons.sh 1>>/var/log/kube-addons.log 2>&1 -end script - -# Wait for 10s to start it again. -post-stop exec sleep 10 - --==================================== MIME-Version: 1.0 Content-Type: text/upstart-job; charset="us-ascii" diff --git a/cluster/saltbase/salt/kube-addons/init.sls b/cluster/saltbase/salt/kube-addons/init.sls index 3989edd838c..3e138d1ae7c 100644 --- a/cluster/saltbase/salt/kube-addons/init.sls +++ b/cluster/saltbase/salt/kube-addons/init.sls @@ -11,13 +11,6 @@ addon-dir-create: - require: - file: addon-dir-delete -/etc/kubernetes/addons/namespace.yaml: - file.managed: - - source: salt://kube-addons/namespace.yaml - - user: root - - group: root - - file_mode: 644 - {% if pillar.get('enable_cluster_monitoring', '').lower() == 'influxdb' %} /etc/kubernetes/addons/cluster-monitoring/influxdb: file.recurse: @@ -156,63 +149,9 @@ addon-dir-create: - file_mode: 644 {% endif %} -/etc/kubernetes/kube-addons.sh: +/etc/kubernetes/manifests/kube-addon-manager.yaml: file.managed: - - source: salt://kube-addons/kube-addons.sh + - source: salt://kube-addons/kube-addon-manager.yaml - user: root - group: root - mode: 755 - -/etc/kubernetes/kube-addon-update.sh: - file.managed: - - source: salt://kube-addons/kube-addon-update.sh - - user: root - - group: root - - mode: 755 - -{% if pillar.get('is_systemd') %} - -{{ pillar.get('systemd_system_path') }}/kube-addons.service: - file.managed: - - source: salt://kube-addons/kube-addons.service - - user: root - - group: root - cmd.wait: - - name: /opt/kubernetes/helpers/services bounce kube-addons - - watch: - - file: {{ pillar.get('systemd_system_path') }}/kube-addons.service - -{% else %} - -/etc/init.d/kube-addons: - file.managed: - - source: salt://kube-addons/initd - - user: root - - group: root - - mode: 755 - -{% endif %} - -# Stop kube-addons service each time salt is executed, just in case -# there was a modification of addons. -# Actually, this should be handled by watching file changes, but -# somehow it doesn't work. -service-kube-addon-stop: - service.dead: - - name: kube-addons - -kube-addons: - service.running: - - enable: True - - require: - - service: service-kube-addon-stop - - watch: -{% if pillar.get('is_systemd') %} - - file: {{ pillar.get('systemd_system_path') }}/kube-addons.service -{% else %} - - file: /etc/init.d/kube-addons -{% endif %} -{% if pillar.get('is_systemd') %} - - provider: - - service: systemd -{%- endif %} diff --git a/cluster/saltbase/salt/kube-addons/initd b/cluster/saltbase/salt/kube-addons/initd deleted file mode 100644 index c9abf854b2b..00000000000 --- a/cluster/saltbase/salt/kube-addons/initd +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash -# -### BEGIN INIT INFO -# Provides: kube-addons -# Required-Start: $local_fs $network $syslog kube-apiserver -# Required-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Kubernetes Addon Object Manager -# Description: -# Enforces installation of Kubernetes Addon Objects -### END INIT INFO - - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="Kubernetes Addon Object Manager" -NAME=kube-addons -DAEMON_LOG_FILE=/var/log/${NAME}.log -PIDFILE=/var/run/${NAME}.pid -SCRIPTNAME=/etc/init.d/${NAME} -KUBE_ADDONS_SH=/etc/kubernetes/kube-addons.sh - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.2-14) to ensure that this file is present -# and status_of_proc is working. -. /lib/lsb/init-functions - - - - -# -# Function that starts the daemon/service -# -do_start() -{ - # use setsid to make sure the new daemon has its own group (I suppose - # start-stop-daemon does create a process group, but let's stay on the - # safe side). - setsid start-stop-daemon --start --verbose --background --no-close --make-pidfile --pidfile "${PIDFILE}" --startas "${KUBE_ADDONS_SH}" > ${DAEMON_LOG_FILE} 2>&1 -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # start-stop-daemon is not used because we have to stop all children - # limitations: - # - stop does not work if the pid file is missing - # - stop does not work if the daemon process is missing (children will not - # be killed) - # This is sufficient - remaining processes will end after a while. - - local pid - pid=$(cat "${PIDFILE}" 2> /dev/null) - if [[ $? != 0 ]]; then - return 1 - fi - local pgrp - # find the process group for the service and kill entire group - # o - output format: pgpg - process group - pgrp=$(ps --no-headers --pid "${pid}" -o pgrp 2>/dev/null) - if [[ $? != 0 ]] || [[ "${pgrp}" == "" ]]; then - return 1 - fi - pgrp=$(echo -e ${pgrp}) # strip whitespaces (that's why there are no quotes around pgrp) - # negative pid is for killing entire group - kill -- -${pgrp} 2> /dev/null - if [[ $? != 0 ]]; then - return 2 - fi - rm -f "${PIDFILE}" - return -} - -case "$1" in - start) - log_daemon_msg "Starting ${DESC}" "${NAME}" - do_start - case "$?" in - 0|1) log_end_msg 0 || exit 0 ;; - 2) log_end_msg 1 || exit 1 ;; - esac - ;; - stop) - log_daemon_msg "Stopping ${DESC}" "${NAME}" - do_stop - case "$?" in - 0|1) log_end_msg 0 || exit 0 ;; - 2) log_end_msg 1 || exit 1 ;; - esac - ;; - status) - status_of_proc -p "${PIDFILE}" "${KUBE_ADDONS_SH}" "${NAME}" - ;; - - restart|force-reload) - log_daemon_msg "Restarting ${DESC}" "${NAME}" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - *) - echo "Usage: ${SCRIPTNAME} {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac diff --git a/cluster/saltbase/salt/kube-addons/kube-addon-manager.yaml b/cluster/saltbase/salt/kube-addons/kube-addon-manager.yaml new file mode 100644 index 00000000000..d46b9888117 --- /dev/null +++ b/cluster/saltbase/salt/kube-addons/kube-addon-manager.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-addon-manager + namespace: kube-system + version: v1 +spec: + hostNetwork: true + containers: + - name: kube-addon-manager + image: gcr.io/google-containers/kube-addon-manager:v1 + resources: + requests: + cpu: 5m + memory: 50Mi + volumeMounts: + - mountPath: /etc/kubernetes/ + name: addons + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/ + name: addons diff --git a/cluster/saltbase/salt/kube-addons/kube-addons.service b/cluster/saltbase/salt/kube-addons/kube-addons.service deleted file mode 100644 index e51e199c693..00000000000 --- a/cluster/saltbase/salt/kube-addons/kube-addons.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Kubernetes Addon Object Manager -Documentation=https://github.com/kubernetes/kubernetes - -[Service] -Environment=HOME=/root -ExecStart=/etc/kubernetes/kube-addons.sh - -[Install] -WantedBy=multi-user.target diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index a6d5a48813b..801f36e2b3b 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -1,4 +1,5 @@ Vagrantfile: node_ip = $node_ips[n] +cluster/addons/addon-manager/kube-addons.sh:# Create admission_control objects if defined before any other addon services. If the limits cluster/addons/registry/images/Dockerfile:ADD run_proxy.sh /usr/bin/run_proxy cluster/addons/registry/images/Dockerfile:CMD ["/usr/bin/run_proxy"] cluster/aws/templates/configure-vm-aws.sh: # We set the hostname_override to the full EC2 private dns name @@ -31,7 +32,6 @@ cluster/photon-controller/util.sh: node_ip=$(${PHOTON} vm networks "${node_id cluster/photon-controller/util.sh: local cert_dir="/srv/kubernetes" cluster/photon-controller/util.sh: node_name=${1} cluster/rackspace/util.sh: local node_ip=$(nova show --minimal ${NODE_NAMES[$i]} \ -cluster/saltbase/salt/kube-addons/kube-addons.sh:# Create admission_control objects if defined before any other addon services. If the limits cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout -%} cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + terminated_pod_gc + " " + cloud_provider + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%} diff --git a/test/e2e/addon_update.go b/test/e2e/addon_update.go index 341459d2253..af8ab6f746b 100644 --- a/test/e2e/addon_update.go +++ b/test/e2e/addon_update.go @@ -207,37 +207,10 @@ var _ = framework.KubeDescribe("Addon update", func() { var err error sshClient, err = getMasterSSHClient() Expect(err).NotTo(HaveOccurred()) - - // Reduce the addon update intervals so that we have faster response - // to changes in the addon directory. - // do not use "service" command because it clears the environment variables - switch framework.TestContext.OSDistro { - case "debian": - sshExecAndVerify(sshClient, "sudo TEST_ADDON_CHECK_INTERVAL_SEC=1 /etc/init.d/kube-addons restart") - case "trusty", "gci": - sshExecAndVerify(sshClient, "sudo initctl restart kube-addons TEST_ADDON_CHECK_INTERVAL_SEC=1") - case "coreos": - sshExecAndVerify(sshClient, "sudo systemctl set-environment TEST_ADDON_CHECK_INTERVAL_SEC=1") - sshExecAndVerify(sshClient, "sudo systemctl restart kubernetes-addons") - default: - framework.Failf("Unsupported OS distro type %s", framework.TestContext.OSDistro) - } }) AfterEach(func() { if sshClient != nil { - // restart addon_update with the default options - switch framework.TestContext.OSDistro { - case "debian": - sshExec(sshClient, "sudo /etc/init.d/kube-addons restart") - case "trusty", "gci": - sshExec(sshClient, "sudo initctl restart kube-addons") - case "coreos": - sshExec(sshClient, "sudo systemctl unset-environment TEST_ADDON_CHECK_INTERVAL_SEC") - sshExec(sshClient, "sudo systemctl restart kubernetes-addons") - default: - framework.Failf("Unsupported OS distro type %s", framework.TestContext.OSDistro) - } sshClient.Close() } })