From b70f1c12206466333f2f61baf25c67078da57e29 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 18 Feb 2016 09:43:09 -0500 Subject: [PATCH 1/2] kube-up: install updates & reboot automatically on boot We run unattened-upgrades manually, and then reboot automatically if we find /var/run/reboot-required; then we check if any services need restarting and restart them automatically using the needrestart tool. This should mean we don't _have_ to build new images on every security update, though we can do so to avoid a reboot. Issue #21382 --- cluster/gce/configure-vm.sh | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index bf850c58e45..b3750ed6487 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -180,6 +180,50 @@ apt-get-update() { done } +# Restart any services that need restarting due to a library upgrade +# Uses needrestart +restart-updated-services() { + if [[ "${AUTO_RESTART_SERVICES:-auto}" == "no" ]]; then + echo "Auto restart of services prevented by AUTO_RESTART_SERVICES=${AUTO_RESTART_SERVICES}" + return + fi + echo "Restarting services with updated libraries (needrestart -r a)" + # The pipes make sure that needrestart doesn't think it is running with a TTY + # Debian bug #803249; fixed but not necessarily in package repos yet + echo "" | needrestart -r a 2>&1 | tee /dev/null +} + +# Reboot the machine if /var/run/reboot-required exists +reboot-if-required() { + if [[ ! -e "/var/run/reboot-required" ]]; then + return + fi + + echo "Reboot is required (/var/run/reboot-required detected)" + if [[ -e "/var/run/reboot-required.pkgs" ]]; then + echo "Packages that triggered reboot:" + cat /var/run/reboot-required.pkgs + fi + + if [[ "${AUTO_REBOOT:-auto}" == "no" ]]; then + echo "Reboot prevented by AUTO_REBOOT=${AUTO_REBOOT}" + return + fi + + rm -f /var/run/reboot-required + rm -f /var/run/reboot-required.pkgs + echo "Triggering reboot" + init 6 +} + +# Install upgrades using unattended-upgrades, then reboot or restart services +auto-upgrade() { + apt-get-install unattended-upgrades needrestart + unattended-upgrade --debug + reboot-if-required # We may reboot the machine right here + restart-updated-services +} + # # Install salt from GCS. See README.md for instructions on how to update these # debs. @@ -794,6 +838,9 @@ if [[ -z "${is_push}" ]]; then ensure-install-dir ensure-packages set-kube-env + if [[ "${AUTO_UPGRADE:-auto}" != "no" ]]; then + auto-upgrade + fi ensure-local-disks [[ "${KUBERNETES_MASTER}" == "true" ]] && mount-master-pd create-salt-pillar From 8b122b102ff531b83b26c6acde40e3869478bc44 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sat, 27 Feb 2016 20:43:51 -0500 Subject: [PATCH 2/2] kube-up: Only upgrade if AUTO_UPGRADE set, but set it on AWS Installing upgrades is kind of dangerous, so make it not the default. But also set AUTO_UPGRADE=true on AWS. --- cluster/aws/util.sh | 2 ++ cluster/gce/configure-vm.sh | 15 ++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 7052cdb285d..f1fc5113e4e 100755 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -996,6 +996,7 @@ function start-master() { echo "cat > kube_env.yaml << __EOF_MASTER_KUBE_ENV_YAML" cat ${KUBE_TEMP}/master-kube-env.yaml + echo "AUTO_UPGRADE: 'true'" # TODO: get rid of these exceptions / harmonize with common or GCE echo "DOCKER_STORAGE: $(yaml-quote ${DOCKER_STORAGE:-})" echo "API_SERVERS: $(yaml-quote ${MASTER_INTERNAL_IP:-})" @@ -1090,6 +1091,7 @@ function start-minions() { echo "cd /var/cache/kubernetes-install" echo "cat > kube_env.yaml << __EOF_KUBE_ENV_YAML" cat ${KUBE_TEMP}/node-kube-env.yaml + echo "AUTO_UPGRADE: 'true'" # TODO: get rid of these exceptions / harmonize with common or GCE echo "DOCKER_STORAGE: $(yaml-quote ${DOCKER_STORAGE:-})" echo "API_SERVERS: $(yaml-quote ${MASTER_INTERNAL_IP:-})" diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index b3750ed6487..e129c16ad43 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -183,7 +183,8 @@ apt-get-update() { # Restart any services that need restarting due to a library upgrade # Uses needrestart restart-updated-services() { - if [[ "${AUTO_RESTART_SERVICES:-auto}" == "no" ]]; then + # We default to restarting services, because this is only done as part of an update + if [[ "${AUTO_RESTART_SERVICES:-true}" != "true" ]]; then echo "Auto restart of services prevented by AUTO_RESTART_SERVICES=${AUTO_RESTART_SERVICES}" return fi @@ -205,7 +206,8 @@ reboot-if-required() { cat /var/run/reboot-required.pkgs fi - if [[ "${AUTO_REBOOT:-auto}" == "no" ]]; then + # We default to rebooting the machine because this is only done as part of an update + if [[ "${AUTO_REBOOT:-true}" != "true" ]]; then echo "Reboot prevented by AUTO_REBOOT=${AUTO_REBOOT}" return fi @@ -218,6 +220,11 @@ reboot-if-required() { # Install upgrades using unattended-upgrades, then reboot or restart services auto-upgrade() { + # We default to not installing upgrades + if [[ "${AUTO_UPGRADE:-false}" != "true" ]]; then + echo "AUTO_UPGRADE not set to true; won't auto-upgrade" + return + fi apt-get-install unattended-upgrades needrestart unattended-upgrade --debug reboot-if-required # We may reboot the machine right here @@ -838,9 +845,7 @@ if [[ -z "${is_push}" ]]; then ensure-install-dir ensure-packages set-kube-env - if [[ "${AUTO_UPGRADE:-auto}" != "no" ]]; then - auto-upgrade - fi + auto-upgrade ensure-local-disks [[ "${KUBERNETES_MASTER}" == "true" ]] && mount-master-pd create-salt-pillar