From 0df6f5d8c5689e9c28e695460da4f8e755627cb4 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 17 Jan 2016 19:20:34 -0500 Subject: [PATCH 1/5] kube-up: move yaml-quote from GCE & AWS to common --- cluster/aws/coreos/util.sh | 3 --- cluster/common.sh | 9 +++++++++ cluster/gce/util.sh | 9 --------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cluster/aws/coreos/util.sh b/cluster/aws/coreos/util.sh index c58144bb02c..602cb5e2514 100644 --- a/cluster/aws/coreos/util.sh +++ b/cluster/aws/coreos/util.sh @@ -53,6 +53,3 @@ function check-minion() { echo "working" } -function yaml-quote { - echo "'$(echo "${@}" | sed -e "s/'/''/g")'" -} diff --git a/cluster/common.sh b/cluster/common.sh index 1e059c790d0..895e51ee0a7 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -410,3 +410,12 @@ function stage-images() { rm -rf "${temp_dir}" return 0 } + +# Quote something appropriate for a yaml string. +# +# TODO(zmerlynn): Note that this function doesn't so much "quote" as +# "strip out quotes", and we really should be using a YAML library for +# this, but PyYAML isn't shipped by default, and *rant rant rant ... SIGH* +function yaml-quote { + echo "'$(echo "${@}" | sed -e "s/'/''/g")'" +} diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 3e4c7061abb..8da57ef6866 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -465,15 +465,6 @@ function add-instance-metadata-from-file { done } -# Quote something appropriate for a yaml string. -# -# TODO(zmerlynn): Note that this function doesn't so much "quote" as -# "strip out quotes", and we really should be using a YAML library for -# this, but PyYAML isn't shipped by default, and *rant rant rant ... SIGH* -function yaml-quote { - echo "'$(echo "${@}" | sed -e "s/'/''/g")'" -} - function write-master-env { # If the user requested that the master be part of the cluster, set the # environment variable to program the master kubelet to register itself. From ae75a7d2594002e1924e8ccbcdead90ff14d4731 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 17 Jan 2016 19:24:03 -0500 Subject: [PATCH 2/5] kube-up: move build-runtime-config from AWS & GCE into common --- cluster/aws/util.sh | 22 ---------------------- cluster/common.sh | 22 ++++++++++++++++++++++ cluster/gce/util.sh | 22 ---------------------- 3 files changed, 22 insertions(+), 44 deletions(-) diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 66be11c09db..7ec9248e8c3 100755 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -1519,25 +1519,3 @@ function get-tokens() { KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) } - -# Builds the RUNTIME_CONFIG var from other feature enable options -function build-runtime-config() { - if [[ "${ENABLE_DEPLOYMENTS}" == "true" ]]; then - if [[ -z "${RUNTIME_CONFIG}" ]]; then - RUNTIME_CONFIG="extensions/v1beta1/deployments=true" - else - if echo "${RUNTIME_CONFIG}" | grep -q -v "extensions/v1beta1/deployments=true"; then - RUNTIME_CONFIG="${RUNTIME_CONFIG},extensions/v1beta1/deployments=true" - fi - fi - fi - if [[ "${ENABLE_DAEMONSETS}" == "true" ]]; then - if [[ -z "${RUNTIME_CONFIG}" ]]; then - RUNTIME_CONFIG="extensions/v1beta1/daemonsets=true" - else - if echo "${RUNTIME_CONFIG}" | grep -q -v "extensions/v1beta1/daemonsets=true"; then - RUNTIME_CONFIG="${RUNTIME_CONFIG},extensions/v1beta1/daemonsets=true" - fi - fi - fi -} diff --git a/cluster/common.sh b/cluster/common.sh index 895e51ee0a7..4bc7ed7e2e9 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -419,3 +419,25 @@ function stage-images() { function yaml-quote { echo "'$(echo "${@}" | sed -e "s/'/''/g")'" } + +# Builds the RUNTIME_CONFIG var from other feature enable options +function build-runtime-config() { + if [[ "${ENABLE_DEPLOYMENTS}" == "true" ]]; then + if [[ -z "${RUNTIME_CONFIG}" ]]; then + RUNTIME_CONFIG="extensions/v1beta1/deployments=true" + else + if echo "${RUNTIME_CONFIG}" | grep -q -v "extensions/v1beta1/deployments=true"; then + RUNTIME_CONFIG="${RUNTIME_CONFIG},extensions/v1beta1/deployments=true" + fi + fi + fi + if [[ "${ENABLE_DAEMONSETS}" == "true" ]]; then + if [[ -z "${RUNTIME_CONFIG}" ]]; then + RUNTIME_CONFIG="extensions/v1beta1/daemonsets=true" + else + if echo "${RUNTIME_CONFIG}" | grep -q -v "extensions/v1beta1/daemonsets=true"; then + RUNTIME_CONFIG="${RUNTIME_CONFIG},extensions/v1beta1/daemonsets=true" + fi + fi + fi +} diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 8da57ef6866..43c12eb9033 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1327,28 +1327,6 @@ function prepare-e2e() { detect-project } -# Builds the RUNTIME_CONFIG var from other feature enable options -function build-runtime-config() { - if [[ "${ENABLE_DEPLOYMENTS}" == "true" ]]; then - if [[ -z "${RUNTIME_CONFIG}" ]]; then - RUNTIME_CONFIG="extensions/v1beta1/deployments=true" - else - if echo "${RUNTIME_CONFIG}" | grep -q -v "extensions/v1beta1/deployments=true"; then - RUNTIME_CONFIG="${RUNTIME_CONFIG},extensions/v1beta1/deployments=true" - fi - fi - fi - if [[ "${ENABLE_DAEMONSETS}" == "true" ]]; then - if [[ -z "${RUNTIME_CONFIG}" ]]; then - RUNTIME_CONFIG="extensions/v1beta1/daemonsets=true" - else - if echo "${RUNTIME_CONFIG}" | grep -q -v "extensions/v1beta1/daemonsets=true"; then - RUNTIME_CONFIG="${RUNTIME_CONFIG},extensions/v1beta1/daemonsets=true" - fi - fi - fi -} - # $1: if 'true', we're building a master yaml, else a node function build-kube-env { local master=$1 From a70412099a187660d80899478f82110265c6d73f Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 17 Jan 2016 21:10:25 -0500 Subject: [PATCH 3/5] AWS: add retries around apt-get install Related to issue #17309 --- cluster/aws/templates/common.sh | 22 ++++++++++++++-------- cluster/aws/templates/format-disks.sh | 10 ++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/cluster/aws/templates/common.sh b/cluster/aws/templates/common.sh index 3e3d2c4d8c4..20be255e216 100644 --- a/cluster/aws/templates/common.sh +++ b/cluster/aws/templates/common.sh @@ -14,9 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +echo "== Refreshing package database ==" +until apt-get update; do + echo "== apt-get update failed, retrying ==" + sleep 5 +done -apt-get update -apt-get install --yes curl +function apt-get-install { + # Forcibly install packages (options borrowed from Salt logs). + until apt-get -q -y -o DPkg::Options::=--force-confold -o DPkg::Options::=--force-confdef install $@; do + echo "== install of packages $@ failed, retrying ==" + sleep 5 + done +} + +apt-get-install curl # Retry a download until we get it. # @@ -44,12 +56,6 @@ install-salt() { return fi - echo "== Refreshing package database ==" - until apt-get update; do - echo "== apt-get update failed, retrying ==" - sleep 5 - done - mkdir -p /var/cache/salt-install cd /var/cache/salt-install diff --git a/cluster/aws/templates/format-disks.sh b/cluster/aws/templates/format-disks.sh index 9f3c7b9a978..1d67816a039 100644 --- a/cluster/aws/templates/format-disks.sh +++ b/cluster/aws/templates/format-disks.sh @@ -47,8 +47,6 @@ done move_docker="" move_kubelet="" -apt-get update - docker_storage=${DOCKER_STORAGE:-aufs} # Format the ephemeral disks @@ -66,7 +64,7 @@ else done if [[ ${docker_storage} == "btrfs" ]]; then - apt-get install --yes btrfs-tools + apt-get-install btrfs-tools if [[ ${#block_devices[@]} == 1 ]]; then echo "One ephemeral block device found; formatting with btrfs" @@ -102,7 +100,7 @@ else # In devicemapper mode, Docker can use LVM directly # Also, fewer code paths are good echo "Using LVM2 and ext4" - apt-get install --yes lvm2 + apt-get-install lvm2 # Don't output spurious "File descriptor X leaked on vgcreate invocation." # Known bug: e.g. Ubuntu #591823 @@ -165,10 +163,10 @@ if [[ ${docker_storage} == "btrfs" ]]; then elif [[ ${docker_storage} == "aufs-nolvm" || ${docker_storage} == "aufs" ]]; then # Install aufs kernel module # Fix issue #14162 with extra-virtual - apt-get install --yes linux-image-extra-$(uname -r) linux-image-extra-virtual + apt-get-install linux-image-extra-$(uname -r) linux-image-extra-virtual # Install aufs tools - apt-get install --yes aufs-tools + apt-get-install aufs-tools DOCKER_OPTS="${DOCKER_OPTS} -s aufs" elif [[ ${docker_storage} == "devicemapper" ]]; then From 4d88a4ed2c76455870abed6541dc83ff5cadb03e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 17 Jan 2016 21:15:05 -0500 Subject: [PATCH 4/5] kube-up: move sha1sum-file into common Also hash the tar files in AWS kube-up --- cluster/aws/util.sh | 5 +++++ cluster/common.sh | 8 ++++++++ cluster/gce/util.sh | 8 -------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 7ec9248e8c3..b2ac5251f11 100755 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -566,10 +566,15 @@ function ensure-temp-dir { # SALT_TAR_URL function upload-server-tars() { SERVER_BINARY_TAR_URL= + SERVER_BINARY_TAR_HASH= SALT_TAR_URL= + SALT_TAR_HASH= ensure-temp-dir + SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}") + SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}") + if [[ -z ${AWS_S3_BUCKET-} ]]; then local project_hash= local key=$(aws configure get aws_access_key_id) diff --git a/cluster/common.sh b/cluster/common.sh index 4bc7ed7e2e9..957439a49b6 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -441,3 +441,11 @@ function build-runtime-config() { fi fi } + +function sha1sum-file() { + if which shasum >/dev/null 2>&1; then + shasum -a1 "$1" | awk '{ print $1 }' + else + sha1sum "$1" | awk '{ print $1 }' + fi +} diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 43c12eb9033..330532cb064 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -115,14 +115,6 @@ function detect-project () { fi } -function sha1sum-file() { - if which shasum >/dev/null 2>&1; then - shasum -a1 "$1" | awk '{ print $1 }' - else - sha1sum "$1" | awk '{ print $1 }' - fi -} - function already-staged() { local -r file=$1 local -r newsum=$2 From c556e8f0f5e75131a8ae909908f550f19de01851 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 17 Jan 2016 21:17:14 -0500 Subject: [PATCH 5/5] kube-up: Refactor kube-env into common; use in AWS Use kube-env on the AWS master also --- cluster/aws/templates/extract-kube-env.sh | 27 ++++ cluster/aws/util.sh | 49 ++---- cluster/common.sh | 180 ++++++++++++++++++++++ cluster/gce/util.sh | 179 --------------------- 4 files changed, 221 insertions(+), 214 deletions(-) create mode 100644 cluster/aws/templates/extract-kube-env.sh diff --git a/cluster/aws/templates/extract-kube-env.sh b/cluster/aws/templates/extract-kube-env.sh new file mode 100644 index 00000000000..5d4152bf612 --- /dev/null +++ b/cluster/aws/templates/extract-kube-env.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apt-get-install python-yaml + +# kube-env has all the environment variables we care about, in a flat yaml format +eval "$(python -c ' +import pipes,sys,yaml + +for k,v in yaml.load(sys.stdin).iteritems(): + print("""readonly {var}={value}""".format(var = k, value = pipes.quote(str(v)))) + print("""export {var}""".format(var = k)) + ' < kube-env.yaml)" + diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index b2ac5251f11..908022e9502 100755 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -53,6 +53,9 @@ if [[ "${KUBE_OS_DISTRIBUTION}" == "ubuntu" ]]; then KUBE_OS_DISTRIBUTION=vivid fi +# For GCE script compatability +OS_DISTRIBUTION=${KUBE_OS_DISTRIBUTION} + case "${KUBE_OS_DISTRIBUTION}" in trusty|wheezy|jessie|vivid|coreos) source "${KUBE_ROOT}/cluster/aws/${KUBE_OS_DISTRIBUTION}/util.sh" @@ -926,48 +929,24 @@ function start-master() { service_ip=$(echo "${octets[*]}" | sed 's/ /./g') MASTER_EXTRA_SANS="IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN},DNS:${MASTER_NAME}" + write-master-env ( # We pipe this to the ami as a startup script in the user-data field. Requires a compatible ami echo "#! /bin/bash" echo "mkdir -p /var/cache/kubernetes-install" echo "cd /var/cache/kubernetes-install" - echo "readonly SALT_MASTER='${MASTER_INTERNAL_IP}'" - echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'" - echo "readonly NODE_INSTANCE_PREFIX='${NODE_INSTANCE_PREFIX}'" - echo "readonly NON_MASQUERADE_CIDR='${NON_MASQUERADE_CIDR:-}'" - echo "readonly CLUSTER_IP_RANGE='${CLUSTER_IP_RANGE}'" - echo "readonly ALLOCATE_NODE_CIDRS='${ALLOCATE_NODE_CIDRS}'" - echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'" - echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" - echo "readonly ZONE='${ZONE}'" - echo "readonly NUM_NODES='${NUM_NODES}'" - echo "readonly KUBE_USER='${KUBE_USER}'" - echo "readonly KUBE_PASSWORD='${KUBE_PASSWORD}'" - echo "readonly SERVICE_CLUSTER_IP_RANGE='${SERVICE_CLUSTER_IP_RANGE}'" - echo "readonly ENABLE_CLUSTER_MONITORING='${ENABLE_CLUSTER_MONITORING:-none}'" - echo "readonly ENABLE_CLUSTER_LOGGING='${ENABLE_CLUSTER_LOGGING:-false}'" - echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'" - echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'" - echo "readonly ELASTICSEARCH_LOGGING_REPLICAS='${ELASTICSEARCH_LOGGING_REPLICAS:-}'" - echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'" - echo "readonly ENABLE_CLUSTER_UI='${ENABLE_CLUSTER_UI:-false}'" - echo "readonly RUNTIME_CONFIG='${RUNTIME_CONFIG}'" - echo "readonly DNS_REPLICAS='${DNS_REPLICAS:-}'" - echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'" - echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'" - echo "readonly ADMISSION_CONTROL='${ADMISSION_CONTROL:-}'" - echo "readonly MASTER_IP_RANGE='${MASTER_IP_RANGE:-}'" - echo "readonly KUBELET_TOKEN='${KUBELET_TOKEN}'" - echo "readonly KUBE_PROXY_TOKEN='${KUBE_PROXY_TOKEN}'" - echo "readonly DOCKER_STORAGE='${DOCKER_STORAGE:-}'" - echo "readonly MASTER_EXTRA_SANS='${MASTER_EXTRA_SANS:-}'" - echo "readonly NETWORK_PROVIDER='${NETWORK_PROVIDER:-}'" - echo "readonly OPENCONTRAIL_TAG='${OPENCONTRAIL_TAG:-}'" - echo "readonly OPENCONTRAIL_KUBERNETES_TAG='${OPENCONTRAIL_KUBERNETES_TAG:-}'" - echo "readonly OPENCONTRAIL_PUBLIC_SUBNET='${OPENCONTRAIL_PUBLIC_SUBNET:-}'" - echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'" + + echo "cat > kube-env.yaml << __EOF_MASTER_KUBE_ENV_YAML" + cat ${KUBE_TEMP}/master-kube-env.yaml + # TODO: get rid of these exceptions / harmonize with common or GCE + echo "SALT_MASTER: $(yaml-quote ${MASTER_INTERNAL_IP:-})" + echo "DOCKER_STORAGE: $(yaml-quote ${DOCKER_STORAGE:-})" + echo "MASTER_EXTRA_SANS: $(yaml-quote ${MASTER_EXTRA_SANS:-})" + echo "__EOF_MASTER_KUBE_ENV_YAML" + grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh" + grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/extract-kube-env.sh" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/setup-master-pd.sh" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/create-dynamic-salt-files.sh" diff --git a/cluster/common.sh b/cluster/common.sh index 957439a49b6..69d0945493c 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -442,6 +442,186 @@ function build-runtime-config() { fi } +function write-master-env { + # If the user requested that the master be part of the cluster, set the + # environment variable to program the master kubelet to register itself. + if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then + KUBELET_APISERVER="${MASTER_NAME}" + fi + + build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" +} + +function write-node-env { + build-kube-env false "${KUBE_TEMP}/node-kube-env.yaml" +} + +# $1: if 'true', we're building a master yaml, else a node +function build-kube-env { + local master=$1 + local file=$2 + + build-runtime-config + + rm -f ${file} + cat >$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file </dev/null 2>&1; then shasum -a1 "$1" | awk '{ print $1 }' diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 330532cb064..b7a60452477 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -457,20 +457,6 @@ function add-instance-metadata-from-file { done } -function write-master-env { - # If the user requested that the master be part of the cluster, set the - # environment variable to program the master kubelet to register itself. - if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then - KUBELET_APISERVER="${MASTER_NAME}" - fi - - build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" -} - -function write-node-env { - build-kube-env false "${KUBE_TEMP}/node-kube-env.yaml" -} - # Create certificate pairs for the cluster. # $1: The public IP for the master. # @@ -1318,168 +1304,3 @@ function restart-apiserver { function prepare-e2e() { detect-project } - -# $1: if 'true', we're building a master yaml, else a node -function build-kube-env { - local master=$1 - local file=$2 - - build-runtime-config - - rm -f ${file} - cat >$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <>$file <