diff --git a/build/common.sh b/build/common.sh index 601afdcd7bd..9b2d5678d7e 100644 --- a/build/common.sh +++ b/build/common.sh @@ -786,8 +786,8 @@ function kube::release::gcs::copy_release_artifacts() { # Having the "template" scripts from the GCE cluster deploy hosted with the # release is useful for GKE. Copy everything from that directory up also. gsutil -m "${gcs_options[@]+${gcs_options[@]}}" cp \ - "${RELEASE_STAGE}/full/kubernetes/cluster/gce/templates/*.sh" \ - "${gcs_destination}extra/gce-templates/" + "${RELEASE_STAGE}/full/kubernetes/cluster/gce/configure-vm.sh" \ + "${gcs_destination}extra/gce/" # Upload the "naked" binaries to GCS. This is useful for install scripts that # download the binaries directly and don't need tars. diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh new file mode 100644 index 00000000000..47a1140b317 --- /dev/null +++ b/cluster/gce/configure-vm.sh @@ -0,0 +1,353 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# If we have any arguments at all, this is a push and not just setup. +is_push=$@ + +function ensure-install-dir() { + INSTALL_DIR="/var/cache/kubernetes-install" + mkdir -p ${INSTALL_DIR} + cd ${INSTALL_DIR} +} + +function set-broken-motd() { + echo -e '\nBroken (or in progress) GCE Kubernetes node setup! Suggested first step:\n tail /var/log/startupscript.log\n' > /etc/motd +} + +function set-good-motd() { + echo -e '\n=== GCE Kubernetes node setup complete ===\n' > /etc/motd +} + +function curl-metadata() { + curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/attributes/${1}" +} + +function set-kube-env() { + local kube_env_yaml="${INSTALL_DIR}/kube_env.yaml" + + until curl-metadata kube-env > "${kube_env_yaml}"; do + echo 'Waiting for kube-env...' + sleep 3 + done + + # kube-env has all the environment variables we care about, in a flat yaml format + eval $(python -c ''' +import pipes,sys,yaml + +for k,v in yaml.load(sys.stdin).iteritems(): + print "readonly {var}={value}".format(var = k, value = pipes.quote(str(v))) +''' < "${kube_env_yaml}") + + # We bake the KUBELET_TOKEN in separately to avoid auth information + # having to be re-communicated on kube-push. (Otherwise the client + # has to keep the bearer token around to handle generating a valid + # kube-env.) + if [[ -z "${KUBELET_TOKEN:-}" ]]; then + until KUBELET_TOKEN=$(curl-metadata kube-token); do + echo 'Waiting for metadata KUBELET_TOKEN...' + sleep 3 + done + fi + + if [[ "${KUBERNETES_MASTER}" == "true" ]]; then + # TODO(zmerlynn): This block of code should disappear once #4561 & #4562 are done + if [[ -z "${KUBERNETES_NODE_NAMES:-}" ]]; then + until KUBERNETES_NODE_NAMES=$(curl-metadata kube-node-names); do + echo 'Waiting for metadata KUBERNETES_NODE_NAMES...' + sleep 3 + done + fi + else + # And this should go away once the master can allocate CIDRs + if [[ -z "${MINION_IP_RANGE:-}" ]]; then + until MINION_IP_RANGE=$(curl-metadata node-ip-range); do + echo 'Waiting for metadata MINION_IP_RANGE...' + sleep 3 + done + fi + fi +} + +function remove-docker-artifacts() { + # Remove docker artifacts on minion nodes, if present + iptables -t nat -F || true + ifconfig docker0 down || true + brctl delbr docker0 || true +} + +# Retry a download until we get it. +# +# $1 is the URL to download +download-or-bust() { + local -r url="$1" + local -r file="${url##*/}" + rm -f "$file" + until [[ -e "${1##*/}" ]]; do + echo "Downloading file ($1)" + curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "$1" + done +} + +# Install salt from GCS. See README.md for instructions on how to update these +# debs. +install-salt() { + apt-get update + + mkdir -p /var/cache/salt-install + cd /var/cache/salt-install + + TARS=( + libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb + python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb + salt-common_2014.1.13+ds-1~bpo70+1_all.deb + salt-minion_2014.1.13+ds-1~bpo70+1_all.deb + ) + URL_BASE="https://storage.googleapis.com/kubernetes-release/salt" + + for tar in "${TARS[@]}"; do + download-or-bust "${URL_BASE}/${tar}" + dpkg -i "${tar}" || true + done + + # This will install any of the unmet dependencies from above. + apt-get install -f -y +} + +# Ensure salt-minion *isn't* running +stop-salt-minion() { + # This ensures it on next reboot + echo manual > /etc/init/salt-minion.override + + service salt-minion stop + while service salt-minion status >/dev/null; do + service salt-minion stop # No, really. + echo "Waiting for salt-minion to shut down" + sleep 1 + done +} + +# Mounts a persistent disk (formatting if needed) to store the persistent data +# on the master -- etcd's data, a few settings, and security certs/keys/tokens. +# +# This function can be reused to mount an existing PD because all of its +# operations modifying the disk are idempotent -- safe_format_and_mount only +# formats an unformatted disk, and mkdir -p will leave a directory be if it +# already exists. +mount-master-pd() { + device_info=$(ls -l /dev/disk/by-id/google-master-pd) + relative_path=${device_info##* } + device_path="/dev/disk/by-id/${relative_path}" + + # Format and mount the disk, create directories on it for all of the master's + # persistent data, and link them to where they're used. + mkdir -p /mnt/master-pd + /usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${device_path}" /mnt/master-pd + # Contains all the data stored in etcd + mkdir -m 700 -p /mnt/master-pd/var/etcd + # Contains the dynamically generated apiserver auth certs and keys + mkdir -p /mnt/master-pd/srv/kubernetes + # Contains the cluster's initial config parameters and auth tokens + mkdir -p /mnt/master-pd/srv/salt-overlay + ln -s /mnt/master-pd/var/etcd /var/etcd + ln -s /mnt/master-pd/srv/kubernetes /srv/kubernetes + ln -s /mnt/master-pd/srv/salt-overlay /srv/salt-overlay + + # This is a bit of a hack to get around the fact that salt has to run after the + # PD and mounted directory are already set up. We can't give ownership of the + # directory to etcd until the etcd user and group exist, but they don't exist + # until salt runs if we don't create them here. We could alternatively make the + # permissions on the directory more permissive, but this seems less bad. + useradd -s /sbin/nologin -d /var/etcd etcd + chown etcd /mnt/master-pd/var/etcd + chgrp etcd /mnt/master-pd/var/etcd +} + +# Create the overlay files for the salt tree. We create these in a separate +# place so that we can blow away the rest of the salt configs on a kube-push and +# re-apply these. +function create-salt-pillar() { + # Always overwrite the cluster-params.sls (even on a push, we have + # these variables) + mkdir -p /srv/salt-overlay/pillar + cat </srv/salt-overlay/pillar/cluster-params.sls +instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")' +node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")' +portal_net: '$(echo "$PORTAL_NET" | sed -e "s/'/''/g")' +enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")' +enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")' +enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")' +enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")' +logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")' +elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")' +enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")' +dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' +dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' +dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' +EOF + + if [[ "${KUBERNETES_MASTER}" == "true" ]]; then + cat <>/srv/salt-overlay/pillar/cluster-params.sls +gce_node_names: '$(echo "$KUBERNETES_NODE_NAMES" | sed -e "s/'/''/g")' +EOF + fi +} + +# This should only happen on cluster initialization +function create-salt-auth() { + mkdir -p /srv/salt-overlay/salt/nginx + echo "${MASTER_HTPASSWD}" > /srv/salt-overlay/salt/nginx/htpasswd + + mkdir -p /srv/salt-overlay/salt/kube-apiserver + known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv" + (umask 077; + echo "${KUBELET_TOKEN},kubelet,kubelet" > "${known_tokens_file}") + + mkdir -p /srv/salt-overlay/salt/kubelet + kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth" + (umask 077; + echo "{\"BearerToken\": \"${KUBELET_TOKEN}\", \"Insecure\": true }" > "${kubelet_auth_file}") +} + +function download-release() { + echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)" + download-or-bust "$SERVER_BINARY_TAR_URL" + + echo "Downloading binary release tar ($SALT_TAR_URL)" + download-or-bust "$SALT_TAR_URL" + + echo "Unpacking Salt tree" + rm -rf kubernetes + tar xzf "${SALT_TAR_URL##*/}" + + echo "Running release install script" + sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}" +} + +function fix-apt-sources() { + sed -i -e "\|^deb.*http://http.debian.net/debian| s/^/#/" /etc/apt/sources.list + sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d/backports.list +} + +function salt-run-local() { + cat </etc/salt/minion.d/local.conf +file_client: local +file_roots: + base: + - /srv/salt +EOF +} + +function salt-debug-log() { + cat </etc/salt/minion.d/log-level-debug.conf +log_level: debug +log_level_logfile: debug +EOF +} + +function salt-master-role() { + cat </etc/salt/minion.d/grains.conf +grains: + roles: + - kubernetes-master + cloud: gce +EOF +} + +function salt-node-role() { + cat </etc/salt/minion.d/grains.conf +grains: + roles: + - kubernetes-pool + cbr-cidr: '$(echo "$MINION_IP_RANGE" | sed -e "s/'/''/g")' + cloud: gce +EOF +} + +function salt-docker-opts() { + DOCKER_OPTS="" + + if [[ -n "${EXTRA_DOCKER_OPTS-}" ]]; then + DOCKER_OPTS="${EXTRA_DOCKER_OPTS}" + fi + + # Decide whether to enable the cache + if [[ "${ENABLE_DOCKER_REGISTRY_CACHE}" == "true" ]]; then + REGION=$(echo "${ZONE}" | cut -f 1,2 -d -) + echo "Enable docker registry cache at region: " $REGION + DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror='https://${REGION}.docker-cache.clustermaster.net'" + fi + + if [[ -n "{DOCKER_OPTS}" ]]; then + cat <>/etc/salt/minion.d/grains.conf + docker_opts: '$(echo "$DOCKER_OPTS" | sed -e "s/'/''/g")' +EOF + fi +} + +function salt-set-apiserver() { + cat <>/etc/salt/minion.d/grains.conf + api_servers: '${KUBERNETES_MASTER_IP}' + apiservers: '${KUBERNETES_MASTER_IP}' +EOF +} + +function configure-salt() { + fix-apt-sources + mkdir -p /etc/salt/minion.d + salt-run-local + if [[ "${KUBERNETES_MASTER}" == "true" ]]; then + salt-master-role + else + salt-node-role + salt-docker-opts + salt-set-apiserver + fi + install-salt + stop-salt-minion +} + +function run-salt() { + salt-call --local state.highstate || true +} + +#################################################################################### + +if [[ -z "${is_push}" ]]; then + echo "== kube-up node config starting ==" + set-broken-motd + ensure-install-dir + set-kube-env + [[ "${KUBERNETES_MASTER}" == "true" ]] && mount-master-pd + create-salt-pillar + create-salt-auth + download-release + configure-salt + remove-docker-artifacts + run-salt + set-good-motd + echo "== kube-up node config done ==" +else + echo "== kube-push node config starting ==" + ensure-install-dir + set-kube-env + create-salt-pillar + run-salt + echo "== kube-push node config done ==" +fi diff --git a/cluster/gce/kube-env.py b/cluster/gce/kube-env.py new file mode 100755 index 00000000000..a48c8799269 --- /dev/null +++ b/cluster/gce/kube-env.py @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import yaml + +def mutate_env(path, var, value): + # Load the existing arguments + if os.path.exists(path): + args = yaml.load(open(path)) + else: + args = {} + args[var] = value + yaml.dump(args, stream=open(path, 'w'), default_flow_style=False) + +if __name__ == "__main__": + mutate_env(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/cluster/gce/templates/README.md b/cluster/gce/templates/README.md deleted file mode 100644 index 967e0ffe3db..00000000000 --- a/cluster/gce/templates/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Updating Salt debs - -We are caching all of the salt debs in GCS for speed and reliability. - -To update them, follow this simple N step process: - -1. Start up a new base image without salt installed. SSH into this image. -2. Install salt via their recommended method: `curl -L https://bootstrap.saltstack.com | sudo sh -s -- -M -X` -3. Find and download the debs that originated at the saltstack.com repo: `aptitude search --disable-columns -F "%p %V" "?installed?origin(saltstack.com)" | xargs aptitude download` -4. Upload these to GCS: `gsutil cp *.deb gs://kubernetes-release/salt/` -5. Make sure that everything is publicly readable: `gsutil acl ch -R -g all:R gs://kubernetes-release/salt/` -6. Test things well :) diff --git a/cluster/gce/templates/common.sh b/cluster/gce/templates/common.sh deleted file mode 100644 index c2b0dde76eb..00000000000 --- a/cluster/gce/templates/common.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Retry a download until we get it. -# -# $1 is the URL to download -download-or-bust() { - local -r url="$1" - local -r file="${url##*/}" - rm -f "$file" - until [[ -e "${1##*/}" ]]; do - echo "Downloading file ($1)" - curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "$1" - md5sum "$file" - done -} - -# Install salt from GCS. See README.md for instructions on how to update these -# debs. -# -# $1 If set to --master, also install the master -install-salt() { - apt-get update - - mkdir -p /var/cache/salt-install - cd /var/cache/salt-install - - TARS=( - libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb - python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb - salt-common_2014.1.13+ds-1~bpo70+1_all.deb - salt-minion_2014.1.13+ds-1~bpo70+1_all.deb - ) - if [[ ${1-} == '--master' ]]; then - TARS+=(salt-master_2014.1.13+ds-1~bpo70+1_all.deb) - fi - URL_BASE="https://storage.googleapis.com/kubernetes-release/salt" - - for tar in "${TARS[@]}"; do - download-or-bust "${URL_BASE}/${tar}" - dpkg -i "${tar}" - done - - # This will install any of the unmet dependencies from above. - apt-get install -f -y - -} diff --git a/cluster/gce/templates/create-dynamic-salt-files.sh b/cluster/gce/templates/create-dynamic-salt-files.sh deleted file mode 100644 index 85c3153968a..00000000000 --- a/cluster/gce/templates/create-dynamic-salt-files.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create the overlay files for the salt tree. We create these in a separate -# place so that we can blow away the rest of the salt configs on a kube-push and -# re-apply these. - -mkdir -p /srv/salt-overlay/pillar -cat </srv/salt-overlay/pillar/cluster-params.sls -instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")' -node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")' -portal_net: '$(echo "$PORTAL_NET" | sed -e "s/'/''/g")' -enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")' -enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")' -enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")' -enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")' -logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")' -elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")' -enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")' -dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' -dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' -dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' -EOF - -mkdir -p /srv/salt-overlay/salt/nginx -echo $MASTER_HTPASSWD > /srv/salt-overlay/salt/nginx/htpasswd - -# Generate and distribute a shared secret (bearer token) to -# apiserver and kubelet so that kubelet can authenticate to -# apiserver to send events. -# This works on CoreOS, so it should work on a lot of distros. -kubelet_token=$(cat /dev/urandom | base64 | tr -d "=+/" | dd bs=32 count=1 2> /dev/null) - -mkdir -p /srv/salt-overlay/salt/kube-apiserver -known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv" -(umask u=rw,go= ; echo "$kubelet_token,kubelet,kubelet" > $known_tokens_file) - -mkdir -p /srv/salt-overlay/salt/kubelet -kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth" -(umask u=rw,go= ; echo "{\"BearerToken\": \"$kubelet_token\", \"Insecure\": true }" > $kubelet_auth_file) diff --git a/cluster/gce/templates/download-release.sh b/cluster/gce/templates/download-release.sh deleted file mode 100755 index 5fadefdb26c..00000000000 --- a/cluster/gce/templates/download-release.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Download and install release - -# This script assumes that the environment variable MASTER_RELEASE_TAR contains -# the release tar to download and unpack. It is meant to be pushed to the -# master and run. - - -echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)" -download-or-bust "$SERVER_BINARY_TAR_URL" - -echo "Downloading binary release tar ($SALT_TAR_URL)" -download-or-bust "$SALT_TAR_URL" - -echo "Unpacking Salt tree" -rm -rf kubernetes -tar xzf "${SALT_TAR_URL##*/}" - -echo "Running release install script" -sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}" diff --git a/cluster/gce/templates/mount-pd.sh b/cluster/gce/templates/mount-pd.sh deleted file mode 100755 index 7233ed1f2a0..00000000000 --- a/cluster/gce/templates/mount-pd.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Mounts a persistent disk (formatting if needed) to store the persistent data -# on the master -- etcd's data, a few settings, and security certs/keys/tokens. -# -# This script can be reused to mount an existing PD because all of its -# operations modifying the disk are idempotent -- safe_format_and_mount only -# formats an unformatted disk, and mkdir -p will leave a directory be if it -# already exists. - -device_info=$(ls -l /dev/disk/by-id/google-master-pd) -relative_path=${device_info##* } -device_path="/dev/disk/by-id/${relative_path}" - -# Format and mount the disk, create directories on it for all of the master's -# persistent data, and link them to where they're used. -mkdir -p /mnt/master-pd -/usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${device_path}" /mnt/master-pd -# Contains all the data stored in etcd -mkdir -m 700 -p /mnt/master-pd/var/etcd -# Contains the dynamically generated apiserver auth certs and keys -mkdir -p /mnt/master-pd/srv/kubernetes -# Contains the cluster's initial config parameters and auth tokens -mkdir -p /mnt/master-pd/srv/salt-overlay -# Contains salt's dynamically generated RSA keys -mkdir -m 770 -p /mnt/master-pd/etc/salt/pki -ln -s /mnt/master-pd/var/etcd /var/etcd -ln -s /mnt/master-pd/srv/kubernetes /srv/kubernetes -ln -s /mnt/master-pd/srv/salt-overlay /srv/salt-overlay -ln -s /mnt/master-pd/etc/salt/pki /etc/salt/pki - -# This is a bit of a hack to get around the fact that salt has to run after the -# PD and mounted directory are already set up. We can't give ownership of the -# directory to etcd until the etcd user and group exist, but they don't exist -# until salt runs if we don't create them here. We could alternatively make the -# permissions on the directory more permissive, but this seems less bad. -useradd -s /sbin/nologin -d /var/etcd etcd -chown etcd /mnt/master-pd/var/etcd -chgrp etcd /mnt/master-pd/var/etcd diff --git a/cluster/gce/templates/salt-master.sh b/cluster/gce/templates/salt-master.sh deleted file mode 100755 index 3eee40eef8a..00000000000 --- a/cluster/gce/templates/salt-master.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -sed -i -e "\|^deb.*http://http.debian.net/debian| s/^/#/" /etc/apt/sources.list -sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d/backports.list - -# Prepopulate the name of the Master -mkdir -p /etc/salt/minion.d -cat </etc/salt/minion.d/master.conf -master: '$(echo "$MASTER_NAME" | sed -e "s/'/''/g")' -EOF - -cat </etc/salt/minion.d/log-level-debug.conf -log_level: debug -log_level_logfile: debug -EOF - -cat </etc/salt/minion.d/grains.conf -grains: - roles: - - kubernetes-master - cloud: gce -EOF - -# Auto accept all keys from minions that try to join -mkdir -p /etc/salt/master.d -cat </etc/salt/master.d/auto-accept.conf -auto_accept: True -EOF - -cat </etc/salt/master.d/reactor.conf -# React to new minions starting by running highstate on them. -reactor: - - 'salt/minion/*/start': - - /srv/reactor/highstate-new.sls -EOF - -cat </etc/salt/master.d/log-level-debug.d -log_level: debug -log_level_logfile: debug -EOF - -install-salt --master - -# Wait a few minutes and trigger another Salt run to better recover from -# any transient errors. -echo "Sleeping 180" -sleep 180 -salt-call state.highstate || true - diff --git a/cluster/gce/templates/salt-minion.sh b/cluster/gce/templates/salt-minion.sh deleted file mode 100755 index 5c57fef9915..00000000000 --- a/cluster/gce/templates/salt-minion.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The repositories are really slow and there are GCE mirrors -sed -i -e "\|^deb.*http://http.debian.net/debian| s/^/#/" /etc/apt/sources.list -sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d/backports.list - -# Prepopulate the name of the Master -mkdir -p /etc/salt/minion.d -cat </etc/salt/minion.d/master.conf -master: '$(echo "$MASTER_NAME" | sed -e "s/'/''/g")' -EOF - -cat </etc/salt/minion.d/log-level-debug.conf -log_level: debug -log_level_logfile: debug -EOF - -# Our minions will have a pool role to distinguish them from the master. -cat </etc/salt/minion.d/grains.conf -grains: - roles: - - kubernetes-pool - cbr-cidr: '$(echo "$MINION_IP_RANGE" | sed -e "s/'/''/g")' - cloud: gce -EOF - -DOCKER_OPTS="" - -if [[ -n "${EXTRA_DOCKER_OPTS-}" ]]; then - DOCKER_OPTS="${EXTRA_DOCKER_OPTS}" -fi - -# Decide if enable the cache -if [[ "${ENABLE_DOCKER_REGISTRY_CACHE}" == "true" ]]; then - REGION=$(echo "${ZONE}" | cut -f 1,2 -d -) - echo "Enable docker registry cache at region: " $REGION - DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror='https://${REGION}.docker-cache.clustermaster.net'" -fi - -if [[ -n "{DOCKER_OPTS}" ]]; then -cat <>/etc/salt/minion.d/grains.conf - docker_opts: '$(echo "$DOCKER_OPTS" | sed -e "s/'/''/g")' -EOF -fi - -install-salt - -# Wait a few minutes and trigger another Salt run to better recover from -# any transient errors. -echo "Sleeping 180" -sleep 180 -salt-call state.highstate || true diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index c6222ea8c9a..d29364e1d6d 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -208,6 +208,7 @@ function detect-minions () { # Vars set: # KUBE_MASTER # KUBE_MASTER_IP +# KUBE_MASTER_IP_INTERNAL function detect-master () { detect-project KUBE_MASTER=${MASTER_NAME} @@ -215,6 +216,9 @@ function detect-master () { KUBE_MASTER_IP=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \ "${MASTER_NAME}" --fields networkInterfaces[0].accessConfigs[0].natIP \ --format=text | awk '{ print $2 }') + KUBE_MASTER_IP_INTERNAL=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \ + "${MASTER_NAME}" --fields networkInterfaces[0].networkIP \ + --format=text | awk '{ print $2 }') fi if [[ -z "${KUBE_MASTER_IP-}" ]]; then echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" >&2 @@ -246,6 +250,14 @@ function get-password { KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))') } +# Set MASTER_HTPASSWD +function set-master-htpasswd { + python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \ + -b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD" + local htpasswd + MASTER_HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd") +} + # Generate authentication token for admin user. Will # read from $HOME/.kubernetes_auth if available. # @@ -332,6 +344,8 @@ function create-route { # $1: The name of the instance template. # $2: The scopes flag. # $3: The minion start script metadata from file. +# $4: The kube-env metadata. +# $5: Raw metadata function create-node-template { detect-project local attempt=0 @@ -347,7 +361,8 @@ function create-node-template { --network "${NETWORK}" \ $2 \ --can-ip-forward \ - --metadata-from-file "$3"; then + --metadata-from-file "$3" "$4" \ + --metadata "$5"; then if (( attempt > 5 )); then echo -e "${color_red}Failed to create instance template $1 ${color_norm}" exit 2 @@ -383,26 +398,94 @@ function add-instance-metadata { done } +# Robustly try to add metadata on an instance, from a file. +# $1: The name of the instace. +# $2: The metadata key=file pair to add. +function add-instance-metadata-from-file { + detect-project + local attempt=0 + while true; do + if ! gcloud compute instances add-metadata "$1" \ + --project "${PROJECT}" \ + --zone "${ZONE}" \ + --metadata-from-file "$2"; then + if (( attempt > 5 )); then + echo -e "${color_red}Failed to add instance metadata in $1 ${color_norm}" + exit 2 + fi + echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in $1. Retrying.${color_norm}" + attempt=$(($attempt+1)) + else + break + fi + done +} + +# Given a yaml file, add or mutate the given env variable +# +# TODO(zmerlynn): Yes, this is an O(n^2) build-up right now. If we end +# up with so many environment variables feeding into Salt that this +# matters, there's probably an issue... +function add-to-env { + ${KUBE_ROOT}/cluster/gce/kube-env.py "$1" "$2" "$3" +} + +# $1: if 'true', we're building a master yaml, else a node +function build-kube-env { + local master=$1 + local file=$2 + + rm -f ${file} + add-to-env ${file} ENV_TIMESTAMP "$(date -uIs)" # Just to track it + add-to-env ${file} KUBERNETES_MASTER "${master}" + add-to-env ${file} INSTANCE_PREFIX "${INSTANCE_PREFIX}" + add-to-env ${file} NODE_INSTANCE_PREFIX "${NODE_INSTANCE_PREFIX}" + add-to-env ${file} SERVER_BINARY_TAR_URL "${SERVER_BINARY_TAR_URL}" + add-to-env ${file} SALT_TAR_URL "${SALT_TAR_URL}" + add-to-env ${file} PORTAL_NET "${PORTAL_NET}" + add-to-env ${file} ENABLE_CLUSTER_MONITORING "${ENABLE_CLUSTER_MONITORING:-false}" + add-to-env ${file} ENABLE_NODE_MONITORING "${ENABLE_NODE_MONITORING:-false}" + add-to-env ${file} ENABLE_CLUSTER_LOGGING "${ENABLE_CLUSTER_LOGGING:-false}" + add-to-env ${file} ENABLE_NODE_LOGGING "${ENABLE_NODE_LOGGING:-false}" + add-to-env ${file} LOGGING_DESTINATION "${LOGGING_DESTINATION:-}" + add-to-env ${file} ELASTICSEARCH_LOGGING_REPLICAS "${ELASTICSEARCH_LOGGING_REPLICAS:-}" + add-to-env ${file} ENABLE_CLUSTER_DNS "${ENABLE_CLUSTER_DNS:-false}" + add-to-env ${file} DNS_REPLICAS "${DNS_REPLICAS:-}" + add-to-env ${file} DNS_SERVER_IP "${DNS_SERVER_IP:-}" + add-to-env ${file} DNS_DOMAIN "${DNS_DOMAIN:-}" + add-to-env ${file} MASTER_HTPASSWD "${MASTER_HTPASSWD}" + if [[ "${master}" != "true" ]]; then + add-to-env ${file} KUBERNETES_MASTER_IP "${KUBE_MASTER_IP_INTERNAL}" + add-to-env ${file} ZONE "${ZONE}" + add-to-env ${file} EXTRA_DOCKER_OPTS "${EXTRA_DOCKER_OPTS}" + add-to-env ${file} ENABLE_DOCKER_REGISTRY_CACHE "${ENABLE_DOCKER_REGISTRY_CACHE:-false}" + fi +} + +function write-master-env { + build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" +} + +function write-node-env { + build-kube-env false "${KUBE_TEMP}/node-kube-env.yaml" +} + # Instantiate a kubernetes cluster # # Assumed vars # KUBE_ROOT # function kube-up { + ensure-temp-dir detect-project + get-password + set-master-htpasswd + # Make sure we have the tar files staged on Google Storage find-release-tars upload-server-tars - ensure-temp-dir - - get-password - python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \ - -b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD" - local htpasswd - htpasswd=$(cat "${KUBE_TEMP}/htpasswd") - if ! gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" &>/dev/null; then echo "Creating new network: ${NETWORK}" # The network needs to be created synchronously or we have a race. The @@ -426,7 +509,7 @@ function kube-up { --allow "tcp:22" & fi - echo "Starting VMs and configuring firewalls" + echo "Starting master and configuring firewalls" gcloud compute firewall-rules create "${MASTER_NAME}-https" \ --project "${PROJECT}" \ --network "${NETWORK}" \ @@ -440,34 +523,13 @@ function kube-up { --zone "${ZONE}" \ --size "10GB" - ( - echo "#! /bin/bash" - echo "mkdir -p /var/cache/kubernetes-install" - echo "cd /var/cache/kubernetes-install" - echo "readonly MASTER_NAME='${MASTER_NAME}'" - echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'" - echo "readonly NODE_INSTANCE_PREFIX='${NODE_INSTANCE_PREFIX}'" - echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'" - echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" - echo "readonly MASTER_HTPASSWD='${htpasswd}'" - echo "readonly PORTAL_NET='${PORTAL_NET}'" - echo "readonly ENABLE_CLUSTER_MONITORING='${ENABLE_CLUSTER_MONITORING:-false}'" - echo "readonly ENABLE_NODE_MONITORING='${ENABLE_NODE_MONITORING:-false}'" - echo "readonly ENABLE_CLUSTER_LOGGING='${ENABLE_CLUSTER_LOGGING:-false}'" - echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'" - echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'" - echo "readonly ELASTICSEARCH_LOGGING_REPLICAS='${ELASTICSEARCH_LOGGING_REPLICAS:-}'" - echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'" - echo "readonly DNS_REPLICAS='${DNS_REPLICAS:-}'" - echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'" - echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'" - grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" - grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/mount-pd.sh" - grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh" - grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh" - grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh" - ) > "${KUBE_TEMP}/master-start.sh" + # Generate a bearer token for this cluster. We push this separately + # from the other cluster variables so that the client (this + # computer) can forget it later. This should disappear with + # https://github.com/GoogleCloudPlatform/kubernetes/issues/3168 + KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + write-master-env gcloud compute instances create "${MASTER_NAME}" \ --project "${PROJECT}" \ --zone "${ZONE}" \ @@ -477,7 +539,9 @@ function kube-up { --tags "${MASTER_TAG}" \ --network "${NETWORK}" \ --scopes "storage-ro" "compute-rw" \ - --metadata-from-file "startup-script=${KUBE_TEMP}/master-start.sh" \ + --metadata-from-file \ + "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" \ + "kube-env=${KUBE_TEMP}/master-kube-env.yaml" \ --disk name="${MASTER_NAME}-pd" device-name=master-pd mode=rw boot=no auto-delete=no & # Create a single firewall rule for all minions. @@ -492,8 +556,12 @@ function kube-up { fi fi - # Wait for last batch of jobs. + # Wait for last batch of jobs wait-for-jobs + detect-master # We need the KUBE_MASTER_IP_INTERNAL for the node startup script + add-instance-metadata "${MASTER_NAME}" "kube-token=${KUBELET_TOKEN}" + + echo "Creating minions." local -a scope_flags=() if (( "${#MINION_SCOPES[@]}" > 0 )); then @@ -502,29 +570,11 @@ function kube-up { scope_flags=("--no-scopes") fi - ( - echo "#! /bin/bash" - echo "ZONE='${ZONE}'" - echo "MASTER_NAME='${MASTER_NAME}'" - echo "until MINION_IP_RANGE=\$(curl --fail --silent -H 'Metadata-Flavor: Google'\\" - echo " http://metadata/computeMetadata/v1/instance/attributes/node-ip-range); do" - echo " echo 'Waiting for metadata MINION_IP_RANGE...'" - echo " sleep 3" - echo "done" - echo "" - echo "# Remove docker artifacts on minion nodes" - echo "iptables -t nat -F" - echo "ifconfig docker0 down" - echo "brctl delbr docker0" - echo "" - echo "EXTRA_DOCKER_OPTS='${EXTRA_DOCKER_OPTS}'" - echo "ENABLE_DOCKER_REGISTRY_CACHE='${ENABLE_DOCKER_REGISTRY_CACHE:-false}'" - grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" - grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-minion.sh" - ) > "${KUBE_TEMP}/minion-start.sh" - + write-node-env create-node-template "${NODE_INSTANCE_PREFIX}-template" "${scope_flags[*]}" \ - "startup-script=${KUBE_TEMP}/minion-start.sh" + "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" \ + "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ + "kube-token=${KUBELET_TOKEN}" gcloud preview managed-instance-groups --zone "${ZONE}" \ create "${NODE_INSTANCE_PREFIX}-group" \ @@ -537,7 +587,17 @@ function kube-up { # to gcloud's deficiency. wait-for-minions-to-run + # Give the master an initial node list (it's waiting in + # startup). This resolves a bit of a chicken-egg issue: The minions + # need to know the master's ip, so we boot the master first. The + # master still needs to know the initial minion list (until all the + # pieces #156 are complete), so we have it wait on the minion + # boot. (The minions further wait until the loop below, where CIDRs + # get filled in.) detect-minion-names + local kube_node_names + kube_node_names=$(IFS=,; echo "${MINION_NAMES[*]}") + add-instance-metadata "${MASTER_NAME}" "kube-node-names=${kube_node_names}" # Create the routes and set IP ranges to instance metadata, 5 instances at a time. for (( i=0; i<${#MINION_NAMES[@]}; i++)); do @@ -550,11 +610,10 @@ function kube-up { fi done + # Wait for last batch of jobs. wait-for-jobs - detect-master - # Reserve the master's IP so that it can later be transferred to another VM # without disrupting the kubelets. IPs are associated with regions, not zones, # so extract the region name, which is the same as the zone but with the final @@ -701,6 +760,14 @@ function kube-down { --delete-disks all \ --zone "${ZONE}" \ "${MASTER_NAME}" || true + + # Delete the master pd (possibly leaked by kube-up if master create failed) + gcloud compute disks delete \ + --project "${PROJECT}" \ + --quiet \ + --zone "${ZONE}" \ + "${MASTER_NAME}"-pd || true + # Find out what minions are running. local -a minions minions=( $(gcloud compute instances list \ @@ -756,27 +823,49 @@ function kube-down { # Update a kubernetes cluster with latest source function kube-push { + OUTPUT=${KUBE_ROOT}/_output/logs + mkdir -p ${OUTPUT} + + ensure-temp-dir detect-project detect-master + detect-minion-names + get-password + set-master-htpasswd # Make sure we have the tar files staged on Google Storage find-release-tars upload-server-tars - ( - echo "#! /bin/bash" - echo "mkdir -p /var/cache/kubernetes-install" - echo "cd /var/cache/kubernetes-install" - echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'" - echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" - grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" - grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh" - echo "echo Executing configuration" - echo "sudo salt '*' mine.update" - echo "sudo salt --force-color '*' state.highstate" - ) | gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "$KUBE_MASTER" --command "sudo bash" + write-master-env + add-instance-metadata-from-file "${KUBE_MASTER}" "kube-env=${KUBE_TEMP}/master-kube-env.yaml" + echo "Pushing to master (log at ${OUTPUT}/kube-push-${KUBE_MASTER}.log) ..." + cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${KUBE_MASTER}" --command "sudo bash -s -- --push" &> ${OUTPUT}/kube-push-"${KUBE_MASTER}".log - get-password + echo "Pushing metadata to minions... " + write-node-env + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + add-instance-metadata-from-file "${MINION_NAMES[$i]}" "kube-env=${KUBE_TEMP}/node-kube-env.yaml" & + done + wait-for-jobs + echo "Done" + + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + echo "Starting push to node (log at ${OUTPUT}/kube-push-${MINION_NAMES[$i]}.log) ..." + cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${MINION_NAMES[$i]}" --command "sudo bash -s -- --push" &> ${OUTPUT}/kube-push-"${MINION_NAMES[$i]}".log & + done + + echo -n "Waiting for node pushes... " + wait-for-jobs + echo "Done" + + # TODO(zmerlynn): Re-create instance-template with the new + # node-kube-env. This isn't important until the node-ip-range issue + # is solved (because that's blocking automatic dynamic nodes from + # working). The node-kube-env has to be composed with the kube-token + # metadata. Ideally we would have + # https://github.com/GoogleCloudPlatform/kubernetes/issues/3168 + # implemented before then, though, so avoiding this mess until then. echo echo "Kubernetes cluster is running. The master is running at:" @@ -785,7 +874,6 @@ function kube-push { echo echo "The user name and password to use is located in ~/.kubernetes_auth." echo - } # ----------------------------------------------------------------------------- diff --git a/cluster/saltbase/pillar/mine.sls b/cluster/saltbase/pillar/mine.sls index f0147a5d032..874dafc3e35 100644 --- a/cluster/saltbase/pillar/mine.sls +++ b/cluster/saltbase/pillar/mine.sls @@ -1,3 +1,6 @@ +# On GCE, there is no Salt mine. We run standalone. +{% if grains.cloud != 'gce' -%} + # Allow everyone to see cached values of who sits at what IP {% set networkInterfaceName = "eth0" %} {% if grains.networkInterfaceName is defined %} @@ -6,3 +9,5 @@ mine_functions: network.ip_addrs: [{{networkInterfaceName}}] grains.items: [] + +{% endif -%} diff --git a/cluster/saltbase/salt/kube-apiserver/default b/cluster/saltbase/salt/kube-apiserver/default index 3391ccb67b3..bedbb68c42d 100644 --- a/cluster/saltbase/salt/kube-apiserver/default +++ b/cluster/saltbase/salt/kube-apiserver/default @@ -22,6 +22,9 @@ {% if grains.etcd_servers is defined -%} {% set etcd_servers = "--etcd_servers=http://" + grains.etcd_servers + ":4001" -%} +{% elif grains.cloud == 'gce' -%} + {# TODO(zmerlynn): I can't see why this isn't generally applicable, but making this change surgical for now. #} + {% set etcd_servers = "--etcd_servers=http://127.0.0.1:4001" -%} {% else -%} {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%} {% set etcd_servers = "--etcd_servers=http://" + ips[0][0] + ":4001" -%} diff --git a/cluster/saltbase/salt/kube-controller-manager/default b/cluster/saltbase/salt/kube-controller-manager/default index 7dcdddba765..8f86939ef7f 100644 --- a/cluster/saltbase/salt/kube-controller-manager/default +++ b/cluster/saltbase/salt/kube-controller-manager/default @@ -21,7 +21,7 @@ {% if grains.cloud is defined -%} {% if grains.cloud == 'gce' -%} {% set cloud_provider = "--cloud_provider=gce" -%} - {% set machines = "--machines=" + ','.join(salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', expr_form='grain').keys()) -%} + {% set machines = "--machines=" + pillar['gce_node_names'] -%} {% endif -%} {% if grains.cloud == 'aws' -%} {% set cloud_provider = "--cloud_provider=aws" -%}