From 5817ca1c7114f83f137e0641d0ced8d41f13e10b Mon Sep 17 00:00:00 2001
From: Yifan Gu <yifan.gu@coreos.com>
Date: Fri, 13 Nov 2015 13:15:32 -0800
Subject: [PATCH] cluster/gce/coreos: Add scripts for configuring the
 master/node.

---
 cluster/gce/coreos/configure-kubelet.sh |  33 +++
 cluster/gce/coreos/configure-node.sh    | 331 ++++++++++++++++++++++++
 2 files changed, 364 insertions(+)
 create mode 100755 cluster/gce/coreos/configure-kubelet.sh
 create mode 100644 cluster/gce/coreos/configure-node.sh

diff --git a/cluster/gce/coreos/configure-kubelet.sh b/cluster/gce/coreos/configure-kubelet.sh
new file mode 100755
index 00000000000..bc5ad667ed8
--- /dev/null
+++ b/cluster/gce/coreos/configure-kubelet.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+MANIFESTS_DIR=/opt/kube-manifests/kubernetes
+
+echo "Configuring hostname"
+hostnamectl set-hostname $(hostname | cut -f1 -d.)
+
+echo "Configuring kubelet"
+mkdir -p /var/lib/kubelet
+mkdir -p /etc/kubernetes/manifests
+src=${MANIFESTS_DIR}/kubelet-config.yaml
+dst=/var/lib/kubelet/kubeconfig
+cp ${src} ${dst}
+sed -i 's/\"/\\\"/g' ${dst} # eval will remove the double quotes if they are not escaped
+eval "echo \"$(< ${dst})\"" > ${dst}
diff --git a/cluster/gce/coreos/configure-node.sh b/cluster/gce/coreos/configure-node.sh
new file mode 100644
index 00000000000..7a4fa1ac177
--- /dev/null
+++ b/cluster/gce/coreos/configure-node.sh
@@ -0,0 +1,331 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+readonly KNOWN_TOKENS_FILE="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
+readonly BASIC_AUTH_FILE="/srv/salt-overlay/salt/kube-apiserver/basic_auth.csv"
+
+# evaluate-manifest evalutes the source manifest with the environment variables.
+function evaluate-manifest() {
+  local src=$1
+  local dst=$2
+  cp ${src} ${dst}
+  sed -i 's/\"/\\\"/g' ${dst} # eval will remove the double quotes if they are not escaped
+  eval "echo \"$(< ${dst})\"" > ${dst}
+}
+
+# evaluate-manifests-dir evalutes the source manifests within $1 and put the result
+# in $2.
+function evaluate-manifests-dir() {
+  local src=$1
+  local dst=$2
+  mkdir -p ${dst}
+
+  for f in ${src}/*
+  do
+    evaluate-manifest $f ${dst}/${f##*/}
+  done
+}
+
+function configure-kube-proxy() {
+  echo "Configuring kube-proxy"
+  mkdir -p /var/lib/kube-proxy
+  evaluate-manifest ${MANIFESTS_DIR}/kubeproxy-config.yaml /var/lib/kube-proxy/kubeconfig
+}
+
+function configure-logging() {
+  if [[ "${LOGGING_DESTINATION}" == "gcp" ]];then
+    echo "Configuring fluentd-gcp"
+    # fluentd-gcp
+    evaluate-manifest ${MANIFESTS_DIR}/fluentd-gcp.yaml /etc/kubernetes/manifests/fluentd-gcp.yaml
+  elif [[ "${LOGGING_DESTINATION}" == "elasticsearch" ]];then
+    echo "Configuring fluentd-es"
+    # fluentd-es
+    evaluate-manifest ${MANIFESTS_DIR}/fluentd-es.yaml /etc/kubernetes/manifests/fluentd-es.yaml
+  fi
+}
+
+function configure-admission-controls() {
+  echo "Configuring admission controls"
+  mkdir -p /etc/kubernetes/admission-controls
+  cp -r ${SALT_DIR}/salt/kube-admission-controls/limit-range /etc/kubernetes/admission-controls/
+}
+
+function configure-etcd() {
+  echo "Configuring etcd"
+  touch /var/log/etcd.log
+  evaluate-manifest ${MANIFESTS_DIR}/etcd.yaml /etc/kubernetes/manifests/etcd.yaml
+}
+
+function configure-etcd-events() {
+  echo "Configuring etcd-events"
+  touch /var/log/etcd-events.log
+  evaluate-manifest ${MANIFESTS_DIR}/etcd-events.yaml /etc/kubernetes/manifests/etcd-events.yaml
+}
+
+function configure-kube-apiserver() {
+  echo "Configuring kube-apiserver"
+  
+  # Wait for etcd to be up.
+  wait-url-up http://127.0.0.1:4001/version
+
+  touch /var/log/kube-apiserver.log
+
+  # Copying known_tokens and basic_auth file.
+  cp ${SALT_OVERLAY}/salt/kube-apiserver/*.csv /srv/kubernetes/
+  evaluate-manifest ${MANIFESTS_DIR}/kube-apiserver.yaml /etc/kubernetes/manifests/kube-apiserver.yaml
+}
+
+function configure-kube-scheduler() {
+  echo "Configuring kube-scheduler"
+  touch /var/log/kube-scheduler.log
+  evaluate-manifest ${MANIFESTS_DIR}/kube-scheduler.yaml /etc/kubernetes/manifests/kube-scheduler.yaml
+}
+
+function configure-kube-controller-manager() {
+  # Wait for api server.
+  wait-url-up http://127.0.0.1:8080/version
+  echo "Configuring kube-controller-manager"
+  touch /var/log/kube-controller-manager.log
+  evaluate-manifest ${MANIFESTS_DIR}/kube-controller-manager.yaml /etc/kubernetes/manifests/kube-controller-manager.yaml
+}
+
+# Wait until $1 become reachable.
+function wait-url-up() {
+  until curl --silent $1
+  do
+    sleep 5
+  done
+}
+
+# Configure addon yamls, and run salt/kube-addons/kube-addon.sh
+function configure-master-addons() {
+  echo "Configuring master addons"
+
+  local addon_dir=/etc/kubernetes/addons
+  mkdir -p ${addon_dir}
+
+  # Copy namespace.yaml
+  evaluate-manifest ${MANIFESTS_DIR}/addons/namespace.yaml ${addon_dir}/namespace.yaml
+
+  if [[ "${ENABLE_L7_LOADBALANCING}" == "glbc" ]]; then
+    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/cluster-loadbalancing/glbc ${addon_dir}/cluster-loadbalancing/glbc
+  fi
+
+  if [[ "${ENABLE_CLUSTER_DNS}" == "true" ]]; then
+    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/dns ${addon_dir}/dns
+  fi
+
+  if [[ "${ENABLE_CLUSTER_UI}" == "true" ]]; then
+    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/kube-ui ${addon_dir}/kube-ui
+  fi
+
+  if [[ "${ENABLE_CLUSTER_MONITORING}" == "influxdb" ]]; then
+    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/cluster-monitoring/influxdb  ${addon_dir}/cluster-monitoring/influxdb
+  elif [[ "${ENABLE_CLUSTER_MONITORING}" == "google" ]]; then
+    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/cluster-monitoring/google  ${addon_dir}/cluster-monitoring/google
+  elif [[ "${ENABLE_CLUSTER_MONITORING}" == "standalone" ]]; then
+    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/cluster-monitoring/standalone  ${addon_dir}/cluster-monitoring/standalone
+  elif [[ "${ENABLE_CLUSTER_MONITORING}" == "googleinfluxdb" ]]; then
+    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/cluster-monitoring/googleinfluxdb  ${addon_dir}/cluster-monitoring/googleinfluxdb
+  fi
+
+  # Note that, KUBE_ENABLE_INSECURE_REGISTRY is not supported yet.
+  if [[ "${ENABLE_CLUSTER_REGISTRY}" == "true" ]]; then
+    CLUSTER_REGISTRY_DISK_SIZE=$(convert-bytes-gce-kube "${CLUSTER_REGISTRY_DISK_SIZE}")
+    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/registry  ${addon_dir}/registry
+  fi
+}
+
+function configure-master-components() {
+  configure-admission-controls
+  configure-etcd
+  configure-etcd-events
+  configure-kube-apiserver
+  configure-kube-scheduler
+  configure-kube-controller-manager
+  configure-master-addons
+}
+
+# TODO(yifan): Merge this with mount-master-pd() in configure-vm.sh
+# Pass ${save_format_and_mount} as an argument.
+function mount-master-pd() {
+  if [[ ! -e /dev/disk/by-id/google-master-pd ]]; then
+    return
+  fi
+  device_info=$(ls -l /dev/disk/by-id/google-master-pd)
+  relative_path=${device_info##* }
+  device_path="/dev/disk/by-id/${relative_path}"
+
+  # Format and mount the disk, create directories on it for all of the master's
+  # persistent data, and link them to where they're used.
+  echo "Mounting master-pd"
+  mkdir -p /mnt/master-pd
+  safe_format_and_mount=${SALT_DIR}/salt/helpers/safe_format_and_mount
+  chmod +x ${safe_format_and_mount}
+  ${safe_format_and_mount} -m "mkfs.ext4 -F" "${device_path}" /mnt/master-pd &>/var/log/master-pd-mount.log || \
+    { echo "!!! master-pd mount failed, review /var/log/master-pd-mount.log !!!"; return 1; }
+  # Contains all the data stored in etcd
+  mkdir -m 700 -p /mnt/master-pd/var/etcd
+  # Contains the dynamically generated apiserver auth certs and keys
+  mkdir -p /mnt/master-pd/srv/kubernetes
+  # Contains the cluster's initial config parameters and auth tokens
+  mkdir -p /mnt/master-pd/srv/salt-overlay
+  # Directory for kube-apiserver to store SSH key (if necessary)
+  mkdir -p /mnt/master-pd/srv/sshproxy
+
+  ln -s -f /mnt/master-pd/var/etcd /var/etcd
+  ln -s -f /mnt/master-pd/srv/kubernetes /srv/kubernetes
+  ln -s -f /mnt/master-pd/srv/sshproxy /srv/sshproxy
+  ln -s -f /mnt/master-pd/srv/salt-overlay /srv/salt-overlay
+
+  # This is a bit of a hack to get around the fact that salt has to run after the
+  # PD and mounted directory are already set up. We can't give ownership of the
+  # directory to etcd until the etcd user and group exist, but they don't exist
+  # until salt runs if we don't create them here. We could alternatively make the
+  # permissions on the directory more permissive, but this seems less bad.
+  if ! id etcd &>/dev/null; then
+    useradd -s /sbin/nologin -d /var/etcd etcd
+  fi
+  chown -R etcd /mnt/master-pd/var/etcd
+  chgrp -R etcd /mnt/master-pd/var/etcd
+}
+
+# The job of this function is simple, but the basic regular expression syntax makes
+# this difficult to read. What we want to do is convert from [0-9]+B, KB, KiB, MB, etc
+# into [0-9]+, Ki, Mi, Gi, etc.
+# This is done in two steps:
+#   1. Convert from [0-9]+X?i?B into [0-9]X? (X denotes the prefix, ? means the field
+#      is optional.
+#   2. Attach an 'i' to the end of the string if we find a letter.
+# The two step process is needed to handle the edge case in which we want to convert
+# a raw byte count, as the result should be a simple number (e.g. 5B -> 5).
+#
+# TODO(yifan): Reuse the one defined in configure-vm.sh to remove duplication.
+function convert-bytes-gce-kube() {
+  local -r storage_space=$1
+  echo "${storage_space}" | sed -e 's/^\([0-9]\+\)\([A-Z]\)\?i\?B$/\1\2/g' -e 's/\([A-Z]\)$/\1i/'
+}
+
+# TODO(yifan): Use create-salt-master-auth() in configure-vm.sh
+function create-salt-master-auth() {
+  if [[ ! -e /srv/kubernetes/ca.crt ]]; then
+    if  [[ ! -z "${CA_CERT:-}" ]] && [[ ! -z "${MASTER_CERT:-}" ]] && [[ ! -z "${MASTER_KEY:-}" ]]; then
+      mkdir -p /srv/kubernetes
+      (umask 077;
+        echo "${CA_CERT}" | base64 -d > /srv/kubernetes/ca.crt;
+        echo "${MASTER_CERT}" | base64 -d > /srv/kubernetes/server.cert;
+        echo "${MASTER_KEY}" | base64 -d > /srv/kubernetes/server.key;
+        # Kubecfg cert/key are optional and included for backwards compatibility.
+        # TODO(roberthbailey): Remove these two lines once GKE no longer requires
+        # fetching clients certs from the master VM.
+        echo "${KUBECFG_CERT:-}" | base64 -d > /srv/kubernetes/kubecfg.crt;
+        echo "${KUBECFG_KEY:-}" | base64 -d > /srv/kubernetes/kubecfg.key)
+    fi
+  fi
+  if [ ! -e "${BASIC_AUTH_FILE}" ]; then
+    mkdir -p /srv/salt-overlay/salt/kube-apiserver
+    (umask 077;
+      echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${BASIC_AUTH_FILE}")
+  fi
+  if [ ! -e "${KNOWN_TOKENS_FILE}" ]; then
+    mkdir -p /srv/salt-overlay/salt/kube-apiserver
+    (umask 077;
+      echo "${KUBE_BEARER_TOKEN},admin,admin" > "${KNOWN_TOKENS_FILE}";
+      echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${KNOWN_TOKENS_FILE}";
+      echo "${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy" >> "${KNOWN_TOKENS_FILE}")
+
+    # Generate tokens for other "service accounts".  Append to known_tokens.
+    #
+    # NB: If this list ever changes, this script actually has to
+    # change to detect the existence of this file, kill any deleted
+    # old tokens and add any new tokens (to handle the upgrade case).
+    local -r service_accounts=("system:scheduler" "system:controller_manager" "system:logging" "system:monitoring" "system:dns")
+    for account in "${service_accounts[@]}"; do
+      token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+      echo "${token},${account},${account}" >> "${KNOWN_TOKENS_FILE}"
+    done
+  fi
+}
+
+# $1 is the directory containing all of the docker images
+function load-docker-images() {
+  local success
+  local restart_docker
+  while true; do
+    success=true
+    restart_docker=false
+    for image in "$1/"*; do
+      timeout 30 docker load -i "${image}" &>/dev/null
+      rc=$?
+      if [[ "$rc" == 124 ]]; then
+        restart_docker=true
+      elif [[ "$rc" != 0 ]]; then
+        success=false
+      fi
+    done
+    if [[ "$success" == "true" ]]; then break; fi
+    if [[ "$restart_docker" == "true" ]]; then systemctl restart docker; fi
+    sleep 15
+  done
+}
+
+
+# TODO(yifan): Making this function more generic for other runtimes.
+function load-master-components-images() {
+  echo "Loading docker images for master components"
+  ${SALT_DIR}/install.sh ${KUBE_BIN_TAR}
+  ${SALT_DIR}/salt/kube-master-addons/kube-master-addons.sh
+
+  # Get the image tags.
+  KUBE_APISERVER_DOCKER_TAG=$(cat ${KUBE_BIN_DIR}/kube-apiserver.docker_tag)
+  KUBE_CONTROLLER_MANAGER_DOCKER_TAG=$(cat ${KUBE_BIN_DIR}/kube-controller-manager.docker_tag)
+  KUBE_SCHEDULER_DOCKER_TAG=$(cat ${KUBE_BIN_DIR}/kube-scheduler.docker_tag)
+}
+
+
+##########
+#  main  #
+##########
+
+KUBE_BIN_TAR=/opt/downloads/kubernetes-server-linux-amd64.tar.gz
+KUBE_BIN_DIR=/opt/kubernetes/server/bin
+SALT_DIR=/opt/kubernetes/saltbase
+SALT_OVERLAY=/srv/salt-overlay
+MANIFESTS_DIR=/opt/kube-manifests/kubernetes
+
+# On CoreOS, the hosts is in /usr/share/baselayout/hosts
+# So we need to manually populdate the hosts file here on gce.
+echo "127.0.0.1 localhost" >> /etc/hosts
+echo "::1 localhost" >> /etc/hosts
+
+if [[ "${KUBERNETES_MASTER}" == "true" ]]; then
+  mount-master-pd
+  create-salt-master-auth
+  load-master-components-images
+  configure-master-components
+else
+  configure-kube-proxy
+fi
+
+if [[ "${ENABLE_NODE_LOGGING}" == "true" ]];then
+  configure-logging
+fi
+  
+echo "Finish configuration successfully!"