kubernetes/cluster/vagrant/util.sh

#!/bin/bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts.

KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/vagrant/${KUBE_CONFIG_FILE-"config-default.sh"}"
source "${KUBE_ROOT}/cluster/common.sh"

function detect-master () {
  KUBE_MASTER_IP=$MASTER_IP
  echo "KUBE_MASTER_IP: ${KUBE_MASTER_IP}" 1>&2
}

# Get node IP addresses and store in KUBE_NODE_IP_ADDRESSES[]
function detect-nodes {
  echo "Nodes already detected" 1>&2
  KUBE_NODE_IP_ADDRESSES=("${NODE_IPS[@]}")
}

# Verify prereqs on host machine  Also sets exports USING_KUBE_SCRIPTS=true so
# that our Vagrantfile doesn't error out.
function verify-prereqs {
  for x in vagrant; do
    if ! which "$x" >/dev/null; then
      echo "Can't find $x in PATH, please fix and retry."
      exit 1
    fi
  done

  local vagrant_plugins=$(vagrant plugin list | sed '-es% .*$%%' '-es%  *% %g' | tr ' ' $'\n')
  local providers=(
      # Format is:
      #   provider_ctl_executable vagrant_provider_name vagrant_provider_plugin_re
      # either provider_ctl_executable or vagrant_provider_plugin_re can
      # be blank (i.e., '') if none is needed by Vagrant (see, e.g.,
      # virtualbox entry)
      '' vmware_fusion vagrant-vmware-fusion
      '' vmware_workstation vagrant-vmware-workstation
      prlctl parallels vagrant-parallels
      VBoxManage virtualbox ''
      virsh libvirt vagrant-libvirt
      '' vsphere vagrant-vsphere
  )
  local provider_found=''
  local provider_bin
  local provider_name
  local provider_plugin_re

  while [ "${#providers[@]}" -gt 0 ]; do
    provider_bin=${providers[0]}
    provider_name=${providers[1]}
    provider_plugin_re=${providers[2]}
    providers=("${providers[@]:3}")

    # If the provider is explicitly set, look only for that provider
    if [ -n "${VAGRANT_DEFAULT_PROVIDER:-}" ] \
        && [ "${VAGRANT_DEFAULT_PROVIDER}" != "${provider_name}" ]; then
      continue
    fi

    if ([ -z "${provider_bin}" ] \
          || which "${provider_bin}" >/dev/null 2>&1) \
        && ([ -z "${provider_plugin_re}" ] \
          || [ -n "$(echo "${vagrant_plugins}" | grep -E "^${provider_plugin_re}$")" ]); then
      provider_found="${provider_name}"
      # Stop after finding the first viable provider
      break
    fi
  done

  if [ -z "${provider_found}" ]; then
    if [ -n "${VAGRANT_DEFAULT_PROVIDER:-}" ]; then
      echo "Can't find the necessary components for the ${VAGRANT_DEFAULT_PROVIDER} vagrant provider."
      echo "Possible reasons could be: "
      echo -e "\t- vmrun utility is not in your path"
      echo -e "\t- Vagrant plugin was not found."
      echo -e "\t- VAGRANT_DEFAULT_PROVIDER is set, but not found."
      echo "Please fix and retry."
    else
      echo "Can't find the necessary components for any viable vagrant providers (e.g., virtualbox), please fix and retry."
    fi

    exit 1
  fi

  # Set VAGRANT_CWD to KUBE_ROOT so that we find the right Vagrantfile no
  # matter what directory the tools are called from.
  export VAGRANT_CWD="${KUBE_ROOT}"

  export USING_KUBE_SCRIPTS=true
}

# Create a set of provision scripts for the master and each of the nodes
function create-provision-scripts {
  kube::util::ensure-temp-dir

  (
    echo "#! /bin/bash"
    echo-kube-env
    echo "NODE_IP='${MASTER_IP}'"
    echo "CONTAINER_ADDR='${MASTER_CONTAINER_ADDR}'"
    echo "CONTAINER_NETMASK='${MASTER_CONTAINER_NETMASK}'"
    awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-utils.sh"
    awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-network-master.sh"
    awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-master.sh"
  ) > "${KUBE_TEMP}/master-start.sh"

  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
    (
      echo "#! /bin/bash"
      echo-kube-env
      echo "NODE_NAME=(${NODE_NAMES[$i]})"
      echo "NODE_IP='${NODE_IPS[$i]}'"
      echo "NODE_ID='$i'"
      echo "CONTAINER_ADDR='${NODE_CONTAINER_ADDRS[$i]}'"
      echo "CONTAINER_NETMASK='${NODE_CONTAINER_NETMASKS[$i]}'"
      awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-utils.sh"
      awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-network-node.sh"
      awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-node.sh"
    ) > "${KUBE_TEMP}/node-start-${i}.sh"
  done
}

function echo-kube-env() {
  echo "KUBE_ROOT=/vagrant"
  echo "INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
  echo "MASTER_NAME='${INSTANCE_PREFIX}-master'"
  echo "MASTER_IP='${MASTER_IP}'"
  echo "NODE_NAMES=(${NODE_NAMES[@]})"
  echo "NODE_IPS=(${NODE_IPS[@]})"
  echo "DEFAULT_NETWORK_IF_NAME=${DEFAULT_NETWORK_IF_NAME}"
  echo "CONTAINER_SUBNET='${CONTAINER_SUBNET}'"
  echo "CLUSTER_IP_RANGE='${CLUSTER_IP_RANGE}'"
  echo "MASTER_CONTAINER_SUBNET='${MASTER_CONTAINER_SUBNET}'"
  echo "NODE_CONTAINER_NETMASKS='${NODE_CONTAINER_NETMASKS[@]}'"
  echo "NODE_CONTAINER_SUBNETS=(${NODE_CONTAINER_SUBNETS[@]})"
  echo "SERVICE_CLUSTER_IP_RANGE='${SERVICE_CLUSTER_IP_RANGE}'"
  echo "MASTER_USER='${MASTER_USER}'"
  echo "MASTER_PASSWD='${MASTER_PASSWD}'"
  echo "KUBE_USER='${KUBE_USER}'"
  echo "KUBE_PASSWORD='${KUBE_PASSWORD}'"
  echo "KUBE_BEARER_TOKEN='${KUBE_BEARER_TOKEN}'"
  echo "ENABLE_CLUSTER_MONITORING='${ENABLE_CLUSTER_MONITORING}'"
  echo "ENABLE_CLUSTER_LOGGING='${ENABLE_CLUSTER_LOGGING:-false}'"
  echo "ELASTICSEARCH_LOGGING_REPLICAS='${ELASTICSEARCH_LOGGING_REPLICAS:-1}'"
  echo "ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'"
  echo "ENABLE_CLUSTER_UI='${ENABLE_CLUSTER_UI}'"
  echo "ENABLE_HOSTPATH_PROVISIONER='${ENABLE_HOSTPATH_PROVISIONER:-false}'"
  echo "LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'"
  echo "ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'"
  echo "DNS_SERVER_IP='${DNS_SERVER_IP:-}'"
  echo "DNS_DOMAIN='${DNS_DOMAIN:-}'"
  echo "RUNTIME_CONFIG='${RUNTIME_CONFIG:-}'"
  echo "ADMISSION_CONTROL='${ADMISSION_CONTROL:-}'"
  echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS:-}'"
  echo "VAGRANT_DEFAULT_PROVIDER='${VAGRANT_DEFAULT_PROVIDER:-}'"
  echo "KUBELET_TOKEN='${KUBELET_TOKEN:-}'"
  echo "KUBE_PROXY_TOKEN='${KUBE_PROXY_TOKEN:-}'"
  echo "MASTER_EXTRA_SANS='${MASTER_EXTRA_SANS:-}'"
  echo "ENABLE_CPU_CFS_QUOTA='${ENABLE_CPU_CFS_QUOTA}'"
  echo "NETWORK_PROVIDER='${NETWORK_PROVIDER:-}'"
  echo "OPENCONTRAIL_TAG='${OPENCONTRAIL_TAG:-}'"
  echo "OPENCONTRAIL_KUBERNETES_TAG='${OPENCONTRAIL_KUBERNETES_TAG:-}'"
  echo "OPENCONTRAIL_PUBLIC_SUBNET='${OPENCONTRAIL_PUBLIC_SUBNET:-}'"
  echo "E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
  echo "CUSTOM_FEDORA_REPOSITORY_URL='${CUSTOM_FEDORA_REPOSITORY_URL:-}'"
  echo "EVICTION_HARD='${EVICTION_HARD:-}'"
}

function verify-cluster {
  # TODO: How does the user know the difference between "tak[ing] some
  # time" and "loop[ing] forever"? Can we give more specific feedback on
  # whether "an error" has occurred?
  echo "Each machine instance has been created/updated."
  echo "  Now waiting for the Salt provisioning process to complete on each machine."
  echo "  This can take some time based on your network, disk, and cpu speed."
  echo "  It is possible for an error to occur during Salt provision of cluster and this could loop forever."

  # verify master has all required daemons
  echo "Validating master"
  local machine="master"
  local -a required_processes=("kube-apiserver" "kube-scheduler" "kube-controller-manager" "kubelet" "docker")
  local validated="1"
  until [[ "$validated" == "0" ]]; do
    validated="0"
    for process in "${required_processes[@]}"; do
      vagrant ssh "${machine}" -c "pgrep -f ${process}" >/dev/null 2>&1 || {
        printf "."
        validated="1"
        sleep 2
      }
    done
  done

  # verify each node has all required daemons
  local i
  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
    echo "Validating ${VAGRANT_NODE_NAMES[$i]}"
    local machine=${VAGRANT_NODE_NAMES[$i]}
    local -a required_processes=("kube-proxy" "kubelet" "docker")
    local validated="1"
    until [[ "${validated}" == "0" ]]; do
      validated="0"
      for process in "${required_processes[@]}"; do
        vagrant ssh "${machine}" -c "pgrep -f ${process}" >/dev/null 2>&1 || {
          printf "."
          validated="1"
          sleep 2
        }
      done
    done
  done

  echo
  echo "Waiting for each node to be registered with cloud provider"
  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
    local validated="0"
    start="$(date +%s)"
    until [[ "$validated" == "1" ]]; do
      now="$(date +%s)"
      # Timeout set to 3 minutes
      if [ $((now - start)) -gt 180 ]; then
        echo "Timeout while waiting for echo node to be registered with cloud provider"
        exit 2
      fi
      local nodes=$("${KUBE_ROOT}/cluster/kubectl.sh" get nodes -o name)
      validated=$(echo $nodes | grep -c "${NODE_NAMES[i]}") || {
        printf "."
        sleep 2
        validated="0"
      }
    done
  done

  # By this time, all kube api calls should work, so no need to loop and retry.
  echo "Validating we can run kubectl commands."
  vagrant ssh master --command "kubectl get pods" || {
    echo "WARNING: kubectl to localhost failed.  This could mean localhost is not bound to an IP"
  }

  (
    # ensures KUBECONFIG is set
    get-kubeconfig-basicauth
    get-kubeconfig-bearertoken
    echo
    echo "Kubernetes cluster is running."
    echo
    echo "The master is running at:"
    echo
    echo "  https://${MASTER_IP}"
    echo
    echo "Administer and visualize its resources using Cockpit:"
    echo
    echo "  https://${MASTER_IP}:9090"
    echo
    echo "For more information on Cockpit, visit http://cockpit-project.org"
    echo
    echo "The user name and password to use is located in ${KUBECONFIG}"
    echo
  )
}

# Instantiate a kubernetes cluster
function kube-up {
  load-or-gen-kube-basicauth
  load-or-gen-kube-bearertoken
  get-tokens
  create-provision-scripts

  vagrant up --no-parallel

  export KUBE_CERT="/tmp/$RANDOM-kubecfg.crt"
  export KUBE_KEY="/tmp/$RANDOM-kubecfg.key"
  export CA_CERT="/tmp/$RANDOM-kubernetes.ca.crt"
  export CONTEXT="vagrant"

  (
   umask 077
   vagrant ssh master -- sudo cat /srv/kubernetes/kubecfg.crt >"${KUBE_CERT}" 2>/dev/null
   vagrant ssh master -- sudo cat /srv/kubernetes/kubecfg.key >"${KUBE_KEY}" 2>/dev/null
   vagrant ssh master -- sudo cat /srv/kubernetes/ca.crt >"${CA_CERT}" 2>/dev/null

   # Update the user's kubeconfig to include credentials for this apiserver.
   create-kubeconfig

   create-kubeconfig-for-federation
  )

  verify-cluster
}

# Delete a kubernetes cluster
function kube-down {
  vagrant destroy -f
}

# Update a kubernetes cluster with latest source
function kube-push {
  get-kubeconfig-basicauth
  get-kubeconfig-bearertoken
  create-provision-scripts
  vagrant provision
}

# Execute prior to running tests to build a release if required for env
function test-build-release {
  # Make a release
  "${KUBE_ROOT}/build/release.sh"
}

# Execute prior to running tests to initialize required structure
function test-setup {
  "${KUBE_ROOT}/cluster/kube-up.sh"
  echo "Vagrant test setup complete" 1>&2
}

# Execute after running tests to perform any required clean-up
function test-teardown {
  kube-down
}

# Find the node name based on the IP address
function find-vagrant-name-by-ip {
  local ip="$1"
  local ip_pattern="${NODE_IP_BASE}(.*)"

  # This is subtle.  We map 10.245.2.2 -> node-1.  We do this by matching a
  # regexp and using the capture to construct the name.
  [[ $ip =~ $ip_pattern ]] || {
    return 1
  }

  echo "node-$((${BASH_REMATCH[1]} - 1))"
}

# Find the vagrant machine name based on the host name of the node
function find-vagrant-name-by-node-name {
  local ip="$1"
  if [[ "$ip" == "${INSTANCE_PREFIX}-master" ]]; then
    echo "master"
    return $?
  fi
  local ip_pattern="${INSTANCE_PREFIX}-node-(.*)"

  [[ $ip =~ $ip_pattern ]] || {
    return 1
  }

  echo "node-${BASH_REMATCH[1]}"
}


# SSH to a node by name or IP ($1) and run a command ($2).
function ssh-to-node {
  local node="$1"
  local cmd="$2"
  local machine

  machine=$(find-vagrant-name-by-ip $node) || true
  [[ -n ${machine-} ]] || machine=$(find-vagrant-name-by-node-name $node) || true
  [[ -n ${machine-} ]] || {
    echo "Cannot find machine to ssh to: $1"
    return 1
  }

  vagrant ssh "${machine}" -c "${cmd}"
}

# Perform preparations required to run e2e tests
function prepare-e2e() {
  echo "Vagrant doesn't need special preparations for e2e tests" 1>&2
}

function get-tokens() {
  KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
  KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
}