From 553a3f049b639b2838f55815a4c62d64b32dde21 Mon Sep 17 00:00:00 2001
From: Mike Danese <mikedanese@google.com>
Date: Wed, 10 Jan 2018 10:00:00 -0800
Subject: [PATCH] remove deprecated photon controller

---
 cluster/photon-controller/config-common.sh    |   72 --
 cluster/photon-controller/config-default.sh   |   94 --
 cluster/photon-controller/config-test.sh      |   20 -
 cluster/photon-controller/setup-prereq.sh     |  239 ----
 cluster/photon-controller/templates/README    |    4 -
 .../templates/create-dynamic-salt-files.sh    |  127 --
 .../photon-controller/templates/hostname.sh   |   22 -
 .../templates/install-release.sh              |   26 -
 .../templates/salt-master.sh                  |   59 -
 .../templates/salt-minion.sh                  |   51 -
 cluster/photon-controller/util.sh             | 1110 -----------------
 11 files changed, 1824 deletions(-)
 delete mode 100644 cluster/photon-controller/config-common.sh
 delete mode 100755 cluster/photon-controller/config-default.sh
 delete mode 100755 cluster/photon-controller/config-test.sh
 delete mode 100755 cluster/photon-controller/setup-prereq.sh
 delete mode 100644 cluster/photon-controller/templates/README
 delete mode 100755 cluster/photon-controller/templates/create-dynamic-salt-files.sh
 delete mode 100755 cluster/photon-controller/templates/hostname.sh
 delete mode 100755 cluster/photon-controller/templates/install-release.sh
 delete mode 100755 cluster/photon-controller/templates/salt-master.sh
 delete mode 100755 cluster/photon-controller/templates/salt-minion.sh
 delete mode 100755 cluster/photon-controller/util.sh

diff --git a/cluster/photon-controller/config-common.sh b/cluster/photon-controller/config-common.sh
deleted file mode 100644
index 412eb26ba2f..00000000000
--- a/cluster/photon-controller/config-common.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-##########################################################
-#
-# These parameters describe objects we are using from
-# Photon Controller. They are all assumed to be pre-existing.
-#
-# Note: if you want help in creating them, you can use 
-# the setup-prereq.sh script, which will create any of these
-# that do not already exist.
-#
-##########################################################
-
-# Pre-created tenant for Kubernetes to use
-PHOTON_TENANT=kube-tenant
-
-# Pre-created project in PHOTON_TENANT for Kubernetes to use
-PHOTON_PROJECT=kube-project
-
-# Pre-created VM flavor for Kubernetes master to use
-# Can be same as master
-# We recommend at least 1GB of memory
-PHOTON_MASTER_FLAVOR=kube-vm
-
-# Pre-created VM flavor for Kubernetes node to use
-# Can be same as master
-# We recommend at least 2GB of memory
-PHOTON_NODE_FLAVOR=kube-vm
-
-# Pre-created disk flavor for Kubernetes to use
-PHOTON_DISK_FLAVOR=kube-disk
-
-# Pre-created Debian 8 image with kube user uploaded to Photon Controller
-# Note: While Photon Controller allows multiple images to have the same
-# name, we assume that there is exactly one image with this name.
-PHOTON_IMAGE=kube
-
-##########################################################
-# 
-# Parameters just for the setup-prereq.sh script: not used
-# elsewhere. If you create the above objects by hand, you 
-# do not need to edit these. 
-# 
-# Note that setup-prereq.sh also creates the objects 
-# above.
-# 
-##########################################################
-
-# The specifications for the master and node flavors
-SETUP_MASTER_FLAVOR_SPEC="vm 1 COUNT, vm.cpu 1 COUNT, vm.memory 2 GB"
-SETUP_NODE_FLAVOR_SPEC=${SETUP_MASTER_FLAVOR_SPEC}
-
-# The specification for the ephemeral disk flavor. 
-SETUP_DISK_FLAVOR_SPEC="ephemeral-disk 1 COUNT"
-
-# The specification for the tenant resource ticket and the project resources
-SETUP_TICKET_SPEC="vm.memory 1000 GB, vm 1000 COUNT"
-SETUP_PROJECT_SPEC="${SETUP_TICKET_SPEC}"
diff --git a/cluster/photon-controller/config-default.sh b/cluster/photon-controller/config-default.sh
deleted file mode 100755
index b2f71dd8c51..00000000000
--- a/cluster/photon-controller/config-default.sh
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-##########################################################
-#
-# Common parameters for Kubernetes
-#
-##########################################################
-
-# Default number of nodes to make. You can change this as needed
-NUM_NODES=3
-
-# Range of IPs assigned to pods
-NODE_IP_RANGES="10.244.0.0/16"
-
-# IPs used by Kubernetes master
-MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
-
-# Range of IPs assigned by Kubernetes to services
-SERVICE_CLUSTER_IP_RANGE="10.244.240.0/20"
-
-##########################################################
-#
-# Advanced parameters for Kubernetes
-#
-##########################################################
-
-# The instance prefix is the beginning of the name given to each VM we create
-# If this is changed, you can have multiple kubernetes clusters per project
-# Note that even if you don't change it, each tenant/project can have its own
-# Kubernetes cluster
-INSTANCE_PREFIX=kubernetes
-
-# Name of the user used to configure the VM
-# We use cloud-init to create the user
-VM_USER=kube
-
-# SSH options for how we connect to the Kubernetes VMs
-# We set the user known hosts file to /dev/null because we are connecting to new VMs.
-# When working in an environment where there is a lot of VM churn, VM IP addresses
-# will be reused, and the ssh keys will be different. This prevents us from seeing error
-# due to this, and it will not save the SSH key to the known_hosts file, so users will
-# still have standard ssh security checks.
-SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=ERROR -C"
-
-# Optional: Enable node logging.
-# Note: currently untested
-ENABLE_NODE_LOGGING=false
-LOGGING_DESTINATION=elasticsearch
-
-# Optional: When set to true, Elasticsearch and Kibana will be setup
-# Note: currently untested
-ENABLE_CLUSTER_LOGGING=false
-ELASTICSEARCH_LOGGING_REPLICAS=1
-
-# Optional: Cluster monitoring to setup as part of the cluster bring up:
-#   none     - No cluster monitoring setup
-#   influxdb - Heapster, InfluxDB, and Grafana
-#   google   - Heapster, Google Cloud Monitoring, and Google Cloud Logging
-# Note: currently untested
-ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}"
-
-# Optional: Install cluster DNS.
-ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"
-DNS_SERVER_IP="10.244.240.240"
-DNS_DOMAIN="cluster.local"
-
-# Optional: Enable DNS horizontal autoscaler
-ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}"
-
-# Optional: Install Kubernetes UI
-ENABLE_CLUSTER_UI=true
-
-# We need to configure subject alternate names (SANs) for the master's certificate
-# we generate.  While users will connect via the external IP, pods (like the UI)
-# will connect via the cluster IP, from the SERVICE_CLUSTER_IP_RANGE.
-# In addition to the extra SANS here, we'll also add one for for the service IP.
-MASTER_EXTRA_SANS="DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN}"
-
-# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
-E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
diff --git a/cluster/photon-controller/config-test.sh b/cluster/photon-controller/config-test.sh
deleted file mode 100755
index 87e68d72f79..00000000000
--- a/cluster/photon-controller/config-test.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-NUM_NODES=2
-NODE_IP_RANGES="10.244.0.0/16"
-MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
-SERVICE_CLUSTER_IP_RANGE="10.244.240.0/20"
diff --git a/cluster/photon-controller/setup-prereq.sh b/cluster/photon-controller/setup-prereq.sh
deleted file mode 100755
index 7212081327b..00000000000
--- a/cluster/photon-controller/setup-prereq.sh
+++ /dev/null
@@ -1,239 +0,0 @@
-#!/bin/bash
-
-# Copyright 2016 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This sets up a Photon Controller with the tenant, project, flavors
-# and image that are needed to deploy Kubernetes with kube-up.
-#
-# This is not meant to be used in production: it creates resource tickets
-# (quotas) that are arbitrary and not likely to work in your environment.
-# However, it may be a quick way to get your environment set up to try out
-# a Kubernetes installation.
-#
-# It uses the names for the tenant, project, and flavors as specified in the
-# config-common.sh file
-#
-# If you want to do this by hand, this script is equivalent to the following
-# Photon Controller commands (assuming you haven't edited config-common.sh
-# to change the names)
-#
-# photon target set https://192.0.2.2
-# photon tenant create kube-tenant
-# photon tenant set kube-tenant
-# photon resource-ticket create --tenant kube-tenant --name kube-resources --limits "vm.memory 1000 GB, vm 1000 COUNT"
-# photon project create --tenant kube-tenant --resource-ticket kube-resources --name kube-project --limits "vm.memory 1000 GB, vm 1000 COUNT"
-# photon project set kube-project
-# photon -n flavor create --name "kube-vm" --kind "vm" --cost "vm 1 COUNT, vm.cpu 1 COUNT, vm.memory 2 GB"
-# photon -n flavor create --name "kube-disk" --kind "ephemeral-disk" --cost "ephemeral-disk 1 COUNT"
-# photon image create kube.vmdk -n kube-image -i EAGER
-#
-# Note that the kube.vmdk can be downloaded as specified in the documentation.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
-# shellcheck source=./util.sh
-source "${KUBE_ROOT}/cluster/photon-controller/util.sh"
-
-function main {
-  verify-cmd-in-path photon
-  set-target
-  create-tenant
-  create-project
-  create-vm-flavor "${PHOTON_MASTER_FLAVOR}" "${SETUP_MASTER_FLAVOR_SPEC}"
-  if [ "${PHOTON_MASTER_FLAVOR}" != "${PHOTON_NODE_FLAVOR}" ]; then
-    create-vm-flavor "${PHOTON_NODE_FLAVOR}" "${SETUP_NODE_FLAVOR_SPEC}"
-  fi
-  create-disk-flavor
-  create-image
-}
-
-function parse-cmd-line {
-  PHOTON_TARGET=${1:-""}
-  PHOTON_VMDK=${2:-""}
-
-  if [[ "${PHOTON_TARGET}" = "" || "${PHOTON_VMDK}" = "" ]]; then
-    echo "Usage: setup-prereq <photon target> <path-to-kube-vmdk>"
-    echo "Target should be a URL like https://192.0.2.1"
-    echo ""
-    echo "This will create the following, based on the configuration in config-common.sh"
-    echo "    * A tenant named ${PHOTON_TENANT}"
-    echo "    * A project named ${PHOTON_PROJECT}"
-    echo "    * A VM flavor named ${PHOTON_MASTER_FLAVOR}"
-    echo "    * A disk flavor named ${PHOTON_DISK_FLAVOR}"
-    echo "It will also upload the Kube VMDK"
-    echo ""
-    echo "It creates the tenant with a resource ticket (quota) that may"
-    echo "be inappropriate for your environment. For a production"
-    echo "environment, you should configure these to match your"
-    echo "environment."
-    exit 1
-  fi
-
-  echo "Photon Target: ${PHOTON_TARGET}"
-  echo "Photon VMDK: ${PHOTON_VMDK}"
-}
-
-function set-target {
-  ${PHOTON} target set "${PHOTON_TARGET}" > /dev/null 2>&1
-}
-
-function create-tenant {
-  local rc=0
-  local output
-
-  ${PHOTON} tenant list | grep -q "\t${PHOTON_TENANT}$" > /dev/null 2>&1 || rc=$?
-  if [[ ${rc} -eq 0 ]]; then
-    echo "Tenant ${PHOTON_TENANT} already made, skipping"
-  else
-    echo "Making tenant ${PHOTON_TENANT}"
-    rc=0
-    output=$(${PHOTON} tenant create "${PHOTON_TENANT}" 2>&1) || {
-      echo "ERROR: Could not create tenant \"${PHOTON_TENANT}\", exiting"
-      echo "Output from tenant creation:"
-      echo "${output}"
-      exit 1
-    }
-  fi
-  ${PHOTON} tenant set "${PHOTON_TENANT}" > /dev/null 2>&1
-}
-
-function create-project {
-  local rc=0
-  local output
-
-  ${PHOTON} project list | grep -q "\t${PHOTON_PROJECT}\t" > /dev/null 2>&1  || rc=$?
-  if [[ ${rc} -eq 0 ]]; then
-    echo "Project ${PHOTON_PROJECT} already made, skipping"
-  else
-    echo "Making project ${PHOTON_PROJECT}"
-    rc=0
-    output=$(${PHOTON} resource-ticket create --tenant "${PHOTON_TENANT}" --name "${PHOTON_TENANT}-resources" --limits "${SETUP_TICKET_SPEC}" 2>&1) || {
-      echo "ERROR: Could not create resource ticket, exiting"
-      echo "Output from resource ticket creation:"
-      echo "${output}"
-      exit 1
-    }
-
-    rc=0
-    output=$(${PHOTON} project create --tenant "${PHOTON_TENANT}" --resource-ticket "${PHOTON_TENANT}-resources" --name "${PHOTON_PROJECT}" --limits "${SETUP_PROJECT_SPEC}" 2>&1) || {
-      echo "ERROR: Could not create project \"${PHOTON_PROJECT}\", exiting"
-      echo "Output from project creation:"
-      echo "${output}"
-      exit 1
-    }
-  fi
-  ${PHOTON} project set "${PHOTON_PROJECT}"
-}
-
-function create-vm-flavor {
-  local flavor_name=${1}
-  local flavor_spec=${2}
-  local rc=0
-  local output
-
-  ${PHOTON} flavor list | grep -q "\t${flavor_name}\t" > /dev/null 2>&1 || rc=$?
-  if [[ ${rc} -eq 0 ]]; then
-    check-flavor-ready "${flavor_name}"
-    echo "Flavor ${flavor_name} already made, skipping"
-  else
-    echo "Making VM flavor ${flavor_name}"
-    rc=0
-    output=$(${PHOTON} -n flavor create --name "${flavor_name}" --kind "vm" --cost "${flavor_spec}" 2>&1) || {
-      echo "ERROR: Could not create vm flavor \"${flavor_name}\", exiting"
-      echo "Output from flavor creation:"
-      echo "${output}"
-      exit 1
-    }
-  fi
-}
-
-function create-disk-flavor {
-  local rc=0
-  local output
-
-  ${PHOTON} flavor list | grep -q "\t${PHOTON_DISK_FLAVOR}\t" > /dev/null 2>&1  || rc=$?
-  if [[ ${rc} -eq 0 ]]; then
-    check-flavor-ready "${PHOTON_DISK_FLAVOR}"
-    echo "Flavor ${PHOTON_DISK_FLAVOR} already made, skipping"
-  else
-    echo "Making disk flavor ${PHOTON_DISK_FLAVOR}"
-    rc=0
-    output=$(${PHOTON} -n flavor create --name "${PHOTON_DISK_FLAVOR}" --kind "ephemeral-disk" --cost "${SETUP_DISK_FLAVOR_SPEC}" 2>&1) || {
-      echo "ERROR: Could not create disk flavor \"${PHOTON_DISK_FLAVOR}\", exiting"
-      echo "Output from flavor creation:"
-      echo "${output}"
-      exit 1
-    }
-  fi
-}
-
-function check-flavor-ready {
-  local flavor_name=${1}
-  local rc=0
-
-  local flavor_id
-  flavor_id=$(${PHOTON} flavor list | grep  "\t${flavor_name}\t" | awk '{print $1}') || {
-    echo "ERROR: Found ${flavor_name} but cannot find it's id"
-    exit 1
-  }
-
-  ${PHOTON} flavor show "${flavor_id}" | grep "\tREADY\$" > /dev/null 2>&1  || {
-    echo "ERROR: Flavor \"${flavor_name}\" already exists but is not READY. Please delete or fix it."
-    exit 1
-  }
-}
-
-function create-image {
-  local rc=0
-  local num_images
-  local output
-
-  ${PHOTON} image list | grep "\t${PHOTON_IMAGE}\t" | grep -q ERROR > /dev/null 2>&1 || rc=$?
-  if [[ ${rc} -eq 0 ]]; then
-    echo "Warning: You have at least one ${PHOTON_IMAGE} image in the ERROR state. You may want to investigate."
-    echo "Images in the ERROR state will be ignored."
-  fi
-
-  rc=0
-  # We don't use grep -c because it exists non-zero when there are no matches, tell shellcheck
-  # shellcheck disable=SC2126 
-  num_images=$(${PHOTON} image list | grep "\t${PHOTON_IMAGE}\t" | grep READY | wc -l)
-  if [[ "${num_images}" -gt 1 ]]; then
-    echo "Warning: You have more than one good ${PHOTON_IMAGE} image. You may want to remove duplicates."
-  fi
-
-  ${PHOTON} image list | grep "\t${PHOTON_IMAGE}\t" | grep -q READY > /dev/null 2>&1 || rc=$?
-  if [[ ${rc} -eq 0 ]]; then
-    echo "Image ${PHOTON_VMDK} already uploaded, skipping"
-  else
-    echo "Uploading image ${PHOTON_VMDK}"
-    rc=0
-    output=$(${PHOTON} image create "${PHOTON_VMDK}" -n "${PHOTON_IMAGE}" -i EAGER 2>&1) || {
-      echo "ERROR: Could not upload image, exiting"
-      echo "Output from image create:"
-      echo "${output}"
-      exit 1
-    }
-  fi
-}
-
-# We don't want silent pipeline failure: we check for failure
-set +o pipefail
-
-parse-cmd-line "$@"
-main
diff --git a/cluster/photon-controller/templates/README b/cluster/photon-controller/templates/README
deleted file mode 100644
index b91d629fa0c..00000000000
--- a/cluster/photon-controller/templates/README
+++ /dev/null
@@ -1,4 +0,0 @@
-The scripts in this directory are not meant to be invoked
-directly. Instead they are partial scripts that are combined into full
-scripts by util.sh and are run on the Kubernetes nodes are part of the
-setup. 
diff --git a/cluster/photon-controller/templates/create-dynamic-salt-files.sh b/cluster/photon-controller/templates/create-dynamic-salt-files.sh
deleted file mode 100755
index 369fdb7095f..00000000000
--- a/cluster/photon-controller/templates/create-dynamic-salt-files.sh
+++ /dev/null
@@ -1,127 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#generate token files
-
-KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
-KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
-known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
-if [[ ! -f "${known_tokens_file}" ]]; then
-
-  mkdir -p /srv/salt-overlay/salt/kube-apiserver
-  known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
-  (umask u=rw,go= ;
-   echo "$KUBELET_TOKEN,kubelet,kubelet" > $known_tokens_file;
-   echo "$KUBE_PROXY_TOKEN,kube_proxy,kube_proxy" >> $known_tokens_file)
-
-  mkdir -p /srv/salt-overlay/salt/kubelet
-  kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
-  (umask u=rw,go= ; echo "{\"BearerToken\": \"$KUBELET_TOKEN\", \"Insecure\": true }" > $kubelet_auth_file)
-  kubelet_kubeconfig_file="/srv/salt-overlay/salt/kubelet/kubeconfig"
-
-  mkdir -p /srv/salt-overlay/salt/kubelet
-  (umask 077;
-  cat > "${kubelet_kubeconfig_file}" << EOF
-apiVersion: v1
-kind: Config
-clusters:
-- cluster:
-    server: https://${KUBE_MASTER_IP}
-    insecure-skip-tls-verify: true
-  name: local
-contexts:
-- context:
-    cluster: local
-    user: kubelet
-  name: service-account-context
-current-context: service-account-context
-users:
-- name: kubelet
-  user:
-    token: ${KUBELET_TOKEN}
-EOF
-)
-
-
-  mkdir -p /srv/salt-overlay/salt/kube-proxy
-  kube_proxy_kubeconfig_file="/srv/salt-overlay/salt/kube-proxy/kubeconfig"
-  # Make a kubeconfig file with the token.
-  # TODO(etune): put apiserver certs into secret too, and reference from authfile,
-  # so that "Insecure" is not needed.
-  (umask 077;
-  cat > "${kube_proxy_kubeconfig_file}" << EOF
-apiVersion: v1
-kind: Config
-clusters:
-- cluster:
-    insecure-skip-tls-verify: true
-  name: local
-contexts:
-- context:
-    cluster: local
-    user: kube-proxy
-  name: service-account-context
-current-context: service-account-context
-users:
-- name: kube-proxy
-  user:
-    token: ${KUBE_PROXY_TOKEN}
-EOF
-)
-
-  # Generate tokens for other "service accounts".  Append to known_tokens.
-  #
-  # NB: If this list ever changes, this script actually has to
-  # change to detect the existence of this file, kill any deleted
-  # old tokens and add any new tokens (to handle the upgrade case).
-  service_accounts=("system:scheduler" "system:controller_manager" "system:logging" "system:monitoring" "system:dns")
-  for account in "${service_accounts[@]}"; do
-    token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
-    echo "${token},${account},${account}" >> "${known_tokens_file}"
-  done
-fi
-
-readonly BASIC_AUTH_FILE="/srv/salt-overlay/salt/kube-apiserver/basic_auth.csv"
-if [[ ! -e "${BASIC_AUTH_FILE}" ]]; then
-  mkdir -p /srv/salt-overlay/salt/kube-apiserver
-  (umask 077;
-    echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${BASIC_AUTH_FILE}")
-fi
-
-
-# Create the overlay files for the salt tree.  We create these in a separate
-# place so that we can blow away the rest of the salt configs on a kube-push and
-# re-apply these.
-
-mkdir -p /srv/salt-overlay/pillar
-cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
-instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")'
-node_instance_prefix: $NODE_INSTANCE_PREFIX
-service_cluster_ip_range: $SERVICE_CLUSTER_IP_RANGE
-enable_cluster_monitoring: "${ENABLE_CLUSTER_MONITORING:-none}"
-enable_cluster_logging: "${ENABLE_CLUSTER_LOGGING:false}"
-enable_cluster_ui: "${ENABLE_CLUSTER_UI:true}"
-enable_node_logging: "${ENABLE_NODE_LOGGING:false}"
-logging_destination: $LOGGING_DESTINATION
-elasticsearch_replicas: $ELASTICSEARCH_LOGGING_REPLICAS
-enable_cluster_dns: "${ENABLE_CLUSTER_DNS:-false}"
-dns_server: $DNS_SERVER_IP
-dns_domain: $DNS_DOMAIN
-e2e_storage_test_environment: "${E2E_STORAGE_TEST_ENVIRONMENT:-false}"
-cluster_cidr: "$NODE_IP_RANGES"
-allocate_node_cidrs: "${ALLOCATE_NODE_CIDRS:-true}"
-admission_control: Initializers,NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota
-EOF
diff --git a/cluster/photon-controller/templates/hostname.sh b/cluster/photon-controller/templates/hostname.sh
deleted file mode 100755
index ae7f4d0f4ef..00000000000
--- a/cluster/photon-controller/templates/hostname.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Remove kube.vm from /etc/hosts
-sed -i -e 's/\b\w\+.vm\b//' /etc/hosts
-
-# Update hostname in /etc/hosts and /etc/hostname
-sed -i -e "s/\\bkube\\b/${MY_NAME}/g" /etc/host{s,name}
-hostname ${MY_NAME}
diff --git a/cluster/photon-controller/templates/install-release.sh b/cluster/photon-controller/templates/install-release.sh
deleted file mode 100755
index 34206a35aab..00000000000
--- a/cluster/photon-controller/templates/install-release.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script assumes that the environment variable SERVER_BINARY_TAR contains
-# the release tar to download and unpack. It is meant to be pushed to the
-# master and run.
-
-echo "Unpacking Salt tree"
-rm -rf kubernetes
-tar xzf "${SALT_TAR}"
-
-echo "Running release install script"
-sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR}"
diff --git a/cluster/photon-controller/templates/salt-master.sh b/cluster/photon-controller/templates/salt-master.sh
deleted file mode 100755
index 19281d008f1..00000000000
--- a/cluster/photon-controller/templates/salt-master.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Use other Debian mirror
-sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
-
-# Prepopulate the name of the Master
-mkdir -p /etc/salt/minion.d
-echo "master: ${MASTER_NAME}" > /etc/salt/minion.d/master.conf
-
-cat <<EOF >/etc/salt/minion.d/grains.conf
-grains:
-  roles:
-    - kubernetes-master
-  cbr-cidr: $MASTER_IP_RANGE
-  cloud: photon-controller
-  master_extra_sans: $MASTER_EXTRA_SANS
-  api_servers: $MASTER_NAME
-  kubelet_kubeconfig: /srv/salt-overlay/salt/kubelet/kubeconfig
-  kube_user: $KUBE_USER
-EOF
-
-# Auto accept all keys from minions that try to join
-mkdir -p /etc/salt/master.d
-cat <<EOF >/etc/salt/master.d/auto-accept.conf
-auto_accept: True
-EOF
-
-cat <<EOF >/etc/salt/master.d/reactor.conf
-# React to new minions starting by running highstate on them.
-reactor:
-  - 'salt/minion/*/start':
-    - /srv/reactor/highstate-new.sls
-    - /srv/reactor/highstate-masters.sls
-    - /srv/reactor/highstate-minions.sls
-EOF
-
-# Install Salt
-#
-# We specify -X to avoid a race condition that can cause minion failure to
-# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
-#
-# -M installs the master
-set +x
-curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -M -X
-set -x
diff --git a/cluster/photon-controller/templates/salt-minion.sh b/cluster/photon-controller/templates/salt-minion.sh
deleted file mode 100755
index 314e5e726d5..00000000000
--- a/cluster/photon-controller/templates/salt-minion.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Use other Debian mirror
-sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
-
-# Resolve hostname of master
-if ! grep -q $KUBE_MASTER /etc/hosts; then
-  echo "Adding host entry for $KUBE_MASTER"
-  echo "${KUBE_MASTER_IP} ${KUBE_MASTER}" >> /etc/hosts
-fi
-
-# Prepopulate the name of the Master
-mkdir -p /etc/salt/minion.d
-echo "master: ${KUBE_MASTER}" > /etc/salt/minion.d/master.conf
-
-# Turn on debugging for salt-minion
-# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion
-
-# Our minions will have a pool role to distinguish them from the master.
-#
-# Setting the "minion_ip" here causes the kubelet to use its IP for
-# identification instead of its hostname.
-#
-cat <<EOF >/etc/salt/minion.d/grains.conf
-grains:
-  hostname_override: $(ip route get 1.1.1.1 | awk '{print $7}')
-  roles:
-    - kubernetes-pool
-    - kubernetes-pool-photon-controller
-  cloud: photon-controller
-EOF
-
-# Install Salt
-#
-# We specify -X to avoid a race condition that can cause minion failure to
-# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
-curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -X
diff --git a/cluster/photon-controller/util.sh b/cluster/photon-controller/util.sh
deleted file mode 100755
index 55ec52ff9cc..00000000000
--- a/cluster/photon-controller/util.sh
+++ /dev/null
@@ -1,1110 +0,0 @@
-#!/bin/bash
-
-# Copyright 2016 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts.
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
-# shellcheck source=./config-common.sh
-source "${KUBE_ROOT}/cluster/photon-controller/config-common.sh"
-# shellcheck source=./config-default.sh
-source "${KUBE_ROOT}/cluster/photon-controller/${KUBE_CONFIG_FILE-"config-default.sh"}"
-# shellcheck source=../common.sh
-source "${KUBE_ROOT}/cluster/common.sh"
-
-readonly PHOTON="photon -n"
-
-# Naming scheme for VMs (masters & nodes)
-readonly MASTER_NAME="${INSTANCE_PREFIX}-master"
-
-# shell check claims this doesn't work because you can't use a variable in a brace
-# range. It does work because we're calling eval.
-# shellcheck disable=SC2051
-readonly NODE_NAMES=($(eval echo "${INSTANCE_PREFIX}"-node-{1.."${NUM_NODES}"}))
-
-#####################################################################
-#
-# Public API
-#
-#####################################################################
-
-#
-# detect-master will query Photon Controller for the Kubernetes master.
-# It assumes that the VM name for the master is unique.
-# It will set KUBE_MASTER_ID to be the VM ID of the master
-# It will set KUBE_MASTER_IP to be the IP address of the master
-# If the silent parameter is passed, it will not print when the master
-# is found: this is used internally just to find the MASTER
-#
-function detect-master {
-  local silent=${1:-""}
-  local tenant_args="--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}"
-
-  KUBE_MASTER=${MASTER_NAME}
-  KUBE_MASTER_ID=${KUBE_MASTER_ID:-""}
-  KUBE_MASTER_IP=${KUBE_MASTER_IP:-""}
-
-  # We don't want silent failure: we check for failure
-  set +o pipefail
-  if [[ -z ${KUBE_MASTER_ID} ]]; then
-    KUBE_MASTER_ID=$(${PHOTON} vm list ${tenant_args} | grep $'\t'"kubernetes-master"$'\t' | awk '{print $1}')
-  fi
-  if [[ -z ${KUBE_MASTER_ID} ]]; then
-    kube::log::error "Could not find Kubernetes master node ID. Make sure you've launched a cluster with kube-up.sh"
-    exit 1
-  fi
-
-  if [[ -z "${KUBE_MASTER_IP-}" ]]; then
-      # Pick out the NICs that have a MAC address owned VMware (with OUI 00:0C:29)
-      # Make sure to ignore lines that have a network interface but no address
-    KUBE_MASTER_IP=$(${PHOTON} vm networks "${KUBE_MASTER_ID}" | grep -i $'\t'"00:0C:29" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
-  fi
-  if [[ -z "${KUBE_MASTER_IP-}" ]]; then
-    kube::log::error "Could not find Kubernetes master node IP. Make sure you've launched a cluster with 'kube-up.sh'" >&2
-    exit 1
-  fi
-  if [[ -z ${silent} ]]; then
-    kube::log::status "Master: $KUBE_MASTER ($KUBE_MASTER_IP)"
-  fi
-  # Reset default set in common.sh
-  set -o pipefail
-}
-
-#
-# detect-nodes will query Photon Controller for the Kubernetes nodes
-# It assumes that the VM name for the nodes are unique.
-# It assumes that NODE_NAMES has been set
-# It will set KUBE_NODE_IP_ADDRESSES to be the VM IPs of the nodes
-# It will set the KUBE_NODE_IDS to be the VM IDs of the nodes
-# If the silent parameter is passed, it will not print when the nodes
-# are found: this is used internally just to find the MASTER
-#
-function detect-nodes {
-  local silent=${1:-""}
-  local failure=0
-  local tenant_args="--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}"
-
-  KUBE_NODE_IP_ADDRESSES=()
-  KUBE_NODE_IDS=()
-  # We don't want silent failure: we check for failure
-  set +o pipefail
-  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
-
-    local node_id
-    node_id=$(${PHOTON} vm list ${tenant_args} | grep $'\t'"${NODE_NAMES[${i}]}"$'\t' | awk '{print $1}')
-    if [[ -z ${node_id} ]]; then
-      kube::log::error "Could not find ${NODE_NAMES[${i}]}"
-      failure=1
-    fi
-    KUBE_NODE_IDS+=("${node_id}")
-
-    # Pick out the NICs that have a MAC address owned VMware (with OUI 00:0C:29)
-    # Make sure to ignore lines that have a network interface but no address
-    node_ip=$(${PHOTON} vm networks "${node_id}" | grep -i $'\t'"00:0C:29" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
-    KUBE_NODE_IP_ADDRESSES+=("${node_ip}")
-
-    if [[ -z ${silent} ]]; then
-      kube::log::status "Node: ${NODE_NAMES[${i}]} (${KUBE_NODE_IP_ADDRESSES[${i}]})"
-    fi
-  done
-
-  if [[ ${failure} -ne 0 ]]; then
-    exit 1
-  fi
-  # Reset default set in common.sh
-  set -o pipefail
-}
-
-# Get node names if they are not static.
-function detect-node-names {
-  echo "TODO: detect-node-names" 1>&2
-}
-
-#
-# Verifies that this computer has sufficient software installed
-# so that it can run the rest of the script.
-#
-function verify-prereqs {
-  verify-cmd-in-path photon
-  verify-cmd-in-path ssh
-  verify-cmd-in-path scp
-  verify-cmd-in-path ssh-add
-  verify-cmd-in-path openssl
-  verify-cmd-in-path mkisofs
-}
-
-#
-# The entry point for bringing up a Kubernetes cluster
-#
-function kube-up {
-  verify-prereqs
-  verify-ssh-prereqs
-  verify-photon-config
-  kube::util::ensure-temp-dir
-
-  find-release-tars
-  find-image-id
-
-  load-or-gen-kube-basicauth
-  gen-cloud-init-iso
-  gen-master-start
-  create-master-vm
-  install-salt-on-master
-
-  gen-node-start
-  install-salt-on-nodes
-
-  detect-nodes -s
-
-  install-kubernetes-on-master
-  install-kubernetes-on-nodes
-
-  wait-master-api
-  wait-node-apis
-
-  setup-pod-routes
-
-  copy-kube-certs
-  kube::log::status "Creating kubeconfig..."
-  create-kubeconfig
-}
-
-# Delete a kubernetes cluster
-function kube-down {
-  detect-master
-  detect-nodes
-
-  pc-delete-vm "${KUBE_MASTER}" "${KUBE_MASTER_ID}"
-  for (( node=0; node<${#KUBE_NODE_IDS[@]}; node++)); do
-    pc-delete-vm "${NODE_NAMES[${node}]}" "${KUBE_NODE_IDS[${node}]}"
-  done
-}
-
-# Update a kubernetes cluster
-function kube-push {
-  echo "TODO: kube-push" 1>&2
-}
-
-# Prepare update a kubernetes component
-function prepare-push {
-  echo "TODO: prepare-push" 1>&2
-}
-
-# Update a kubernetes master
-function push-master {
-  echo "TODO: push-master" 1>&2
-}
-
-# Update a kubernetes node
-function push-node {
-  echo "TODO: push-node" 1>&2
-}
-
-# Execute prior to running tests to build a release if required for env
-function test-build-release {
-  echo "TODO: test-build-release" 1>&2
-}
-
-# Execute prior to running tests to initialize required structure
-function test-setup {
-  echo "TODO: test-setup" 1>&2
-}
-
-# Execute after running tests to perform any required clean-up
-function test-teardown {
-  echo "TODO: test-teardown" 1>&2
-}
-
-#####################################################################
-#
-# Internal functions
-#
-#####################################################################
-
-#
-# Uses Photon Controller to make a VM
-# Takes two parameters:
-#   - The name of the VM (Assumed to be unique)
-#   - The name of the flavor to create the VM (Assumed to be unique)
-#
-# It assumes that the variables in config-common.sh (PHOTON_TENANT, etc)
-# are set correctly.
-#
-# It also assumes the cloud-init ISO has been generated
-#
-# When it completes, it sets two environment variables for use by the
-# caller: _VM_ID (the ID of the created VM) and _VM_IP (the IP address
-# of the created VM)
-#
-function pc-create-vm {
-  local vm_name="${1}"
-  local vm_flavor="${2}"
-  local rc=0
-  local i=0
-
-  # Create the VM
-  local tenant_args="--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}"
-  local vm_args="--name ${vm_name} --image ${PHOTON_IMAGE_ID} --flavor ${vm_flavor}"
-  local disk_args="disk-1 ${PHOTON_DISK_FLAVOR} boot=true"
-
-  rc=0
-  _VM_ID=$(${PHOTON} vm create ${tenant_args} ${vm_args} --disks "${disk_args}" 2>&1) || rc=$?
-  if [[ ${rc} -ne 0 ]]; then
-    kube::log::error "Failed to create VM. Error output:"
-    echo "${_VM_ID}"
-    exit 1
-  fi
-  kube::log::status "Created VM ${vm_name}: ${_VM_ID}"
-
-  # Start the VM
-  # Note that the VM has cloud-init in it, and we attach an ISO that
-  # contains a user-data.txt file for cloud-init. When the VM starts,
-  # cloud-init will temporarily mount the ISO and configure the VM
-  # Our user-data will configure the 'kube' user and set up the ssh
-  # authorized keys to allow us to ssh to the VM and do further work.
-  run-cmd "${PHOTON} vm attach-iso -p ${KUBE_TEMP}/cloud-init.iso ${_VM_ID}"
-  run-cmd "${PHOTON} vm start ${_VM_ID}"
-  kube::log::status "Started VM ${vm_name}, waiting for network address..."
-
-   # Wait for the VM to be started and connected to the network
-  have_network=0
-  for i in {1..120}; do
-    # photon -n vm networks print several fields:
-    # NETWORK MAC IP GATEWAY CONNECTED?
-    # We wait until CONNECTED is True
-    rc=0
-    networks=$(${PHOTON} vm networks "${_VM_ID}") || rc=$?
-    if [[ ${rc} -ne 0 ]]; then
-      kube::log::error "'${PHOTON} vm networks ${_VM_ID}' failed. Error output: "
-      echo "${networks}"
-    fi
-    networks=$(echo "${networks}" | grep True) || rc=$?
-    if [[ ${rc} -eq 0 ]]; then
-      have_network=1
-      break;
-    fi
-    sleep 1
-  done
-
-  # Fail if the VM didn't come up
-  if [[ ${have_network} -eq 0 ]]; then
-    kube::log::error "VM ${vm_name} failed to start up: no IP was found"
-    exit 1
-  fi
-
-  # Find the IP address of the VM
-  _VM_IP=$(${PHOTON} vm networks "${_VM_ID}" | head -1 | awk -F'\t' '{print $3}')
-  kube::log::status "VM ${vm_name} has IP: ${_VM_IP}"
-}
-
-#
-# Delete one of our VMs
-# If it is STARTED, it will be stopped first.
-#
-function pc-delete-vm {
-  local vm_name="${1}"
-  local vm_id="${2}"
-  local rc=0
-
-  kube::log::status "Deleting VM ${vm_name}"
-  # In some cases, head exits before photon, so the pipline exits with
-  # SIGPIPE. We disable the pipefile option to hide that failure.
-  set +o pipefail
-  ${PHOTON} vm show "${vm_id}" | head -1 | grep STARTED > /dev/null 2>&1 || rc=$?
-  set +o pipefail
-  if [[ ${rc} -eq 0 ]]; then
-    ${PHOTON} vm stop "${vm_id}" > /dev/null 2>&1 || rc=$?
-    if [[ ${rc} -ne 0 ]]; then
-      kube::log::error "Error: could not stop ${vm_name} ($vm_id)"
-      kube::log::error "Please investigate and stop manually"
-      return
-    fi
-  fi
-
-  rc=0
-  ${PHOTON} vm delete "${vm_id}" > /dev/null 2>&1 || rc=$?
-  if [[ ${rc} -ne 0 ]]; then
-    kube::log::error "Error: could not delete ${vm_name} ($vm_id)"
-    kube::log::error "Please investigate and delete manually"
-  fi
-}
-
-#
-# Looks for the image named PHOTON_IMAGE
-# Sets PHOTON_IMAGE_ID to be the id of that image.
-# We currently assume there is exactly one image with name
-#
-function find-image-id {
-  local rc=0
-  PHOTON_IMAGE_ID=$(${PHOTON} image list | grep $'\t'"${PHOTON_IMAGE}"$'\t' | head -1 | grep READY | awk -F'\t' '{print $1}')
-  if [[ ${rc} -ne 0 ]]; then
-    kube::log::error "Cannot find image \"${PHOTON_IMAGE}\""
-    fail=1
-  fi
-}
-
-#
-# Generate an ISO with a single file called user-data.txt
-# This ISO will be used to configure cloud-init (which is already
-# on the VM). We will tell cloud-init to create the kube user/group
-# and give ourselves the ability to ssh to the VM with ssh. We also
-# allow people to ssh with the same password that was randomly
-# generated for access to Kubernetes as a backup method.
-#
-# Assumes environment variables:
-#   - VM_USER
-#   - KUBE_PASSWORD (randomly generated password)
-#
-function gen-cloud-init-iso {
-  local password_hash
-  password_hash=$(openssl passwd -1 "${KUBE_PASSWORD}")
-
-  local ssh_key
-  ssh_key=$(ssh-add -L | head -1)
-
-  # Make the user-data file that will be used by cloud-init
-  (
-    echo "#cloud-config"
-    echo ""
-    echo "groups:"
-    echo "  - ${VM_USER}"
-    echo ""
-    echo "users:"
-    echo "  - name: ${VM_USER}"
-    echo "    gecos: Kubernetes"
-    echo "    primary-group: ${VM_USER}"
-    echo "    lock-passwd: false"
-    echo "    passwd: ${password_hash}"
-    echo "    ssh-authorized-keys: "
-    echo "      - ${ssh_key}"
-    echo "    sudo: ALL=(ALL) NOPASSWD:ALL"
-    echo "    shell: /bin/bash"
-    echo ""
-    echo "hostname:"
-    echo "  - hostname: kube"
-  ) > "${KUBE_TEMP}/user-data.txt"
-
-  # Make the ISO that will contain the user-data
-  # The -rock option means that we'll generate real filenames (long and with case)
-  run-cmd "mkisofs -rock -o ${KUBE_TEMP}/cloud-init.iso ${KUBE_TEMP}/user-data.txt"
-}
-
-#
-# Generate a script used to install salt on the master
-# It is placed into $KUBE_TEMP/master-start.sh
-#
-function gen-master-start {
-  python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
-    -b -c "${KUBE_TEMP}/htpasswd" "${KUBE_USER}" "${KUBE_PASSWORD}"
-  local htpasswd
-  htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
-
-  # This calculation of the service IP should work, but if you choose an
-  # alternate subnet, there's a small chance you'd need to modify the
-  # service_ip, below.  We'll choose an IP like 10.244.240.1 by taking
-  # the first three octets of the SERVICE_CLUSTER_IP_RANGE and tacking
-  # on a .1
-  local octets
-  local service_ip
-  octets=($(echo "${SERVICE_CLUSTER_IP_RANGE}" | sed -e 's|/.*||' -e 's/\./ /g'))
-  ((octets[3]+=1))
-  service_ip=$(echo "${octets[*]}" | sed 's/ /./g')
-  MASTER_EXTRA_SANS="IP:${service_ip},DNS:${MASTER_NAME},${MASTER_EXTRA_SANS}"
-
-  (
-    echo "#! /bin/bash"
-    echo "readonly MY_NAME=${MASTER_NAME}"
-    grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/hostname.sh"
-    echo "cd /home/kube/cache/kubernetes-install"
-    echo "readonly KUBE_MASTER_IP='{$KUBE_MASTER_IP}'"
-    echo "readonly MASTER_NAME='${MASTER_NAME}'"
-    echo "readonly MASTER_IP_RANGE='${MASTER_IP_RANGE}'"
-    echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
-    echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-node'"
-    echo "readonly NODE_IP_RANGES='${NODE_IP_RANGES}'"
-    echo "readonly SERVICE_CLUSTER_IP_RANGE='${SERVICE_CLUSTER_IP_RANGE}'"
-    echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'"
-    echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'"
-    echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'"
-    echo "readonly ENABLE_CLUSTER_UI='${ENABLE_CLUSTER_UI:-false}'"
-    echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'"
-    echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'"
-    echo "readonly KUBE_USER='${KUBE_USER:-}'"
-    echo "readonly KUBE_PASSWORD='${KUBE_PASSWORD:-}'"
-    echo "readonly SERVER_BINARY_TAR='${SERVER_BINARY_TAR##*/}'"
-    echo "readonly SALT_TAR='${SALT_TAR##*/}'"
-    echo "readonly MASTER_HTPASSWD='${htpasswd}'"
-    echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
-    echo "readonly MASTER_EXTRA_SANS='${MASTER_EXTRA_SANS:-}'"
-    grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/create-dynamic-salt-files.sh"
-    grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/install-release.sh"
-    grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/salt-master.sh"
-  ) > "${KUBE_TEMP}/master-start.sh"
-}
-
-#
-# Generate the scripts for each node to install salt
-#
-function gen-node-start {
-  local i
-  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
-    (
-      echo "#! /bin/bash"
-      echo "readonly MY_NAME=${NODE_NAMES[${i}]}"
-      grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/hostname.sh"
-      echo "KUBE_MASTER=${KUBE_MASTER}"
-      echo "KUBE_MASTER_IP=${KUBE_MASTER_IP}"
-      echo "NODE_IP_RANGE=$NODE_IP_RANGES"
-      grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/salt-minion.sh"
-    ) > "${KUBE_TEMP}/node-start-${i}.sh"
-  done
-}
-
-#
-# Create a script that will run on the Kubernetes master and will run salt
-# to configure the master. We make it a script instead of just running a
-# single ssh command so that we can get logging.
-#
-function gen-master-salt {
-  gen-salt "kubernetes-master"
-}
-
-#
-# Create scripts that will be run on the Kubernetes master. Each of these
-# will invoke salt to configure one of the nodes
-#
-function gen-node-salt {
-  local i
-  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
-    gen-salt "${NODE_NAMES[${i}]}"
-  done
-}
-
-#
-# Shared implementation for gen-master-salt and gen-node-salt
-# Writes a script that installs Kubernetes with salt
-# The core of the script is simple (run 'salt ... state.highstate')
-# We also do a bit of logging so we can debug problems
-#
-# There is also a funky workaround for an issue with docker 1.9
-# (elsewhere we peg ourselves to docker 1.9). It's fixed in 1.10,
-# so we should be able to remove it in the future
-# https://github.com/docker/docker/issues/18113
-# The problem is that sometimes the install (with apt-get) of
-# docker fails. Deleting a file and retrying fixes it.
-#
-# Tell shellcheck to ignore our variables within single quotes:
-# We're writing a script, not executing it, so this is normal
-# shellcheck disable=SC2016
-function gen-salt {
-  node_name=${1}
-    (
-      echo '#!/bin/bash'
-      echo ''
-      echo "node=${node_name}"
-      echo 'out=/tmp/${node}-salt.out'
-      echo 'log=/tmp/${node}-salt.log'
-      echo ''
-      echo 'echo $(date) >> $log'
-      echo 'salt ${node} state.highstate -t 30 --no-color > ${out}'
-      echo 'grep -E "Failed:[[:space:]]+0" ${out}'
-      echo 'success=$?'
-      echo 'cat ${out} >> ${log}'
-      echo ''
-      echo 'if [[ ${success} -ne 0 ]]; then'
-      echo '  # Did we try to install docker-engine?'
-      echo '  attempted=$(grep docker-engine ${out} | wc -l)'
-      echo '  # Is docker-engine installed?'
-      echo '  installed=$(salt --output=txt ${node} pkg.version docker-engine | wc -l)'
-      echo '  if [[ ${attempted} -ne 0 && ${installed} -eq 0 ]]; then'
-      echo '    echo "Unwedging docker-engine install" >> ${log}'
-      echo '    salt ${node} cmd.run "rm -f /var/lib/docker/network/files/local-kv.db"'
-      echo '  fi'
-      echo 'fi'
-      echo 'exit ${success}'
-    ) > "${KUBE_TEMP}/${node_name}-salt.sh"
-}
-
-#
-# Generate a script to add a route to a host (master or node)
-# The script will do two things:
-# 1. Add the route immediately with the route command
-# 2. Persist the route by saving it in /etc/network/interfaces
-# This was done with a script because it was easier to get the quoting right
-# and make it clear.
-#
-function gen-add-route {
-  route=${1}
-  gateway=${2}
-  (
-      echo '#!/bin/bash'
-      echo ''
-      echo '# Immediately add route'
-      echo "sudo route add -net ${route} gw ${gateway}"
-      echo ''
-      echo '# Persist route so it lasts over restarts'
-      echo 'sed -in "s|^iface eth0.*|&\n    post-up route add -net' "${route} gw ${gateway}|"'" /etc/network/interfaces'
-  ) > "${KUBE_TEMP}/add-route.sh"
-}
-
-#
-# Create the Kubernetes master VM
-# Sets global variables:
-# - KUBE_MASTER    (Name)
-# - KUBE_MASTER_ID (Photon VM ID)
-# - KUBE_MASTER_IP (IP address)
-#
-function create-master-vm {
-  kube::log::status "Starting master VM..."
-  pc-create-vm "${MASTER_NAME}" "${PHOTON_MASTER_FLAVOR}"
-  KUBE_MASTER=${MASTER_NAME}
-  KUBE_MASTER_ID=${_VM_ID}
-  KUBE_MASTER_IP=${_VM_IP}
-}
-
-#
-# Install salt on the Kubernetes master
-# Relies on the master-start.sh script created in gen-master-start
-#
-function install-salt-on-master {
-  kube::log::status "Installing salt on master..."
-  upload-server-tars "${MASTER_NAME}" "${KUBE_MASTER_IP}"
-  run-script-remotely "${KUBE_MASTER_IP}" "${KUBE_TEMP}/master-start.sh"
-}
-
-#
-# Installs salt on Kubernetes nodes in parallel
-# Relies on the node-start script created in gen-node-start
-#
-function install-salt-on-nodes {
-  kube::log::status "Creating nodes and installing salt on them..."
-
-  # Start each of the VMs in parallel
-  # In the future, we'll batch this because it doesn't scale well
-  # past 10 or 20 nodes
-  local node
-  for (( node=0; node<${#NODE_NAMES[@]}; node++)); do
-  (
-    pc-create-vm "${NODE_NAMES[${node}]}" "${PHOTON_NODE_FLAVOR}"
-    run-script-remotely "${_VM_IP}" "${KUBE_TEMP}/node-start-${node}.sh"
-  ) &
-  done
-
-  # Wait for the node VM startups to complete
-  local fail=0
-  local job
-  for job in $(jobs -p); do
-    wait "${job}" || fail=$((fail + 1))
-  done
-  if (( fail != 0 )); then
-    kube::log::error "Failed to start ${fail}/${NUM_NODES} nodes"
-    exit 1
-  fi
-}
-
-#
-# Install Kubernetes on the master.
-# This uses the kubernetes-master-salt.sh script created by gen-master-salt
-# That script uses salt to install Kubernetes
-#
-function install-kubernetes-on-master {
-  # Wait until salt-master is running: it may take a bit
-  try-until-success-ssh "${KUBE_MASTER_IP}" \
-    "Waiting for salt-master to start on ${KUBE_MASTER}" \
-    "pgrep salt-master"
-  gen-master-salt
-  copy-file-to-vm "${_VM_IP}" "${KUBE_TEMP}/kubernetes-master-salt.sh" "/tmp/kubernetes-master-salt.sh"
-  try-until-success-ssh "${KUBE_MASTER_IP}" \
-    "Installing Kubernetes on ${KUBE_MASTER} via salt" \
-    "sudo /bin/bash /tmp/kubernetes-master-salt.sh"
-}
-
-#
-# Install Kubernetes on the nodes in parallel
-# This uses the kubernetes-master-salt.sh script created by gen-node-salt
-# That script uses salt to install Kubernetes
-#
-function install-kubernetes-on-nodes {
-  gen-node-salt
-
-  # Run in parallel to bring up the cluster faster
-  # TODO: Batch this so that we run up to N in parallel, so
-  # we don't overload this machine or the salt master
-  local node
-  for (( node=0; node<${#NODE_NAMES[@]}; node++)); do
-    (
-      copy-file-to-vm "${_VM_IP}" "${KUBE_TEMP}/${NODE_NAMES[${node}]}-salt.sh" "/tmp/${NODE_NAMES[${node}]}-salt.sh"
-      try-until-success-ssh "${KUBE_NODE_IP_ADDRESSES[${node}]}" \
-        "Waiting for salt-master to start on ${NODE_NAMES[${node}]}" \
-        "pgrep salt-minion"
-      try-until-success-ssh "${KUBE_MASTER_IP}" \
-        "Installing Kubernetes on ${NODE_NAMES[${node}]} via salt" \
-        "sudo /bin/bash /tmp/${NODE_NAMES[${node}]}-salt.sh"
-    ) &
-  done
-
-  # Wait for the Kubernetes installations to complete
-  local fail=0
-  local job
-  for job in $(jobs -p); do
-    wait "${job}" || fail=$((fail + 1))
-  done
-  if (( fail != 0 )); then
-    kube::log::error "Failed to start install Kubernetes on ${fail} out of ${NUM_NODES} nodess"
-    exit 1
-  fi
-}
-
-#
-# Upload the Kubernetes tarballs to the master
-#
-function upload-server-tars {
-  vm_name=${1}
-  vm_ip=${2}
-
-  run-ssh-cmd "${vm_ip}" "mkdir -p /home/kube/cache/kubernetes-install"
-
-  local tar
-  for tar in "${SERVER_BINARY_TAR}" "${SALT_TAR}"; do
-    local base_tar
-    base_tar=$(basename "${tar}")
-    kube::log::status "Uploading ${base_tar} to ${vm_name}..."
-    copy-file-to-vm "${vm_ip}" "${tar}" "/home/kube/cache/kubernetes-install/${tar##*/}"
-  done
-}
-
-#
-# Wait for the Kubernets healthz API to be responsive on the master
-#
-function wait-master-api {
-  local curl_creds="--insecure --user ${KUBE_USER}:${KUBE_PASSWORD}"
-  local curl_output="--fail --output /dev/null --silent"
-  local curl_net="--max-time 1"
-
-  try-until-success "Waiting for Kubernetes API on ${KUBE_MASTER}" \
-    "curl ${curl_creds} ${curl_output} ${curl_net} https://${KUBE_MASTER_IP}/healthz"
-}
-
-#
-# Wait for the Kubernetes healthz API to be responsive on each node
-#
-function wait-node-apis {
-  local curl_output="--fail --output /dev/null --silent"
-  local curl_net="--max-time 1"
-
-  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
-    try-until-success "Waiting for Kubernetes API on ${NODE_NAMES[${i}]}..." \
-      "curl ${curl_output} ${curl_net} http://${KUBE_NODE_IP_ADDRESSES[${i}]}:10250/healthz"
-  done
-}
-
-#
-# Configure the nodes so the pods can communicate
-# Each node will have a bridge named cbr0 for the NODE_IP_RANGES
-# defined in config-default.sh. This finds the IP subnet (assigned
-# by Kubernetes) to nodes and configures routes so they can communicate
-#
-# Also configure the master to be able to talk to the nodes. This is
-# useful so that you can get to the UI from the master.
-#
-function setup-pod-routes {
-  local node
-
-  KUBE_NODE_BRIDGE_NETWORK=()
-  for (( node=0; node<${#NODE_NAMES[@]}; node++)); do
-
-    # This happens in two steps (wait for an address, wait for a non 172.x.x.x address)
-    # because it's both simpler and more clear what's happening.
-    try-until-success-ssh "${KUBE_NODE_IP_ADDRESSES[${node}]}" \
-      "Waiting for cbr0 bridge on ${NODE_NAMES[${node}]} to have an address"  \
-      'sudo ifconfig cbr0  | grep -oP "inet addr:\K\S+"'
-
-    try-until-success-ssh "${KUBE_NODE_IP_ADDRESSES[${node}]}" \
-      "Waiting for cbr0 bridge on ${NODE_NAMES[${node}]} to have correct address"  \
-      'sudo ifconfig cbr0  | grep -oP "inet addr:\K\S+" | grep -v  "^172."'
-
-    run-ssh-cmd "${KUBE_NODE_IP_ADDRESSES[${node}]}" 'sudo ip route show | grep -E "dev cbr0" | cut -d " " -f1'
-    KUBE_NODE_BRIDGE_NETWORK+=(${_OUTPUT})
-    kube::log::status "cbr0 on ${NODE_NAMES[${node}]} is ${_OUTPUT}"
-  done
-
-  local i
-  local j
-  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
-    kube::log::status "Configuring pod routes on ${NODE_NAMES[${i}]}..."
-    gen-add-route "${KUBE_NODE_BRIDGE_NETWORK[${i}]}" "${KUBE_NODE_IP_ADDRESSES[${i}]}"
-    run-script-remotely "${KUBE_MASTER_IP}" "${KUBE_TEMP}/add-route.sh"
-
-    for (( j=0; j<${#NODE_NAMES[@]}; j++)); do
-      if [[ "${i}" != "${j}" ]]; then
-        gen-add-route "${KUBE_NODE_BRIDGE_NETWORK[${j}]}" "${KUBE_NODE_IP_ADDRESSES[${j}]}"
-        run-script-remotely "${KUBE_NODE_IP_ADDRESSES[${i}]}" "${KUBE_TEMP}/add-route.sh"
-      fi
-    done
-  done
-}
-
-#
-# Copy the certificate/key from the Kubernetes master
-# These are used to create the kubeconfig file, which allows
-# users to use kubectl easily
-#
-# We also set KUBE_CERT, KUBE_KEY, CA_CERT, and CONTEXT because they
-# are needed by create-kubeconfig from common.sh to generate
-# the kube config file.
-#
-function copy-kube-certs {
-  local cert="kubecfg.crt"
-  local key="kubecfg.key"
-  local ca="ca.crt"
-  local cert_dir="/srv/kubernetes"
-
-  kube::log::status "Copying credentials from ${KUBE_MASTER}"
-
-  # Set global environment variables: needed by create-kubeconfig
-  # in common.sh
-  export KUBE_CERT="${KUBE_TEMP}/${cert}"
-  export KUBE_KEY="${KUBE_TEMP}/${key}"
-  export CA_CERT="${KUBE_TEMP}/${ca}"
-  export CONTEXT="photon-${INSTANCE_PREFIX}"
-
-  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 644 ${cert_dir}/${cert}"
-  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 644 ${cert_dir}/${key}"
-  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 644 ${cert_dir}/${ca}"
-
-  copy-file-from-vm "${KUBE_MASTER_IP}" "${cert_dir}/${cert}" "${KUBE_CERT}"
-  copy-file-from-vm "${KUBE_MASTER_IP}" "${cert_dir}/${key}"  "${KUBE_KEY}"
-  copy-file-from-vm "${KUBE_MASTER_IP}" "${cert_dir}/${ca}"   "${CA_CERT}"
-
-  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 600 ${cert_dir}/${cert}"
-  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 600 ${cert_dir}/${key}"
-  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 600 ${cert_dir}/${ca}"
-}
-
-#
-# Copies a script to a VM and runs it
-# Parameters:
-#   - IP of VM
-#   - Path to local file
-#
-function run-script-remotely {
-  local vm_ip=${1}
-  local local_file="${2}"
-  local base_file
-  local remote_file
-
-  base_file=$(basename "${local_file}")
-  remote_file="/tmp/${base_file}"
-
-  copy-file-to-vm "${vm_ip}" "${local_file}" "${remote_file}"
-  run-ssh-cmd "${vm_ip}" "chmod 700 ${remote_file}"
-  run-ssh-cmd "${vm_ip}" "nohup sudo ${remote_file} < /dev/null 1> ${remote_file}.out 2>&1 &"
-}
-
-#
-# Runs an command on a VM using ssh
-# Parameters:
-#   - (optional) -i to ignore failure
-#   - IP address of the VM
-#   - Command to run
-# Assumes environment variables:
-#   - VM_USER
-#   - SSH_OPTS
-#
-function run-ssh-cmd {
-  local ignore_failure=""
-  if [[ "${1}" = "-i" ]]; then
-    ignore_failure="-i"
-    shift
-  fi
-
-  local vm_ip=${1}
-  shift
-  local cmd=${1}
-
-
-  run-cmd ${ignore_failure} "ssh ${SSH_OPTS} $VM_USER@${vm_ip} $1"
-}
-
-#
-# Uses scp to copy file to VM
-# Parameters:
-#   - IP address of the VM
-#   - Path to local file
-#   - Path to remote file
-# Assumes environment variables:
-#   - VM_USER
-#   - SSH_OPTS
-#
-function copy-file-to-vm {
-  local vm_ip=${1}
-  local local_file=${2}
-  local remote_file=${3}
-
-  run-cmd "scp ${SSH_OPTS} ${local_file} ${VM_USER}@${vm_ip}:${remote_file}"
-}
-
-function copy-file-from-vm {
-  local vm_ip=${1}
-  local remote_file=${2}
-  local local_file=${3}
-
-  run-cmd "scp ${SSH_OPTS} ${VM_USER}@${vm_ip}:${remote_file} ${local_file}"
-}
-
-#
-# Run a command, print nice error output
-# Used by copy-file-to-vm and run-ssh-cmd
-#
-function run-cmd {
-  local rc=0
-  local ignore_failure=""
-  if [[ "${1}" = "-i" ]]; then
-    ignore_failure=${1}
-    shift
-  fi
-
-  local cmd=$1
-  local output
-  output=$(${cmd} 2>&1) || rc=$?
-  if [[ ${rc} -ne 0 ]]; then
-    if [[ -z "${ignore_failure}" ]]; then
-      kube::log::error "Failed to run command: ${cmd} Output:"
-      echo "${output}"
-      exit 1
-    fi
-  fi
-  _OUTPUT=${output}
-  return ${rc}
-}
-
-#
-# After the initial VM setup, we use SSH with keys to access the VMs
-# This requires an SSH agent, so we verify that it's running
-#
-function verify-ssh-prereqs {
-  kube::log::status "Validating SSH configuration..."
-  local rc
-
-  rc=0
-  ssh-add -L 1> /dev/null 2> /dev/null || rc=$?
-  # "Could not open a connection to your authentication agent."
-  if [[ "${rc}" -eq 2 ]]; then
-    # ssh agent wasn't running, so start it and ensure we stop it
-    eval "$(ssh-agent)" > /dev/null
-    trap-add "kill ${SSH_AGENT_PID}" EXIT
-  fi
-
-  rc=0
-  ssh-add -L 1> /dev/null 2> /dev/null || rc=$?
-  # "The agent has no identities."
-  if [[ "${rc}" -eq 1 ]]; then
-  # Try adding one of the default identities, with or without passphrase.
-    ssh-add || true
-  fi
-
-  # Expect at least one identity to be available.
-  if ! ssh-add -L 1> /dev/null 2> /dev/null; then
-    kube::log::error "Could not find or add an SSH identity."
-    kube::log::error "Please start ssh-agent, add your identity, and retry."
-    exit 1
-  fi
-}
-
-#
-# Verify that Photon Controller has been configured in the way we expect. Specifically
-# - Have the flavors been created?
-# - Has the image been uploaded?
-# TODO: Check the tenant and project as well.
-function verify-photon-config {
-  kube::log::status "Validating Photon configuration..."
-
-  # We don't want silent failure: we check for failure
-  set +o pipefail
-
-  verify-photon-flavors
-  verify-photon-image
-  verify-photon-tenant
-
-  # Reset default set in common.sh
-  set -o pipefail
-}
-
-#
-# Verify that the VM and disk flavors have been created
-#
-function verify-photon-flavors {
-  local rc=0
-
-  ${PHOTON} flavor list | awk -F'\t' '{print $2}' | grep -q "^${PHOTON_MASTER_FLAVOR}$" > /dev/null 2>&1 || rc=$?
-  if [[ ${rc} -ne 0 ]]; then
-    kube::log::error "ERROR: Cannot find VM flavor named ${PHOTON_MASTER_FLAVOR}"
-    exit 1
-  fi
-
-  if [[ "${PHOTON_MASTER_FLAVOR}" != "${PHOTON_NODE_FLAVOR}" ]]; then
-    rc=0
-    ${PHOTON} flavor list | awk -F'\t' '{print $2}' | grep -q "^${PHOTON_NODE_FLAVOR}$" > /dev/null 2>&1 || rc=$?
-    if [[ ${rc} -ne 0 ]]; then
-      kube::log::error "ERROR: Cannot find VM flavor named ${PHOTON_NODE_FLAVOR}"
-      exit 1
-    fi
-  fi
-
-  ${PHOTON} flavor list | awk -F'\t' '{print $2}' | grep -q "^${PHOTON_DISK_FLAVOR}$" > /dev/null 2>&1 || rc=$?
-  if [[ ${rc} -ne 0 ]]; then
-    kube::log::error "ERROR: Cannot find disk flavor named ${PHOTON_DISK_FLAVOR}"
-    exit 1
-  fi
-}
-
-#
-# Verify that we have the image we need, and it's not in error state or
-# multiple copies
-#
-function verify-photon-image {
-  local rc
-
-  rc=0
-  ${PHOTON} image list | grep -q $'\t'"${PHOTON_IMAGE}"$'\t'  > /dev/null 2>&1 || rc=$?
-  if [[ ${rc} -ne 0 ]]; then
-    kube::log::error "ERROR: Cannot find image \"${PHOTON_IMAGE}\""
-    exit 1
-  fi
-
-  rc=0
-  ${PHOTON} image list | grep $'\t'"${PHOTON_IMAGE}"$'\t' | grep ERROR > /dev/null 2>&1 || rc=$?
-  if [[ ${rc} -eq 0 ]]; then
-    echo "Warning: You have at least one ${PHOTON_IMAGE} image in the ERROR state. You may want to investigate."
-    echo "Images in the ERROR state will be ignored."
-  fi
-
-  rc=0
-  num_images=$(${PHOTON} image list | grep $'\t'"${PHOTON_IMAGE}"$'\t' | grep -c READY)
-  if [[ "${num_images}" -gt 1 ]]; then
-    echo "ERROR: You have more than one READY ${PHOTON_IMAGE} image. Ensure there is only one"
-    exit 1
-  fi
-}
-
-function verify-photon-tenant {
-  local rc
-
-  rc=0
-  ${PHOTON} tenant list | grep -q $'\t'"${PHOTON_TENANT}"  > /dev/null 2>&1 || rc=$?
-  if [[ ${rc} -ne 0 ]]; then
-    echo "ERROR: Cannot find tenant \"${PHOTON_TENANT}\""
-    exit 1
-  fi
-
-  ${PHOTON} project list --tenant "${PHOTON_TENANT}" | grep -q $'\t'"${PHOTON_PROJECT}"$'\t' > /dev/null 2>&1  || rc=$?
-  if [[ ${rc} -ne 0 ]]; then
-    echo "ERROR: Cannot find project \"${PHOTON_PROJECT}\""
-    exit 1
-  fi
-}
-
-#
-# Verifies that a given command is in the PATH
-#
-function verify-cmd-in-path {
-  cmd=${1}
-  which "${cmd}" >/dev/null || {
-    kube::log::error "Can't find ${cmd} in PATH, please install and retry."
-    exit 1
-  }
-}
-
-#
-# Repeatedly try a command over ssh until it succeeds or until five minutes have passed
-# The timeout isn't exact, since we assume the command runs instantaneously, and
-# it doesn't.
-#
-function try-until-success-ssh {
-  local vm_ip=${1}
-  local cmd_description=${2}
-  local cmd=${3}
-  local timeout=600
-  local sleep_time=5
-  local max_attempts
-
-  ((max_attempts=timeout/sleep_time))
-
-  kube::log::status "${cmd_description} for up to 10 minutes..."
-  local attempt=0
-  while true; do
-    local rc=0
-    run-ssh-cmd -i "${vm_ip}" "${cmd}" || rc=1
-    if [[ ${rc} != 0 ]]; then
-      if (( attempt == max_attempts )); then
-        kube::log::error "Failed, cannot proceed: you may need to retry to log into the VM to debug"
-        exit 1
-      fi
-    else
-      break
-    fi
-    attempt=$((attempt+1))
-    sleep ${sleep_time}
-  done
-}
-
-function try-until-success {
-  local cmd_description=${1}
-  local cmd=${2}
-  local timeout=600
-  local sleep_time=5
-  local max_attempts
-
-  ((max_attempts=timeout/sleep_time))
-
-  kube::log::status "${cmd_description} for up to 10 minutes..."
-  local attempt=0
-  while true; do
-    local rc=0
-    run-cmd -i "${cmd}" || rc=1
-    if [[ ${rc} != 0 ]]; then
-      if (( attempt == max_attempts )); then
-        kube::log::error "Failed, cannot proceed"
-        exit 1
-      fi
-    else
-      break
-    fi
-    attempt=$((attempt+1))
-    sleep ${sleep_time}
-  done
-}
-
-#
-# Sets up a trap handler
-#
-function trap-add {
-  local handler="${1}"
-  local signal="${2-EXIT}"
-  local cur
-
-  cur="$(eval "sh -c 'echo \$3' -- $(trap -p ${signal})")"
-  if [[ -n "${cur}" ]]; then
-    handler="${cur}; ${handler}"
-  fi
-
-  # We want ${handler} to expand now, so tell shellcheck
-  # shellcheck disable=SC2064
-  trap "${handler}" ${signal}
-}