diff --git a/cluster/pre-existing/util.sh b/cluster/pre-existing/util.sh index 06ef5ee9985..7dd3c632fb6 100644 --- a/cluster/pre-existing/util.sh +++ b/cluster/pre-existing/util.sh @@ -42,15 +42,4 @@ function detect-project() { function create-certs { execute-cmd-on-pre-existing-master-with-retries 'sudo cat /etc/kubernetes/admin.conf' > /tmp/kubeconfig - - # CA_CERT_BASE64, KUBELET_CERT_BASE64 and KUBELET_KEY_BASE64 might be used - # in test/kubemark/iks/util.sh - # If it becomes clear that the variables are not used anywhere, then we can - # remove them: - CA_CERT_BASE64=$(grep certificate-authority /tmp/kubeconfig | awk '{print $2}' | head -n 1) - KUBELET_CERT_BASE64=$(grep client-certificate-data /tmp/kubeconfig | awk '{print $2}' | head -n 1) - KUBELET_KEY_BASE64=$(grep client-key-data /tmp/kubeconfig | awk '{print $2}' | head -n 1) - export CA_CERT_BASE64 - export KUBELET_CERT_BASE64 - export KUBELET_KEY_BASE64 } diff --git a/test/kubemark/iks/shutdown.sh b/test/kubemark/iks/shutdown.sh deleted file mode 100644 index 9b0b2477afe..00000000000 --- a/test/kubemark/iks/shutdown.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script that destroys the clusters used, namespace, and deployment. - -KUBECTL=kubectl -KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" -RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" - -# Login to cloud services -complete-login - -# Remove resources created for kubemark -# shellcheck disable=SC2154 # Color defined in sourced script -echo -e "${color_yellow}REMOVING RESOURCES${color_norm}" -spawn-config -"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/addons" &> /dev/null || true -"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" &> /dev/null || true -"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" &> /dev/null || true -rm -rf "${RESOURCE_DIRECTORY}/addons" - "${RESOURCE_DIRECTORY}/hollow-node.yaml" &> /dev/null || true - -# Remove clusters, namespaces, and deployments -delete-clusters -if [[ -f "${RESOURCE_DIRECTORY}/iks-namespacelist.sh" ]] ; then - bash "${RESOURCE_DIRECTORY}/iks-namespacelist.sh" - rm -f "${RESOURCE_DIRECTORY}/iks-namespacelist.sh" -fi -# shellcheck disable=SC2154 # Color defined in sourced script -echo -e "${color_blue}EXECUTION COMPLETE${color_norm}" -exit 0 diff --git a/test/kubemark/iks/startup.sh b/test/kubemark/iks/startup.sh deleted file mode 100644 index a330aa91e61..00000000000 --- a/test/kubemark/iks/startup.sh +++ /dev/null @@ -1,320 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script that creates a Kubemark cluster for IBM cloud. - -KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" -KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" -RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" - -# Generate secret and configMap for the hollow-node pods to work, prepare -# manifests of the hollow-node and heapster replication controllers from -# templates, and finally create these resources through kubectl. -function create-kube-hollow-node-resources { - # Create kubeconfig for Kubelet. - KUBELET_KUBECONFIG_CONTENTS="$(cat < "${RESOURCE_DIRECTORY}/addons/heapster.json" - metrics_mem_per_node=4 - metrics_mem=$((200 + metrics_mem_per_node*NUM_NODES)) - sed -i'' -e "s/{{METRICS_MEM}}/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" - metrics_cpu_per_node_numerator=${NUM_NODES} - metrics_cpu_per_node_denominator=2 - metrics_cpu=$((80 + metrics_cpu_per_node_numerator / metrics_cpu_per_node_denominator)) - sed -i'' -e "s/{{METRICS_CPU}}/${metrics_cpu}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" - eventer_mem_per_node=500 - eventer_mem=$((200 * 1024 + eventer_mem_per_node*NUM_NODES)) - sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" - - # Cluster Autoscaler. - if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-}" == "true" ]]; then - echo "Setting up Cluster Autoscaler" - KUBEMARK_AUTOSCALER_MIG_NAME="${KUBEMARK_AUTOSCALER_MIG_NAME:-${NODE_INSTANCE_PREFIX}-group}" - KUBEMARK_AUTOSCALER_MIN_NODES="${KUBEMARK_AUTOSCALER_MIN_NODES:-0}" - KUBEMARK_AUTOSCALER_MAX_NODES="${KUBEMARK_AUTOSCALER_MAX_NODES:-${DESIRED_NODES}}" - NUM_NODES=${KUBEMARK_AUTOSCALER_MAX_NODES} - echo "Setting maximum cluster size to ${NUM_NODES}." - KUBEMARK_MIG_CONFIG="autoscaling.k8s.io/nodegroup: ${KUBEMARK_AUTOSCALER_MIG_NAME}" - sed "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/cluster-autoscaler_template.json" > "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" - sed -i'' -e "s/{{kubemark_autoscaler_mig_name}}/${KUBEMARK_AUTOSCALER_MIG_NAME}/g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" - sed -i'' -e "s/{{kubemark_autoscaler_min_nodes}}/${KUBEMARK_AUTOSCALER_MIN_NODES}/g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" - sed -i'' -e "s/{{kubemark_autoscaler_max_nodes}}/${KUBEMARK_AUTOSCALER_MAX_NODES}/g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" - fi - - # Kube DNS. - if [[ "${ENABLE_KUBEMARK_KUBE_DNS:-}" == "true" ]]; then - echo "Setting up kube-dns" - sed "s/{{dns_domain}}/${KUBE_DNS_DOMAIN}/g" "${RESOURCE_DIRECTORY}/kube_dns_template.yaml" > "${RESOURCE_DIRECTORY}/addons/kube_dns.yaml" - fi - - "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" - set-registry-secrets - - # Create the replication controller for hollow-nodes. - # We allow to override the NUM_REPLICAS when running Cluster Autoscaler. - NUM_REPLICAS=${NUM_REPLICAS:-${KUBEMARK_NUM_NODES}} - sed "s/{{numreplicas}}/${NUM_REPLICAS}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml" - proxy_cpu=20 - if [ "${NUM_NODES}" -gt 1000 ]; then - proxy_cpu=50 - fi - proxy_mem_per_node=50 - proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES)) - hollow_kubelet_params=$(eval "for param in ${HOLLOW_KUBELET_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done") - hollow_kubelet_params=${hollow_kubelet_params%?} - hollow_proxy_params=$(eval "for param in ${HOLLOW_PROXY_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done") - hollow_proxy_params=${hollow_proxy_params%?} - - sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s'{{kubemark_image_registry}}'${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s/{{kubemark_image_tag}}/${KUBEMARK_IMAGE_TAG}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s/{{hollow_kubelet_params}}/${hollow_kubelet_params}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s/{{hollow_proxy_params}}/${hollow_proxy_params}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s'{{kubemark_mig_config}}'${KUBEMARK_MIG_CONFIG:-}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark" - - echo "Created secrets, configMaps, replication-controllers required for hollow-nodes." -} - -# Wait until all hollow-nodes are running or there is a timeout. -function wait-for-hollow-nodes-to-run-or-timeout { - echo -n "Waiting for all hollow-nodes to become Running" - start=$(date +%s) - # IKS uses a real cluster for hollow master, so need to exclude the real worker nodes - nodes=$("${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node | grep hollow-node 2> /dev/null) || true - ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1)) - until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do - echo -n "." - sleep 1 - now=$(date +%s) - # Fail it if it already took more than 30 minutes. - if [ $((now - start)) -gt 1800 ]; then - echo "" - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}" - # Try listing nodes again - if it fails it means that API server is not responding - if "${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node &> /dev/null; then - echo "Found only ${ready} ready hollow-nodes while waiting for ${NUM_NODES}." - else - echo "Got error while trying to list hollow-nodes. Probably API server is down." - fi - spawn-config - pods=$("${KUBECTL}" get pods -l name=hollow-node --namespace=kubemark) || true - running=$(($(echo "${pods}" | grep -c "Running"))) - echo "${running} hollow-nodes are reported as 'Running'" - not_running=$(($(echo "${pods}" | grep -vc "Running") - 1)) - echo "${not_running} hollow-nodes are reported as NOT 'Running'" - echo "${pods}" | grep -v "Running" - exit 1 - fi - nodes=$("${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node 2> /dev/null) || true - ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1)) - done - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_green} Done!${color_norm}" -} - -############################### Main Function ######################################## -# In order for the cluster autoscalar to function, the template file must be changed so that the ":443" -# is removed. This is because the port is already given with the MASTER_IP. - - -# Create clusters and populate with hollow nodes -complete-login -build-kubemark-image -choose-clusters -generate-values -set-hollow-master -echo "Creating kube hollow node resources" -create-kube-hollow-node-resources -master-config -# shellcheck disable=SC2154 # Color defined in sourced script -echo -e "${color_blue}EXECUTION COMPLETE${color_norm}" - -# Check status of Kubemark -# shellcheck disable=SC2154 # Color defined in sourced script -echo -e "${color_yellow}CHECKING STATUS${color_norm}" -wait-for-hollow-nodes-to-run-or-timeout - -# Celebrate -echo "" -# shellcheck disable=SC2154 # Color defined in sourced script -echo -e "${color_blue}SUCCESS${color_norm}" -clean-repo -exit 0 \ No newline at end of file diff --git a/test/kubemark/iks/util.sh b/test/kubemark/iks/util.sh deleted file mode 100644 index 554095888a5..00000000000 --- a/test/kubemark/iks/util.sh +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../.. - -# Creates a new kube-spawn cluster -function create-clusters { - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_yellow}CHECKING CLUSTERS${color_norm}" - if ibmcloud ks clusters | grep -Fq 'deleting'; then - echo -n "Deleting old clusters" - fi - while ibmcloud ks clusters | grep -Fq 'deleting' - do - echo -n "." - sleep 10 - done - echo "" - ibmcloud ks region-set us-east >/dev/null - ibmcloud ks vlans wdc06 >/dev/null - PRIVLAN=$(ibmcloud ks vlans wdc06 --json | jq '. | .[] | select(.type == "private") | .id' | sed -e "s/\"//g") - PUBVLAN=$(ibmcloud ks vlans wdc06 --json | jq '. | .[] | select(.type == "public") | .id' | sed -e "s/\"//g") - if ! ibmcloud ks clusters | grep -Fq 'kubeSpawnTester'; then - echo "Creating spawning cluster" - # make number and spec of node workers configurable - # otherwise it can't afford tests like kubemark-5000 - # TODO: dynamically adjust the number and spec - ibmcloud ks cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers "${NUM_NODES:-2}" --machine-type "${NODE_SIZE}" --name kubeSpawnTester - fi - if ! ibmcloud ks clusters | grep -Fq 'kubeMasterTester'; then - echo "Creating master cluster" - # if we can't make it a bare master (workers = 0) - # then make workers = 1 with the smallest machine spec - ibmcloud ks cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers 1 --machine-type u2c.2x4 --name kubeMasterTester - fi - push-image - if ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'normal'; then - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_cyan}Warning: new clusters may take up to 60 minutes to be ready${color_norm}" - echo -n "Clusters loading" - fi - while ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'normal' - do - echo -n "." - sleep 5 - done - while ! ibmcloud ks clusters | grep 'kubeMasterTester' | grep -Fq 'normal' - do - echo -n "." - sleep 5 - done - echo -e "${color_yellow}CLUSTER CREATION COMPLETE${color_norm}" -} - -# Builds and pushes image to registry -function push-image { - if [[ "${ISBUILD}" = "y" ]]; then - if ! ibmcloud cr namespaces | grep -Fq "${KUBE_NAMESPACE}"; then - echo "Creating registry namespace" - ibmcloud cr namespace-add "${KUBE_NAMESPACE}" - echo "ibmcloud cr namespace-rm ${KUBE_NAMESPACE}" >> "${RESOURCE_DIRECTORY}/iks-namespacelist.sh" - fi - docker build -t "${KUBEMARK_INIT_TAG}" "${KUBEMARK_IMAGE_LOCATION}" - docker tag "${KUBEMARK_INIT_TAG}" "${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG}" - docker push "${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG}" - echo "Image pushed" - else - KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-brandondr96}" - KUBE_NAMESPACE="" - fi -} - -# Allow user to use existing clusters if desired -function choose-clusters { - echo -n -e "Do you want to use custom clusters? [y/N]${color_cyan}>${color_norm} " - read -r USE_EXISTING - if [[ "${USE_EXISTING}" = "y" ]]; then - echo -e "${color_yellow}Enter path for desired hollow-node spawning cluster kubeconfig file:${color_norm}" - read -r CUSTOM_SPAWN_CONFIG - echo -e "${color_yellow}Enter path for desired hollow-node hosting cluster kubeconfig file:${color_norm}" - read -r CUSTOM_MASTER_CONFIG - push-image - elif [[ "${USE_EXISTING}" = "N" ]]; then - create-clusters - else - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_red}Invalid response, please try again:${color_norm}" - choose-clusters - fi -} - -# Ensure secrets are correctly set -function set-registry-secrets { - spawn-config - kubectl get secret bluemix-default-secret-regional -o yaml | sed 's/default/kubemark/g' | kubectl -n kubemark create -f - - kubectl patch serviceaccount -n kubemark default -p '{"imagePullSecrets": [{"name": "bluemix-kubemark-secret"}]}' - kubectl -n kubemark get serviceaccounts default -o json | jq 'del(.metadata.resourceVersion)' | jq 'setpath(["imagePullSecrets"];[{"name":"bluemix-kubemark-secret-regional"}])' | kubectl -n kubemark replace serviceaccount default -f - -} - -# Sets the hollow-node master -# Exported variables: -# MASTER_IP - IP Address of the Kubemark master -function set-hollow-master { - echo -e "${color_yellow}CONFIGURING MASTER${color_norm}" - master-config - MASTER_IP=$(grep server "$KUBECONFIG" | awk -F "/" '{print $3}') - export MASTER_IP -} - -# Set up master cluster environment -# Exported variables: -# KUBECONFIG - Overrides default kube config for the purpose of setting up the Kubemark master components. -function master-config { - if [[ "${USE_EXISTING}" = "y" ]]; then - export KUBECONFIG=${CUSTOM_MASTER_CONFIG} - else - eval "$(ibmcloud ks cluster-config kubeMasterTester --admin | grep export)" - fi -} - -# Set up spawn cluster environment -# Exported variables: -# KUBECONFIG - Overrides default kube config for the purpose of setting up the hollow-node cluster. -function spawn-config { - if [[ "${USE_EXISTING}" = "y" ]]; then - export KUBECONFIG=${CUSTOM_SPAWN_CONFIG} - else - eval "$(ibmcloud ks cluster-config kubeSpawnTester --admin | grep export)" - fi -} - -# Deletes existing clusters -function delete-clusters { - echo "DELETING CLUSTERS" - ibmcloud ks cluster-rm kubeSpawnTester - ibmcloud ks cluster-rm kubeMasterTester - while ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'deleting' - do - sleep 5 - done - while ! ibmcloud ks clusters | grep 'kubeMasterTester' | grep -Fq 'deleting' - do - sleep 5 - done - kubectl delete ns kubemark -} - -# Login to cloud services -function complete-login { - echo -e "${color_yellow}LOGGING INTO CLOUD SERVICES${color_norm}" - echo -n -e "Do you have a federated IBM cloud login? [y/N]${color_cyan}>${color_norm} " - read -r ISFED - if [[ "${ISFED}" = "y" ]]; then - ibmcloud login --sso -a "${REGISTRY_LOGIN_URL}" - elif [[ "${ISFED}" = "N" ]]; then - ibmcloud login -a "${REGISTRY_LOGIN_URL}" - else - echo -e "${color_red}Invalid response, please try again:${color_norm}" - complete-login - fi - ibmcloud cr login -} - -# Generate values to fill the hollow-node configuration templates. -# Exported variables: -# KUBECTL - The name or path to the kubernetes client binary. -function generate-values { - echo "Generating values" - master-config - KUBECTL=kubectl - export KUBECTL - KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" - RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" - CONFIGPATH=${KUBECONFIG%/*} - KUBELET_CERT_BASE64="${KUBELET_CERT_BASE64:-$(base64 "${CONFIGPATH}/admin.pem" | tr -d '\r\n')}" - KUBELET_KEY_BASE64="${KUBELET_KEY_BASE64:-$(base64 "${CONFIGPATH}/admin-key.pem" | tr -d '\r\n')}" - CA_CERT_BASE64="${CA_CERT_BASE64:-$( base64 "$(find "${CONFIGPATH}" -name "*ca*" | head -n 1)" | tr -d '\r\n')}" - -} - -# Build image for kubemark -function build-kubemark-image { - echo -n -e "Do you want to build the kubemark image? [y/N]${color_cyan}>${color_norm} " - read -r ISBUILD - if [[ "${ISBUILD}" = "y" ]]; then - echo -e "${color_yellow}BUILDING IMAGE${color_norm}" - "${KUBE_ROOT}/build/run.sh" make kubemark - cp "${KUBE_ROOT}/_output/dockerized/bin/linux/amd64/kubemark" "${KUBEMARK_IMAGE_LOCATION}" - elif [[ "${ISBUILD}" = "N" ]]; then - echo -n "" - else - echo -e "${color_red}Invalid response, please try again:${color_norm}" - build-kubemark-image - fi -} - -# Clean up repository -function clean-repo { - echo -n -e "Do you want to remove build output and binary? [y/N]${color_cyan}>${color_norm} " - read -r ISCLEAN - if [[ "${ISCLEAN}" = "y" ]]; then - echo -e "${color_yellow}CLEANING REPO${color_norm}" - rm -rf "${KUBE_ROOT}/_output" - rm -f "${KUBEMARK_IMAGE_LOCATION}/kubemark" - elif [[ "${ISCLEAN}" = "N" ]]; then - echo -n "" - else - echo -e "${color_red}Invalid response, please try again:${color_norm}" - clean-repo - fi -}