From b538f18c0e1d98249d8779fec7a5093cbd9723ef Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 17:00:03 -0600 Subject: [PATCH 01/14] Add color-coding to kubemark scripts. --- test/kubemark/skeleton/util.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/kubemark/skeleton/util.sh b/test/kubemark/skeleton/util.sh index 144f947372f..0620dc0496c 100644 --- a/test/kubemark/skeleton/util.sh +++ b/test/kubemark/skeleton/util.sh @@ -74,3 +74,20 @@ function copy-files() { function delete-master-instance-and-resources { echo "Deleting master instance and its allocated resources" 1>&2 } + +# Common colors used throughout the kubemark scripts +if [[ -z "${color_start-}" ]]; then + declare -r color_start="\033[" + # shellcheck disable=SC2034 + declare -r color_red="${color_start}0;31m" + # shellcheck disable=SC2034 + declare -r color_yellow="${color_start}0;33m" + # shellcheck disable=SC2034 + declare -r color_green="${color_start}0;32m" + # shellcheck disable=SC2034 + declare -r color_blue="${color_start}1;34m" + # shellcheck disable=SC2034 + declare -r color_cyan="${color_start}1;36m" + # shellcheck disable=SC2034 + declare -r color_norm="${color_start}0m" +fi \ No newline at end of file From ce4c85e3fdcb7e532f4e2d47b35f5f6f636f89ef Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 17:13:09 -0600 Subject: [PATCH 02/14] Fix shellcheck lint errors in cluster/kubemark/gce/config-default.sh --- cluster/kubemark/gce/config-default.sh | 4 +++- hack/.shellcheck_failures | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cluster/kubemark/gce/config-default.sh b/cluster/kubemark/gce/config-default.sh index 6e66cd73e08..46cd1fef848 100644 --- a/cluster/kubemark/gce/config-default.sh +++ b/cluster/kubemark/gce/config-default.sh @@ -19,6 +19,8 @@ # gce/util.sh script which assumes config filename), but if some things that # are enabled by default should not run in hollow clusters, they should be disabled here. +# shellcheck disable=SC2034 # Variables sourced in other scripts. + source "${KUBE_ROOT}/cluster/gce/config-common.sh" GCLOUD=gcloud @@ -115,7 +117,7 @@ ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-false} # (e.g. kubemark master, Heapster) enough resources to handle maximum cluster size. if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER}" == "true" ]]; then NUM_REPLICAS=1 - if [[ ! -z "$NUM_NODES" ]]; then + if [[ -n "$NUM_NODES" ]]; then echo "WARNING: Using Cluster Autoscaler, ignoring NUM_NODES parameter. Set KUBEMARK_AUTOSCALER_MAX_NODES to specify maximum size of the cluster." fi fi diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index d805a941c33..f66781abb49 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -19,7 +19,6 @@ ./cluster/gce/upgrade.sh ./cluster/gce/util.sh ./cluster/images/conformance/run_e2e.sh -./cluster/kubemark/gce/config-default.sh ./cluster/kubemark/iks/config-default.sh ./cluster/kubemark/util.sh ./cluster/log-dump/log-dump.sh From 9a58913e8fe96c5aa85e74f126bf498d897c89f3 Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 17:18:14 -0600 Subject: [PATCH 03/14] Fix shellcheck lint errors in cluster/kubemark/iks/config-default.sh --- cluster/kubemark/iks/config-default.sh | 1 + hack/.shellcheck_failures | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/kubemark/iks/config-default.sh b/cluster/kubemark/iks/config-default.sh index b74a6ab5d81..62c041a6faf 100644 --- a/cluster/kubemark/iks/config-default.sh +++ b/cluster/kubemark/iks/config-default.sh @@ -16,6 +16,7 @@ # Cloud information RANDGEN=$(dd if=/dev/urandom bs=64 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=16 count=1 2>/dev/null | sed 's/[A-Z]//g') +# shellcheck disable=2034 # Variable sourced in other scripts. KUBE_NAMESPACE="kubemark_${RANDGEN}" KUBEMARK_IMAGE_TAG="${KUBEMARK_IMAGE_TAG:-2}" KUBEMARK_IMAGE_LOCATION="${KUBEMARK_IMAGE_LOCATION:-${KUBE_ROOT}/cluster/images/kubemark}" diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index f66781abb49..a5de8ff948e 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -19,7 +19,6 @@ ./cluster/gce/upgrade.sh ./cluster/gce/util.sh ./cluster/images/conformance/run_e2e.sh -./cluster/kubemark/iks/config-default.sh ./cluster/kubemark/util.sh ./cluster/log-dump/log-dump.sh ./cluster/pre-existing/util.sh From 9a4f4878f5769c370c672b77c53b744540289de3 Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 17:22:05 -0600 Subject: [PATCH 04/14] Fix shellcheck lint errors in cluster/kubemark/util.sh --- cluster/kubemark/util.sh | 8 ++++---- hack/.shellcheck_failures | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cluster/kubemark/util.sh b/cluster/kubemark/util.sh index 348f3d009d0..52ecbdf5daf 100644 --- a/cluster/kubemark/util.sh +++ b/cluster/kubemark/util.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. -source ${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh -source ${KUBE_ROOT}/cluster/${CLOUD_PROVIDER}/util.sh -source ${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. +source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" +source "${KUBE_ROOT}/cluster/${CLOUD_PROVIDER}/util.sh" +source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index a5de8ff948e..bb215102045 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -19,7 +19,6 @@ ./cluster/gce/upgrade.sh ./cluster/gce/util.sh ./cluster/images/conformance/run_e2e.sh -./cluster/kubemark/util.sh ./cluster/log-dump/log-dump.sh ./cluster/pre-existing/util.sh ./cluster/restore-from-backup.sh From 5f4b919887787daff38ec3d66ff60fa74388d358 Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 17:28:08 -0600 Subject: [PATCH 05/14] Fix shellcheck lint errors in test/kubemark/common/util.sh --- hack/.shellcheck_failures | 1 - test/kubemark/common/util.sh | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index bb215102045..77b53dd1d7e 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -145,7 +145,6 @@ ./test/images/volume/rbd/create_block.sh ./test/images/volume/rbd/mon.sh ./test/images/volume/rbd/osd.sh -./test/kubemark/common/util.sh ./test/kubemark/gce/util.sh ./test/kubemark/iks/shutdown.sh ./test/kubemark/iks/startup.sh diff --git a/test/kubemark/common/util.sh b/test/kubemark/common/util.sh index 171397bcbe2..05cdba4c77c 100644 --- a/test/kubemark/common/util.sh +++ b/test/kubemark/common/util.sh @@ -17,7 +17,7 @@ # Running cmd $RETRIES times in case of failures. function run-cmd-with-retries { RETRIES="${RETRIES:-3}" - for attempt in $(seq 1 ${RETRIES}); do + for attempt in $(seq 1 "${RETRIES}"); do local ret_val=0 exec 5>&1 # Duplicate &1 to &5 for use below. # We don't use 'local' to declare result as then ret_val always gets value 0. @@ -26,19 +26,24 @@ function run-cmd-with-retries { if [[ "${ret_val:-0}" -ne "0" ]]; then if [[ $(echo "${result}" | grep -c "already exists") -gt 0 ]]; then if [[ "${attempt}" == 1 ]]; then + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_red}Failed to $1 $2 ${3:-} as the resource hasn't been deleted from a previous run.${color_norm}" >& 2 exit 1 fi + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_yellow}Succeeded to $1 $2 ${3:-} in the previous attempt, but status response wasn't received.${color_norm}" return 0 fi + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_yellow}Attempt $attempt failed to $1 $2 ${3:-}. Retrying.${color_norm}" >& 2 - sleep $(($attempt * 5)) + sleep $((attempt * 5)) else + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_green}Succeeded to $1 $2 ${3:-}.${color_norm}" return 0 fi done + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_red}Failed to $1 $2 ${3:-}.${color_norm}" >& 2 exit 1 } From 186b83fe5bb7b1be9c469b24be2bc5ca573df8d3 Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 17:36:53 -0600 Subject: [PATCH 06/14] Fix shellcheck lint errors in test/kubemark/gce/util.sh --- hack/.shellcheck_failures | 1 - test/kubemark/gce/util.sh | 28 ++++++++++++++-------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index 77b53dd1d7e..a643ca0a37e 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -145,7 +145,6 @@ ./test/images/volume/rbd/create_block.sh ./test/images/volume/rbd/mon.sh ./test/images/volume/rbd/osd.sh -./test/kubemark/gce/util.sh ./test/kubemark/iks/shutdown.sh ./test/kubemark/iks/startup.sh ./test/kubemark/iks/util.sh diff --git a/test/kubemark/gce/util.sh b/test/kubemark/gce/util.sh index a922ab7f411..d4ba2c00861 100644 --- a/test/kubemark/gce/util.sh +++ b/test/kubemark/gce/util.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../.. +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../.. source "${KUBE_ROOT}/test/kubemark/common/util.sh" @@ -47,7 +47,7 @@ function get-or-create-master-ip { } function create-master-instance-with-resources { - GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}" + GCLOUD_COMMON_ARGS=(--project "${PROJECT}" --zone "${ZONE}") # Override the master image project to cos-cloud for COS images staring with `cos` string prefix. DEFAULT_GCI_PROJECT=google-containers if [[ "${GCI_VERSION}" == "cos"* ]]; then @@ -56,13 +56,13 @@ function create-master-instance-with-resources { MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-${DEFAULT_GCI_PROJECT}} run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} \ + "${GCLOUD_COMMON_ARGS[@]}" \ --type "${MASTER_DISK_TYPE}" \ --size "${MASTER_DISK_SIZE}" & if [ "${EVENT_PD:-}" == "true" ]; then run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \ - ${GCLOUD_COMMON_ARGS} \ + "${GCLOUD_COMMON_ARGS[@]}" \ --type "${MASTER_DISK_TYPE}" \ --size "${MASTER_DISK_SIZE}" & fi @@ -72,7 +72,7 @@ function create-master-instance-with-resources { wait run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \ - ${GCLOUD_COMMON_ARGS} \ + "${GCLOUD_COMMON_ARGS[@]}" \ --address "${MASTER_IP}" \ --machine-type "${MASTER_SIZE}" \ --image-project="${MASTER_IMAGE_PROJECT}" \ @@ -84,13 +84,13 @@ function create-master-instance-with-resources { --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" run-gcloud-compute-with-retries instances add-metadata "${MASTER_NAME}" \ - ${GCLOUD_COMMON_ARGS} \ + "${GCLOUD_COMMON_ARGS[@]}" \ --metadata-from-file startup-script="${KUBE_ROOT}/test/kubemark/resources/start-kubemark-master.sh" & if [ "${EVENT_PD:-}" == "true" ]; then echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}" run-gcloud-compute-with-retries instances attach-disk "${MASTER_NAME}" \ - ${GCLOUD_COMMON_ARGS} \ + "${GCLOUD_COMMON_ARGS[@]}" \ --disk "${MASTER_NAME}-event-pd" \ --device-name="master-event-pd" & fi @@ -112,20 +112,20 @@ function execute-cmd-on-master-with-retries() { } function copy-files() { - run-gcloud-compute-with-retries scp --recurse --zone="${ZONE}" --project="${PROJECT}" $@ + run-gcloud-compute-with-retries scp --recurse --zone="${ZONE}" --project="${PROJECT}" "$@" } function delete-master-instance-and-resources { - GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE} --quiet" + GCLOUD_COMMON_ARGS=(--project "${PROJECT}" --zone "${ZONE}" --quiet) gcloud compute instances delete "${MASTER_NAME}" \ - ${GCLOUD_COMMON_ARGS} || true + "${GCLOUD_COMMON_ARGS[@]}" || true gcloud compute disks delete "${MASTER_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} || true + "${GCLOUD_COMMON_ARGS[@]}" || true gcloud compute disks delete "${MASTER_NAME}-event-pd" \ - ${GCLOUD_COMMON_ARGS} &> /dev/null || true + "${GCLOUD_COMMON_ARGS[@]}" &> /dev/null || true gcloud compute addresses delete "${MASTER_NAME}-ip" \ --project "${PROJECT}" \ @@ -138,9 +138,9 @@ function delete-master-instance-and-resources { if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then gcloud compute instances delete "${EVENT_STORE_NAME}" \ - ${GCLOUD_COMMON_ARGS} || true + "${GCLOUD_COMMON_ARGS[@]}" || true gcloud compute disks delete "${EVENT_STORE_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} || true + "${GCLOUD_COMMON_ARGS[@]}" || true fi } From 6310305d3b76a1f3cfc4b46f60823e895595e5aa Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 17:40:52 -0600 Subject: [PATCH 07/14] Fix shellcheck lint errors in test/kubemark/iks/shutdown.sh --- hack/.shellcheck_failures | 1 - test/kubemark/iks/shutdown.sh | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index a643ca0a37e..26a0a9acab6 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -145,7 +145,6 @@ ./test/images/volume/rbd/create_block.sh ./test/images/volume/rbd/mon.sh ./test/images/volume/rbd/osd.sh -./test/kubemark/iks/shutdown.sh ./test/kubemark/iks/startup.sh ./test/kubemark/iks/util.sh ./test/kubemark/master-log-dump.sh diff --git a/test/kubemark/iks/shutdown.sh b/test/kubemark/iks/shutdown.sh index 723ff35429a..9b0b2477afe 100644 --- a/test/kubemark/iks/shutdown.sh +++ b/test/kubemark/iks/shutdown.sh @@ -24,6 +24,7 @@ RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" complete-login # Remove resources created for kubemark +# shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_yellow}REMOVING RESOURCES${color_norm}" spawn-config "${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/addons" &> /dev/null || true @@ -35,8 +36,9 @@ rm -rf "${RESOURCE_DIRECTORY}/addons" # Remove clusters, namespaces, and deployments delete-clusters if [[ -f "${RESOURCE_DIRECTORY}/iks-namespacelist.sh" ]] ; then - bash ${RESOURCE_DIRECTORY}/iks-namespacelist.sh - rm -f ${RESOURCE_DIRECTORY}/iks-namespacelist.sh + bash "${RESOURCE_DIRECTORY}/iks-namespacelist.sh" + rm -f "${RESOURCE_DIRECTORY}/iks-namespacelist.sh" fi +# shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_blue}EXECUTION COMPLETE${color_norm}" exit 0 From e3e2a96521afbacf5521f01e7f6aa7b4d16d7d45 Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 17:47:06 -0600 Subject: [PATCH 08/14] Fix shellcheck lint errors in test/kubemark/iks/startup.sh --- hack/.shellcheck_failures | 1 - test/kubemark/iks/startup.sh | 66 ++++++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index 26a0a9acab6..51f35750a30 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -145,7 +145,6 @@ ./test/images/volume/rbd/create_block.sh ./test/images/volume/rbd/mon.sh ./test/images/volume/rbd/osd.sh -./test/kubemark/iks/startup.sh ./test/kubemark/iks/util.sh ./test/kubemark/master-log-dump.sh ./test/kubemark/resources/start-kubemark-master.sh diff --git a/test/kubemark/iks/startup.sh b/test/kubemark/iks/startup.sh index 6a643943444..c5bcd9f3f1a 100644 --- a/test/kubemark/iks/startup.sh +++ b/test/kubemark/iks/startup.sh @@ -25,7 +25,8 @@ RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" # templates, and finally create these resources through kubectl. function create-kube-hollow-node-resources { # Create kubeconfig for Kubelet. - KUBELET_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 + KUBELET_KUBECONFIG_CONTENTS="$(cat < "${RESOURCE_DIRECTORY}/addons/heapster.json" metrics_mem_per_node=4 - metrics_mem=$((200 + ${metrics_mem_per_node}*${NUM_NODES})) + metrics_mem=$((200 + metrics_mem_per_node*NUM_NODES)) sed -i'' -e "s/{{METRICS_MEM}}/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" metrics_cpu_per_node_numerator=${NUM_NODES} metrics_cpu_per_node_denominator=2 metrics_cpu=$((80 + metrics_cpu_per_node_numerator / metrics_cpu_per_node_denominator)) sed -i'' -e "s/{{METRICS_CPU}}/${metrics_cpu}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" eventer_mem_per_node=500 - eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES})) + eventer_mem=$((200 * 1024 + eventer_mem_per_node*NUM_NODES)) sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" # Cluster Autoscaler. @@ -216,7 +234,7 @@ current-context: kubemark-context") proxy_cpu=50 fi proxy_mem_per_node=50 - proxy_mem=$((100 * 1024 + ${proxy_mem_per_node}*${NUM_NODES})) + proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES)) sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" sed -i'' -e "s'{{kubemark_image_registry}}'${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" @@ -236,8 +254,7 @@ function wait-for-hollow-nodes-to-run-or-timeout { echo -n "Waiting for all hollow-nodes to become Running" start=$(date +%s) nodes=$("${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node 2> /dev/null) || true - ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1)) - + ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1)) until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do echo -n "." sleep 1 @@ -245,6 +262,7 @@ function wait-for-hollow-nodes-to-run-or-timeout { # Fail it if it already took more than 30 minutes. if [ $((now - start)) -gt 1800 ]; then echo "" + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}" # Try listing nodes again - if it fails it means that API server is not responding if "${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node &> /dev/null; then @@ -254,16 +272,17 @@ function wait-for-hollow-nodes-to-run-or-timeout { fi spawn-config pods=$("${KUBECTL}" get pods -l name=hollow-node --namespace=kubemark) || true - running=$(($(echo "${pods}" | grep "Running" | wc -l))) + running=$(($(echo "${pods}" | grep -c "Running"))) echo "${running} hollow-nodes are reported as 'Running'" - not_running=$(($(echo "${pods}" | grep -v "Running" | wc -l) - 1)) + not_running=$(($(echo "${pods}" | grep -vc "Running") - 1)) echo "${not_running} hollow-nodes are reported as NOT 'Running'" - echo $(echo "${pods}" | grep -v "Running") + echo "${pods}" | grep -v "Running" exit 1 fi nodes=$("${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node 2> /dev/null) || true - ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1)) + ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1)) done + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_green} Done!${color_norm}" } @@ -281,14 +300,17 @@ set-hollow-master echo "Creating kube hollow node resources" create-kube-hollow-node-resources master-config +# shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_blue}EXECUTION COMPLETE${color_norm}" # Check status of Kubemark +# shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_yellow}CHECKING STATUS${color_norm}" wait-for-hollow-nodes-to-run-or-timeout # Celebrate echo "" +# shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_blue}SUCCESS${color_norm}" clean-repo -exit 0 +exit 0 \ No newline at end of file From cb59cb33ff8739af2444687ae38d66d289f035c2 Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 18:10:14 -0600 Subject: [PATCH 09/14] Fix shellcheck lint errors in test/kubemark/iks/util.sh --- hack/.shellcheck_failures | 1 - test/kubemark/iks/util.sh | 76 +++++++++++++++++++++++---------------- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index 51f35750a30..9279f5ff236 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -145,7 +145,6 @@ ./test/images/volume/rbd/create_block.sh ./test/images/volume/rbd/mon.sh ./test/images/volume/rbd/osd.sh -./test/kubemark/iks/util.sh ./test/kubemark/master-log-dump.sh ./test/kubemark/resources/start-kubemark-master.sh ./test/kubemark/run-e2e-tests.sh diff --git a/test/kubemark/iks/util.sh b/test/kubemark/iks/util.sh index af48292d0fb..de91618f9a8 100644 --- a/test/kubemark/iks/util.sh +++ b/test/kubemark/iks/util.sh @@ -14,10 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../.. +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../.. # Creates a new kube-spawn cluster function create-clusters { + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_yellow}CHECKING CLUSTERS${color_norm}" if bx cs clusters | grep -Fq 'deleting'; then echo -n "Deleting old clusters" @@ -34,14 +35,15 @@ function create-clusters { PUBVLAN=$(bx cs vlans wdc06 --json | jq '. | .[] | select(.type == "public") | .id' | sed -e "s/\"//g") if ! bx cs clusters | grep -Fq 'kubeSpawnTester'; then echo "Creating spawning cluster" - bx cs cluster-create --location ${CLUSTER_LOCATION} --public-vlan ${PUBVLAN} --private-vlan ${PRIVLAN} --workers 2 --machine-type u2c.2x4 --name kubeSpawnTester + bx cs cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers 2 --machine-type u2c.2x4 --name kubeSpawnTester fi if ! bx cs clusters | grep -Fq 'kubeMasterTester'; then echo "Creating master cluster" - bx cs cluster-create --location ${CLUSTER_LOCATION} --public-vlan ${PUBVLAN} --private-vlan ${PRIVLAN} --workers 2 --machine-type u2c.2x4 --name kubeMasterTester + bx cs cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers 2 --machine-type u2c.2x4 --name kubeMasterTester fi push-image if ! bx cs clusters | grep 'kubeSpawnTester' | grep -Fq 'normal'; then + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_cyan}Warning: new clusters may take up to 60 minutes to be ready${color_norm}" echo -n "Clusters loading" fi @@ -61,17 +63,17 @@ function create-clusters { # Builds and pushes image to registry function push-image { if [[ "${ISBUILD}" = "y" ]]; then - if ! bx cr namespaces | grep -Fq ${KUBE_NAMESPACE}; then + if ! bx cr namespaces | grep -Fq "${KUBE_NAMESPACE}"; then echo "Creating registry namespace" - bx cr namespace-add ${KUBE_NAMESPACE} - echo "bx cr namespace-rm ${KUBE_NAMESPACE}" >> ${RESOURCE_DIRECTORY}/iks-namespacelist.sh + bx cr namespace-add "${KUBE_NAMESPACE}" + echo "bx cr namespace-rm ${KUBE_NAMESPACE}" >> "${RESOURCE_DIRECTORY}/iks-namespacelist.sh" fi - docker build -t ${KUBEMARK_INIT_TAG} ${KUBEMARK_IMAGE_LOCATION} - docker tag ${KUBEMARK_INIT_TAG} ${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG} - docker push ${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG} + docker build -t "${KUBEMARK_INIT_TAG}" "${KUBEMARK_IMAGE_LOCATION}" + docker tag "${KUBEMARK_INIT_TAG}" "${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG}" + docker push "${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG}" echo "Image pushed" else - KUBEMARK_IMAGE_REGISTRY=$(echo "brandondr96") + KUBEMARK_IMAGE_REGISTRY="brandondr96" KUBE_NAMESPACE="" fi } @@ -79,16 +81,17 @@ function push-image { # Allow user to use existing clusters if desired function choose-clusters { echo -n -e "Do you want to use custom clusters? [y/N]${color_cyan}>${color_norm} " - read USE_EXISTING + read -r USE_EXISTING if [[ "${USE_EXISTING}" = "y" ]]; then echo -e "${color_yellow}Enter path for desired hollow-node spawning cluster kubeconfig file:${color_norm}" - read CUSTOM_SPAWN_CONFIG + read -r CUSTOM_SPAWN_CONFIG echo -e "${color_yellow}Enter path for desired hollow-node hosting cluster kubeconfig file:${color_norm}" - read CUSTOM_MASTER_CONFIG + read -r CUSTOM_MASTER_CONFIG push-image elif [[ "${USE_EXISTING}" = "N" ]]; then create-clusters else + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_red}Invalid response, please try again:${color_norm}" choose-clusters fi @@ -102,28 +105,35 @@ function set-registry-secrets { kubectl -n kubemark get serviceaccounts default -o json | jq 'del(.metadata.resourceVersion)' | jq 'setpath(["imagePullSecrets"];[{"name":"bluemix-kubemark-secret-regional"}])' | kubectl -n kubemark replace serviceaccount default -f - } -# Sets hollow nodes spawned under master +# Sets the hollow-node master +# Exported variables: +# MASTER_IP - IP Address of the Kubemark master function set-hollow-master { echo -e "${color_yellow}CONFIGURING MASTER${color_norm}" master-config - MASTER_IP=$(cat $KUBECONFIG | grep server | awk -F "/" '{print $3}') + MASTER_IP=$(grep server "$KUBECONFIG" | awk -F "/" '{print $3}') + export MASTER_IP } # Set up master cluster environment +# Exported variables: +# KUBECONFIG - Overrides default kube config for the purpose of setting up the Kubemark master components. function master-config { if [[ "${USE_EXISTING}" = "y" ]]; then export KUBECONFIG=${CUSTOM_MASTER_CONFIG} else - $(bx cs cluster-config kubeMasterTester --admin | grep export) + eval "$(bx cs cluster-config kubeMasterTester --admin | grep export)" fi } # Set up spawn cluster environment +# Exported variables: +# KUBECONFIG - Overrides default kube config for the purpose of setting up the hollow-node cluster. function spawn-config { if [[ "${USE_EXISTING}" = "y" ]]; then export KUBECONFIG=${CUSTOM_SPAWN_CONFIG} else - $(bx cs cluster-config kubeSpawnTester --admin | grep export) + eval "$(bx cs cluster-config kubeSpawnTester --admin | grep export)" fi } @@ -147,11 +157,11 @@ function delete-clusters { function complete-login { echo -e "${color_yellow}LOGGING INTO CLOUD SERVICES${color_norm}" echo -n -e "Do you have a federated IBM cloud login? [y/N]${color_cyan}>${color_norm} " - read ISFED + read -r ISFED if [[ "${ISFED}" = "y" ]]; then - bx login --sso -a ${REGISTRY_LOGIN_URL} + bx login --sso -a "${REGISTRY_LOGIN_URL}" elif [[ "${ISFED}" = "N" ]]; then - bx login -a ${REGISTRY_LOGIN_URL} + bx login -a "${REGISTRY_LOGIN_URL}" else echo -e "${color_red}Invalid response, please try again:${color_norm}" complete-login @@ -159,28 +169,34 @@ function complete-login { bx cr login } -# Generate values to fill the hollow-node configuration +# Generate values to fill the hollow-node configuration templates. +# Exported variables: +# KUBECTL - The name or path to the kubernetes client binary. +# TEST_CLUSTER_API_CONTENT_TYPE - Defines the content-type of the requests used by the Kubemark components. function generate-values { echo "Generating values" master-config KUBECTL=kubectl + export KUBECTL KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" TEST_CLUSTER_API_CONTENT_TYPE="bluemix" #Determine correct usage of this + export TEST_CLUSTER_API_CONTENT_TYPE CONFIGPATH=${KUBECONFIG%/*} - KUBELET_CERT_BASE64="${KUBELET_CERT_BASE64:-$(cat ${CONFIGPATH}/admin.pem | base64 | tr -d '\r\n')}" - KUBELET_KEY_BASE64="${KUBELET_KEY_BASE64:-$(cat ${CONFIGPATH}/admin-key.pem | base64 | tr -d '\r\n')}" - CA_CERT_BASE64="${CA_CERT_BASE64:-$(cat `find ${CONFIGPATH} -name *ca*` | base64 | tr -d '\r\n')}" + KUBELET_CERT_BASE64="${KUBELET_CERT_BASE64:-$(base64 "${CONFIGPATH}/admin.pem" | tr -d '\r\n')}" + KUBELET_KEY_BASE64="${KUBELET_KEY_BASE64:-$(base64 "${CONFIGPATH}/admin-key.pem" | tr -d '\r\n')}" + CA_CERT_BASE64="${CA_CERT_BASE64:-$( base64 "$(find "${CONFIGPATH}" -name "*ca*" | head -n 1)" | tr -d '\r\n')}" + } # Build image for kubemark function build-kubemark-image { echo -n -e "Do you want to build the kubemark image? [y/N]${color_cyan}>${color_norm} " - read ISBUILD + read -r ISBUILD if [[ "${ISBUILD}" = "y" ]]; then echo -e "${color_yellow}BUILDING IMAGE${color_norm}" - ${KUBE_ROOT}/build/run.sh make kubemark - cp ${KUBE_ROOT}/_output/dockerized/bin/linux/amd64/kubemark ${KUBEMARK_IMAGE_LOCATION} + "${KUBE_ROOT}/build/run.sh" make kubemark + cp "${KUBE_ROOT}/_output/dockerized/bin/linux/amd64/kubemark" "${KUBEMARK_IMAGE_LOCATION}" elif [[ "${ISBUILD}" = "N" ]]; then echo -n "" else @@ -192,11 +208,11 @@ function build-kubemark-image { # Clean up repository function clean-repo { echo -n -e "Do you want to remove build output and binary? [y/N]${color_cyan}>${color_norm} " - read ISCLEAN + read -r ISCLEAN if [[ "${ISCLEAN}" = "y" ]]; then echo -e "${color_yellow}CLEANING REPO${color_norm}" - rm -rf ${KUBE_ROOT}/_output - rm -f ${KUBEMARK_IMAGE_LOCATION}/kubemark + rm -rf "${KUBE_ROOT}/_output" + rm -f "${KUBEMARK_IMAGE_LOCATION}/kubemark" elif [[ "${ISCLEAN}" = "N" ]]; then echo -n "" else From f72ac1f5b7911b0b17598dab32f02488fbac48d8 Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 18:11:36 -0600 Subject: [PATCH 10/14] Fix shellcheck lint errors in test/kubemark/master-log-dump.sh --- hack/.shellcheck_failures | 1 - test/kubemark/master-log-dump.sh | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index 9279f5ff236..5a8e1bb028e 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -145,7 +145,6 @@ ./test/images/volume/rbd/create_block.sh ./test/images/volume/rbd/mon.sh ./test/images/volume/rbd/osd.sh -./test/kubemark/master-log-dump.sh ./test/kubemark/resources/start-kubemark-master.sh ./test/kubemark/run-e2e-tests.sh ./test/kubemark/start-kubemark.sh diff --git a/test/kubemark/master-log-dump.sh b/test/kubemark/master-log-dump.sh index beaf8ed732f..70eae3d6eba 100755 --- a/test/kubemark/master-log-dump.sh +++ b/test/kubemark/master-log-dump.sh @@ -15,12 +15,12 @@ # limitations under the License. REPORT_DIR="${1:-_artifacts}" -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. -source ${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh -source ${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh +source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" +source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" export KUBEMARK_MASTER_NAME="${MASTER_NAME}" echo "Dumping logs for kubemark master: ${KUBEMARK_MASTER_NAME}" -DUMP_ONLY_MASTER_LOGS=true ${KUBE_ROOT}/cluster/log-dump/log-dump.sh "${REPORT_DIR}" +DUMP_ONLY_MASTER_LOGS=true "${KUBE_ROOT}/cluster/log-dump/log-dump.sh" "${REPORT_DIR}" From adf4bf17415a1fc581e8e013e336c54ef8affe7d Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 18:15:36 -0600 Subject: [PATCH 11/14] Fix shellcheck lint errors in test/kubemark/resources/start-kubemark-master.sh --- hack/.shellcheck_failures | 1 - .../resources/start-kubemark-master.sh | 29 +++++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index 5a8e1bb028e..403301efd1c 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -145,7 +145,6 @@ ./test/images/volume/rbd/create_block.sh ./test/images/volume/rbd/mon.sh ./test/images/volume/rbd/osd.sh -./test/kubemark/resources/start-kubemark-master.sh ./test/kubemark/run-e2e-tests.sh ./test/kubemark/start-kubemark.sh ./test/kubemark/stop-kubemark.sh diff --git a/test/kubemark/resources/start-kubemark-master.sh b/test/kubemark/resources/start-kubemark-master.sh index 1eaf178bf61..b4b097eb83d 100755 --- a/test/kubemark/resources/start-kubemark-master.sh +++ b/test/kubemark/resources/start-kubemark-master.sh @@ -107,7 +107,7 @@ function find-attached-pd() { if [[ ! -e /dev/disk/by-id/${pd_name} ]]; then echo "" fi - device_info=$(ls -l /dev/disk/by-id/${pd_name}) + device_info=$(ls -l "/dev/disk/by-id/${pd_name}") relative_path=${device_info##* } echo "/dev/disk/by-id/${relative_path}" } @@ -288,9 +288,9 @@ function start-kubelet { # # $1 is the file to create. function prepare-log-file { - touch $1 - chmod 644 $1 - chown root:root $1 + touch "$1" + chmod 644 "$1" + chown root:root "$1" } # A helper function for copying addon manifests and set dir/files @@ -301,10 +301,13 @@ function prepare-log-file { function setup-addon-manifests { local -r src_dir="${KUBE_ROOT}/$2" local -r dst_dir="/etc/kubernetes/$1/$2" + if [[ ! -d "${dst_dir}" ]]; then mkdir -p "${dst_dir}" fi - local files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml") + + local files + files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml") if [[ -n "${files}" ]]; then cp "${src_dir}/"*.yaml "${dst_dir}" fi @@ -508,7 +511,7 @@ function compute-kube-apiserver-params { params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv" params+=" --secure-port=443" params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv" - params+=" --target-ram-mb=$((${NUM_NODES} * 60))" + params+=" --target-ram-mb=$((NUM_NODES * 60))" params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" params+=" --admission-control=${CUSTOM_ADMISSION_PLUGINS}" params+=" --authorization-mode=Node,RBAC" @@ -598,7 +601,7 @@ function start-kubemaster-component() { local -r component=$1 prepare-log-file /var/log/"${component}".log local -r src_file="${KUBE_ROOT}/${component}.yaml" - local -r params=$(compute-${component}-params) + local -r params=$("compute-${component}-params") # Evaluate variables. sed -i -e "s@{{params}}@${params}@g" "${src_file}" @@ -609,18 +612,18 @@ function start-kubemaster-component() { elif [ "${component}" == "kube-addon-manager" ]; then setup-addon-manifests "addons" "kubemark-rbac-bindings" else - local -r component_docker_tag=$(cat ${KUBE_BINDIR}/${component}.docker_tag) + local -r component_docker_tag=$(cat "${KUBE_BINDIR}/${component}.docker_tag") sed -i -e "s@{{${component}_docker_tag}}@${component_docker_tag}@g" "${src_file}" if [ "${component}" == "kube-apiserver" ]; then local audit_policy_config_mount="" local audit_policy_config_volume="" if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then - read -d '' audit_policy_config_mount << EOF + read -r -d '' audit_policy_config_mount << EOF - name: auditpolicyconfigmount mountPath: ${audit_policy_file} readOnly: true EOF - read -d '' audit_policy_config_volume << EOF + read -r -d '' audit_policy_config_volume << EOF - name: auditpolicyconfigmount hostPath: path: ${audit_policy_file} @@ -669,7 +672,8 @@ fi main_etcd_mount_point="/mnt/disks/master-pd" mount-pd "google-master-pd" "${main_etcd_mount_point}" # Contains all the data stored in etcd. - mkdir -m 700 -p "${main_etcd_mount_point}/var/etcd" + mkdir -p "${main_etcd_mount_point}/var/etcd" + chmod 700 "${main_etcd_mount_point}/var/etcd" ln -s -f "${main_etcd_mount_point}/var/etcd" /var/etcd mkdir -p /etc/srv # Setup the dynamically generated apiserver auth certs and keys to pd. @@ -692,7 +696,8 @@ fi event_etcd_mount_point="/mnt/disks/master-event-pd" mount-pd "google-master-event-pd" "${event_etcd_mount_point}" # Contains all the data stored in event etcd. - mkdir -m 700 -p "${event_etcd_mount_point}/var/etcd/events" + mkdir -p "${event_etcd_mount_point}/var/etcd/events" + chmod 700 "${event_etcd_mount_point}/var/etcd/events" ln -s -f "${event_etcd_mount_point}/var/etcd/events" /var/etcd/events fi } From 46333a01b4f0c793ecbf1586271b6663ff4ca7de Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 18:20:32 -0600 Subject: [PATCH 12/14] Fix shellcheck lint errors in test/kubemark/run-e2e-tests.sh --- hack/.shellcheck_failures | 1 - test/kubemark/run-e2e-tests.sh | 20 +++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index 403301efd1c..dcf3424064b 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -145,7 +145,6 @@ ./test/images/volume/rbd/create_block.sh ./test/images/volume/rbd/mon.sh ./test/images/volume/rbd/osd.sh -./test/kubemark/run-e2e-tests.sh ./test/kubemark/start-kubemark.sh ./test/kubemark/stop-kubemark.sh ./third_party/forked/shell2junit/sh2ju.sh diff --git a/test/kubemark/run-e2e-tests.sh b/test/kubemark/run-e2e-tests.sh index d110c34b87a..c9260cb19f6 100755 --- a/test/kubemark/run-e2e-tests.sh +++ b/test/kubemark/run-e2e-tests.sh @@ -17,10 +17,10 @@ export KUBERNETES_PROVIDER="kubemark" export KUBE_CONFIG_FILE="config-default.sh" -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. # We need an absolute path to KUBE_ROOT -ABSOLUTE_ROOT=$(readlink -f ${KUBE_ROOT}) +ABSOLUTE_ROOT=$(readlink -f "${KUBE_ROOT}") source "${KUBE_ROOT}/cluster/kubemark/util.sh" @@ -32,21 +32,23 @@ export KUBE_MASTER_URL="https://${KUBE_MASTER_IP}" export KUBECONFIG="${ABSOLUTE_ROOT}/test/kubemark/resources/kubeconfig.kubemark" export E2E_MIN_STARTUP_PODS=0 -if [[ -z "$@" ]]; then - ARGS='--ginkgo.focus=[Feature:Performance]' +if [[ -z "$*" ]]; then + ARGS=('--ginkgo.focus=[Feature:Performance]') else - ARGS=$@ + ARGS=("$@") fi if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER}" == "true" ]]; then - ARGS="${ARGS} --kubemark-external-kubeconfig=${DEFAULT_KUBECONFIG}" + ARGS+=("--kubemark-external-kubeconfig=${DEFAULT_KUBECONFIG}") fi if [[ -f /.dockerenv ]]; then # Running inside a dockerized runner. - go run ./hack/e2e.go -- --check-version-skew=false --test --test_args="--e2e-verify-service-account=false --dump-logs-on-failure=false ${ARGS}" + go run ./hack/e2e.go -- --check-version-skew=false --test --test_args="--e2e-verify-service-account=false --dump-logs-on-failure=false ${ARGS[*]}" else # Running locally. - ARGS=$(echo $ARGS | sed 's/\[/\\\[/g' | sed 's/\]/\\\]/g') - ${KUBE_ROOT}/hack/ginkgo-e2e.sh "--e2e-verify-service-account=false" "--dump-logs-on-failure=false" $ARGS + for ((i=0; i < ${ARGS[@]}; i++)); do + ARGS[$i]="$(echo "ARGS[$i]" | sed -e 's/\[/\\\[/g' -e 's/\]/\\\]/g' )" + done + "${KUBE_ROOT}/hack/ginkgo-e2e.sh" "--e2e-verify-service-account=false" "--dump-logs-on-failure=false" "${ARGS[@]}" fi From b8aae458a112d971ca3a487e98cf3bfa3857a0ad Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 18:25:12 -0600 Subject: [PATCH 13/14] Fix shellcheck lint errors in test/kubemark/start-kubemark.sh --- hack/.shellcheck_failures | 1 - test/kubemark/start-kubemark.sh | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index dcf3424064b..cdf19a3516d 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -145,7 +145,6 @@ ./test/images/volume/rbd/create_block.sh ./test/images/volume/rbd/mon.sh ./test/images/volume/rbd/osd.sh -./test/kubemark/start-kubemark.sh ./test/kubemark/stop-kubemark.sh ./third_party/forked/shell2junit/sh2ju.sh ./third_party/intemp/intemp.sh diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index e1712f169cb..cf685b0f5a3 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -20,7 +20,7 @@ set -o errexit set -o nounset set -o pipefail -TMP_ROOT="$(dirname "${BASH_SOURCE}")/../.." +TMP_ROOT="$(dirname "${BASH_SOURCE[@]}")/../.." KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}") source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh" @@ -373,14 +373,14 @@ current-context: kubemark-context" mkdir -p "${RESOURCE_DIRECTORY}/addons" sed "s/{{MASTER_IP}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json" metrics_mem_per_node=4 - metrics_mem=$((200 + ${metrics_mem_per_node}*${NUM_NODES})) + metrics_mem=$((200 + metrics_mem_per_node*NUM_NODES)) sed -i'' -e "s/{{METRICS_MEM}}/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" metrics_cpu_per_node_numerator=${NUM_NODES} metrics_cpu_per_node_denominator=2 metrics_cpu=$((80 + metrics_cpu_per_node_numerator / metrics_cpu_per_node_denominator)) sed -i'' -e "s/{{METRICS_CPU}}/${metrics_cpu}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" eventer_mem_per_node=500 - eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES})) + eventer_mem=$((200 * 1024 + eventer_mem_per_node*NUM_NODES)) sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" # Cluster Autoscaler. @@ -415,7 +415,7 @@ current-context: kubemark-context" proxy_cpu=50 fi proxy_mem_per_node=50 - proxy_mem=$((100 * 1024 + ${proxy_mem_per_node}*${NUM_NODES})) + proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES)) sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" sed -i'' -e "s'{{kubemark_image_registry}}'${KUBEMARK_IMAGE_REGISTRY}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" @@ -434,7 +434,7 @@ function wait-for-hollow-nodes-to-run-or-timeout { echo -n "Waiting for all hollow-nodes to become Running" start=$(date +%s) nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true - ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1)) + ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1)) until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do echo -n "." @@ -443,6 +443,7 @@ function wait-for-hollow-nodes-to-run-or-timeout { # Fail it if it already took more than 30 minutes. if [ $((now - start)) -gt 1800 ]; then echo "" + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}" # Try listing nodes again - if it fails it means that API server is not responding if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then @@ -451,16 +452,17 @@ function wait-for-hollow-nodes-to-run-or-timeout { echo "Got error while trying to list hollow-nodes. Probably API server is down." fi pods=$("${KUBECTL}" get pods -l name=hollow-node --namespace=kubemark) || true - running=$(($(echo "${pods}" | grep "Running" | wc -l))) + running=$(($(echo "${pods}" | grep -c "Running"))) echo "${running} hollow-nodes are reported as 'Running'" - not_running=$(($(echo "${pods}" | grep -v "Running" | wc -l) - 1)) + not_running=$(($(echo "${pods}" | grep -vc "Running") - 1)) echo "${not_running} hollow-nodes are reported as NOT 'Running'" echo "${pods}" | grep -v Running exit 1 fi nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true - ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1)) + ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1)) done + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_green} Done!${color_norm}" } @@ -475,6 +477,7 @@ write-local-kubeconfig # Setup for master. function start-master { + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_yellow}STARTING SETUP FOR MASTER${color_norm}" create-master-environment-file create-master-instance-with-resources From e137f4702a471d1263dab3081e0f83d4f2b9ea51 Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Mon, 4 Feb 2019 18:26:57 -0600 Subject: [PATCH 14/14] Fix shellcheck lint errors in test/kubemark/stop-kubemark.sh --- hack/.shellcheck_failures | 1 - test/kubemark/stop-kubemark.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index cdf19a3516d..a95b78e1eae 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -145,7 +145,6 @@ ./test/images/volume/rbd/create_block.sh ./test/images/volume/rbd/mon.sh ./test/images/volume/rbd/osd.sh -./test/kubemark/stop-kubemark.sh ./third_party/forked/shell2junit/sh2ju.sh ./third_party/intemp/intemp.sh ./third_party/multiarch/qemu-user-static/register/qemu-binfmt-conf.sh diff --git a/test/kubemark/stop-kubemark.sh b/test/kubemark/stop-kubemark.sh index b3b3c7865a0..75ef920b201 100755 --- a/test/kubemark/stop-kubemark.sh +++ b/test/kubemark/stop-kubemark.sh @@ -16,7 +16,7 @@ # Script that destroys Kubemark cluster and deletes all master resources. -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh" source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"