From 3ac0e22f624f2baff1872aae0ec71f6dfe42ed94 Mon Sep 17 00:00:00 2001 From: Shyam Jeedigunta Date: Tue, 7 Feb 2017 13:07:23 +0100 Subject: [PATCH] Refactored kubemark code into provider-specific and provider-independent parts [Part-3] --- cluster/kubemark/gce/config-default.sh | 1 + test/kubemark/cloud-provider-config.sh | 3 + test/kubemark/common.sh | 74 ------------------- test/kubemark/gce/util.sh | 59 ++++++++------- test/kubemark/get-real-pod-for-hollow-node.sh | 18 ----- .../resources/hollow-node_template.json | 4 +- test/kubemark/run-e2e-tests.sh | 4 +- test/kubemark/start-kubemark.sh | 28 +++++-- test/kubemark/stop-kubemark.sh | 9 ++- 9 files changed, 67 insertions(+), 133 deletions(-) delete mode 100644 test/kubemark/common.sh delete mode 100755 test/kubemark/get-real-pod-for-hollow-node.sh diff --git a/cluster/kubemark/gce/config-default.sh b/cluster/kubemark/gce/config-default.sh index e92c5de99d6..d08b9d3bbe6 100644 --- a/cluster/kubemark/gce/config-default.sh +++ b/cluster/kubemark/gce/config-default.sh @@ -43,6 +43,7 @@ NETWORK=${KUBE_GCE_NETWORK:-default} INSTANCE_PREFIX="${INSTANCE_PREFIX:-"default"}" MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master" MASTER_TAG="kubemark-master" +EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.240.0.0/11}" RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" diff --git a/test/kubemark/cloud-provider-config.sh b/test/kubemark/cloud-provider-config.sh index 9b3475548d3..a3aab1088ef 100644 --- a/test/kubemark/cloud-provider-config.sh +++ b/test/kubemark/cloud-provider-config.sh @@ -15,3 +15,6 @@ # limitations under the License. CLOUD_PROVIDER="${CLOUD_PROVIDER:-gce}" +CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-gcr.io}" +PROJECT="${PROJECT:-}" +KUBEMARK_IMAGE_MAKE_TARGET="${KUBEMARK_IMAGE_MAKE_TARGET:-gcloudpush}" diff --git a/test/kubemark/common.sh b/test/kubemark/common.sh deleted file mode 100644 index 0c2d4294a89..00000000000 --- a/test/kubemark/common.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" -source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" -source "${KUBE_ROOT}/cluster/kubemark/util.sh" -source "${KUBE_ROOT}/cluster/lib/util.sh" - -# hack/lib/init.sh will ovewrite ETCD_VERSION if this is unset -# what what is default in hack/lib/etcd.sh -# To avoid it, if it is empty, we set it to 'avoid-overwrite' and -# clean it after that. -if [ -z "${ETCD_VERSION:-}" ]; then - ETCD_VERSION="avoid-overwrite" -fi -source "${KUBE_ROOT}/hack/lib/init.sh" -if [ "${ETCD_VERSION:-}" == "avoid-overwrite" ]; then - ETCD_VERSION="" -fi - -detect-project &> /dev/null -export PROJECT -find-release-tars - -MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master" -MASTER_TAG="kubemark-master" -EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store" - -export KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" -export KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" -export RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" - -# Runs gcloud compute command with the given parameters. Up to $RETRIES will be made -# to execute the command. -# arguments: -# $@: all stuff that goes after 'gcloud compute' -function run-gcloud-compute-with-retries { - RETRIES="${RETRIES:-3}" - for attempt in $(seq 1 ${RETRIES}); do - local -r gcloud_result=$(gcloud compute "$@" 2>&1) - local -r ret_val="$?" - echo "${gcloud_result}" - if [[ "${ret_val}" -ne "0" ]]; then - if [[ $(echo "${gcloud_result}" | grep -c "already exists") -gt 0 ]]; then - if [[ "${attempt}" == 1 ]]; then - echo -e "${color_red}Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2 - exit 1 - fi - echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}" - return 0 - fi - echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2 - sleep $(($attempt * 5)) - else - echo -e "${color_green}Succeeded to gcloud compute $1 $2 $3.${color_norm}" - return 0 - fi - done - echo -e "${color_red}Failed to $1 $2 $3.${color_norm}" >& 2 - exit 1 -} diff --git a/test/kubemark/gce/util.sh b/test/kubemark/gce/util.sh index cc41a9af80b..fa65061b3ef 100644 --- a/test/kubemark/gce/util.sh +++ b/test/kubemark/gce/util.sh @@ -14,6 +14,35 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Wrapper for gcloud compute, running it $RETRIES times in case of failures. +# Args: +# $@: all stuff that goes after 'gcloud compute' +function run-gcloud-compute-with-retries { + RETRIES="${RETRIES:-3}" + for attempt in $(seq 1 ${RETRIES}); do + local -r gcloud_result=$(gcloud compute "$@" 2>&1) + local -r ret_val="$?" + echo "${gcloud_result}" + if [[ "${ret_val}" -ne "0" ]]; then + if [[ $(echo "${gcloud_result}" | grep -c "already exists") -gt 0 ]]; then + if [[ "${attempt}" == 1 ]]; then + echo -e "${color_red}Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2 + exit 1 + fi + echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}" + return 0 + fi + echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2 + sleep $(($attempt * 5)) + else + echo -e "${color_green}Succeeded to gcloud compute $1 $2 $3.${color_norm}" + return 0 + fi + done + echo -e "${color_red}Failed to $1 $2 $3.${color_norm}" >& 2 + exit 1 +} + function create-master-instance-with-resources { GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}" @@ -103,33 +132,3 @@ function delete-master-instance-and-resources { ${GCLOUD_COMMON_ARGS} || true fi } - -function delete-master-instance-and-resources { - GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE} --quiet" - - gcloud compute instances delete "${MASTER_NAME}" \ - ${GCLOUD_COMMON_ARGS} || true - - gcloud compute disks delete "${MASTER_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} || true - - gcloud compute disks delete "${MASTER_NAME}-event-pd" \ - ${GCLOUD_COMMON_ARGS} &> /dev/null || true - - gcloud compute addresses delete "${MASTER_NAME}-ip" \ - --project "${PROJECT}" \ - --region "${REGION}" \ - --quiet || true - - gcloud compute firewall-rules delete "${MASTER_NAME}-https" \ - --project "${PROJECT}" \ - --quiet || true - - if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then - gcloud compute instances delete "${EVENT_STORE_NAME}" \ - ${GCLOUD_COMMON_ARGS} || true - - gcloud compute disks delete "${EVENT_STORE_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} || true - fi -} diff --git a/test/kubemark/get-real-pod-for-hollow-node.sh b/test/kubemark/get-real-pod-for-hollow-node.sh deleted file mode 100755 index 76580f7df59..00000000000 --- a/test/kubemark/get-real-pod-for-hollow-node.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kubectl get pod --namespace=kubemark -o json | grep -C100 $1 | grep hollow | head -n1 | cut -d: -f2 | cut -d, -f1 | tr -d \" | tr -d '[[:space:]]' -echo "" diff --git a/test/kubemark/resources/hollow-node_template.json b/test/kubemark/resources/hollow-node_template.json index 7cd2e09ad0c..e229f1b7c60 100644 --- a/test/kubemark/resources/hollow-node_template.json +++ b/test/kubemark/resources/hollow-node_template.json @@ -52,7 +52,7 @@ "containers": [ { "name": "hollow-kubelet", - "image": "gcr.io/{{project}}/kubemark:latest", + "image": "{{registry}}/{{project}}/kubemark:latest", "ports": [ {"containerPort": 4194}, {"containerPort": 10250}, @@ -106,7 +106,7 @@ }, { "name": "hollow-proxy", - "image": "gcr.io/{{project}}/kubemark:latest", + "image": "{{registry}}/{{project}}/kubemark:latest", "env": [ { "name": "CONTENT_TYPE", diff --git a/test/kubemark/run-e2e-tests.sh b/test/kubemark/run-e2e-tests.sh index 1a1fccdd670..e911c15ac5c 100755 --- a/test/kubemark/run-e2e-tests.sh +++ b/test/kubemark/run-e2e-tests.sh @@ -22,9 +22,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. # We need an absolute path to KUBE_ROOT ABSOLUTE_ROOT=$(readlink -f ${KUBE_ROOT}) -source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" source "${KUBE_ROOT}/cluster/kubemark/util.sh" -source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" echo "Kubemark master name: ${MASTER_NAME}" @@ -35,7 +33,7 @@ export KUBECONFIG="${ABSOLUTE_ROOT}/test/kubemark/resources/kubeconfig.kubemark" export E2E_MIN_STARTUP_PODS=0 if [[ -z "$@" ]]; then - ARGS='--ginkgo.focus=should\sallow\sstarting\s30\spods\sper\snode' + ARGS='--ginkgo.focus=\[Feature:performance\]' else ARGS=$@ fi diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 4bab969b0d2..ff57f9f7bab 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -19,10 +19,27 @@ TMP_ROOT="$(dirname "${BASH_SOURCE}")/../.." KUBE_ROOT=$(readlink -e ${TMP_ROOT} 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' ${TMP_ROOT}) -source "${KUBE_ROOT}/test/kubemark/common.sh" source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh" source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh" +source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" +source "${KUBE_ROOT}/cluster/kubemark/util.sh" + +# hack/lib/init.sh will ovewrite ETCD_VERSION if this is unset +# what what is default in hack/lib/etcd.sh +# To avoid it, if it is empty, we set it to 'avoid-overwrite' and +# clean it after that. +if [ -z "${ETCD_VERSION:-}" ]; then + ETCD_VERSION="avoid-overwrite" +fi +source "${KUBE_ROOT}/hack/lib/init.sh" +if [ "${ETCD_VERSION:-}" == "avoid-overwrite" ]; then + ETCD_VERSION="" +fi + +KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" +KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" +RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" # Write all environment variables that we need to pass to the kubemark master, # locally to the file ${RESOURCE_DIRECTORY}/kubemark-master-env.sh. @@ -153,7 +170,6 @@ EOF # Finds the right kubemark binary for 'linux/amd64' platform and uses it to # create a docker image for hollow-node and upload it to the appropriate # docker container registry for the cloud provider. -# TODO(shyamjvs): Make the image upload URL and makefile variable w.r.t. provider. function create-and-upload-hollow-node-image { MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark" KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)" @@ -168,7 +184,7 @@ function create-and-upload-hollow-node-image { cd "${MAKE_DIR}" RETRIES=3 for attempt in $(seq 1 ${RETRIES}); do - if ! make; then + if ! REGISTRY="${CONTAINER_REGISTRY}" PROJECT="${PROJECT}" make "${KUBEMARK_IMAGE_MAKE_TARGET}"; then if [[ $((attempt)) -eq "${RETRIES}" ]]; then echo "${color_red}Make failed. Exiting.${color_norm}" exit 1 @@ -281,7 +297,6 @@ current-context: kubemark-context") --from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}" # Create addon pods. - # TODO(shyamjvs): Make path to docker image variable in heapster_template.json. mkdir -p "${RESOURCE_DIRECTORY}/addons" sed "s/{{MASTER_IP}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json" metrics_mem_per_node=4 @@ -293,8 +308,8 @@ current-context: kubemark-context") "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" # Create the replication controller for hollow-nodes. - # TODO(shyamjvs): Make path to docker image variable in hollow-node_template.json. sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json" + sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark" @@ -338,8 +353,11 @@ function wait-for-hollow-nodes-to-run-or-timeout { } ############################### Main Function ######################################## +detect-project &> /dev/null + # Setup for master. echo -e "${color_yellow}STARTING SETUP FOR MASTER${color_norm}" +find-release-tars create-master-environment-file create-master-instance-with-resources generate-pki-config diff --git a/test/kubemark/stop-kubemark.sh b/test/kubemark/stop-kubemark.sh index 29d760cb852..855274eab63 100755 --- a/test/kubemark/stop-kubemark.sh +++ b/test/kubemark/stop-kubemark.sh @@ -18,10 +18,17 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. -source "${KUBE_ROOT}/test/kubemark/common.sh" source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh" source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh" +source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" +source "${KUBE_ROOT}/cluster/kubemark/util.sh" + +KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" +KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" +RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" + +detect-project &> /dev/null "${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/addons" &> /dev/null || true "${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/hollow-node.json" &> /dev/null || true