Merge pull request #76909 from Huang-Wei/fix-iks-kubemark

kubemark: fix and enhance kubemark scripts for IKS
This commit is contained in:
Kubernetes Prow Robot 2019-04-29 06:30:23 -07:00 committed by GitHub
commit 9b437f9520
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 47 additions and 34 deletions

View File

@ -25,8 +25,16 @@ CLUSTER_LOCATION="${CLUSTER_LOCATION:-wdc06}"
REGISTRY_LOGIN_URL="${REGISTRY_LOGIN_URL:-https://api.ng.bluemix.net}"
# User defined
# number of real workers in spawnTester cluster
NUM_NODES="${NUM_NODES:-2}"
# spec of real workers in spawnTester cluster
NODE_SIZE=${NODE_SIZE:-u2c.2x4}
DESIRED_NODES="${DESIRED_NODES:-10}"
# number of hollow nodes
# TODO: once IKS supports `kubetest`, i.e. the base cluster provisioning implemented in `kubetest`
# we can use NUM_NODES=${KUBEMARK_NUM_NODES:-10} to avoid usage of KUBEMARK_NUM_NODES
# more context, see https://github.com/kubernetes/kubernetes/pull/76909#discussion_r277735942
KUBEMARK_NUM_NODES="${KUBEMARK_NUM_NODES:-10}"
ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-true}"
ENABLE_KUBEMARK_KUBE_DNS="${ENABLE_KUBEMARK_KUBE_DNS:-false}"
KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-"--v=2"}"

View File

@ -227,7 +227,7 @@ EOF
# Create the replication controller for hollow-nodes.
# We allow to override the NUM_REPLICAS when running Cluster Autoscaler.
NUM_REPLICAS=${NUM_REPLICAS:-${NUM_NODES}}
NUM_REPLICAS=${NUM_REPLICAS:-${KUBEMARK_NUM_NODES}}
sed "s/{{numreplicas}}/${NUM_REPLICAS}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml"
proxy_cpu=20
if [ "${NUM_NODES}" -gt 1000 ]; then
@ -240,9 +240,8 @@ EOF
sed -i'' -e "s'{{kubemark_image_registry}}'${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{kubemark_image_tag}}/${KUBEMARK_IMAGE_TAG}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{kubelet_verbosity_level}}/${KUBELET_TEST_LOG_LEVEL}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{kubeproxy_verbosity_level}}/${KUBEPROXY_TEST_LOG_LEVEL}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{use_real_proxier}}/${USE_REAL_PROXIER}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{hollow_kubelet_params}}/${HOLLOW_KUBELET_TEST_ARGS:-}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{hollow_proxy_params}}/${HOLLOW_PROXY_TEST_ARGS:-}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s'{{kubemark_mig_config}}'${KUBEMARK_MIG_CONFIG:-}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"
@ -253,7 +252,8 @@ EOF
function wait-for-hollow-nodes-to-run-or-timeout {
echo -n "Waiting for all hollow-nodes to become Running"
start=$(date +%s)
nodes=$("${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node 2> /dev/null) || true
# IKS uses a real cluster for hollow master, so need to exclude the real worker nodes
nodes=$("${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node | grep hollow-node 2> /dev/null) || true
ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))
until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do
echo -n "."

View File

@ -20,39 +20,44 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../..
function create-clusters {
# shellcheck disable=SC2154 # Color defined in sourced script
echo -e "${color_yellow}CHECKING CLUSTERS${color_norm}"
if bx cs clusters | grep -Fq 'deleting'; then
if ibmcloud ks clusters | grep -Fq 'deleting'; then
echo -n "Deleting old clusters"
fi
while bx cs clusters | grep -Fq 'deleting'
while ibmcloud ks clusters | grep -Fq 'deleting'
do
echo -n "."
sleep 10
done
echo ""
bx cs region-set us-east >/dev/null
bx cs vlans wdc06 >/dev/null
PRIVLAN=$(bx cs vlans wdc06 --json | jq '. | .[] | select(.type == "private") | .id' | sed -e "s/\"//g")
PUBVLAN=$(bx cs vlans wdc06 --json | jq '. | .[] | select(.type == "public") | .id' | sed -e "s/\"//g")
if ! bx cs clusters | grep -Fq 'kubeSpawnTester'; then
ibmcloud ks region-set us-east >/dev/null
ibmcloud ks vlans wdc06 >/dev/null
PRIVLAN=$(ibmcloud ks vlans wdc06 --json | jq '. | .[] | select(.type == "private") | .id' | sed -e "s/\"//g")
PUBVLAN=$(ibmcloud ks vlans wdc06 --json | jq '. | .[] | select(.type == "public") | .id' | sed -e "s/\"//g")
if ! ibmcloud ks clusters | grep -Fq 'kubeSpawnTester'; then
echo "Creating spawning cluster"
bx cs cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers 2 --machine-type u2c.2x4 --name kubeSpawnTester
# make number and spec of node workers configurable
# otherwise it can't afford tests like kubemark-5000
# TODO: dynamically adjust the number and spec
ibmcloud ks cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers "${NUM_NODES:-2}" --machine-type "${NODE_SIZE}" --name kubeSpawnTester
fi
if ! bx cs clusters | grep -Fq 'kubeMasterTester'; then
if ! ibmcloud ks clusters | grep -Fq 'kubeMasterTester'; then
echo "Creating master cluster"
bx cs cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers 2 --machine-type u2c.2x4 --name kubeMasterTester
# if we can't make it a bare master (workers = 0)
# then make workers = 1 with the smallest machine spec
ibmcloud ks cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers 1 --machine-type u2c.2x4 --name kubeMasterTester
fi
push-image
if ! bx cs clusters | grep 'kubeSpawnTester' | grep -Fq 'normal'; then
if ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'normal'; then
# shellcheck disable=SC2154 # Color defined in sourced script
echo -e "${color_cyan}Warning: new clusters may take up to 60 minutes to be ready${color_norm}"
echo -n "Clusters loading"
fi
while ! bx cs clusters | grep 'kubeSpawnTester' | grep -Fq 'normal'
while ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'normal'
do
echo -n "."
sleep 5
done
while ! bx cs clusters | grep 'kubeMasterTester' | grep -Fq 'normal'
while ! ibmcloud ks clusters | grep 'kubeMasterTester' | grep -Fq 'normal'
do
echo -n "."
sleep 5
@ -63,17 +68,17 @@ function create-clusters {
# Builds and pushes image to registry
function push-image {
if [[ "${ISBUILD}" = "y" ]]; then
if ! bx cr namespaces | grep -Fq "${KUBE_NAMESPACE}"; then
if ! ibmcloud cr namespaces | grep -Fq "${KUBE_NAMESPACE}"; then
echo "Creating registry namespace"
bx cr namespace-add "${KUBE_NAMESPACE}"
echo "bx cr namespace-rm ${KUBE_NAMESPACE}" >> "${RESOURCE_DIRECTORY}/iks-namespacelist.sh"
ibmcloud cr namespace-add "${KUBE_NAMESPACE}"
echo "ibmcloud cr namespace-rm ${KUBE_NAMESPACE}" >> "${RESOURCE_DIRECTORY}/iks-namespacelist.sh"
fi
docker build -t "${KUBEMARK_INIT_TAG}" "${KUBEMARK_IMAGE_LOCATION}"
docker tag "${KUBEMARK_INIT_TAG}" "${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG}"
docker push "${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG}"
echo "Image pushed"
else
KUBEMARK_IMAGE_REGISTRY="brandondr96"
KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-brandondr96}"
KUBE_NAMESPACE=""
fi
}
@ -122,7 +127,7 @@ function master-config {
if [[ "${USE_EXISTING}" = "y" ]]; then
export KUBECONFIG=${CUSTOM_MASTER_CONFIG}
else
eval "$(bx cs cluster-config kubeMasterTester --admin | grep export)"
eval "$(ibmcloud ks cluster-config kubeMasterTester --admin | grep export)"
fi
}
@ -133,20 +138,20 @@ function spawn-config {
if [[ "${USE_EXISTING}" = "y" ]]; then
export KUBECONFIG=${CUSTOM_SPAWN_CONFIG}
else
eval "$(bx cs cluster-config kubeSpawnTester --admin | grep export)"
eval "$(ibmcloud ks cluster-config kubeSpawnTester --admin | grep export)"
fi
}
# Deletes existing clusters
function delete-clusters {
echo "DELETING CLUSTERS"
bx cs cluster-rm kubeSpawnTester
bx cs cluster-rm kubeMasterTester
while ! bx cs clusters | grep 'kubeSpawnTester' | grep -Fq 'deleting'
do
sleep 5
done
while ! bx cs clusters | grep 'kubeMasterTester' | grep -Fq 'deleting'
ibmcloud ks cluster-rm kubeSpawnTester
ibmcloud ks cluster-rm kubeMasterTester
while ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'deleting'
do
sleep 5
done
while ! ibmcloud ks clusters | grep 'kubeMasterTester' | grep -Fq 'deleting'
do
sleep 5
done
@ -159,14 +164,14 @@ function complete-login {
echo -n -e "Do you have a federated IBM cloud login? [y/N]${color_cyan}>${color_norm} "
read -r ISFED
if [[ "${ISFED}" = "y" ]]; then
bx login --sso -a "${REGISTRY_LOGIN_URL}"
ibmcloud login --sso -a "${REGISTRY_LOGIN_URL}"
elif [[ "${ISFED}" = "N" ]]; then
bx login -a "${REGISTRY_LOGIN_URL}"
ibmcloud login -a "${REGISTRY_LOGIN_URL}"
else
echo -e "${color_red}Invalid response, please try again:${color_norm}"
complete-login
fi
bx cr login
ibmcloud cr login
}
# Generate values to fill the hollow-node configuration templates.