diff --git a/scripts/integration b/scripts/integration index 26ee78a9..b10ccb59 100755 --- a/scripts/integration +++ b/scripts/integration @@ -1,11 +1,11 @@ #!/bin/bash function kubectlinstall { - K8SVERSION=$1 - if [ ! -f /usr/local/bin/kubectl-${K8SVERSION} ]; then - echo_with_time "Downloading kubectl ${K8SVERSION}" - curl -sL -o /usr/local/bin/kubectl-${K8SVERSION} https://storage.googleapis.com/kubernetes-release/release/$K8SVERSION/bin/linux/amd64/kubectl - chmod +x /usr/local/bin/kubectl-${K8SVERSION} + k8sversion=$1 + if [ ! -f "/usr/local/bin/kubectl-${k8sversion}" ]; then + echo_with_time "Downloading kubectl ${k8sversion}" + curl -sL -o "/usr/local/bin/kubectl-${k8sversion}" "https://storage.googleapis.com/kubernetes-release/release/${k8sversion}/bin/linux/amd64/kubectl" + chmod +x "/usr/local/bin/kubectl-${k8sversion}" fi } @@ -25,12 +25,21 @@ trap cleanup TERM EXIT pids="" tail_pids="" +pids_upgrade="" +tail_pids_upgrade="" RESULT="0" +UPGRADE_RESULT="0" declare -A versions_to_test declare -A pids_to_version +declare -A pids_upgrade_to_version declare -A pids_results declare -A rollout_results declare -A rkeup_results +declare -A pids_upgrade_results +declare -A rollout_upgrade_results +declare -A rkeup_upgrade_results +declare -A version_to_upgraded_version + echo_with_time "INFO - Running $0" source $(dirname $0)/version @@ -56,12 +65,13 @@ for ver in $all_versions; do done for ver in "${!versions_to_test[@]}"; do - echo_with_time "Testing version ${versions_to_test["${ver}"]}" + version_to_test=${versions_to_test["${ver}"]} + echo_with_time "Testing version ${version_to_test}" # Create cluster yaml with random node names node=$(cat /dev/urandom | tr -dc a-z | head -c${1:-8}) - cat << EOF > "./bin/cluster-${versions_to_test["${ver}"]}.yml" -kubernetes_version: ${versions_to_test["${ver}"]} + cat << EOF > "./bin/cluster-${version_to_test}.yml" +kubernetes_version: ${version_to_test} nodes: - address: rke-node-${node} role: [etcd, controlplane, worker] @@ -70,20 +80,20 @@ EOF # Run rke - output to logs and track results. export DOCKER_API_VERSION=1.24 - ./bin/rke up --dind --config "./bin/cluster-${versions_to_test["${ver}"]}.yml" 2>&1 >"./bin/cluster-${versions_to_test["${ver}"]}.log" & + ./bin/rke up --dind --config "./bin/cluster-${version_to_test}.yml" 2>&1 >"./bin/cluster-${version_to_test}.log" & pids="$pids $!" - pid_to_version["$!"]="${versions_to_test["${ver}"]}" + pids_to_version["$!"]="${version_to_test}" # Tail logs. sleep 1 - tail -f "./bin/cluster-${versions_to_test["${ver}"]}.log" & + tail -f "./bin/cluster-${version_to_test}.log" & tail_pids="$tail_pids $!" done # Wait for rke to finish for pid in $pids; do wait $pid - echo_with_time "PID ${pid} for ${pid_to_version["${pid}"]} is done" + echo_with_time "PID ${pid} for ${pids_to_version["${pid}"]} is done" pid_results["${pid}"]="$?" done @@ -93,13 +103,14 @@ for pid in $tail_pids; do done for pid in "${!pid_results[@]}"; do + pid_to_version=${pids_to_version["${pid}"]} + clusterk8sversion=$(echo "${pid_to_version}" | awk -F'-' '{ print $1 }') + kubectlinstall $clusterk8sversion if [ "${pid_results["${pid}"]}" == "0" ]; then rkeup_results["${pid}"]="0" - CLUSTERK8SVERSION=$(echo "${pid_to_version["${pid}"]}" | awk -F'-' '{ print $1 }') - echo_with_time "[OK] rke up successful for ${pid_to_version["${pid}"]}" - kubectlinstall $CLUSTERK8SVERSION - echo_with_time "Checking rollout status of all deployments/daemonsets for ${pid_to_version["${pid}"]}" - for namespace in $(/usr/local/bin/kubectl-${CLUSTERK8SVERSION} --kubeconfig "./bin/kube_config_cluster-${pid_to_version["${pid}"]}.yml" get namespace -o custom-columns=NAME:.metadata.name --no-headers); do + echo_with_time "[OK] rke up successful for ${pid_to_version}" + echo_with_time "Checking rollout status of all deployments/daemonsets for ${pid_to_version}" + for namespace in $(/usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_to_version}.yml" get namespace -o custom-columns=NAME:.metadata.name --no-headers); do if [ "${rollout_results["${pid}"]}" = "1" ]; then continue fi @@ -110,21 +121,23 @@ for pid in "${!pid_results[@]}"; do if [ "${rollout_results["${pid}"]}" = "1" ]; then continue fi - echo_with_time "Checking rollout status of $kind $name in namespace $namespace for ${pid_to_version["${pid}"]}" - /usr/local/bin/kubectl-${CLUSTERK8SVERSION} --kubeconfig "./bin/kube_config_cluster-${pid_to_version["${pid}"]}.yml" -n $namespace rollout status $kind $name -w --timeout=5m >/dev/null 2>&1 + echo_with_time "Checking rollout status of $kind $name in namespace $namespace for ${pid_to_version}" + /usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_to_version}.yml" -n $namespace rollout status $kind $name -w --timeout=5m >/dev/null 2>&1 if [ $? -ne 0 ]; then - echo_with_time "[FAIL] Rollout of $kind $name in namespace $namespace for ${pid_to_version["${pid}"]} did not complete in 5 minutes" + echo_with_time "[FAIL] Rollout of $kind $name in namespace $namespace for ${pid_to_version} did not complete in 5 minutes" rollout_results["${pid}"]="1" - /usr/local/bin/kubectl-${CLUSTERK8SVERSION} --kubeconfig "./bin/kube_config_cluster-${pid_to_version["${pid}"]}.yml" get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{end}}' - /usr/local/bin/kubectl-${CLUSTERK8SVERSION} --kubeconfig "./bin/kube_config_cluster-${pid_to_version["${pid}"]}.yml" -n kube-system get pods + /usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_to_version}.yml" get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{end}}' + /usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_to_version}.yml" -n kube-system get pods else - echo_with_time "[OK] Rollout of $kind $name in namespace $namespace for ${pid_to_version["${pid}"]} complete" + echo_with_time "[OK] Rollout of $kind $name in namespace $namespace for ${pid_to_version} complete" rollout_results["${pid}"]="0" fi - done <<<$(/usr/local/bin/kubectl-${CLUSTERK8SVERSION} --kubeconfig "./bin/kube_config_cluster-${pid_to_version["${pid}"]}.yml" -n $namespace get deploy,daemonset --no-headers -o custom-columns=NAME:.metadata.name,KIND:.kind --no-headers) + done <<<$(/usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_to_version}.yml" -n $namespace get deploy,daemonset --no-headers -o custom-columns=NAME:.metadata.name,KIND:.kind --no-headers) done else rkeup_results["${pid}"]="1" + /usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_to_version}.yml" get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{end}}' + /usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_to_version}.yml" -n kube-system get pods fi done @@ -132,18 +145,127 @@ echo "" echo_with_time "---- TEST RESULTS ----" for pid in "${!pid_results[@]}"; do if [ "${rkeup_results["${pid}"]}" != "0" ]; then - echo_with_time "[FAIL] ${pid_to_version["${pid}"]} (rke up)" + echo_with_time "[FAIL] ${pids_to_version["${pid}"]} (rke up)" RESULT=1 continue fi if [ "${rollout_results["${pid}"]}" != "0" ]; then - echo_with_time "[FAIL] ${pid_to_version["${pid}"]} (rollout status)" + echo_with_time "[FAIL] ${pids_to_version["${pid}"]} (rollout status)" RESULT=1 continue fi - echo_with_time "[PASS] ${pid_to_version["${pid}"]}" + echo_with_time "[PASS] ${pids_to_version["${pid}"]}" done if [ "$RESULT" = "1" ]; then + echo_with_time "[FAIL] One or more clusters failed to provision" + exit 1 +fi + +echo "" +echo_with_time "Testing upgrading clusters" +for ver in "${!versions_to_test[@]}"; do + echo_with_time "Checking if we can upgrade cluster with version ${versions_to_test["${ver}"]}" + + # This gets the next version in the list + # Example $all_versions: "v1.16.15-rancher1-2 v1.17.12-rancher1-1 v1.18.9-rancher1-1 v1.19.2-rancher1-1" + upgrade_to_version=$(echo $all_versions | grep -oP '(?<='"${versions_to_test["${ver}"]}"' )[^ ]*') + + if [ "${upgrade_to_version}" = "" ]; then + echo_with_time "No newer version found for ${versions_to_test["${ver}"]} to upgrade to" + continue + fi + + # Saving the original version -> upgrade version to print in the logs + version_to_upgraded_version[${versions_to_test["${ver}"]}]="${upgrade_to_version}" + + # Change original cluster yaml with new version to upgrade to + sed -i 's/'"${versions_to_test["${ver}"]}"'/'"${upgrade_to_version}"'/g' "./bin/cluster-${versions_to_test["${ver}"]}.yml" + + echo_with_time "Upgrading cluster version ${versions_to_test["${ver}"]} to version ${upgrade_to_version}" + # Run rke - output to logs and track results. + export DOCKER_API_VERSION=1.24 + ./bin/rke up --dind --config "./bin/cluster-${versions_to_test["${ver}"]}.yml" 2>&1 >"./bin/cluster-upgrade-${versions_to_test["${ver}"]}-${upgrade_to_version}.log" & + pids_upgrade="$pids_upgrade $!" + pids_upgrade_to_version["$!"]="${versions_to_test["${ver}"]}" + + # Tail logs. + sleep 1 + tail -f "./bin/cluster-upgrade-${versions_to_test["${ver}"]}-${upgrade_to_version}.log" & + tail_pids_upgrade="$tail_pids_upgrade $!" +done + +# Wait for rke to finish +for pid_upgrade in $pids_upgrade; do + wait $pid_upgrade + echo_with_time "PID ${pid_upgrade} for upgrade of ${pids_upgrade_to_version["${pid_upgrade}"]} to "${version_to_upgraded_version["${pids_upgrade_to_version["${pid_upgrade}"]}"]}" is done" + pid_upgrade_results["${pid_upgrade}"]="$?" +done + +# Stop tailing the logs +for pid_upgrade in $tail_pids_upgrade; do + kill $pid_upgrade 2>&1 >/dev/null +done + +for pid in "${!pid_upgrade_results[@]}"; do + # This is the initially provisioned version of the cluster, retrieved from the pid + pid_upgrade_to_version=${pids_upgrade_to_version["${pid}"]} + # This is the version the cluster was upgraded to + upgraded_version=${version_to_upgraded_version["${pid_upgrade_to_version}"]} + clusterk8sversion=$(echo "${pid_upgrade_to_version}" | awk -F'-' '{ print $1 }') + kubectlinstall $clusterk8sversion + if [ "${pid_upgrade_results["${pid}"]}" == "0" ]; then + rkeup_upgrade_results["${pid}"]="0" + echo_with_time "[OK] rke up successful for upgrade of ${pid_upgrade_to_version}" + echo_with_time "Checking rollout status of all deployments/daemonsets for upgrade of ${pid_upgrade_to_version} to "${upgraded_version}"" + for namespace in $(/usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_upgrade_to_version}.yml" get namespace -o custom-columns=NAME:.metadata.name --no-headers); do + if [ "${rollout_upgrade_results["${pid}"]}" = "1" ]; then + continue + fi + while read name kind; do + if [ "$name" = "" ] || [ "$kind" = "" ]; then + continue + fi + if [ "${rollout_upgrade_results["${pid}"]}" = "1" ]; then + continue + fi + echo_with_time "Checking rollout status of $kind $name in namespace $namespace for upgrade ${pid_upgrade_to_version} to "${upgraded_version}"" + /usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_upgrade_to_version}.yml" -n $namespace rollout status $kind $name -w --timeout=5m >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo_with_time "[FAIL] Rollout of $kind $name in namespace $namespace for upgrade ${pid_upgrade_to_version} to "${upgraded_version}" did not complete in 5 minutes" + rollout_upgrade_results["${pid}"]="1" + /usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_upgrade_to_version}.yml" get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{end}}' + /usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_upgrade_to_version}.yml" -n kube-system get pods + else + echo_with_time "[OK] Rollout of $kind $name in namespace $namespace for upgrade ${pid_upgrade_to_version} to "${upgraded_version}" complete" + rollout_upgrade_results["${pid}"]="0" + fi + done <<<$(/usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pid_upgrade_to_version}.yml" -n $namespace get deploy,daemonset --no-headers -o custom-columns=NAME:.metadata.name,KIND:.kind --no-headers) + done + else + rkeup_upgrade_results["${pid}"]="1" + /usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pids_to_version["${pid}"]}.yml" get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{end}}' + /usr/local/bin/kubectl-${clusterk8sversion} --kubeconfig "./bin/kube_config_cluster-${pids_to_version["${pid}"]}.yml" -n kube-system get pods + fi +done + +echo "" +echo_with_time "---- UPGRADE TEST RESULTS ----" +for pid in "${!pid_upgrade_results[@]}"; do + if [ "${rkeup_upgrade_results["${pid}"]}" != "0" ]; then + echo_with_time "[FAIL] upgrade ${pids_upgrade_to_version["${pid}"]} to "${version_to_upgraded_version["${pids_upgrade_to_version["${pid}"]}"]}" (rke up)" + UPGRADE_RESULT=1 + continue + fi + if [ "${rollout_upgrade_results["${pid}"]}" != "0" ]; then + echo_with_time "[FAIL] upgrade ${pids_upgrade_to_version["${pid}"]} to "${version_to_upgraded_version["${pids_upgrade_to_version["${pid}"]}"]}" (rollout status)" + UPGRADE_RESULT=1 + continue + fi + echo_with_time "[PASS] upgrade ${pids_upgrade_to_version["${pid}"]} to "${version_to_upgraded_version["${pids_upgrade_to_version["${pid}"]}"]}"" +done + +if [ "$UPGRADE_RESULT" = "1" ]; then + echo_with_time "[FAIL] One or more clusters failed to upgrade" exit 1 fi