From 68267a399620dc8ef5f62bdbc2d980915006c1f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Mon, 18 Sep 2023 08:13:52 -0700 Subject: [PATCH 1/3] ci: Create clusters in individual resource groups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes it so that each AKS cluster is created in its own individual resource group, rather than using the "kataCI" resource group for all test clusters. This is to accommodate a tool that we recently introduced in our Azure subscription which automatically deletes resource groups after a set amount of time, in order to keep spending under control. The tool will automatically delete any resource group, unless it has a tag SkipAutoDeleteTill = YYYY-MM-DD. When this tag is present, the resource group will be retained until the specified date. Note that I tagged all current resource groups in our subscription with SkipAutoDeleteTill = 2043-01-01 so that we don't lose any existing resources. Fixes: #7982 Signed-off-by: Aurélien Bombo --- tests/gha-run-k8s-common.sh | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index fd9b981090..cbd79c01c4 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -11,7 +11,6 @@ set -o pipefail tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${tests_dir}/common.bash" -AZ_RG="${AZ_RG:-kataCI}" K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" function _print_cluster_name() { @@ -21,6 +20,12 @@ function _print_cluster_name() { echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64" } +function _print_rg_name() { + test_type="${1:-k8s}" + + echo "${AZ_RG:-"kataCI-$(_print_cluster_name ${test_type})"}" +} + function install_azure_cli() { curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash # The aks-preview extension is required while the Mariner Kata host is in preview. @@ -38,9 +43,15 @@ function login_azure() { function create_cluster() { test_type="${1:-k8s}" - # First, ensure that the cluster didn't fail to get cleaned up from a previous run. + # First ensure it didn't fail to get cleaned up from a previous run. delete_cluster "${test_type}" || true + local rg="$(_print_rg_name ${test_type})" + + az group create \ + -l eastus2 \ + -n "${rg}" + local instance_type="" case ${K8S_TEST_HOST_TYPE} in small) @@ -52,7 +63,8 @@ function create_cluster() { esac az aks create \ - -g "${AZ_RG}" \ + -g "${rg}" \ + --node-resource-group "node-${rg}" \ -n "$(_print_cluster_name ${test_type})" \ -s "${instance_type}" \ --node-count 1 \ @@ -79,16 +91,15 @@ function get_cluster_credentials() { test_type="${1:-k8s}" az aks get-credentials \ - -g "${AZ_RG}" \ + -g "$(_print_rg_name ${test_type})" \ -n "$(_print_cluster_name ${test_type})" } function delete_cluster() { test_type="${1:-k8s}" - az aks delete \ - -g "${AZ_RG}" \ - -n "$(_print_cluster_name ${test_type})" \ + az group delete \ + -g "$(_print_rg_name ${test_type})" \ --yes } From d9ef1352af139030d760bedbaa860080a5f1d7aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Mon, 18 Sep 2023 09:23:30 -0700 Subject: [PATCH 2/3] ci: Add first letter of the K8S_TEST_HOST_TYPE to resource group name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ideally we'd add the instance_type or the full K8S_TEST_HOST_TYPE but that exceeds the maximum amount of characteres allowed for the cluster name. With this in mind, let's use the first letter of K8S_TEST_HOST_TYPE instead. Signed-off-by: Aurélien Bombo --- tests/gha-run-k8s-common.sh | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index cbd79c01c4..5b8193c88f 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -13,11 +13,25 @@ source "${tests_dir}/common.bash" K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" +function _print_instance_type() { + case ${K8S_TEST_HOST_TYPE} in + small) + echo "Standard_D2s_v5" + ;; + normal) + echo "Standard_D4s_v5" + ;; + *) + echo "Unknown instance type '${K8S_TEST_HOST_TYPE}'" >&2 + exit 1 + esac +} + function _print_cluster_name() { test_type="${1:-k8s}" short_sha="$(git rev-parse --short=12 HEAD)" - echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64" + echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64-${K8S_TEST_HOST_TYPE:0:1}" } function _print_rg_name() { @@ -52,21 +66,11 @@ function create_cluster() { -l eastus2 \ -n "${rg}" - local instance_type="" - case ${K8S_TEST_HOST_TYPE} in - small) - instance_type="Standard_D2s_v5" - ;; - normal) - instance_type="Standard_D4s_v5" - ;; - esac - az aks create \ -g "${rg}" \ --node-resource-group "node-${rg}" \ -n "$(_print_cluster_name ${test_type})" \ - -s "${instance_type}" \ + -s "$(_print_instance_type)" \ --node-count 1 \ --generate-ssh-keys \ $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation") From 486fe14c99c437a302be2810fb6361c91122e2a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 18 Sep 2023 11:10:27 -0700 Subject: [PATCH 3/3] ci: Properly set K8S_TEST_UNION MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise only the first test will be executed Signed-off-by: Aurélien Bombo Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/run_kubernetes_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 165cadaf66..b0686180a0 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -66,10 +66,10 @@ else case ${K8S_TEST_HOST_TYPE} in small) - K8S_TEST_UNION=($K8S_TEST_SMALL_HOST_UNION) + K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]}) ;; normal) - K8S_TEST_UNION=($K8S_TEST_NORMAL_HOST_UNION) + K8S_TEST_UNION=(${K8S_TEST_NORMAL_HOST_UNION[@]}) ;; baremetal) K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]} ${K8S_TEST_NORMAL_HOST_UNION[@]})