From 7c4a0f7facabab2af2ada581a57efa9a555d0a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 11:47:14 +0200 Subject: [PATCH] ci: Use variable size of VMs depending on the tests running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let me start with a fair warning that this commit is hard to split into different parts that could be easily tested (or not tested, just ignored) without breaking pieces. Now, about the commit itself, as we're on the run to reduce costs related to our sponsorship on Azure, we can split the k8s tests we run in 2 simple groups: * Tests that can be run in the smaller Azure instance (D2s_v5) * Tests that required the normal Azure instance (D4s_v5) With this in mind, we're now passing to the tests which type of host we're using, which allows us to select to run either one of the two types of tests, or even both in case of running the tests on a baremetal system. Fixes: #7972 Signed-off-by: Fabiano Fidêncio (cherry picked from commit c69a1e33bde402ec095aa47cdd27f75940fc8cfa) --- .github/workflows/run-k8s-tests-on-aks.yaml | 4 +++ .github/workflows/run-k8s-tests-on-garm.yaml | 11 +++++++- .github/workflows/run-k8s-tests-on-sev.yaml | 1 + .github/workflows/run-k8s-tests-on-snp.yaml | 1 + .github/workflows/run-k8s-tests-on-tdx.yaml | 1 + tests/gha-run-k8s-common.sh | 13 +++++++++- tests/integration/kubernetes/gha-run.sh | 1 + .../kubernetes/run_kubernetes_tests.sh | 26 ++++++++++++++++++- 8 files changed, 55 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index c7b2f90811..23439e3f1b 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -33,6 +33,9 @@ jobs: - clh - dragonball - qemu + instance-type: + - small + - normal include: - host_os: cbl-mariner vmm: clh @@ -46,6 +49,7 @@ jobs: KATA_HYPERVISOR: ${{ matrix.vmm }} KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index 3fdb8302d6..1fd4b00ee1 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -35,7 +35,15 @@ jobs: - devmapper k8s: - k3s - runs-on: garm-ubuntu-2004-smaller + instance: + - garm-ubuntu-2004 + - garm-ubuntu-2004-smaller + include: + - instance: garm-ubuntu-2004 + instance-type: normal + - instance: garm-ubuntu-2004-smaller + instance-type: small + runs-on: ${{ matrix.instance }} env: DOCKER_REGISTRY: ${{ inputs.registry }} DOCKER_REPO: ${{ inputs.repo }} @@ -45,6 +53,7 @@ jobs: KUBERNETES: ${{ matrix.k8s }} SNAPSHOTTER: ${{ matrix.snapshotter }} USING_NFD: "false" + K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index df105974a7..a720e24869 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -39,6 +39,7 @@ jobs: KUBECONFIG: /home/kata/.kube/config KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index 80e146795a..33ae57d3a8 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -39,6 +39,7 @@ jobs: KUBECONFIG: /home/kata/.kube/config KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index f9b14bc9d0..940fd0a396 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -38,6 +38,7 @@ jobs: KATA_HYPERVISOR: ${{ matrix.vmm }} KUBERNETES: "k3s" USING_NFD: "true" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 40a0903b9b..fd9b981090 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -12,6 +12,7 @@ tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${tests_dir}/common.bash" AZ_RG="${AZ_RG:-kataCI}" +K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" function _print_cluster_name() { test_type="${1:-k8s}" @@ -40,10 +41,20 @@ function create_cluster() { # First, ensure that the cluster didn't fail to get cleaned up from a previous run. delete_cluster "${test_type}" || true + local instance_type="" + case ${K8S_TEST_HOST_TYPE} in + small) + instance_type="Standard_D2s_v5" + ;; + normal) + instance_type="Standard_D4s_v5" + ;; + esac + az aks create \ -g "${AZ_RG}" \ -n "$(_print_cluster_name ${test_type})" \ - -s "Standard_D2s_v5" \ + -s "${instance_type}" \ --node-count 1 \ --generate-ssh-keys \ $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation") diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index c53583aff0..1cb11c4569 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -233,6 +233,7 @@ function cleanup() { function main() { export KATA_HOST_OS="${KATA_HOST_OS:-}" + export K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-}" action="${1:-}" diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index aff825b47b..0d4c40cfa6 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -13,11 +13,12 @@ source "${kubernetes_dir}/../../common.bash" TARGET_ARCH="${TARGET_ARCH:-x86_64}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" K8S_TEST_DEBUG="${K8S_TEST_DEBUG:-false}" +K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" if [ -n "${K8S_TEST_UNION:-}" ]; then K8S_TEST_UNION=($K8S_TEST_UNION) else - K8S_TEST_UNION=( \ + K8S_TEST_SMALL_HOST_UNION=( \ "k8s-confidential.bats" \ "k8s-attach-handlers.bats" \ "k8s-caps.bats" \ @@ -53,6 +54,29 @@ else "k8s-volume.bats" \ "k8s-nginx-connectivity.bats" \ ) + + K8S_TEST_NORMAL_HOST_UNION=( \ + "k8s-number-cpus.bats" \ + "k8s-parallel.bats" \ + "k8s-scale-nginx.bats" \ + ) + + case ${K8S_TEST_HOST_TYPE} in + small) + K8S_TEST_UNION=($K8S_TEST_SMALL_HOST_UNION) + ;; + normal) + K8S_TEST_UNION=($K8S_TEST_NORMAL_HOST_UNION) + ;; + baremetal) + K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]} ${K8S_TEST_NORMAL_HOST_UNION[@]}) + + ;; + *) + echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | baremetal" + return 1 + ;; + esac fi # we may need to skip a few test cases when running on non-x86_64 arch