ci: Use variable size of VMs depending on the tests running

Let me start with a fair warning that this commit is hard to split into
different parts that could be easily tested (or not tested, just
ignored) without breaking pieces.

Now, about the commit itself, as we're on the run to reduce costs
related to our sponsorship on Azure, we can split the k8s tests we run
in 2 simple groups:
* Tests that can be run in the smaller Azure instance (D2s_v5)
* Tests that required the normal Azure instance (D4s_v5)

With this in mind, we're now passing to the tests which type of host
we're using, which allows us to select to run either one of the two
types of tests, or even both in case of running the tests on a baremetal
system.

Fixes: #7972

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
(cherry picked from commit c69a1e33bd)

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -33,6 +33,9 @@ jobs:
           - clh
           - dragonball
           - qemu
+        instance-type:
+          - small
+          - normal
         include:
           - host_os: cbl-mariner
             vmm: clh
@@ -46,6 +49,7 @@ jobs:
       KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBERNETES: "vanilla"
       USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/run-k8s-tests-on-garm.yaml

@@ -35,7 +35,15 @@ jobs:
           - devmapper
         k8s:
           - k3s
-    runs-on: garm-ubuntu-2004-smaller
+        instance:
+          - garm-ubuntu-2004
+          - garm-ubuntu-2004-smaller
+        include:
+          - instance: garm-ubuntu-2004
+            instance-type: normal
+          - instance: garm-ubuntu-2004-smaller
+            instance-type: small
+    runs-on: ${{ matrix.instance }}
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
@@ -45,6 +53,7 @@ jobs:
       KUBERNETES: ${{ matrix.k8s }}
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -39,6 +39,7 @@ jobs:
       KUBECONFIG: /home/kata/.kube/config
       KUBERNETES: "vanilla"
       USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: "baremetal"
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -39,6 +39,7 @@ jobs:
       KUBECONFIG: /home/kata/.kube/config
       KUBERNETES: "vanilla"
       USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: "baremetal"
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -38,6 +38,7 @@ jobs:
       KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBERNETES: "k3s"
       USING_NFD: "true"
+      K8S_TEST_HOST_TYPE: "baremetal"
     steps:
       - uses: actions/checkout@v3
         with:

tests/gha-run-k8s-common.sh

@@ -12,6 +12,7 @@ tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${tests_dir}/common.bash"
 
 AZ_RG="${AZ_RG:-kataCI}"
+K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}"
 
 function _print_cluster_name() {
     test_type="${1:-k8s}"
@@ -40,10 +41,20 @@ function create_cluster() {
     # First, ensure that the cluster didn't fail to get cleaned up from a previous run.
     delete_cluster "${test_type}" || true
 
+    local instance_type=""
+    case ${K8S_TEST_HOST_TYPE} in
+	    small)
+		    instance_type="Standard_D2s_v5"
+		    ;;
+	    normal)
+		    instance_type="Standard_D4s_v5"
+		    ;;
+    esac
+
     az aks create \
         -g "${AZ_RG}" \
         -n "$(_print_cluster_name ${test_type})" \
-        -s "Standard_D2s_v5" \
+        -s "${instance_type}" \
         --node-count 1 \
         --generate-ssh-keys \
         $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation")

tests/integration/kubernetes/gha-run.sh

@@ -233,6 +233,7 @@ function cleanup() {
 
 function main() {
     export KATA_HOST_OS="${KATA_HOST_OS:-}"
+    export K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-}"
 
     action="${1:-}"
 

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -13,11 +13,12 @@ source "${kubernetes_dir}/../../common.bash"
 TARGET_ARCH="${TARGET_ARCH:-x86_64}"
 KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
 K8S_TEST_DEBUG="${K8S_TEST_DEBUG:-false}"
+K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}"
 
 if [ -n "${K8S_TEST_UNION:-}" ]; then
 	K8S_TEST_UNION=($K8S_TEST_UNION)
 else
-	K8S_TEST_UNION=( \
+	K8S_TEST_SMALL_HOST_UNION=( \
 		"k8s-confidential.bats" \
 		"k8s-attach-handlers.bats" \
 		"k8s-caps.bats" \
@@ -53,6 +54,29 @@ else
 		"k8s-volume.bats" \
 		"k8s-nginx-connectivity.bats" \
 	)
+
+	K8S_TEST_NORMAL_HOST_UNION=( \
+		"k8s-number-cpus.bats" \
+		"k8s-parallel.bats" \
+		"k8s-scale-nginx.bats" \
+	)
+
+	case ${K8S_TEST_HOST_TYPE} in
+		small)
+			K8S_TEST_UNION=($K8S_TEST_SMALL_HOST_UNION)
+			;;
+		normal)
+			K8S_TEST_UNION=($K8S_TEST_NORMAL_HOST_UNION)
+			;;
+		baremetal)
+			K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]} ${K8S_TEST_NORMAL_HOST_UNION[@]})
+
+			;;
+		*)
+			echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | baremetal"
+			return 1
+			;;
+	esac
 fi
 
 # we may need to skip a few test cases when running on non-x86_64 arch