diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index c7b2f90811..23439e3f1b 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -33,6 +33,9 @@ jobs: - clh - dragonball - qemu + instance-type: + - small + - normal include: - host_os: cbl-mariner vmm: clh @@ -46,6 +49,7 @@ jobs: KATA_HYPERVISOR: ${{ matrix.vmm }} KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index 3fdb8302d6..1fd4b00ee1 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -35,7 +35,15 @@ jobs: - devmapper k8s: - k3s - runs-on: garm-ubuntu-2004-smaller + instance: + - garm-ubuntu-2004 + - garm-ubuntu-2004-smaller + include: + - instance: garm-ubuntu-2004 + instance-type: normal + - instance: garm-ubuntu-2004-smaller + instance-type: small + runs-on: ${{ matrix.instance }} env: DOCKER_REGISTRY: ${{ inputs.registry }} DOCKER_REPO: ${{ inputs.repo }} @@ -45,6 +53,7 @@ jobs: KUBERNETES: ${{ matrix.k8s }} SNAPSHOTTER: ${{ matrix.snapshotter }} USING_NFD: "false" + K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index df105974a7..a720e24869 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -39,6 +39,7 @@ jobs: KUBECONFIG: /home/kata/.kube/config KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index 80e146795a..33ae57d3a8 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -39,6 +39,7 @@ jobs: KUBECONFIG: /home/kata/.kube/config KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index f9b14bc9d0..940fd0a396 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -38,6 +38,7 @@ jobs: KATA_HYPERVISOR: ${{ matrix.vmm }} KUBERNETES: "k3s" USING_NFD: "true" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 40a0903b9b..fd9b981090 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -12,6 +12,7 @@ tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${tests_dir}/common.bash" AZ_RG="${AZ_RG:-kataCI}" +K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" function _print_cluster_name() { test_type="${1:-k8s}" @@ -40,10 +41,20 @@ function create_cluster() { # First, ensure that the cluster didn't fail to get cleaned up from a previous run. delete_cluster "${test_type}" || true + local instance_type="" + case ${K8S_TEST_HOST_TYPE} in + small) + instance_type="Standard_D2s_v5" + ;; + normal) + instance_type="Standard_D4s_v5" + ;; + esac + az aks create \ -g "${AZ_RG}" \ -n "$(_print_cluster_name ${test_type})" \ - -s "Standard_D2s_v5" \ + -s "${instance_type}" \ --node-count 1 \ --generate-ssh-keys \ $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation") diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index c53583aff0..1cb11c4569 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -233,6 +233,7 @@ function cleanup() { function main() { export KATA_HOST_OS="${KATA_HOST_OS:-}" + export K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-}" action="${1:-}" diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index aff825b47b..0d4c40cfa6 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -13,11 +13,12 @@ source "${kubernetes_dir}/../../common.bash" TARGET_ARCH="${TARGET_ARCH:-x86_64}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" K8S_TEST_DEBUG="${K8S_TEST_DEBUG:-false}" +K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" if [ -n "${K8S_TEST_UNION:-}" ]; then K8S_TEST_UNION=($K8S_TEST_UNION) else - K8S_TEST_UNION=( \ + K8S_TEST_SMALL_HOST_UNION=( \ "k8s-confidential.bats" \ "k8s-attach-handlers.bats" \ "k8s-caps.bats" \ @@ -53,6 +54,29 @@ else "k8s-volume.bats" \ "k8s-nginx-connectivity.bats" \ ) + + K8S_TEST_NORMAL_HOST_UNION=( \ + "k8s-number-cpus.bats" \ + "k8s-parallel.bats" \ + "k8s-scale-nginx.bats" \ + ) + + case ${K8S_TEST_HOST_TYPE} in + small) + K8S_TEST_UNION=($K8S_TEST_SMALL_HOST_UNION) + ;; + normal) + K8S_TEST_UNION=($K8S_TEST_NORMAL_HOST_UNION) + ;; + baremetal) + K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]} ${K8S_TEST_NORMAL_HOST_UNION[@]}) + + ;; + *) + echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | baremetal" + return 1 + ;; + esac fi # we may need to skip a few test cases when running on non-x86_64 arch