diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index c7b2f90811..23439e3f1b 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -33,6 +33,9 @@ jobs: - clh - dragonball - qemu + instance-type: + - small + - normal include: - host_os: cbl-mariner vmm: clh @@ -46,6 +49,7 @@ jobs: KATA_HYPERVISOR: ${{ matrix.vmm }} KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index 3fdb8302d6..1fd4b00ee1 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -35,7 +35,15 @@ jobs: - devmapper k8s: - k3s - runs-on: garm-ubuntu-2004-smaller + instance: + - garm-ubuntu-2004 + - garm-ubuntu-2004-smaller + include: + - instance: garm-ubuntu-2004 + instance-type: normal + - instance: garm-ubuntu-2004-smaller + instance-type: small + runs-on: ${{ matrix.instance }} env: DOCKER_REGISTRY: ${{ inputs.registry }} DOCKER_REPO: ${{ inputs.repo }} @@ -45,6 +53,7 @@ jobs: KUBERNETES: ${{ matrix.k8s }} SNAPSHOTTER: ${{ matrix.snapshotter }} USING_NFD: "false" + K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index df105974a7..a720e24869 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -39,6 +39,7 @@ jobs: KUBECONFIG: /home/kata/.kube/config KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index 80e146795a..33ae57d3a8 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -39,6 +39,7 @@ jobs: KUBECONFIG: /home/kata/.kube/config KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index f9b14bc9d0..940fd0a396 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -38,6 +38,7 @@ jobs: KATA_HYPERVISOR: ${{ matrix.vmm }} KUBERNETES: "k3s" USING_NFD: "true" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 40a0903b9b..fd9b981090 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -12,6 +12,7 @@ tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${tests_dir}/common.bash" AZ_RG="${AZ_RG:-kataCI}" +K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" function _print_cluster_name() { test_type="${1:-k8s}" @@ -40,10 +41,20 @@ function create_cluster() { # First, ensure that the cluster didn't fail to get cleaned up from a previous run. delete_cluster "${test_type}" || true + local instance_type="" + case ${K8S_TEST_HOST_TYPE} in + small) + instance_type="Standard_D2s_v5" + ;; + normal) + instance_type="Standard_D4s_v5" + ;; + esac + az aks create \ -g "${AZ_RG}" \ -n "$(_print_cluster_name ${test_type})" \ - -s "Standard_D2s_v5" \ + -s "${instance_type}" \ --node-count 1 \ --generate-ssh-keys \ $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation") diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index bbe7acc27a..d510bc2956 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -245,6 +245,7 @@ function cleanup() { function main() { export KATA_HOST_OS="${KATA_HOST_OS:-}" + export K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-}" action="${1:-}" diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 28abedc2df..165cadaf66 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -13,6 +13,7 @@ source "${kubernetes_dir}/../../common.bash" TARGET_ARCH="${TARGET_ARCH:-x86_64}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" K8S_TEST_DEBUG="${K8S_TEST_DEBUG:-false}" +K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" # String generated using "base64 -w 0 kata-containers/src/kata-opa/allow-all.rego" ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY:-"cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEV4ZWNQcm9jZXNzUmVxdWVzdCA9IHRydWUKZGVmYXVsdCBHZXRNZXRyaWNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgR2V0T09NRXZlbnRSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHdWVzdERldGFpbHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBMaXN0SW50ZXJmYWNlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IExpc3RSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBNZW1Ib3RwbHVnQnlQcm9iZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE9ubGluZUNQVU1lbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFBhdXNlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUHVsbEltYWdlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVhZFN0cmVhbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlbW92ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlbW92ZVN0YWxlVmlydGlvZnNTaGFyZU1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlc2VlZFJhbmRvbURldlJlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBSZXN1bWVDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTZXRHdWVzdERhdGVUaW1lUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2V0UG9saWN5UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2lnbmFsUHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0Q29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhcnRUcmFjaW5nUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhdHNDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdG9wVHJhY2luZ1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFR0eVdpblJlc2l6ZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUVwaGVtZXJhbE1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUludGVyZmFjZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZVJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdhaXRQcm9jZXNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgV3JpdGVTdHJlYW1SZXF1ZXN0IDo9IHRydWUK"}" @@ -20,7 +21,7 @@ ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY:-"cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQ if [ -n "${K8S_TEST_UNION:-}" ]; then K8S_TEST_UNION=($K8S_TEST_UNION) else - K8S_TEST_UNION=( \ + K8S_TEST_SMALL_HOST_UNION=( \ "k8s-confidential.bats" \ "k8s-attach-handlers.bats" \ "k8s-caps.bats" \ @@ -56,6 +57,29 @@ else "k8s-volume.bats" \ "k8s-nginx-connectivity.bats" \ ) + + K8S_TEST_NORMAL_HOST_UNION=( \ + "k8s-number-cpus.bats" \ + "k8s-parallel.bats" \ + "k8s-scale-nginx.bats" \ + ) + + case ${K8S_TEST_HOST_TYPE} in + small) + K8S_TEST_UNION=($K8S_TEST_SMALL_HOST_UNION) + ;; + normal) + K8S_TEST_UNION=($K8S_TEST_NORMAL_HOST_UNION) + ;; + baremetal) + K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]} ${K8S_TEST_NORMAL_HOST_UNION[@]}) + + ;; + *) + echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | baremetal" + return 1 + ;; + esac fi policy_tests_enabled() {