From c69a1e33bde402ec095aa47cdd27f75940fc8cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 11:47:14 +0200 Subject: [PATCH] ci: Use variable size of VMs depending on the tests running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let me start with a fair warning that this commit is hard to split into different parts that could be easily tested (or not tested, just ignored) without breaking pieces. Now, about the commit itself, as we're on the run to reduce costs related to our sponsorship on Azure, we can split the k8s tests we run in 2 simple groups: * Tests that can be run in the smaller Azure instance (D2s_v5) * Tests that required the normal Azure instance (D4s_v5) With this in mind, we're now passing to the tests which type of host we're using, which allows us to select to run either one of the two types of tests, or even both in case of running the tests on a baremetal system. Fixes: #7972 Signed-off-by: Fabiano Fidêncio --- .github/workflows/run-k8s-tests-on-aks.yaml | 4 +++ .github/workflows/run-k8s-tests-on-garm.yaml | 11 +++++++- .github/workflows/run-k8s-tests-on-sev.yaml | 1 + .github/workflows/run-k8s-tests-on-snp.yaml | 1 + .github/workflows/run-k8s-tests-on-tdx.yaml | 1 + tests/gha-run-k8s-common.sh | 13 +++++++++- tests/integration/kubernetes/gha-run.sh | 1 + .../kubernetes/run_kubernetes_tests.sh | 26 ++++++++++++++++++- 8 files changed, 55 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index c7b2f90811..23439e3f1b 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -33,6 +33,9 @@ jobs: - clh - dragonball - qemu + instance-type: + - small + - normal include: - host_os: cbl-mariner vmm: clh @@ -46,6 +49,7 @@ jobs: KATA_HYPERVISOR: ${{ matrix.vmm }} KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index 3fdb8302d6..1fd4b00ee1 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -35,7 +35,15 @@ jobs: - devmapper k8s: - k3s - runs-on: garm-ubuntu-2004-smaller + instance: + - garm-ubuntu-2004 + - garm-ubuntu-2004-smaller + include: + - instance: garm-ubuntu-2004 + instance-type: normal + - instance: garm-ubuntu-2004-smaller + instance-type: small + runs-on: ${{ matrix.instance }} env: DOCKER_REGISTRY: ${{ inputs.registry }} DOCKER_REPO: ${{ inputs.repo }} @@ -45,6 +53,7 @@ jobs: KUBERNETES: ${{ matrix.k8s }} SNAPSHOTTER: ${{ matrix.snapshotter }} USING_NFD: "false" + K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index df105974a7..a720e24869 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -39,6 +39,7 @@ jobs: KUBECONFIG: /home/kata/.kube/config KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index 80e146795a..33ae57d3a8 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -39,6 +39,7 @@ jobs: KUBECONFIG: /home/kata/.kube/config KUBERNETES: "vanilla" USING_NFD: "false" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index f9b14bc9d0..940fd0a396 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -38,6 +38,7 @@ jobs: KATA_HYPERVISOR: ${{ matrix.vmm }} KUBERNETES: "k3s" USING_NFD: "true" + K8S_TEST_HOST_TYPE: "baremetal" steps: - uses: actions/checkout@v3 with: diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 40a0903b9b..fd9b981090 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -12,6 +12,7 @@ tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${tests_dir}/common.bash" AZ_RG="${AZ_RG:-kataCI}" +K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" function _print_cluster_name() { test_type="${1:-k8s}" @@ -40,10 +41,20 @@ function create_cluster() { # First, ensure that the cluster didn't fail to get cleaned up from a previous run. delete_cluster "${test_type}" || true + local instance_type="" + case ${K8S_TEST_HOST_TYPE} in + small) + instance_type="Standard_D2s_v5" + ;; + normal) + instance_type="Standard_D4s_v5" + ;; + esac + az aks create \ -g "${AZ_RG}" \ -n "$(_print_cluster_name ${test_type})" \ - -s "Standard_D2s_v5" \ + -s "${instance_type}" \ --node-count 1 \ --generate-ssh-keys \ $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation") diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index bbe7acc27a..d510bc2956 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -245,6 +245,7 @@ function cleanup() { function main() { export KATA_HOST_OS="${KATA_HOST_OS:-}" + export K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-}" action="${1:-}" diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 28abedc2df..165cadaf66 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -13,6 +13,7 @@ source "${kubernetes_dir}/../../common.bash" TARGET_ARCH="${TARGET_ARCH:-x86_64}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" K8S_TEST_DEBUG="${K8S_TEST_DEBUG:-false}" +K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" # String generated using "base64 -w 0 kata-containers/src/kata-opa/allow-all.rego" ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY:-"cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEV4ZWNQcm9jZXNzUmVxdWVzdCA9IHRydWUKZGVmYXVsdCBHZXRNZXRyaWNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgR2V0T09NRXZlbnRSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHdWVzdERldGFpbHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBMaXN0SW50ZXJmYWNlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IExpc3RSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBNZW1Ib3RwbHVnQnlQcm9iZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE9ubGluZUNQVU1lbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFBhdXNlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUHVsbEltYWdlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVhZFN0cmVhbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlbW92ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlbW92ZVN0YWxlVmlydGlvZnNTaGFyZU1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlc2VlZFJhbmRvbURldlJlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBSZXN1bWVDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTZXRHdWVzdERhdGVUaW1lUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2V0UG9saWN5UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2lnbmFsUHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0Q29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhcnRUcmFjaW5nUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhdHNDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdG9wVHJhY2luZ1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFR0eVdpblJlc2l6ZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUVwaGVtZXJhbE1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUludGVyZmFjZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZVJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdhaXRQcm9jZXNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgV3JpdGVTdHJlYW1SZXF1ZXN0IDo9IHRydWUK"}" @@ -20,7 +21,7 @@ ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY:-"cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQ if [ -n "${K8S_TEST_UNION:-}" ]; then K8S_TEST_UNION=($K8S_TEST_UNION) else - K8S_TEST_UNION=( \ + K8S_TEST_SMALL_HOST_UNION=( \ "k8s-confidential.bats" \ "k8s-attach-handlers.bats" \ "k8s-caps.bats" \ @@ -56,6 +57,29 @@ else "k8s-volume.bats" \ "k8s-nginx-connectivity.bats" \ ) + + K8S_TEST_NORMAL_HOST_UNION=( \ + "k8s-number-cpus.bats" \ + "k8s-parallel.bats" \ + "k8s-scale-nginx.bats" \ + ) + + case ${K8S_TEST_HOST_TYPE} in + small) + K8S_TEST_UNION=($K8S_TEST_SMALL_HOST_UNION) + ;; + normal) + K8S_TEST_UNION=($K8S_TEST_NORMAL_HOST_UNION) + ;; + baremetal) + K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]} ${K8S_TEST_NORMAL_HOST_UNION[@]}) + + ;; + *) + echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | baremetal" + return 1 + ;; + esac fi policy_tests_enabled() {