ci: Use variable size of VMs depending on the tests running

Let me start with a fair warning that this commit is hard to split into
different parts that could be easily tested (or not tested, just
ignored) without breaking pieces.

Now, about the commit itself, as we're on the run to reduce costs
related to our sponsorship on Azure, we can split the k8s tests we run
in 2 simple groups:
* Tests that can be run in the smaller Azure instance (D2s_v5)
* Tests that required the normal Azure instance (D4s_v5)

With this in mind, we're now passing to the tests which type of host
we're using, which allows us to select to run either one of the two
types of tests, or even both in case of running the tests on a baremetal
system.

Fixes: #7972

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
(cherry picked from commit c69a1e33bd)
This commit is contained in:
Fabiano Fidêncio 2023-09-15 11:47:14 +02:00
parent 7019a25f25
commit 7c4a0f7fac
8 changed files with 55 additions and 3 deletions

View File

@ -33,6 +33,9 @@ jobs:
- clh - clh
- dragonball - dragonball
- qemu - qemu
instance-type:
- small
- normal
include: include:
- host_os: cbl-mariner - host_os: cbl-mariner
vmm: clh vmm: clh
@ -46,6 +49,7 @@ jobs:
KATA_HYPERVISOR: ${{ matrix.vmm }} KATA_HYPERVISOR: ${{ matrix.vmm }}
KUBERNETES: "vanilla" KUBERNETES: "vanilla"
USING_NFD: "false" USING_NFD: "false"
K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:

View File

@ -35,7 +35,15 @@ jobs:
- devmapper - devmapper
k8s: k8s:
- k3s - k3s
runs-on: garm-ubuntu-2004-smaller instance:
- garm-ubuntu-2004
- garm-ubuntu-2004-smaller
include:
- instance: garm-ubuntu-2004
instance-type: normal
- instance: garm-ubuntu-2004-smaller
instance-type: small
runs-on: ${{ matrix.instance }}
env: env:
DOCKER_REGISTRY: ${{ inputs.registry }} DOCKER_REGISTRY: ${{ inputs.registry }}
DOCKER_REPO: ${{ inputs.repo }} DOCKER_REPO: ${{ inputs.repo }}
@ -45,6 +53,7 @@ jobs:
KUBERNETES: ${{ matrix.k8s }} KUBERNETES: ${{ matrix.k8s }}
SNAPSHOTTER: ${{ matrix.snapshotter }} SNAPSHOTTER: ${{ matrix.snapshotter }}
USING_NFD: "false" USING_NFD: "false"
K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:

View File

@ -39,6 +39,7 @@ jobs:
KUBECONFIG: /home/kata/.kube/config KUBECONFIG: /home/kata/.kube/config
KUBERNETES: "vanilla" KUBERNETES: "vanilla"
USING_NFD: "false" USING_NFD: "false"
K8S_TEST_HOST_TYPE: "baremetal"
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:

View File

@ -39,6 +39,7 @@ jobs:
KUBECONFIG: /home/kata/.kube/config KUBECONFIG: /home/kata/.kube/config
KUBERNETES: "vanilla" KUBERNETES: "vanilla"
USING_NFD: "false" USING_NFD: "false"
K8S_TEST_HOST_TYPE: "baremetal"
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:

View File

@ -38,6 +38,7 @@ jobs:
KATA_HYPERVISOR: ${{ matrix.vmm }} KATA_HYPERVISOR: ${{ matrix.vmm }}
KUBERNETES: "k3s" KUBERNETES: "k3s"
USING_NFD: "true" USING_NFD: "true"
K8S_TEST_HOST_TYPE: "baremetal"
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:

View File

@ -12,6 +12,7 @@ tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${tests_dir}/common.bash" source "${tests_dir}/common.bash"
AZ_RG="${AZ_RG:-kataCI}" AZ_RG="${AZ_RG:-kataCI}"
K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}"
function _print_cluster_name() { function _print_cluster_name() {
test_type="${1:-k8s}" test_type="${1:-k8s}"
@ -40,10 +41,20 @@ function create_cluster() {
# First, ensure that the cluster didn't fail to get cleaned up from a previous run. # First, ensure that the cluster didn't fail to get cleaned up from a previous run.
delete_cluster "${test_type}" || true delete_cluster "${test_type}" || true
local instance_type=""
case ${K8S_TEST_HOST_TYPE} in
small)
instance_type="Standard_D2s_v5"
;;
normal)
instance_type="Standard_D4s_v5"
;;
esac
az aks create \ az aks create \
-g "${AZ_RG}" \ -g "${AZ_RG}" \
-n "$(_print_cluster_name ${test_type})" \ -n "$(_print_cluster_name ${test_type})" \
-s "Standard_D2s_v5" \ -s "${instance_type}" \
--node-count 1 \ --node-count 1 \
--generate-ssh-keys \ --generate-ssh-keys \
$([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation") $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation")

View File

@ -233,6 +233,7 @@ function cleanup() {
function main() { function main() {
export KATA_HOST_OS="${KATA_HOST_OS:-}" export KATA_HOST_OS="${KATA_HOST_OS:-}"
export K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-}"
action="${1:-}" action="${1:-}"

View File

@ -13,11 +13,12 @@ source "${kubernetes_dir}/../../common.bash"
TARGET_ARCH="${TARGET_ARCH:-x86_64}" TARGET_ARCH="${TARGET_ARCH:-x86_64}"
KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
K8S_TEST_DEBUG="${K8S_TEST_DEBUG:-false}" K8S_TEST_DEBUG="${K8S_TEST_DEBUG:-false}"
K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}"
if [ -n "${K8S_TEST_UNION:-}" ]; then if [ -n "${K8S_TEST_UNION:-}" ]; then
K8S_TEST_UNION=($K8S_TEST_UNION) K8S_TEST_UNION=($K8S_TEST_UNION)
else else
K8S_TEST_UNION=( \ K8S_TEST_SMALL_HOST_UNION=( \
"k8s-confidential.bats" \ "k8s-confidential.bats" \
"k8s-attach-handlers.bats" \ "k8s-attach-handlers.bats" \
"k8s-caps.bats" \ "k8s-caps.bats" \
@ -53,6 +54,29 @@ else
"k8s-volume.bats" \ "k8s-volume.bats" \
"k8s-nginx-connectivity.bats" \ "k8s-nginx-connectivity.bats" \
) )
K8S_TEST_NORMAL_HOST_UNION=( \
"k8s-number-cpus.bats" \
"k8s-parallel.bats" \
"k8s-scale-nginx.bats" \
)
case ${K8S_TEST_HOST_TYPE} in
small)
K8S_TEST_UNION=($K8S_TEST_SMALL_HOST_UNION)
;;
normal)
K8S_TEST_UNION=($K8S_TEST_NORMAL_HOST_UNION)
;;
baremetal)
K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]} ${K8S_TEST_NORMAL_HOST_UNION[@]})
;;
*)
echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | baremetal"
return 1
;;
esac
fi fi
# we may need to skip a few test cases when running on non-x86_64 arch # we may need to skip a few test cases when running on non-x86_64 arch