From 91a0b3b4067d36c1b50122b61228860c988f2802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Tue, 25 Jul 2023 11:21:30 -0700 Subject: [PATCH 1/5] tests: aks: Simply delete cluster when cleaning up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we're going to delete the cluster anyway, no need to call kata-cleanup. Fixes: #7454 Signed-off-by: Aurélien Bombo --- tests/integration/kubernetes/gha-run.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index c873a0d79c..adcbccfaef 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -121,6 +121,11 @@ function cleanup() { echo "Gather information about the nodes and pods before cleaning up the node" get_nodes_and_pods_info "yes" + if [ "${platform}" = "aks" ]; then + delete_cluster + return + fi + # Switch back to the default namespace and delete the tests one kubectl config set-context --current --namespace=default kubectl delete namespace kata-containers-k8s-tests @@ -145,10 +150,6 @@ function cleanup() { kubectl delete ${cleanup_spec} kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" - - if [ "${platform}" = "aks" ]; then - delete_cluster - fi } function delete_cluster() { From bdde6aa948db1de06abd822a3d638605bbb6620e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Tue, 25 Jul 2023 11:51:03 -0700 Subject: [PATCH 2/5] tests: k8s: Split deployment and testing commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This splits deploying Kata and running the tests into separate commands to make it possible to rerun tests locally without having to redeploy Kata each time. Signed-off-by: Aurélien Bombo --- .github/workflows/run-k8s-tests-on-aks.yaml | 6 +++++- .github/workflows/run-k8s-tests-on-sev.yaml | 6 +++++- .github/workflows/run-k8s-tests-on-snp.yaml | 8 ++++++-- .github/workflows/run-k8s-tests-on-tdx.yaml | 8 ++++++-- tests/integration/kubernetes/gha-run.sh | 19 +++++++++++-------- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 6b3b16aeba..d561dae50f 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -67,9 +67,13 @@ jobs: - name: Download credentials for the Kubernetes CLI to use them run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks + - name: Run tests timeout-minutes: 60 - run: bash tests/integration/kubernetes/gha-run.sh run-tests-aks + run: bash tests/integration/kubernetes/gha-run.sh run-tests - name: Delete AKS cluster if: always() diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index 3b43751bc9..bec1f4edf6 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -34,9 +34,13 @@ jobs: with: ref: ${{ inputs.commit-hash }} + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev + - name: Run tests timeout-minutes: 30 - run: bash tests/integration/kubernetes/gha-run.sh run-tests-sev + run: bash tests/integration/kubernetes/gha-run.sh run-tests - name: Delete kata-deploy if: always() diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index 49c55fb36d..cb65128268 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -34,10 +34,14 @@ jobs: with: ref: ${{ inputs.commit-hash }} + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp + - name: Run tests timeout-minutes: 30 - run: bash tests/integration/kubernetes/gha-run.sh run-tests-snp - + run: bash tests/integration/kubernetes/gha-run.sh run-tests + - name: Delete kata-deploy if: always() run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index 791d7cba84..e7373e3de3 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -33,10 +33,14 @@ jobs: with: ref: ${{ inputs.commit-hash }} + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx + - name: Run tests timeout-minutes: 30 - run: bash tests/integration/kubernetes/gha-run.sh run-tests-tdx - + run: bash tests/integration/kubernetes/gha-run.sh run-tests + - name: Delete kata-deploy if: always() run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index adcbccfaef..4d8f9a6f2c 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -59,16 +59,13 @@ function get_cluster_credentials() { -n "$(_print_cluster_name)" } -function run_tests() { +function deploy_kata() { platform="${1}" ensure_yq # Emsure we're in the default namespace kubectl config set-context --current --namespace=default - # Delete any spurious tests namespace that was left behind - kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Enable debug for Kata Containers @@ -101,6 +98,11 @@ function run_tests() { else sleep 60s fi +} + +function run_tests() { + # Delete any spurious tests namespace that was left behind + kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true # Create a new namespace for the tests and switch to it kubectl apply -f ${kubernetes_dir}/runtimeclass_workloads/tests-namespace.yaml @@ -190,10 +192,11 @@ function main() { install-bats) install_bats ;; install-kubectl) install_kubectl ;; get-cluster-credentials) get_cluster_credentials ;; - run-tests-aks) run_tests "aks" ;; - run-tests-sev) run_tests "sev" ;; - run-tests-snp) run_tests "snp" ;; - run-tests-tdx) run_tests "tdx" ;; + deploy-kata-aks) deploy_kata "aks" ;; + deploy-kata-sev) deploy_kata "sev" ;; + deploy-kata-snp) deploy_kata "snp" ;; + deploy-kata-tdx) deploy_kata "tdx" ;; + run-tests) run_tests ;; cleanup-sev) cleanup "sev" ;; cleanup-snp) cleanup "snp" ;; cleanup-tdx) cleanup "tdx" ;; From d7f04a64a049e9aa747a63dc534d77943826baf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Tue, 25 Jul 2023 12:06:57 -0700 Subject: [PATCH 3/5] tests: k8s: Leave `runtimeclass_workloads/` alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes it so that `setup.sh` doesn't make changes in `runtimeclass_workloads/` directly. Instead we treat that as a template directory and we use the new directory `runtimeclass_workloads_work/` as a work dir. This has two advantages: * Allows rerunning tests without the assumption that `setup.sh` must be idempotent. E.g. the `set_runtime_class()` step would break. * Doesn't pollute your git environment with a bunch of changes when developing. Signed-off-by: Aurélien Bombo --- tests/.gitignore | 1 + tests/integration/kubernetes/setup.sh | 12 +++++++++--- tests/integration/kubernetes/tests_common.sh | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 tests/.gitignore diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000000..122d160715 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +integration/kubernetes/runtimeclass_workloads_work/ diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index 01e7d8c51a..fb68e47cbd 100755 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -10,25 +10,31 @@ set -o pipefail kubernetes_dir=$(dirname "$(readlink -f "$0")") source "${kubernetes_dir}/../../common.bash" +reset_workloads_work_dir() { + rm -rf ${kubernetes_dir}/runtimeclass_workloads_work + cp -R ${kubernetes_dir}/runtimeclass_workloads ${kubernetes_dir}/runtimeclass_workloads_work +} + set_runtime_class() { - sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml + sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads_work/*.yaml } set_kernel_path() { if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then mariner_kernel_path="/usr/share/cloud-hypervisor/vmlinux.bin" - find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.kernel]' "${mariner_kernel_path}" \; + find ${kubernetes_dir}/runtimeclass_workloads_work/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.kernel]' "${mariner_kernel_path}" \; fi } set_initrd_path() { if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then initrd_path="/opt/kata/share/kata-containers/kata-containers-initrd-mariner.img" - find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.initrd]' "${initrd_path}" \; + find ${kubernetes_dir}/runtimeclass_workloads_work/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.initrd]' "${initrd_path}" \; fi } main() { + reset_workloads_work_dir set_runtime_class set_kernel_path set_initrd_path diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 481cf4a570..bcf686241c 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -34,6 +34,6 @@ dragonball_limitations="https://github.com/kata-containers/kata-containers/issue export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" get_pod_config_dir() { - pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads" + pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads_work" info "k8s configured to use runtimeclass" } From 350f3f70b7c83e17f5d12b92b53fd6e7fc1b020a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Tue, 25 Jul 2023 14:41:26 -0700 Subject: [PATCH 4/5] tests: Import `common.bash` in `run_kubernetes_tests.sh` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not sure why this works in GHA, but the `info` call on line 65 would fail locally. Signed-off-by: Aurélien Bombo --- tests/integration/kubernetes/run_kubernetes_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index db1e16633c..0a4e8cbed5 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -8,6 +8,7 @@ set -e kubernetes_dir=$(dirname "$(readlink -f "$0")") +source "${kubernetes_dir}/../../common.bash" TARGET_ARCH="${TARGET_ARCH:-x86_64}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" From 4703434b120a010060732928093a3204ca9e5455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Tue, 25 Jul 2023 14:48:34 -0700 Subject: [PATCH 5/5] tests: k8s: Allow using custom resource group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This simply allows setting a custom resource group when debugging locally, so as to prevent name collisions and not pollute the namespace. Signed-off-by: Aurélien Bombo --- tests/integration/kubernetes/gha-run.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 4d8f9a6f2c..ee4d26be34 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -12,6 +12,8 @@ kubernetes_dir="$(dirname "$(readlink -f "$0")")" source "${kubernetes_dir}/../../common.bash" tools_dir="${repo_root_dir}/tools" +AZ_RG="${AZ_RG:-kataCI}" + function _print_cluster_name() { short_sha="$(git rev-parse --short=12 HEAD)" echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64" @@ -36,7 +38,7 @@ function create_cluster() { delete_cluster || true az aks create \ - -g "kataCI" \ + -g "${AZ_RG}" \ -n "$(_print_cluster_name)" \ -s "Standard_D4s_v5" \ --node-count 1 \ @@ -55,7 +57,7 @@ function install_kubectl() { function get_cluster_credentials() { az aks get-credentials \ - -g "kataCI" \ + -g "${AZ_RG}" \ -n "$(_print_cluster_name)" } @@ -156,7 +158,7 @@ function cleanup() { function delete_cluster() { az aks delete \ - -g "kataCI" \ + -g "${AZ_RG}" \ -n "$(_print_cluster_name)" \ --yes }