Merge pull request #7455 from sprt/local-tests

tests: QoL improvements for running tests locally

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -67,9 +67,13 @@ jobs:
       - name: Download credentials for the Kubernetes CLI to use them
         run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials
 
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks
+      
       - name: Run tests
         timeout-minutes: 60
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests-aks
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete AKS cluster
         if: always()

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -34,9 +34,13 @@ jobs:
         with:
           ref: ${{ inputs.commit-hash }}
 
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
+  
       - name: Run tests
         timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests-sev
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete kata-deploy
         if: always()

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -34,10 +34,14 @@ jobs:
         with:
           ref: ${{ inputs.commit-hash }}
 
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
+  
       - name: Run tests
         timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests-snp
-
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+      
       - name: Delete kata-deploy
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -33,10 +33,14 @@ jobs:
         with:
           ref: ${{ inputs.commit-hash }}
 
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
+  
       - name: Run tests
         timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests-tdx
-
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+  
       - name: Delete kata-deploy
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx

tests/.gitignore

@@ -0,0 +1 @@
+integration/kubernetes/runtimeclass_workloads_work/

tests/integration/kubernetes/gha-run.sh

@@ -12,6 +12,8 @@ kubernetes_dir="$(dirname "$(readlink -f "$0")")"
 source "${kubernetes_dir}/../../common.bash"
 tools_dir="${repo_root_dir}/tools"
 
+AZ_RG="${AZ_RG:-kataCI}"
+
 function _print_cluster_name() {
     short_sha="$(git rev-parse --short=12 HEAD)"
     echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64"
@@ -36,7 +38,7 @@ function create_cluster() {
     delete_cluster || true
 
     az aks create \
-        -g "kataCI" \
+        -g "${AZ_RG}" \
         -n "$(_print_cluster_name)" \
         -s "Standard_D4s_v5" \
         --node-count 1 \
@@ -55,20 +57,17 @@ function install_kubectl() {
 
 function get_cluster_credentials() {
     az aks get-credentials \
-        -g "kataCI" \
+        -g "${AZ_RG}" \
         -n "$(_print_cluster_name)"
 }
 
-function run_tests() {
+function deploy_kata() {
     platform="${1}"
     ensure_yq
 
     # Emsure we're in the default namespace
     kubectl config set-context --current --namespace=default
 
-    # Delete any spurious tests namespace that was left behind
-    kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true
-
     sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 
     # Enable debug for Kata Containers
@@ -101,6 +100,11 @@ function run_tests() {
     else
         sleep 60s
     fi
+}
+
+function run_tests() {
+    # Delete any spurious tests namespace that was left behind
+    kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true
 
     # Create a new namespace for the tests and switch to it
     kubectl apply -f ${kubernetes_dir}/runtimeclass_workloads/tests-namespace.yaml
@@ -121,6 +125,11 @@ function cleanup() {
     echo "Gather information about the nodes and pods before cleaning up the node"
     get_nodes_and_pods_info "yes"
 
+    if [ "${platform}" = "aks" ]; then
+        delete_cluster
+        return
+    fi
+
     # Switch back to the default namespace and delete the tests one
     kubectl config set-context --current --namespace=default
     kubectl delete namespace kata-containers-k8s-tests
@@ -145,15 +154,11 @@ function cleanup() {
     kubectl delete ${cleanup_spec}
     kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
     kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml"
-
-   if [ "${platform}" = "aks" ]; then
-	   delete_cluster
-   fi
 }
 
 function delete_cluster() {
     az aks delete \
-        -g "kataCI" \
+        -g "${AZ_RG}" \
         -n "$(_print_cluster_name)" \
         --yes
 }
@@ -189,10 +194,11 @@ function main() {
         install-bats) install_bats ;;
         install-kubectl) install_kubectl ;;
         get-cluster-credentials) get_cluster_credentials ;;
-        run-tests-aks) run_tests "aks" ;;
-        run-tests-sev) run_tests "sev" ;;
-        run-tests-snp) run_tests "snp" ;;
-        run-tests-tdx) run_tests "tdx" ;;
+        deploy-kata-aks) deploy_kata "aks" ;;
+        deploy-kata-sev) deploy_kata "sev" ;;
+        deploy-kata-snp) deploy_kata "snp" ;;
+        deploy-kata-tdx) deploy_kata "tdx" ;;
+        run-tests) run_tests ;;
         cleanup-sev) cleanup "sev" ;;
         cleanup-snp) cleanup "snp" ;;
         cleanup-tdx) cleanup "tdx" ;;

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -8,6 +8,7 @@
 set -e
 
 kubernetes_dir=$(dirname "$(readlink -f "$0")")
+source "${kubernetes_dir}/../../common.bash"
 
 TARGET_ARCH="${TARGET_ARCH:-x86_64}"
 KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"

tests/integration/kubernetes/setup.sh

@@ -10,25 +10,31 @@ set -o pipefail
 kubernetes_dir=$(dirname "$(readlink -f "$0")")
 source "${kubernetes_dir}/../../common.bash"
 
+reset_workloads_work_dir() {
+    rm -rf ${kubernetes_dir}/runtimeclass_workloads_work
+    cp -R ${kubernetes_dir}/runtimeclass_workloads ${kubernetes_dir}/runtimeclass_workloads_work
+}
+
 set_runtime_class() {
-    sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml
+    sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads_work/*.yaml
 }
 
 set_kernel_path() {
     if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
         mariner_kernel_path="/usr/share/cloud-hypervisor/vmlinux.bin"
-        find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.kernel]' "${mariner_kernel_path}" \;
+        find ${kubernetes_dir}/runtimeclass_workloads_work/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.kernel]' "${mariner_kernel_path}" \;
     fi
 }
 
 set_initrd_path() {
     if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
         initrd_path="/opt/kata/share/kata-containers/kata-containers-initrd-mariner.img"
-        find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.initrd]' "${initrd_path}" \;
+        find ${kubernetes_dir}/runtimeclass_workloads_work/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.initrd]' "${initrd_path}" \;
     fi
 }
 
 main() {
+    reset_workloads_work_dir
     set_runtime_class
     set_kernel_path
     set_initrd_path

tests/integration/kubernetes/tests_common.sh

@@ -34,6 +34,6 @@ dragonball_limitations="https://github.com/kata-containers/kata-containers/issue
 export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}"
 
 get_pod_config_dir() {
-	pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
+	pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads_work"
 	info "k8s configured to use runtimeclass"
 }