diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 6b3b16aeba..d561dae50f 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -67,9 +67,13 @@ jobs: - name: Download credentials for the Kubernetes CLI to use them run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks + - name: Run tests timeout-minutes: 60 - run: bash tests/integration/kubernetes/gha-run.sh run-tests-aks + run: bash tests/integration/kubernetes/gha-run.sh run-tests - name: Delete AKS cluster if: always() diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index 3b43751bc9..bec1f4edf6 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -34,9 +34,13 @@ jobs: with: ref: ${{ inputs.commit-hash }} + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev + - name: Run tests timeout-minutes: 30 - run: bash tests/integration/kubernetes/gha-run.sh run-tests-sev + run: bash tests/integration/kubernetes/gha-run.sh run-tests - name: Delete kata-deploy if: always() diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index 49c55fb36d..cb65128268 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -34,10 +34,14 @@ jobs: with: ref: ${{ inputs.commit-hash }} + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp + - name: Run tests timeout-minutes: 30 - run: bash tests/integration/kubernetes/gha-run.sh run-tests-snp - + run: bash tests/integration/kubernetes/gha-run.sh run-tests + - name: Delete kata-deploy if: always() run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index 791d7cba84..e7373e3de3 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -33,10 +33,14 @@ jobs: with: ref: ${{ inputs.commit-hash }} + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx + - name: Run tests timeout-minutes: 30 - run: bash tests/integration/kubernetes/gha-run.sh run-tests-tdx - + run: bash tests/integration/kubernetes/gha-run.sh run-tests + - name: Delete kata-deploy if: always() run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000000..122d160715 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +integration/kubernetes/runtimeclass_workloads_work/ diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index c873a0d79c..ee4d26be34 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -12,6 +12,8 @@ kubernetes_dir="$(dirname "$(readlink -f "$0")")" source "${kubernetes_dir}/../../common.bash" tools_dir="${repo_root_dir}/tools" +AZ_RG="${AZ_RG:-kataCI}" + function _print_cluster_name() { short_sha="$(git rev-parse --short=12 HEAD)" echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64" @@ -36,7 +38,7 @@ function create_cluster() { delete_cluster || true az aks create \ - -g "kataCI" \ + -g "${AZ_RG}" \ -n "$(_print_cluster_name)" \ -s "Standard_D4s_v5" \ --node-count 1 \ @@ -55,20 +57,17 @@ function install_kubectl() { function get_cluster_credentials() { az aks get-credentials \ - -g "kataCI" \ + -g "${AZ_RG}" \ -n "$(_print_cluster_name)" } -function run_tests() { +function deploy_kata() { platform="${1}" ensure_yq # Emsure we're in the default namespace kubectl config set-context --current --namespace=default - # Delete any spurious tests namespace that was left behind - kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Enable debug for Kata Containers @@ -101,6 +100,11 @@ function run_tests() { else sleep 60s fi +} + +function run_tests() { + # Delete any spurious tests namespace that was left behind + kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true # Create a new namespace for the tests and switch to it kubectl apply -f ${kubernetes_dir}/runtimeclass_workloads/tests-namespace.yaml @@ -121,6 +125,11 @@ function cleanup() { echo "Gather information about the nodes and pods before cleaning up the node" get_nodes_and_pods_info "yes" + if [ "${platform}" = "aks" ]; then + delete_cluster + return + fi + # Switch back to the default namespace and delete the tests one kubectl config set-context --current --namespace=default kubectl delete namespace kata-containers-k8s-tests @@ -145,15 +154,11 @@ function cleanup() { kubectl delete ${cleanup_spec} kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" - - if [ "${platform}" = "aks" ]; then - delete_cluster - fi } function delete_cluster() { az aks delete \ - -g "kataCI" \ + -g "${AZ_RG}" \ -n "$(_print_cluster_name)" \ --yes } @@ -189,10 +194,11 @@ function main() { install-bats) install_bats ;; install-kubectl) install_kubectl ;; get-cluster-credentials) get_cluster_credentials ;; - run-tests-aks) run_tests "aks" ;; - run-tests-sev) run_tests "sev" ;; - run-tests-snp) run_tests "snp" ;; - run-tests-tdx) run_tests "tdx" ;; + deploy-kata-aks) deploy_kata "aks" ;; + deploy-kata-sev) deploy_kata "sev" ;; + deploy-kata-snp) deploy_kata "snp" ;; + deploy-kata-tdx) deploy_kata "tdx" ;; + run-tests) run_tests ;; cleanup-sev) cleanup "sev" ;; cleanup-snp) cleanup "snp" ;; cleanup-tdx) cleanup "tdx" ;; diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index db1e16633c..0a4e8cbed5 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -8,6 +8,7 @@ set -e kubernetes_dir=$(dirname "$(readlink -f "$0")") +source "${kubernetes_dir}/../../common.bash" TARGET_ARCH="${TARGET_ARCH:-x86_64}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index 01e7d8c51a..fb68e47cbd 100755 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -10,25 +10,31 @@ set -o pipefail kubernetes_dir=$(dirname "$(readlink -f "$0")") source "${kubernetes_dir}/../../common.bash" +reset_workloads_work_dir() { + rm -rf ${kubernetes_dir}/runtimeclass_workloads_work + cp -R ${kubernetes_dir}/runtimeclass_workloads ${kubernetes_dir}/runtimeclass_workloads_work +} + set_runtime_class() { - sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml + sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads_work/*.yaml } set_kernel_path() { if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then mariner_kernel_path="/usr/share/cloud-hypervisor/vmlinux.bin" - find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.kernel]' "${mariner_kernel_path}" \; + find ${kubernetes_dir}/runtimeclass_workloads_work/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.kernel]' "${mariner_kernel_path}" \; fi } set_initrd_path() { if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then initrd_path="/opt/kata/share/kata-containers/kata-containers-initrd-mariner.img" - find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.initrd]' "${initrd_path}" \; + find ${kubernetes_dir}/runtimeclass_workloads_work/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.initrd]' "${initrd_path}" \; fi } main() { + reset_workloads_work_dir set_runtime_class set_kernel_path set_initrd_path diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 481cf4a570..bcf686241c 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -34,6 +34,6 @@ dragonball_limitations="https://github.com/kata-containers/kata-containers/issue export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" get_pod_config_dir() { - pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads" + pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads_work" info "k8s configured to use runtimeclass" }