diff --git a/tests/common.bash b/tests/common.bash index 5e5f833ef3..08b82601bf 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -338,7 +338,7 @@ function ensure_yq() { export GOPATH export PATH="${GOPATH}/bin:${PATH}" INSTALL_IN_GOPATH=true "${repo_root_dir}/ci/install_yq.sh" - hash -d yq # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache. + hash -d yq 2> /dev/null || true # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache. } # dependency: What we want to get the version from the versions.yaml file diff --git a/tests/functional/kata-deploy/gha-run.sh b/tests/functional/kata-deploy/gha-run.sh index ef362f40e0..8e1c9a1d05 100755 --- a/tests/functional/kata-deploy/gha-run.sh +++ b/tests/functional/kata-deploy/gha-run.sh @@ -10,27 +10,57 @@ set -o nounset set -o pipefail kata_deploy_dir="$(dirname "$(readlink -f "$0")")" -source "$kata_deploy_dir}/../../gha-run-k8s-common.sh" -tools_dir="${repo_root_dir}/tools" +source "${kata_deploy_dir}/../../gha-run-k8s-common.sh" function run_tests() { - return 0 + cleanup_runtimeclasses || true + + pushd "${kata_deploy_dir}" + bash run-kata-deploy-tests.sh + popd +} + +function cleanup_runtimeclasses() { + # Cleanup any runtime class that was left behind in the cluster, in + # case of a test failure, apart from the default one that comes from + # AKS + for rc in `kubectl get runtimeclass -o name | grep -v "kata-mshv-vm-isolation" | sed 's|runtimeclass.node.k8s.io/||'` + do + kubectl delete runtimeclass $rc; + done +} + +function cleanup() { + platform="${1}" + test_type="${2:-k8s}" + + cleanup_runtimeclasses || true + + if [ "${platform}" = "aks" ]; then + delete_cluster ${test_type} + fi } function main() { export KATA_HOST_OS="${KATA_HOST_OS:-}" + platform="aks" + if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then + platform="tdx" + fi + export platform + action="${1:-}" case "${action}" in install-azure-cli) install_azure_cli ;; login-azure) login_azure ;; - create-cluster) create_cluster ;; + create-cluster) create_cluster "kata-deploy" ;; install-bats) install_bats ;; install-kubectl) install_kubectl ;; - get-cluster-credentials) get_cluster_credentials ;; + get-cluster-credentials) get_cluster_credentials "kata-deploy" ;; run-tests) run_tests ;; - delete-cluster) cleanup "aks" ;; + delete-cluster) cleanup "aks" "kata-deploy" ;; *) >&2 echo "Invalid argument"; exit 2 ;; esac } diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats new file mode 100644 index 0000000000..84faf8fe14 --- /dev/null +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -0,0 +1,106 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/../../common.bash" + +setup() { + repo_root_dir="${BATS_TEST_DIRNAME}/../../../" + ensure_yq + + # We expect 2 runtime classes because: + # * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`. + # * `kata-${KATA_HYPERVISOR}` is the other one + # * As part of the tests we're only deploying the specific runtimeclass that will be used, instead of all of them. + expected_runtime_classes=2 + + # We expect both runtime classes to have the same handler: kata-${KATA_HYPERVISOR} + expected_handlers_re=( \ + "kata\s+kata-${KATA_HYPERVISOR}" \ + "kata-${KATA_HYPERVISOR}\s+kata-${KATA_HYPERVISOR}" \ + ) + + # Set the latest image, the one generated as part of the PR, to be used as part of the tests + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + + # Enable debug for Kata Containers + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[1].value' --tag '!!str' "true" + # Create the runtime class only for the shim that's being tested + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[2].value' "${KATA_HYPERVISOR}" + # Set the tested hypervisor as the default `kata` shim + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}" + # Let the `kata-deploy` script take care of the runtime class creation / removal + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true" + # Let the `kata-deploy` create the default `kata` runtime class + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true" + + if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS" + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}" + fi + + echo "::group::Final kata-deploy.yaml that is used in the test" + cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image" + echo "::endgroup::" + + kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + if [ "${platform}" = "tdx" ]; then + kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s" + else + kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + fi + kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod + + # Give some time for the pod to finish what's doing and have the + # runtimeclasses properly created + sleep 30s +} + +@test "Test runtimeclasses are being properly created" { + # We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy + current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l) + [[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]] + + for handler_re in ${expected_handlers_re[@]} + do + kubectl get runtimeclass | grep -E "${handler_re}" + done +} + +teardown() { + kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation" + + if [ "${platform}" = "tdx" ]; then + deploy_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"" + cleanup_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s"" + else + deploy_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"" + cleanup_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"" + fi + + kubectl delete ${deploy_spec} + kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + + # Let the `kata-deploy` script take care of the runtime class creation / removal + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true" + # Create the runtime class only for the shim that's being tested + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[2].value' "${KATA_HYPERVISOR}" + # Set the tested hypervisor as the default `kata` shim + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}" + # Let the `kata-deploy` create the default `kata` runtime class + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true" + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image" + + kubectl apply ${cleanup_spec} + sleep 30s + + kubectl delete ${cleanup_spec} + kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" +} diff --git a/tests/functional/kata-deploy/run-kata-deploy-tests.sh b/tests/functional/kata-deploy/run-kata-deploy-tests.sh new file mode 100644 index 0000000000..859bebf2e1 --- /dev/null +++ b/tests/functional/kata-deploy/run-kata-deploy-tests.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +kata_deploy_dir=$(dirname "$(readlink -f "$0")") +source "${kata_deploy_dir}/../../common.bash" + +if [ -n "${KATA_DEPLOY_TEST_UNION:-}" ]; then + KATA_DEPLOY_TEST_UNION=($KATA_DEPLOY_TEST_UNION) +else + KATA_DEPLOY_TEST_UNION=( \ + "kata-deploy.bats" \ + ) +fi + +info "Run tests" +for KATA_DEPLOY_TEST_ENTRY in ${KATA_DEPLOY_TEST_UNION[@]} +do + bats "${KATA_DEPLOY_TEST_ENTRY}" +done diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index cb727a7b72..bf0d202e53 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -14,8 +14,10 @@ source "${tests_dir}/common.bash" AZ_RG="${AZ_RG:-kataCI}" function _print_cluster_name() { + test_type="${1:-k8s}" + short_sha="$(git rev-parse --short=12 HEAD)" - echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64" + echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64" } function install_azure_cli() { @@ -33,12 +35,14 @@ function login_azure() { } function create_cluster() { + test_type="${1:-k8s}" + # First, ensure that the cluster didn't fail to get cleaned up from a previous run. - delete_cluster || true + delete_cluster "${test_type}" || true az aks create \ -g "${AZ_RG}" \ - -n "$(_print_cluster_name)" \ + -n "$(_print_cluster_name ${test_type})" \ -s "Standard_D4s_v5" \ --node-count 1 \ --generate-ssh-keys \ @@ -61,15 +65,19 @@ function install_kubectl() { } function get_cluster_credentials() { + test_type="${1:-k8s}" + az aks get-credentials \ -g "${AZ_RG}" \ - -n "$(_print_cluster_name)" + -n "$(_print_cluster_name ${test_type})" } function delete_cluster() { + test_type="${1:-k8s}" + az aks delete \ -g "${AZ_RG}" \ - -n "$(_print_cluster_name)" \ + -n "$(_print_cluster_name ${test_type})" \ --yes } diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 07a16a1963..10c7aa483a 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -84,13 +84,14 @@ function run_tests() { function cleanup() { platform="${1}" + test_type="${2:-k8s}" ensure_yq echo "Gather information about the nodes and pods before cleaning up the node" get_nodes_and_pods_info if [ "${platform}" = "aks" ]; then - delete_cluster + delete_cluster ${test_type} return fi diff --git a/tests/integration/kubernetes/kata-deploy-ensure-runtimec-classes-created.bats b/tests/integration/kubernetes/kata-deploy-ensure-runtimec-classes-created.bats deleted file mode 100644 index b2ec3c330b..0000000000 --- a/tests/integration/kubernetes/kata-deploy-ensure-runtimec-classes-created.bats +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bats -# -# Copyright (c) 2023 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -load "${BATS_TEST_DIRNAME}/../../common.bash" -load "${BATS_TEST_DIRNAME}/tests_common.sh" - -setup() { - # We expect 2 runtime classes because: - # * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`. - # * `kata-${KATA_HYPERVISOR}` is the other one - # * As part of the tests we're only deploying the specific runtimeclass that will be used, instead of all of them. - expected_runtime_classes=2 - - # We expect both runtime classes to have the same handler: kata-${KATA_HYPERVISOR} - expected_handlers_re=( \ - "kata\s+kata-${KATA_HYPERVISOR}" \ - "kata-${KATA_HYPERVISOR}\s+kata-${KATA_HYPERVISOR}" \ - ) -} - -@test "Test runtimeclasses are being properly created" { - # We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy - current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l) - [[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]] - - for handler_re in ${expected_handlers_re[@]} - do - [[ $(kubectl get runtimeclass | grep -E "${handler_re}") ]] - done -} - -teardown() { - kubectl get runtimeclasses -} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index e46cbc9fe0..f8b635d220 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -18,7 +18,6 @@ if [ -n "${K8S_TEST_UNION:-}" ]; then K8S_TEST_UNION=($K8S_TEST_UNION) else K8S_TEST_UNION=( \ - "kata-deploy-ensure-runtimec-classes-created.bats" \ "k8s-attach-handlers.bats" \ "k8s-caps.bats" \ "k8s-configmap.bats" \