From 2c5db14a1ad44a6986b0186b37c809be2d79c55b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 15 Aug 2023 16:30:23 +0200 Subject: [PATCH] gha: kata-deploy: Add the first kata-deploy test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test, at least for now, only checks whether the runtimeclasses have been properly created. This is just a migration from a test we had as part of the k8s suite. Signed-off-by: Fabiano Fidêncio (cherry picked from commit 4ffc2c86f3e32f518d87f465ee98c789b1ffb273) --- tests/functional/kata-deploy/gha-run.sh | 13 +- tests/functional/kata-deploy/kata-deploy.bats | 119 ++++++++++++++++++ .../kata-deploy/run-kata-deploy-tests.sh | 3 +- 3 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 tests/functional/kata-deploy/kata-deploy.bats diff --git a/tests/functional/kata-deploy/gha-run.sh b/tests/functional/kata-deploy/gha-run.sh index 0e3f2057a1..211990482f 100755 --- a/tests/functional/kata-deploy/gha-run.sh +++ b/tests/functional/kata-deploy/gha-run.sh @@ -10,16 +10,23 @@ set -o nounset set -o pipefail kata_deploy_dir="$(dirname "$(readlink -f "$0")")" -source "$kata_deploy_dir}/../../gha-run-k8s-common.sh" -tools_dir="${repo_root_dir}/tools" +source "${kata_deploy_dir}/../../gha-run-k8s-common.sh" function run_tests() { - return 0 + pushd "${kata_deploy_dir}" + bash run-kata-deploy-tests.sh + popd } function main() { export KATA_HOST_OS="${KATA_HOST_OS:-}" + platform="aks" + if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then + platform="tdx" + fi + export platform + action="${1:-}" case "${action}" in diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats new file mode 100644 index 0000000000..531c3eb64c --- /dev/null +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -0,0 +1,119 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/../../common.bash" + +setup() { + repo_root_dir="${BATS_TEST_DIRNAME}/../../../" + ensure_yq + + # Cleanup any runtimeclass already present in the cluster + # apart from the default one that comes from AKS + for rc in `kubectl get runtimeclass -o name | grep -v "kata-mshv-vm-isolation" | sed 's|runtimeclass.node.k8s.io/||'`; do + kubectl delete runtimeclass $rc; + done + + # We expect 2 runtime classes because: + # * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`. + # * `kata-${KATA_HYPERVISOR}` is the other one + # * As part of the tests we're only deploying the specific runtimeclass that will be used, instead of all of them. + expected_runtime_classes=2 + + # We expect both runtime classes to have the same handler: kata-${KATA_HYPERVISOR} + expected_handlers_re=( \ + "kata\s+kata-${KATA_HYPERVISOR}" \ + "kata-${KATA_HYPERVISOR}\s+kata-${KATA_HYPERVISOR}" \ + ) + + # Set the latest image, the one generated as part of the PR, to be used as part of the tests + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + + # Enable debug for Kata Containers + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[1].value' --tag '!!str' "true" + # Create the runtime class only for the shim that's being tested + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[2].value' "${KATA_HYPERVISOR}" + # Set the tested hypervisor as the default `kata` shim + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}" + # Let the `kata-deploy` script take care of the runtime class creation / removal + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true" + # Let the `kata-deploy` create the default `kata` runtime class + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true" + + if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS" + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}" + fi + + echo "::group::Final kata-deploy.yaml that is used in the test" + cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image" + echo "::endgroup::" + + kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + if [ "${platform}" = "tdx" ]; then + kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s" + else + kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + fi + kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod + + # Give some time for the pod to finish what's doing and have the + # runtimeclasses properly created + sleep 30s +} + +@test "Test runtimeclasses are being properly created" { + # We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy + current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l) + [[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]] + + for handler_re in ${expected_handlers_re[@]} + do + kubectl get runtimeclass | grep -E "${handler_re}" + done +} + +teardown() { + kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation" + + if [ "${platform}" = "tdx" ]; then + deploy_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"" + cleanup_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s"" + else + deploy_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"" + cleanup_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"" + fi + + kubectl delete ${deploy_spec} + kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + + # Let the `kata-deploy` script take care of the runtime class creation / removal + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true" + # Create the runtime class only for the shim that's being tested + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[2].value' "${KATA_HYPERVISOR}" + # Set the tested hypervisor as the default `kata` shim + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}" + # Let the `kata-deploy` create the default `kata` runtime class + yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true" + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image" + + kubectl apply ${cleanup_spec} + sleep 30s + + kubectl delete ${cleanup_spec} + kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + + # Cleanup any runtime class that was left behind in the cluster, in + # case of a test failure, apart from the default one that comes from + # AKS + for rc in `kubectl get runtimeclass -o name | grep -v "kata-mshv-vm-isolation" | sed 's|runtimeclass.node.k8s.io/||'`; do + kubectl delete runtimeclass $rc; + done +} diff --git a/tests/functional/kata-deploy/run-kata-deploy-tests.sh b/tests/functional/kata-deploy/run-kata-deploy-tests.sh index f66206869b..859bebf2e1 100644 --- a/tests/functional/kata-deploy/run-kata-deploy-tests.sh +++ b/tests/functional/kata-deploy/run-kata-deploy-tests.sh @@ -14,11 +14,12 @@ if [ -n "${KATA_DEPLOY_TEST_UNION:-}" ]; then KATA_DEPLOY_TEST_UNION=($KATA_DEPLOY_TEST_UNION) else KATA_DEPLOY_TEST_UNION=( \ + "kata-deploy.bats" \ ) fi info "Run tests" for KATA_DEPLOY_TEST_ENTRY in ${KATA_DEPLOY_TEST_UNION[@]} do - #bats "${KATA_DEPLOY_TEST_ENTRY}" + bats "${KATA_DEPLOY_TEST_ENTRY}" done