Merge pull request #7649 from fidencio/topic/k8s-tests-remove-kata-deploy-tests

gha: k8s: kata-deploy: Move kata-deploy specific tests from integration/kubernetes to functional/kata-deploy

tests/common.bash

@@ -338,7 +338,7 @@ function ensure_yq() {
     export GOPATH
     export PATH="${GOPATH}/bin:${PATH}"
     INSTALL_IN_GOPATH=true "${repo_root_dir}/ci/install_yq.sh"
-    hash -d yq # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache.
+    hash -d yq 2> /dev/null || true # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache.
 }
 
 # dependency: What we want to get the version from the versions.yaml file

tests/functional/kata-deploy/gha-run.sh

@@ -10,27 +10,57 @@ set -o nounset
 set -o pipefail
 
 kata_deploy_dir="$(dirname "$(readlink -f "$0")")"
-source "$kata_deploy_dir}/../../gha-run-k8s-common.sh"
+source "${kata_deploy_dir}/../../gha-run-k8s-common.sh"
-tools_dir="${repo_root_dir}/tools"
 
 function run_tests() {
-	return 0
+	cleanup_runtimeclasses || true
+
+	pushd "${kata_deploy_dir}"
+	bash run-kata-deploy-tests.sh
+	popd
+}
+
+function cleanup_runtimeclasses() {
+	# Cleanup any runtime class that was left behind in the cluster, in
+	# case of a test failure, apart from the default one that comes from
+	# AKS
+	for rc in `kubectl get runtimeclass -o name | grep -v "kata-mshv-vm-isolation" | sed 's|runtimeclass.node.k8s.io/||'`
+	do
+		kubectl delete runtimeclass $rc;
+	done
+}
+
+function cleanup() {
+	platform="${1}"
+	test_type="${2:-k8s}"
+
+	cleanup_runtimeclasses || true
+
+	if [ "${platform}" = "aks" ]; then
+		delete_cluster ${test_type}
+	fi
 }
 
 function main() {
     export KATA_HOST_OS="${KATA_HOST_OS:-}"
 
+    platform="aks"
+    if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then
+	    platform="tdx"
+    fi
+    export platform
+
     action="${1:-}"
 
     case "${action}" in
         install-azure-cli) install_azure_cli ;;
         login-azure) login_azure ;;
-        create-cluster) create_cluster ;;
+        create-cluster) create_cluster "kata-deploy" ;;
         install-bats) install_bats ;;
         install-kubectl) install_kubectl ;;
-        get-cluster-credentials) get_cluster_credentials ;;
+        get-cluster-credentials) get_cluster_credentials "kata-deploy" ;;
         run-tests) run_tests ;;
-        delete-cluster) cleanup "aks" ;;
+        delete-cluster) cleanup "aks" "kata-deploy" ;;
         *) >&2 echo "Invalid argument"; exit 2 ;;
     esac
 }

tests/functional/kata-deploy/kata-deploy.bats

@@ -0,0 +1,106 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/../../common.bash"
+
+setup() {
+	repo_root_dir="${BATS_TEST_DIRNAME}/../../../"
+	ensure_yq
+
+	# We expect 2 runtime classes because:
+	# * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`.
+	# * `kata-${KATA_HYPERVISOR}` is the other one
+	#    * As part of the tests we're only deploying the specific runtimeclass that will be used, instead of all of them.
+	expected_runtime_classes=2
+
+	# We expect both runtime classes to have the same handler: kata-${KATA_HYPERVISOR}
+	expected_handlers_re=( \
+		"kata\s+kata-${KATA_HYPERVISOR}" \
+		"kata-${KATA_HYPERVISOR}\s+kata-${KATA_HYPERVISOR}" \
+	)
+
+	# Set the latest image, the one generated as part of the PR, to be used as part of the tests
+	sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+
+	# Enable debug for Kata Containers
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[1].value' --tag '!!str' "true"
+	# Create the runtime class only for the shim that's being tested
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[2].value' "${KATA_HYPERVISOR}"
+	# Set the tested hypervisor as the default `kata` shim
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}"
+	# Let the `kata-deploy` script take care of the runtime class creation / removal
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
+	# Let the `kata-deploy` create the default `kata` runtime class
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true"
+	
+	if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
+		yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
+		yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
+	fi
+	
+	echo "::group::Final kata-deploy.yaml that is used in the test"
+	cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
+	echo "::endgroup::"
+	
+	kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
+	if [ "${platform}" = "tdx" ]; then
+		kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"
+	else
+		kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	fi
+	kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
+
+	# Give some time for the pod to finish what's doing and have the
+	# runtimeclasses properly created
+	sleep 30s
+}
+
+@test "Test runtimeclasses are being properly created" {
+	# We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy
+	current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l)
+	[[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]]
+
+	for handler_re in ${expected_handlers_re[@]}
+	do
+		kubectl get runtimeclass | grep -E "${handler_re}"
+	done
+}
+
+teardown() {
+	kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation"
+
+	if [ "${platform}" = "tdx" ]; then
+		deploy_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s""
+		cleanup_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s""
+	else
+		deploy_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml""
+		cleanup_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml""
+	fi
+
+	kubectl delete ${deploy_spec}
+	kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
+	
+	# Let the `kata-deploy` script take care of the runtime class creation / removal
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
+	# Create the runtime class only for the shim that's being tested
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[2].value' "${KATA_HYPERVISOR}"
+	# Set the tested hypervisor as the default `kata` shim
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}"
+	# Let the `kata-deploy` create the default `kata` runtime class
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true"
+	
+	sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
+	cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
+	grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"
+
+	kubectl apply ${cleanup_spec}
+	sleep 30s
+	
+	kubectl delete ${cleanup_spec}
+	kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
+}

tests/functional/kata-deploy/run-kata-deploy-tests.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+kata_deploy_dir=$(dirname "$(readlink -f "$0")")
+source "${kata_deploy_dir}/../../common.bash"
+
+if [ -n "${KATA_DEPLOY_TEST_UNION:-}" ]; then
+	KATA_DEPLOY_TEST_UNION=($KATA_DEPLOY_TEST_UNION)
+else
+	KATA_DEPLOY_TEST_UNION=( \
+		"kata-deploy.bats" \
+	)
+fi
+
+info "Run tests"
+for KATA_DEPLOY_TEST_ENTRY in ${KATA_DEPLOY_TEST_UNION[@]}
+do
+	bats "${KATA_DEPLOY_TEST_ENTRY}"
+done

tests/gha-run-k8s-common.sh

@@ -14,8 +14,10 @@ source "${tests_dir}/common.bash"
 AZ_RG="${AZ_RG:-kataCI}"
 
 function _print_cluster_name() {
+    test_type="${1:-k8s}"
+
     short_sha="$(git rev-parse --short=12 HEAD)"
-    echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64"
+    echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64"
 }
 
 function install_azure_cli() {
@@ -33,12 +35,14 @@ function login_azure() {
 }
 
 function create_cluster() {
+    test_type="${1:-k8s}"
+
     # First, ensure that the cluster didn't fail to get cleaned up from a previous run.
-    delete_cluster || true
+    delete_cluster "${test_type}" || true
 
     az aks create \
         -g "${AZ_RG}" \
-        -n "$(_print_cluster_name)" \
+        -n "$(_print_cluster_name ${test_type})" \
         -s "Standard_D4s_v5" \
         --node-count 1 \
         --generate-ssh-keys \
@@ -61,15 +65,19 @@ function install_kubectl() {
 }
 
 function get_cluster_credentials() {
+    test_type="${1:-k8s}"
+
     az aks get-credentials \
         -g "${AZ_RG}" \
-        -n "$(_print_cluster_name)"
+        -n "$(_print_cluster_name ${test_type})"
 }
 
 function delete_cluster() {
+    test_type="${1:-k8s}"
+
     az aks delete \
         -g "${AZ_RG}" \
-        -n "$(_print_cluster_name)" \
+        -n "$(_print_cluster_name ${test_type})" \
         --yes
 }
 

tests/integration/kubernetes/gha-run.sh

@@ -84,13 +84,14 @@ function run_tests() {
 
 function cleanup() {
     platform="${1}"
+    test_type="${2:-k8s}"
     ensure_yq
 
     echo "Gather information about the nodes and pods before cleaning up the node"
     get_nodes_and_pods_info
 
     if [ "${platform}" = "aks" ]; then
-        delete_cluster
+        delete_cluster ${test_type}
         return
     fi
 

tests/integration/kubernetes/kata-deploy-ensure-runtimec-classes-created.bats

@@ -1,38 +0,0 @@
-#!/usr/bin/env bats
-#
-# Copyright (c) 2023 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-load "${BATS_TEST_DIRNAME}/../../common.bash"
-load "${BATS_TEST_DIRNAME}/tests_common.sh"
-
-setup() {
-	# We expect 2 runtime classes because:
-	# * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`.
-	# * `kata-${KATA_HYPERVISOR}` is the other one
-	#    * As part of the tests we're only deploying the specific runtimeclass that will be used, instead of all of them.
-	expected_runtime_classes=2
-
-	# We expect both runtime classes to have the same handler: kata-${KATA_HYPERVISOR}
-	expected_handlers_re=( \
-		"kata\s+kata-${KATA_HYPERVISOR}" \
-		"kata-${KATA_HYPERVISOR}\s+kata-${KATA_HYPERVISOR}" \
-	)
-}
-
-@test "Test runtimeclasses are being properly created" {
-	# We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy
-	current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l)
-	[[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]]
-
-	for handler_re in ${expected_handlers_re[@]}
-	do
-		[[ $(kubectl get runtimeclass | grep -E "${handler_re}") ]]
-	done
-}
-
-teardown() {
-	kubectl get runtimeclasses
-}

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -18,7 +18,6 @@ if [ -n "${K8S_TEST_UNION:-}" ]; then
 	K8S_TEST_UNION=($K8S_TEST_UNION)
 else
 	K8S_TEST_UNION=( \
-		"kata-deploy-ensure-runtimec-classes-created.bats" \
 		"k8s-attach-handlers.bats" \
 		"k8s-caps.bats" \
 		"k8s-configmap.bats" \