mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-08-30 23:37:45 +00:00
Merge pull request #7649 from fidencio/topic/k8s-tests-remove-kata-deploy-tests
gha: k8s: kata-deploy: Move kata-deploy specific tests from integration/kubernetes to functional/kata-deploy
This commit is contained in:
Fabiano Fidêncio
GitHub
@@ -338,7 +338,7 @@ function ensure_yq() {
|
||||
export GOPATH
|
||||
export PATH="${GOPATH}/bin:${PATH}"
|
||||
INSTALL_IN_GOPATH=true "${repo_root_dir}/ci/install_yq.sh"
|
||||
hash -d yq # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache.
|
||||
hash -d yq 2> /dev/null || true # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache.
|
||||
}
|
||||
|
||||
# dependency: What we want to get the version from the versions.yaml file
|
||||
|
||||
@@ -10,27 +10,57 @@ set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
kata_deploy_dir="$(dirname "$(readlink -f "$0")")"
|
||||
source "$kata_deploy_dir}/../../gha-run-k8s-common.sh"
|
||||
tools_dir="${repo_root_dir}/tools"
|
||||
source "${kata_deploy_dir}/../../gha-run-k8s-common.sh"
|
||||
|
||||
function run_tests() {
|
||||
return 0
|
||||
cleanup_runtimeclasses || true
|
||||
|
||||
pushd "${kata_deploy_dir}"
|
||||
bash run-kata-deploy-tests.sh
|
||||
popd
|
||||
}
|
||||
|
||||
function cleanup_runtimeclasses() {
|
||||
# Cleanup any runtime class that was left behind in the cluster, in
|
||||
# case of a test failure, apart from the default one that comes from
|
||||
# AKS
|
||||
for rc in `kubectl get runtimeclass -o name | grep -v "kata-mshv-vm-isolation" | sed 's|runtimeclass.node.k8s.io/||'`
|
||||
do
|
||||
kubectl delete runtimeclass $rc;
|
||||
done
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
platform="${1}"
|
||||
test_type="${2:-k8s}"
|
||||
|
||||
cleanup_runtimeclasses || true
|
||||
|
||||
if [ "${platform}" = "aks" ]; then
|
||||
delete_cluster ${test_type}
|
||||
fi
|
||||
}
|
||||
|
||||
function main() {
|
||||
export KATA_HOST_OS="${KATA_HOST_OS:-}"
|
||||
|
||||
platform="aks"
|
||||
if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then
|
||||
platform="tdx"
|
||||
fi
|
||||
export platform
|
||||
|
||||
action="${1:-}"
|
||||
|
||||
case "${action}" in
|
||||
install-azure-cli) install_azure_cli ;;
|
||||
login-azure) login_azure ;;
|
||||
create-cluster) create_cluster ;;
|
||||
create-cluster) create_cluster "kata-deploy" ;;
|
||||
install-bats) install_bats ;;
|
||||
install-kubectl) install_kubectl ;;
|
||||
get-cluster-credentials) get_cluster_credentials ;;
|
||||
get-cluster-credentials) get_cluster_credentials "kata-deploy" ;;
|
||||
run-tests) run_tests ;;
|
||||
delete-cluster) cleanup "aks" ;;
|
||||
delete-cluster) cleanup "aks" "kata-deploy" ;;
|
||||
*) >&2 echo "Invalid argument"; exit 2 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
106
tests/functional/kata-deploy/kata-deploy.bats
Normal file
106
tests/functional/kata-deploy/kata-deploy.bats
Normal file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bats
|
||||
#
|
||||
# Copyright (c) 2023 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
load "${BATS_TEST_DIRNAME}/../../common.bash"
|
||||
|
||||
setup() {
|
||||
repo_root_dir="${BATS_TEST_DIRNAME}/../../../"
|
||||
ensure_yq
|
||||
|
||||
# We expect 2 runtime classes because:
|
||||
# * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`.
|
||||
# * `kata-${KATA_HYPERVISOR}` is the other one
|
||||
# * As part of the tests we're only deploying the specific runtimeclass that will be used, instead of all of them.
|
||||
expected_runtime_classes=2
|
||||
|
||||
# We expect both runtime classes to have the same handler: kata-${KATA_HYPERVISOR}
|
||||
expected_handlers_re=( \
|
||||
"kata\s+kata-${KATA_HYPERVISOR}" \
|
||||
"kata-${KATA_HYPERVISOR}\s+kata-${KATA_HYPERVISOR}" \
|
||||
)
|
||||
|
||||
# Set the latest image, the one generated as part of the PR, to be used as part of the tests
|
||||
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
|
||||
# Enable debug for Kata Containers
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[1].value' --tag '!!str' "true"
|
||||
# Create the runtime class only for the shim that's being tested
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[2].value' "${KATA_HYPERVISOR}"
|
||||
# Set the tested hypervisor as the default `kata` shim
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}"
|
||||
# Let the `kata-deploy` script take care of the runtime class creation / removal
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
|
||||
# Let the `kata-deploy` create the default `kata` runtime class
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true"
|
||||
|
||||
if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
|
||||
fi
|
||||
|
||||
echo "::group::Final kata-deploy.yaml that is used in the test"
|
||||
cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
|
||||
echo "::endgroup::"
|
||||
|
||||
kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
|
||||
if [ "${platform}" = "tdx" ]; then
|
||||
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"
|
||||
else
|
||||
kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
fi
|
||||
kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
|
||||
|
||||
# Give some time for the pod to finish what's doing and have the
|
||||
# runtimeclasses properly created
|
||||
sleep 30s
|
||||
}
|
||||
|
||||
@test "Test runtimeclasses are being properly created" {
|
||||
# We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy
|
||||
current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l)
|
||||
[[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]]
|
||||
|
||||
for handler_re in ${expected_handlers_re[@]}
|
||||
do
|
||||
kubectl get runtimeclass | grep -E "${handler_re}"
|
||||
done
|
||||
}
|
||||
|
||||
teardown() {
|
||||
kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation"
|
||||
|
||||
if [ "${platform}" = "tdx" ]; then
|
||||
deploy_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s""
|
||||
cleanup_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s""
|
||||
else
|
||||
deploy_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml""
|
||||
cleanup_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml""
|
||||
fi
|
||||
|
||||
kubectl delete ${deploy_spec}
|
||||
kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
|
||||
|
||||
# Let the `kata-deploy` script take care of the runtime class creation / removal
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
|
||||
# Create the runtime class only for the shim that's being tested
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[2].value' "${KATA_HYPERVISOR}"
|
||||
# Set the tested hypervisor as the default `kata` shim
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}"
|
||||
# Let the `kata-deploy` create the default `kata` runtime class
|
||||
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true"
|
||||
|
||||
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"
|
||||
|
||||
kubectl apply ${cleanup_spec}
|
||||
sleep 30s
|
||||
|
||||
kubectl delete ${cleanup_spec}
|
||||
kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
|
||||
}
|
||||
25
tests/functional/kata-deploy/run-kata-deploy-tests.sh
Normal file
25
tests/functional/kata-deploy/run-kata-deploy-tests.sh
Normal file
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2023 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
kata_deploy_dir=$(dirname "$(readlink -f "$0")")
|
||||
source "${kata_deploy_dir}/../../common.bash"
|
||||
|
||||
if [ -n "${KATA_DEPLOY_TEST_UNION:-}" ]; then
|
||||
KATA_DEPLOY_TEST_UNION=($KATA_DEPLOY_TEST_UNION)
|
||||
else
|
||||
KATA_DEPLOY_TEST_UNION=( \
|
||||
"kata-deploy.bats" \
|
||||
)
|
||||
fi
|
||||
|
||||
info "Run tests"
|
||||
for KATA_DEPLOY_TEST_ENTRY in ${KATA_DEPLOY_TEST_UNION[@]}
|
||||
do
|
||||
bats "${KATA_DEPLOY_TEST_ENTRY}"
|
||||
done
|
||||
@@ -14,8 +14,10 @@ source "${tests_dir}/common.bash"
|
||||
AZ_RG="${AZ_RG:-kataCI}"
|
||||
|
||||
function _print_cluster_name() {
|
||||
test_type="${1:-k8s}"
|
||||
|
||||
short_sha="$(git rev-parse --short=12 HEAD)"
|
||||
echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64"
|
||||
echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64"
|
||||
}
|
||||
|
||||
function install_azure_cli() {
|
||||
@@ -33,12 +35,14 @@ function login_azure() {
|
||||
}
|
||||
|
||||
function create_cluster() {
|
||||
test_type="${1:-k8s}"
|
||||
|
||||
# First, ensure that the cluster didn't fail to get cleaned up from a previous run.
|
||||
delete_cluster || true
|
||||
delete_cluster "${test_type}" || true
|
||||
|
||||
az aks create \
|
||||
-g "${AZ_RG}" \
|
||||
-n "$(_print_cluster_name)" \
|
||||
-n "$(_print_cluster_name ${test_type})" \
|
||||
-s "Standard_D4s_v5" \
|
||||
--node-count 1 \
|
||||
--generate-ssh-keys \
|
||||
@@ -61,15 +65,19 @@ function install_kubectl() {
|
||||
}
|
||||
|
||||
function get_cluster_credentials() {
|
||||
test_type="${1:-k8s}"
|
||||
|
||||
az aks get-credentials \
|
||||
-g "${AZ_RG}" \
|
||||
-n "$(_print_cluster_name)"
|
||||
-n "$(_print_cluster_name ${test_type})"
|
||||
}
|
||||
|
||||
function delete_cluster() {
|
||||
test_type="${1:-k8s}"
|
||||
|
||||
az aks delete \
|
||||
-g "${AZ_RG}" \
|
||||
-n "$(_print_cluster_name)" \
|
||||
-n "$(_print_cluster_name ${test_type})" \
|
||||
--yes
|
||||
}
|
||||
|
||||
|
||||
@@ -84,13 +84,14 @@ function run_tests() {
|
||||
|
||||
function cleanup() {
|
||||
platform="${1}"
|
||||
test_type="${2:-k8s}"
|
||||
ensure_yq
|
||||
|
||||
echo "Gather information about the nodes and pods before cleaning up the node"
|
||||
get_nodes_and_pods_info
|
||||
|
||||
if [ "${platform}" = "aks" ]; then
|
||||
delete_cluster
|
||||
delete_cluster ${test_type}
|
||||
return
|
||||
fi
|
||||
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env bats
|
||||
#
|
||||
# Copyright (c) 2023 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
load "${BATS_TEST_DIRNAME}/../../common.bash"
|
||||
load "${BATS_TEST_DIRNAME}/tests_common.sh"
|
||||
|
||||
setup() {
|
||||
# We expect 2 runtime classes because:
|
||||
# * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`.
|
||||
# * `kata-${KATA_HYPERVISOR}` is the other one
|
||||
# * As part of the tests we're only deploying the specific runtimeclass that will be used, instead of all of them.
|
||||
expected_runtime_classes=2
|
||||
|
||||
# We expect both runtime classes to have the same handler: kata-${KATA_HYPERVISOR}
|
||||
expected_handlers_re=( \
|
||||
"kata\s+kata-${KATA_HYPERVISOR}" \
|
||||
"kata-${KATA_HYPERVISOR}\s+kata-${KATA_HYPERVISOR}" \
|
||||
)
|
||||
}
|
||||
|
||||
@test "Test runtimeclasses are being properly created" {
|
||||
# We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy
|
||||
current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l)
|
||||
[[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]]
|
||||
|
||||
for handler_re in ${expected_handlers_re[@]}
|
||||
do
|
||||
[[ $(kubectl get runtimeclass | grep -E "${handler_re}") ]]
|
||||
done
|
||||
}
|
||||
|
||||
teardown() {
|
||||
kubectl get runtimeclasses
|
||||
}
|
||||
@@ -18,7 +18,6 @@ if [ -n "${K8S_TEST_UNION:-}" ]; then
|
||||
K8S_TEST_UNION=($K8S_TEST_UNION)
|
||||
else
|
||||
K8S_TEST_UNION=( \
|
||||
"kata-deploy-ensure-runtimec-classes-created.bats" \
|
||||
"k8s-attach-handlers.bats" \
|
||||
"k8s-caps.bats" \
|
||||
"k8s-configmap.bats" \
|
||||
|
||||
Reference in New Issue
Block a user