github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-07-05 19:47:53 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #9721 from fidencio/topic/ci-kata-deploy-improvements-and-fixes

Browse Source 
kata-deploy / kata-cleanup / ci: Fixes and improvements to kata-deploy / kata-cleanup and its usage in the CI
This commit is contained in:
Fabiano Fidêncio 2024-05-27 12:29:40 +02:00 committed by GitHub
commit 3276bb52b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 107 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
tests/functional/kata-deploy/kata-deploy.bats
View File

@ -11,6 +11,8 @@ setup() {
repo_root_dir="${BATS_TEST_DIRNAME}/../../../"
ensure_yq
pushd "${repo_root_dir}"
# We expect 2 runtime classes because:
# * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`.
# * `kata-${KATA_HYPERVISOR}` is the other one
@ -24,67 +26,77 @@ setup() {
)
# Set the latest image, the one generated as part of the PR, to be used as part of the tests
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
# Enable debug for Kata Containers
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[1].value' \
--tag '!!str' "true"
# Create the runtime class only for the shim that's being tested
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[2].value' \
"${KATA_HYPERVISOR}"
# Set the tested hypervisor as the default `kata` shim
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[3].value' \
"${KATA_HYPERVISOR}"
# Let the `kata-deploy` script take care of the runtime class creation / removal
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[4].value' \
--tag '!!str' "true"
# Let the `kata-deploy` create the default `kata` runtime class
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[5].value' \
--tag '!!str' "true"
if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[+].name' \
"HOST_OS"
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[-1].value' \
"${KATA_HOST_OS}"
fi
echo "::group::Final kata-deploy.yaml that is used in the test"
cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
cat "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
echo "::endgroup::"
kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
kubectl apply -f "tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
if [ "${KUBERNETES}" = "k0s" ]; then
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s"
kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/k0s"
elif [ "${KUBERNETES}" = "k3s" ]; then
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"
kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/k3s"
elif [ "${KUBERNETES}" = "rke2" ]; then
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2"
kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/rke2"
else
kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
kubectl apply -f "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
fi
local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
waitForProcess 600 10 "$cmd"
waitForProcess 240 10 "$cmd"
# Give some time for the pod to finish what's doing and have the
# runtimeclasses properly created
sleep 30s
echo "::group::kata-deploy logs"
kubectl -n kube-system logs --tail=100 -l name=kata-deploy
echo "::endgroup::"
echo "::group::Runtime classes"
kubectl get runtimeclass
echo "::endgroup::"
popd
}
@test "Test runtimeclasses are being properly created and container runtime not broken" {
@ -109,24 +121,23 @@ setup() {
# Check that the container runtime verison doesn't have unknown, which happens when containerd can't start properly
container_runtime_version=$(kubectl get nodes --no-headers -o custom-columns=CONTAINER_RUNTIME:.status.nodeInfo.containerRuntimeVersion)
[[ ${container_runtime_version} != *"containerd://Unknown"* ]]
}
teardown() {
kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation"
pushd "${repo_root_dir}"
if [ "${KUBERNETES}" = "k0s" ]; then
deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s\""
cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\""
deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/k0s\""
cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\""
elif [ "${KUBERNETES}" = "k3s" ]; then
deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s\""
cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\""
deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/k3s\""
cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\""
elif [ "${KUBERNETES}" = "rke2" ]; then
deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2\""
cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\""
deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/rke2\""
cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\""
else
deploy_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\""
cleanup_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\""
deploy_spec="-f \"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\""
cleanup_spec="-f \"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\""
fi
kubectl delete ${deploy_spec}
@ -134,32 +145,34 @@ teardown() {
# Let the `kata-deploy` script take care of the runtime class creation / removal
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
'spec.template.spec.containers[0].env[4].value' \
--tag '!!str' "true"
# Create the runtime class only for the shim that's being tested
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
'spec.template.spec.containers[0].env[2].value' \
"${KATA_HYPERVISOR}"
# Set the tested hypervisor as the default `kata` shim
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
'spec.template.spec.containers[0].env[3].value' \
"${KATA_HYPERVISOR}"
# Let the `kata-deploy` create the default `kata` runtime class
yq write -i \
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[5].value' \
--tag '!!str' "true"
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
cat "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"
kubectl apply ${cleanup_spec}
sleep 30s
kubectl delete ${cleanup_spec}
kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
kubectl delete -f "tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
popd
}

2
tests/functional/kata-deploy/run-kata-deploy-tests.sh
View File

@ -21,5 +21,5 @@ fi
info "Run tests"
for KATA_DEPLOY_TEST_ENTRY in ${KATA_DEPLOY_TEST_UNION[@]}
do
bats "${KATA_DEPLOY_TEST_ENTRY}"
bats --show-output-of-passing-tests "${KATA_DEPLOY_TEST_ENTRY}"
done

32
tests/integration/kubernetes/gha-run.sh
View File

@ -151,7 +151,9 @@ function deploy_kata() {
[ "$platform" = "kcli" ] && \
export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
if [ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]; then
cleanup_kata_deploy || true
fi
set_default_cluster_namespace
@ -235,11 +237,12 @@ function deploy_kata() {
echo "::endgroup::"
kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
if [ "${KUBERNETES}" = "k3s" ]; then
kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"
else
kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
fi
case "${KUBERNETES}" in
k0s) kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s" ;;
k3s) kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" ;;
rke2) kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2" ;;
*) kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
esac
local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "$cmd"
@ -388,13 +391,24 @@ function collect_artifacts() {
function cleanup_kata_deploy() {
ensure_yq
if [ "${KUBERNETES}" = "k3s" ]; then
case "${KUBERNETES}" in
k0s)
deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s""
cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k0s""
;;
k3s)
deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s""
cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s""
else
;;
rke2)
deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2""
cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/rke2""
;;
*)
deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml""
cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml""
fi
;;
esac
# shellcheck disable=2086
kubectl delete ${deploy_spec}
@ -487,7 +501,9 @@ function deploy_nydus_snapshotter() {
git clone -b "${nydus_snapshotter_version}" "${nydus_snapshotter_url}" "${nydus_snapshotter_install_dir}"
pushd "$nydus_snapshotter_install_dir"
if [ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]; then
cleanup_nydus_snapshotter || true
fi
if [ "${PULL_TYPE}" == "guest-pull" ]; then
# Enable guest pull feature in nydus snapshotter
yq write -i \

4
tests/integration/kubernetes/run_kubernetes_tests.sh
View File

@ -86,12 +86,12 @@ else
normal)
K8S_TEST_UNION=(${K8S_TEST_NORMAL_HOST_UNION[@]})
;;
baremetal)
all|baremetal)
K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]} ${K8S_TEST_NORMAL_HOST_UNION[@]})
;;
*)
echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | baremetal"
echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | all | baremetal"
return 1
;;
esac

7
tools/packaging/kata-deploy/kata-cleanup/overlays/k0s/kustomization.yaml Normal file
View File

@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../base
patches:
- path: mount_k0s_conf.yaml

17
tools/packaging/kata-deploy/kata-cleanup/overlays/k0s/mount_k0s_conf.yaml Normal file
View File

@ -0,0 +1,17 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kubelet-kata-cleanup
namespace: kube-system
spec:
template:
spec:
containers:
- name: kube-kata-cleanup
volumeMounts:
- name: containerd-conf
mountPath: /etc/containerd/
volumes:
- name: containerd-conf
hostPath:
path: /etc/k0s/containerd.d/

2
tools/packaging/kata-deploy/scripts/kata-deploy.sh
View File

@ -465,6 +465,7 @@ function configure_containerd_runtime() {
if grep -q "version = 2\>" $containerd_conf_file || [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
pluginid=\"io.containerd.grpc.v1.cri\"
fi
local runtime_table=".plugins.${pluginid}.containerd.runtimes.\"${runtime}\""
local runtime_options_table="${runtime_table}.options"
local runtime_type=\"io.containerd."${runtime}".v2\"
@ -640,6 +641,7 @@ function main() {
# This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets.
# k0s will automatically pick up these files and adds these in containerd configuration imports list.
containerd_conf_file="/etc/containerd/kata-containers.toml"
touch "$containerd_conf_file"
else
# runtime == containerd
if [ ! -f "$containerd_conf_file" ] && [ -d $(dirname "$containerd_conf_file") ] && \