mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-07-05 03:26:37 +00:00
Merge pull request #9721 from fidencio/topic/ci-kata-deploy-improvements-and-fixes
kata-deploy / kata-cleanup / ci: Fixes and improvements to kata-deploy / kata-cleanup and its usage in the CI
This commit is contained in:
Fabiano Fidêncio
GitHub
commit
3276bb52b6
@ -11,6 +11,8 @@ setup() {
|
||||
repo_root_dir="${BATS_TEST_DIRNAME}/../../../"
|
||||
ensure_yq
|
||||
|
||||
pushd "${repo_root_dir}"
|
||||
|
||||
# We expect 2 runtime classes because:
|
||||
# * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`.
|
||||
# * `kata-${KATA_HYPERVISOR}` is the other one
|
||||
@ -24,67 +26,77 @@ setup() {
|
||||
)
|
||||
|
||||
# Set the latest image, the one generated as part of the PR, to be used as part of the tests
|
||||
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
|
||||
# Enable debug for Kata Containers
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
'spec.template.spec.containers[0].env[1].value' \
|
||||
--tag '!!str' "true"
|
||||
# Create the runtime class only for the shim that's being tested
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
'spec.template.spec.containers[0].env[2].value' \
|
||||
"${KATA_HYPERVISOR}"
|
||||
# Set the tested hypervisor as the default `kata` shim
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
'spec.template.spec.containers[0].env[3].value' \
|
||||
"${KATA_HYPERVISOR}"
|
||||
# Let the `kata-deploy` script take care of the runtime class creation / removal
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
'spec.template.spec.containers[0].env[4].value' \
|
||||
--tag '!!str' "true"
|
||||
# Let the `kata-deploy` create the default `kata` runtime class
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
'spec.template.spec.containers[0].env[5].value' \
|
||||
--tag '!!str' "true"
|
||||
|
||||
if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
'spec.template.spec.containers[0].env[+].name' \
|
||||
"HOST_OS"
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
'spec.template.spec.containers[0].env[-1].value' \
|
||||
"${KATA_HOST_OS}"
|
||||
fi
|
||||
|
||||
echo "::group::Final kata-deploy.yaml that is used in the test"
|
||||
cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
|
||||
cat "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
|
||||
echo "::endgroup::"
|
||||
|
||||
kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
|
||||
kubectl apply -f "tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
|
||||
if [ "${KUBERNETES}" = "k0s" ]; then
|
||||
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s"
|
||||
kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/k0s"
|
||||
elif [ "${KUBERNETES}" = "k3s" ]; then
|
||||
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"
|
||||
kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/k3s"
|
||||
elif [ "${KUBERNETES}" = "rke2" ]; then
|
||||
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2"
|
||||
kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/rke2"
|
||||
else
|
||||
kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
kubectl apply -f "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
fi
|
||||
|
||||
local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
|
||||
waitForProcess 600 10 "$cmd"
|
||||
waitForProcess 240 10 "$cmd"
|
||||
|
||||
# Give some time for the pod to finish what's doing and have the
|
||||
# runtimeclasses properly created
|
||||
sleep 30s
|
||||
|
||||
echo "::group::kata-deploy logs"
|
||||
kubectl -n kube-system logs --tail=100 -l name=kata-deploy
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::Runtime classes"
|
||||
kubectl get runtimeclass
|
||||
echo "::endgroup::"
|
||||
|
||||
popd
|
||||
}
|
||||
|
||||
@test "Test runtimeclasses are being properly created and container runtime not broken" {
|
||||
@ -109,24 +121,23 @@ setup() {
|
||||
# Check that the container runtime verison doesn't have unknown, which happens when containerd can't start properly
|
||||
container_runtime_version=$(kubectl get nodes --no-headers -o custom-columns=CONTAINER_RUNTIME:.status.nodeInfo.containerRuntimeVersion)
|
||||
[[ ${container_runtime_version} != *"containerd://Unknown"* ]]
|
||||
|
||||
}
|
||||
|
||||
teardown() {
|
||||
kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation"
|
||||
pushd "${repo_root_dir}"
|
||||
|
||||
if [ "${KUBERNETES}" = "k0s" ]; then
|
||||
deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s\""
|
||||
cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\""
|
||||
deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/k0s\""
|
||||
cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\""
|
||||
elif [ "${KUBERNETES}" = "k3s" ]; then
|
||||
deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s\""
|
||||
cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\""
|
||||
deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/k3s\""
|
||||
cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\""
|
||||
elif [ "${KUBERNETES}" = "rke2" ]; then
|
||||
deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2\""
|
||||
cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\""
|
||||
deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/rke2\""
|
||||
cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\""
|
||||
else
|
||||
deploy_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\""
|
||||
cleanup_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\""
|
||||
deploy_spec="-f \"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\""
|
||||
cleanup_spec="-f \"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\""
|
||||
fi
|
||||
|
||||
kubectl delete ${deploy_spec}
|
||||
@ -134,32 +145,34 @@ teardown() {
|
||||
|
||||
# Let the `kata-deploy` script take care of the runtime class creation / removal
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
|
||||
'spec.template.spec.containers[0].env[4].value' \
|
||||
--tag '!!str' "true"
|
||||
# Create the runtime class only for the shim that's being tested
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
|
||||
'spec.template.spec.containers[0].env[2].value' \
|
||||
"${KATA_HYPERVISOR}"
|
||||
# Set the tested hypervisor as the default `kata` shim
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
|
||||
'spec.template.spec.containers[0].env[3].value' \
|
||||
"${KATA_HYPERVISOR}"
|
||||
# Let the `kata-deploy` create the default `kata` runtime class
|
||||
yq write -i \
|
||||
"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
|
||||
'spec.template.spec.containers[0].env[5].value' \
|
||||
--tag '!!str' "true"
|
||||
|
||||
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"
|
||||
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
cat "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"
|
||||
|
||||
kubectl apply ${cleanup_spec}
|
||||
sleep 30s
|
||||
|
||||
kubectl delete ${cleanup_spec}
|
||||
kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
|
||||
kubectl delete -f "tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
|
||||
|
||||
popd
|
||||
}
|
||||
|
||||
@ -21,5 +21,5 @@ fi
|
||||
info "Run tests"
|
||||
for KATA_DEPLOY_TEST_ENTRY in ${KATA_DEPLOY_TEST_UNION[@]}
|
||||
do
|
||||
bats "${KATA_DEPLOY_TEST_ENTRY}"
|
||||
bats --show-output-of-passing-tests "${KATA_DEPLOY_TEST_ENTRY}"
|
||||
done
|
||||
|
||||
@ -151,7 +151,9 @@ function deploy_kata() {
|
||||
[ "$platform" = "kcli" ] && \
|
||||
export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
|
||||
|
||||
cleanup_kata_deploy || true
|
||||
if [ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]; then
|
||||
cleanup_kata_deploy || true
|
||||
fi
|
||||
|
||||
set_default_cluster_namespace
|
||||
|
||||
@ -235,11 +237,12 @@ function deploy_kata() {
|
||||
echo "::endgroup::"
|
||||
|
||||
kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
|
||||
if [ "${KUBERNETES}" = "k3s" ]; then
|
||||
kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"
|
||||
else
|
||||
kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
fi
|
||||
case "${KUBERNETES}" in
|
||||
k0s) kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s" ;;
|
||||
k3s) kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" ;;
|
||||
rke2) kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2" ;;
|
||||
*) kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
esac
|
||||
|
||||
local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
|
||||
waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "$cmd"
|
||||
@ -388,13 +391,24 @@ function collect_artifacts() {
|
||||
function cleanup_kata_deploy() {
|
||||
ensure_yq
|
||||
|
||||
if [ "${KUBERNETES}" = "k3s" ]; then
|
||||
deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s""
|
||||
cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s""
|
||||
else
|
||||
deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml""
|
||||
cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml""
|
||||
fi
|
||||
case "${KUBERNETES}" in
|
||||
k0s)
|
||||
deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s""
|
||||
cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k0s""
|
||||
;;
|
||||
k3s)
|
||||
deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s""
|
||||
cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s""
|
||||
;;
|
||||
rke2)
|
||||
deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2""
|
||||
cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/rke2""
|
||||
;;
|
||||
*)
|
||||
deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml""
|
||||
cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml""
|
||||
;;
|
||||
esac
|
||||
|
||||
# shellcheck disable=2086
|
||||
kubectl delete ${deploy_spec}
|
||||
@ -487,7 +501,9 @@ function deploy_nydus_snapshotter() {
|
||||
git clone -b "${nydus_snapshotter_version}" "${nydus_snapshotter_url}" "${nydus_snapshotter_install_dir}"
|
||||
|
||||
pushd "$nydus_snapshotter_install_dir"
|
||||
cleanup_nydus_snapshotter || true
|
||||
if [ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]; then
|
||||
cleanup_nydus_snapshotter || true
|
||||
fi
|
||||
if [ "${PULL_TYPE}" == "guest-pull" ]; then
|
||||
# Enable guest pull feature in nydus snapshotter
|
||||
yq write -i \
|
||||
|
||||
@ -86,12 +86,12 @@ else
|
||||
normal)
|
||||
K8S_TEST_UNION=(${K8S_TEST_NORMAL_HOST_UNION[@]})
|
||||
;;
|
||||
baremetal)
|
||||
all|baremetal)
|
||||
K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]} ${K8S_TEST_NORMAL_HOST_UNION[@]})
|
||||
|
||||
;;
|
||||
*)
|
||||
echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | baremetal"
|
||||
echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | all | baremetal"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ../../base
|
||||
patches:
|
||||
- path: mount_k0s_conf.yaml
|
||||
@ -0,0 +1,17 @@
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: kubelet-kata-cleanup
|
||||
namespace: kube-system
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: kube-kata-cleanup
|
||||
volumeMounts:
|
||||
- name: containerd-conf
|
||||
mountPath: /etc/containerd/
|
||||
volumes:
|
||||
- name: containerd-conf
|
||||
hostPath:
|
||||
path: /etc/k0s/containerd.d/
|
||||
@ -459,12 +459,13 @@ function configure_containerd_runtime() {
|
||||
local runtime="kata-${shim}"
|
||||
local configuration="configuration-${shim}"
|
||||
local pluginid=cri
|
||||
|
||||
|
||||
# if we are running k0s auto containerd.toml generation, the base template is by default version 2
|
||||
# we can safely assume to reference the newer version of cri
|
||||
if grep -q "version = 2\>" $containerd_conf_file || [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
|
||||
pluginid=\"io.containerd.grpc.v1.cri\"
|
||||
fi
|
||||
|
||||
local runtime_table=".plugins.${pluginid}.containerd.runtimes.\"${runtime}\""
|
||||
local runtime_options_table="${runtime_table}.options"
|
||||
local runtime_type=\"io.containerd."${runtime}".v2\"
|
||||
@ -640,6 +641,7 @@ function main() {
|
||||
# This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets.
|
||||
# k0s will automatically pick up these files and adds these in containerd configuration imports list.
|
||||
containerd_conf_file="/etc/containerd/kata-containers.toml"
|
||||
touch "$containerd_conf_file"
|
||||
else
|
||||
# runtime == containerd
|
||||
if [ ! -f "$containerd_conf_file" ] && [ -d $(dirname "$containerd_conf_file") ] && \
|
||||
|
||||
Loading…
Reference in New Issue
Block a user