github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-09-12 12:29:11 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #8679 from stevenhorsman/kata-deploy-containerd-config-fix

Browse Source 
gha: kata-deploy: Revert containerd config break
This commit is contained in:
Greg Kurz
2023-12-20 12:58:56 +01:00
committed by GitHub
parent 7e5868a55f ee5fa08a27
commit ce094ecdc2
2 changed files with 60 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
tests/functional/kata-deploy/kata-deploy.bats
View File

@@ -36,17 +36,17 @@ setup() {
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
# Let the `kata-deploy` create the default `kata` runtime class
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true"
if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
fi
echo "::group::Final kata-deploy.yaml that is used in the test"
cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
echo "::endgroup::"
kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
if [ "${KUBERNETES}" = "k0s" ]; then
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s"
@@ -64,7 +64,7 @@ setup() {
sleep 30s
}
@test "Test runtimeclasses are being properly created" {
@test "Test runtimeclasses are being properly created and container runtime not broken" {
# We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy
current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l)
[[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]]
@@ -73,6 +73,20 @@ setup() {
do
kubectl get runtimeclass | grep -E "${handler_re}"
done
# Ensure that kata-deploy didn't corrupt containerd config, by trying to get the container runtime and node status
echo "::group::kubectl node debug"
kubectl get node -o wide
kubectl describe nodes
echo "::endgroup::"
# Wait to see if the nodes get back into Ready state - if not then containerd might be having issues
kubectl wait nodes --timeout=60s --all --for condition=Ready=True
# Check that the container runtime verison doesn't have unknown, which happens when containerd can't start properly
container_runtime_version=$(kubectl get nodes --no-headers -o custom-columns=CONTAINER_RUNTIME:.status.nodeInfo.containerRuntimeVersion)
[[ ${container_runtime_version} != *"containerd://Unknown"* ]]
}
teardown() {
@@ -94,7 +108,7 @@ teardown() {
kubectl delete ${deploy_spec}
kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
# Let the `kata-deploy` script take care of the runtime class creation / removal
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
# Create the runtime class only for the shim that's being tested
@@ -103,14 +117,14 @@ teardown() {
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}"
# Let the `kata-deploy` create the default `kata` runtime class
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true"
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"
kubectl apply ${cleanup_spec}
sleep 30s
kubectl delete ${cleanup_spec}
kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
}

54
tools/packaging/kata-deploy/scripts/kata-deploy.sh
View File

@@ -94,8 +94,8 @@ function get_container_runtime() {
else
echo "k3s"
fi
# Note: we assumed you used a conventional k0s setup and k0s will generate a systemd entry k0scontroller.service and k0sworker.service respectively
# and it is impossible to run this script without a kubelet, so this k0s controller must also have worker mode enabled
# Note: we assumed you used a conventional k0s setup and k0s will generate a systemd entry k0scontroller.service and k0sworker.service respectively
# and it is impossible to run this script without a kubelet, so this k0s controller must also have worker mode enabled
elif host_systemctl is-active --quiet k0scontroller; then
echo "k0s-controller"
elif host_systemctl is-active --quiet k0sworker; then
@@ -344,24 +344,48 @@ function configure_containerd_runtime() {
local runtime="kata-${shim}"
local configuration="configuration-${shim}"
local pluginid=cri
# if we are running k0s auto containerd.toml generation, the base template is by default version 2
# we can safely assume to reference the newer version of cri
if grep -q "version = 2\>" $containerd_conf_file || [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
pluginid=\"io.containerd.grpc.v1.cri\"
fi
local runtime_table=".plugins.${pluginid}.containerd.runtimes.\"${runtime}\""
local runtime_options_table="${runtime_table}.options"
local runtime_type=\"io.containerd."${runtime}".v2\"
local runtime_config_path=\"$(get_kata_containers_config_path "${shim}")/${configuration}.toml\"
tomlq -i -t $(printf '%s.runtime_type=%s' ${runtime_table} ${runtime_type}) ${containerd_conf_file}
tomlq -i -t $(printf '%s.privileged_without_host_devices=true' ${runtime_table}) ${containerd_conf_file}
tomlq -i -t $(printf '%s.pod_annotations=["io.katacontainers.*"]' ${runtime_table}) ${containerd_conf_file}
tomlq -i -t $(printf '%s.ConfigPath=%s' ${runtime_options_table} ${runtime_config_path}) ${containerd_conf_file}
local runtime_table="plugins.${pluginid}.containerd.runtimes.$runtime"
local runtime_type="io.containerd.$runtime.v2"
local options_table="$runtime_table.options"
local config_path="$(get_kata_containers_config_path "${shim}")/$configuration.toml"
if grep -q "\[$runtime_table\]" $containerd_conf_file; then
echo "Configuration exists for $runtime_table, overwriting"
sed -i "/\[$runtime_table\]/,+1s#runtime_type.*#runtime_type = \"${runtime_type}\"#" $containerd_conf_file
else
cat <<EOF | tee -a "$containerd_conf_file"
[$runtime_table]
runtime_type = "${runtime_type}"
privileged_without_host_devices = true
pod_annotations = ["io.katacontainers.*"]
EOF
fi
if grep -q "\[$options_table\]" $containerd_conf_file; then
echo "Configuration exists for $options_table, overwriting"
sed -i "/\[$options_table\]/,+1s#ConfigPath.*#ConfigPath = \"${config_path}\"#" $containerd_conf_file
else
cat <<EOF | tee -a "$containerd_conf_file"
[$options_table]
ConfigPath = "${config_path}"
EOF
fi
if [ "${DEBUG}" == "true" ]; then
tomlq -i -t '.debug.level = "debug"' ${containerd_conf_file}
if grep -q "\[debug\]" $containerd_conf_file; then
sed -i 's/level.*/level = \"debug\"/' $containerd_conf_file
else
cat <<EOF | tee -a "$containerd_conf_file"
[debug]
level = "debug"
EOF
fi
fi
}
@@ -461,7 +485,7 @@ function main() {
containerd_conf_file="${containerd_conf_tmpl_file}"
containerd_conf_file_backup="${containerd_conf_file}.bak"
elif [ "$runtime" == "k0s-worker" ] || [ "$runtime" == "k0s-controller" ]; then
# From 1.27.1 onwards k0s enables dynamic configuration on containerd CRI runtimes.
# From 1.27.1 onwards k0s enables dynamic configuration on containerd CRI runtimes.
# This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets.
# k0s will automatically pick up these files and adds these in containerd configuration imports list.
containerd_conf_file="/etc/containerd/kata-containers.toml"