From 9e718b4e2345892132e4c0fadc0cbcf7534f66ae Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Fri, 15 Dec 2023 14:04:30 +0000 Subject: [PATCH 1/2] gha: kata-deploy: Add containerd status check After kata-deploy has installed, check that the worker nodes are still in Ready state and don't have a containerd://Unknown container runtime versions, identicating that container isn't working to ensure that we didn't corrupt the containerd config during kata-deploy's edits Fixes: #8678 Signed-off-by: stevenhorsman --- tests/functional/kata-deploy/kata-deploy.bats | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats index d4f957d054..6b50e87c21 100644 --- a/tests/functional/kata-deploy/kata-deploy.bats +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -36,17 +36,17 @@ setup() { yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true" # Let the `kata-deploy` create the default `kata` runtime class yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true" - + if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS" yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}" fi - + echo "::group::Final kata-deploy.yaml that is used in the test" cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image" echo "::endgroup::" - + kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" if [ "${KUBERNETES}" = "k0s" ]; then kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s" @@ -64,7 +64,7 @@ setup() { sleep 30s } -@test "Test runtimeclasses are being properly created" { +@test "Test runtimeclasses are being properly created and container runtime not broken" { # We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l) [[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]] @@ -73,6 +73,20 @@ setup() { do kubectl get runtimeclass | grep -E "${handler_re}" done + + # Ensure that kata-deploy didn't corrupt containerd config, by trying to get the container runtime and node status + echo "::group::kubectl node debug" + kubectl get node -o wide + kubectl describe nodes + echo "::endgroup::" + + # Wait to see if the nodes get back into Ready state - if not then containerd might be having issues + kubectl wait nodes --timeout=60s --all --for condition=Ready=True + + # Check that the container runtime verison doesn't have unknown, which happens when containerd can't start properly + container_runtime_version=$(kubectl get nodes --no-headers -o custom-columns=CONTAINER_RUNTIME:.status.nodeInfo.containerRuntimeVersion) + [[ ${container_runtime_version} != *"containerd://Unknown"* ]] + } teardown() { @@ -94,7 +108,7 @@ teardown() { kubectl delete ${deploy_spec} kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - + # Let the `kata-deploy` script take care of the runtime class creation / removal yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true" # Create the runtime class only for the shim that's being tested @@ -103,14 +117,14 @@ teardown() { yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}" # Let the `kata-deploy` create the default `kata` runtime class yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true" - + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image" kubectl apply ${cleanup_spec} sleep 30s - + kubectl delete ${cleanup_spec} kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" } From ee5fa08a270467bf78a6382ac6376213b7f44ed8 Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Mon, 18 Dec 2023 17:49:55 +0000 Subject: [PATCH 2/2] Revert "kata-deploy: Use tomlq to configure containerd" This reverts commit dd9f5b07b9243ca152c3d7fe9df472e2f31eb103. Signed-off-by: stevenhorsman --- .../kata-deploy/scripts/kata-deploy.sh | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 7ff1b54d8a..208b3c96ce 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -94,8 +94,8 @@ function get_container_runtime() { else echo "k3s" fi - # Note: we assumed you used a conventional k0s setup and k0s will generate a systemd entry k0scontroller.service and k0sworker.service respectively - # and it is impossible to run this script without a kubelet, so this k0s controller must also have worker mode enabled + # Note: we assumed you used a conventional k0s setup and k0s will generate a systemd entry k0scontroller.service and k0sworker.service respectively + # and it is impossible to run this script without a kubelet, so this k0s controller must also have worker mode enabled elif host_systemctl is-active --quiet k0scontroller; then echo "k0s-controller" elif host_systemctl is-active --quiet k0sworker; then @@ -344,24 +344,48 @@ function configure_containerd_runtime() { local runtime="kata-${shim}" local configuration="configuration-${shim}" local pluginid=cri - + # if we are running k0s auto containerd.toml generation, the base template is by default version 2 # we can safely assume to reference the newer version of cri if grep -q "version = 2\>" $containerd_conf_file || [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then pluginid=\"io.containerd.grpc.v1.cri\" fi - local runtime_table=".plugins.${pluginid}.containerd.runtimes.\"${runtime}\"" - local runtime_options_table="${runtime_table}.options" - local runtime_type=\"io.containerd."${runtime}".v2\" - local runtime_config_path=\"$(get_kata_containers_config_path "${shim}")/${configuration}.toml\" - - tomlq -i -t $(printf '%s.runtime_type=%s' ${runtime_table} ${runtime_type}) ${containerd_conf_file} - tomlq -i -t $(printf '%s.privileged_without_host_devices=true' ${runtime_table}) ${containerd_conf_file} - tomlq -i -t $(printf '%s.pod_annotations=["io.katacontainers.*"]' ${runtime_table}) ${containerd_conf_file} - tomlq -i -t $(printf '%s.ConfigPath=%s' ${runtime_options_table} ${runtime_config_path}) ${containerd_conf_file} - + local runtime_table="plugins.${pluginid}.containerd.runtimes.$runtime" + local runtime_type="io.containerd.$runtime.v2" + local options_table="$runtime_table.options" + local config_path="$(get_kata_containers_config_path "${shim}")/$configuration.toml" + if grep -q "\[$runtime_table\]" $containerd_conf_file; then + echo "Configuration exists for $runtime_table, overwriting" + sed -i "/\[$runtime_table\]/,+1s#runtime_type.*#runtime_type = \"${runtime_type}\"#" $containerd_conf_file + else + cat <