From 6cc6ca5a7fe1c6f074140295293e641a2dec7735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 13 Dec 2023 14:13:34 +0100 Subject: [PATCH] kata-deploy: Allow setting up snapshotters per runtime handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since containerd 1.7.0 we can easily set a specific snapshotter to be used with a runtime handler, and we should take advantage of this, mostly as it'll help setting up any runtime using devmapper or nydus snapshotters. This implementation here has a few caveats: * The format expected for the SNAPSHOTTER_HANDLER_MAPPING is: `shim:snapshotter,shim:snapshotter,...` * It only works with containerd 1.7 or newer * We **never** change the default containerd snapshotter * We don't do any check on our side to verify whether the snapshotter required is properly deployed * Users will have to add an annotation to their pods, in order to use the snapshotter set up per runtime handler * Example: ``` metadata: ... annotations: io.containerd.cri.runtime-handler: kata-fc ``` Fixes: #8615 Signed-off-by: Fabiano FidĂȘncio --- .../kata-deploy/base/kata-deploy.yaml | 2 + .../kata-deploy/scripts/kata-deploy.sh | 65 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index e746c6f32b..0360847a59 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -41,6 +41,8 @@ spec: value: "false" - name: ALLOWED_HYPERVISOR_ANNOTATIONS value: "" + - name: SNAPSHOTTER_HANDLER_MAPPING + value: "" securityContext: privileged: true volumeMounts: diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 7ff1b54d8a..4d551e6528 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -25,6 +25,10 @@ for allowed_hypervisor_annotation in "${non_formatted_allowed_hypervisor_annotat done allowed_hypervisor_annotations=$(echo $allowed_hypervisor_annotations | sed 's/,$//') +SNAPSHOTTER_HANDLER_MAPPING="${SNAPSHOTTER_HANDLER_MAPPING:-}" +IFS=',' read -a snapshotters <<< "$SNAPSHOTTER_HANDLER_MAPPING" +snapshotters_delimiter=':' + # If we fail for any reason a message will be displayed die() { msg="$*" @@ -363,6 +367,20 @@ function configure_containerd_runtime() { if [ "${DEBUG}" == "true" ]; then tomlq -i -t '.debug.level = "debug"' ${containerd_conf_file} fi + + if [ -n "${SNAPSHOTTER_HANDLER_MAPPING}" ]; then + for m in ${snapshotters[@]}; do + key="${m%$snapshotters_delimiter*}" + + if [ "${key}" != "${shim}" ]; then + continue + fi + + value="${m#*$snapshotters_delimiter}" + tomlq -i -t $(printf '%s.snapshotter=%s' ${shim} ${value}) ${containerd_conf_file} + break + done + fi } function configure_containerd() { @@ -431,6 +449,49 @@ function reset_runtime() { wait_till_node_is_ready } +function containerd_snapshotter_version_check() { + local container_runtime_version=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}') + local containerd_prefix="containerd://" + local containerd_version_to_avoid="1.6" + local containerd_version=${container_runtime_version#$containerd_prefix} + + if grep -q ^$containerd_version_to_avoid <<< $containerd_version; then + if [ -n "${SNAPSHOTTER_HANDLER_MAPPING}" ]; then + die "kata-deploy only supports snapshotter configuration with containerd 1.7 or newer" + fi + fi +} + +function snapshotter_handler_mapping_validation_check() { + echo "Validating the snapshotter-handler mapping: \"${SNAPSHOTTER_HANDLER_MAPPING}\"" + if [ -z "${SNAPSHOTTER_HANDLER_MAPPING}" ]; then + echo "No snapshotter has been requested, using the default value from containerd" + return + fi + + for m in ${snapshotters[@]}; do + shim="${m%$snapshotters_delimiter*}" + snapshotter="${m#*$snapshotters_delimiter}" + + if [ -z "$shim"]; then + die "The snapshotter must follow the \"shim:snapshotter,shim:snapshotter,...\" format, but at least one shim is empty" + fi + + if [ -z "$snapshotter"]; then + die "The snapshotter must follow the \"shim:snapshotter,shim:snapshotter,...\" format, but at least one snapshotter is empty" + fi + + if ! grep -q " $shim " <<< " $shims "; then + die "\"$shim\" is not part of \"$SHIMS\"" + fi + + matches=$(grep -o "$shim$snapshotters_delimiter" <<< "${SNAPSHOTTER_HANDLER_MAPPING}" | wc -l) + if [ $matches -ne 1 ]; then + die "One, and only one, entry per shim is required" + fi + done +} + function main() { echo "Environment variables passed to this script" echo "* NODE_NAME: ${NODE_NAME}" @@ -481,6 +542,10 @@ function main() { # only install / remove / update if we are dealing with CRIO or containerd if [[ "$runtime" =~ ^(crio|containerd|k3s|k3s-agent|rke2-agent|rke2-server|k0s-worker|k0s-controller)$ ]]; then + if [ "$runtime" != "crio" ]; then + containerd_snapshotter_version_check + snapshotter_handler_mapping_validation_check + fi case "$action" in install)