kata-deploy: scripts: Allow deploying nydus-snapshotter

Let's introduce a new EXPERIMENTAL_SETUP_SNAPSHOTTER environemnt
variable that, when set, allows kata-deploy to put the nydus snapshotter
in the correct place, and configure containerd accordingly.

Mind, this is a stop gap till the nydus-snapshotter helm chart is ready
to be used and behaving well enough to become a weak dependency of our
helm chart.  When that happens this code can be deleted entirely.

Users can have nydus-snapshotter deployed and configured for the
guest-pull use case by simply passing:
```
EXPERIMENTAL_SETUP_SNAPSHOTTER="nydus"
```

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>

tools/packaging/kata-deploy/Dockerfile

@@ -4,11 +4,37 @@
 
 ARG BASE_IMAGE_NAME=alpine
 ARG BASE_IMAGE_TAG=3.22
-FROM $BASE_IMAGE_NAME:$BASE_IMAGE_TAG
+FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} AS base
+
+#### Nydus snapshotter & nydus image
+
+FROM golang:1.24-alpine AS nydus-binary-downloader
+
+# Keep the version here aligned with "ndyus-snapshotter.version"
+# in versions.yaml
+ARG NYDUS_SNAPSHOTTER_VERSION=v0.15.2
+ARG NYDUS_SNAPSHOTTER_REPO=https://github.com/containerd/nydus-snapshotter
+
+RUN \
+	mkdir -p /opt/nydus-snapshotter && \
+	ARCH=$(uname -m) && \
+	if [[ "${ARCH}" == "x86_64" ]]; then ARCH=amd64 ; fi && \
+	if [[ "${ARCH}" == "aarch64" ]]; then ARCH=arm64; fi && \
+	apk add --no-cache curl && \
+	curl -fOL --progress-bar ${NYDUS_SNAPSHOTTER_REPO}/releases/download/${NYDUS_SNAPSHOTTER_VERSION}/nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz && \
+	tar xvzpf nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz -C /opt/nydus-snapshotter && \
+	rm nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz
+
+
+#### kata-deploy main image
+
+# kata-deploy args
+FROM base
+
 ARG KATA_ARTIFACTS=./kata-static.tar.zst
 ARG DESTINATION=/opt/kata-artifacts
 
-COPY ${KATA_ARTIFACTS} ${WORKDIR}
+COPY ${KATA_ARTIFACTS} /
 
 # I understand that in order to be on the safer side, it'd
 # be good to have the alpine packages pointing to a very
@@ -37,4 +63,7 @@ RUN \
 	pip install --no-cache-dir yq==3.2.3 --break-system-packages
 
 COPY scripts ${DESTINATION}/scripts
+COPY nydus-snapshotter ${DESTINATION}/nydus-snapshotter
+COPY --from=nydus-binary-downloader /opt/nydus-snapshotter/bin/containerd-nydus-grpc ${DESTINATION}/nydus-snapshotter/
+COPY --from=nydus-binary-downloader /opt/nydus-snapshotter/bin/nydus-overlayfs ${DESTINATION}/nydus-snapshotter/
 COPY runtimeclasses ${DESTINATION}/runtimeclasses

tools/packaging/kata-deploy/nydus-snapshotter/config-guest-pulling.toml

@@ -0,0 +1,15 @@
+version = 1
+
+# Snapshotter's own home directory where it stores and creates necessary resources
+root = "/var/lib/containerd-nydus"
+
+# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
+address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+
+[daemon]
+# Enable proxy mode
+fs_driver = "proxy"
+
+[snapshot]
+# Insert Kata volume information to `Mount.Options`
+enable_kata_volume = true

tools/packaging/kata-deploy/nydus-snapshotter/nydus-snapshotter.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Nydus snapshotter
+After=network.target local-fs.target
+Before=containerd.service
+
+[Service]
+ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus-snapshotter/config-guest-pulling.toml --log-to-stdout
+
+[Install]
+RequiredBy=containerd.service

tools/packaging/kata-deploy/scripts/kata-deploy.sh

@@ -62,6 +62,8 @@ AGENT_NO_PROXY="${AGENT_NO_PROXY:-}"
 PULL_TYPE_MAPPING="${PULL_TYPE_MAPPING:-}"
 IFS=',' read -a pull_types <<< "$PULL_TYPE_MAPPING"
 
+EXPERIMENTAL_SETUP_SNAPSHOTTER="${EXPERIMENTAL_SETUP_SNAPSHOTTER:-}"
+
 INSTALLATION_PREFIX="${INSTALLATION_PREFIX:-}"
 default_dest_dir="/opt/kata"
 dest_dir="${default_dest_dir}"
@@ -497,26 +499,33 @@ function wait_till_node_is_ready() {
 	done
 }
 
+function restart_runtime() {
+	local runtime="${1}"
+
+	if [ "${runtime}" == "k0s-worker" ] || [ "${runtime}" == "k0s-controller" ]; then
+		# do nothing, k0s will automatically load the config on the fly
+		:
+	elif [ "${runtime}" == "microk8s" ]; then
+		host_systemctl restart snap.microk8s.daemon-containerd.service
+	else
+		host_systemctl daemon-reload
+		host_systemctl restart "${runtime}"
+	fi
+
+	wait_till_node_is_ready
+}
+
 function configure_cri_runtime() {
-	case $1 in
+	local runtime="${1}"
+
+	case "${runtime}" in
 	crio)
 		configure_crio
 		;;
 	containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker | microk8s)
-		configure_containerd "$1"
+		configure_containerd "${runtime}"
 		;;
 	esac
-	if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
-		# do nothing, k0s will automatically load the config on the fly
-		:
-	elif [ "$1" == "microk8s" ]; then
-		host_systemctl restart snap.microk8s.daemon-containerd.service
-	else
-		host_systemctl daemon-reload
-		host_systemctl restart "$1"
-	fi
-
-	wait_till_node_is_ready
 }
 
 function configure_crio_runtime() {
@@ -799,6 +808,90 @@ function snapshotter_handler_mapping_validation_check() {
 	done
 }
 
+function configure_nydus_snapshotter() {
+	info "Configuring nydus-snapshotter"
+
+	configuration_file="${1}"
+	pluginid="${2}"
+
+	tomlq -i -t $(printf '.plugins.%s.disable_snapshot_annotations=false' ${pluginid}) ${configuration_file}
+
+	tomlq -i -t $(printf '.proxy_plugins.nydus.type="snapshot"') ${configuration_file}
+	tomlq -i -t $(printf '.proxy_plugins.nydus.address="/run/containerd-nydus/containerd-nydus-grpc.sock"') ${configuration_file}
+}
+
+function configure_snapshotter() {
+	snapshotter="${1}"
+
+	local runtime="$(get_container_runtime)"
+	local pluginid="\"io.containerd.grpc.v1.cri\".containerd" # version = 2
+	local configuration_file="${containerd_conf_file}"
+
+	# Properly set the configuration file in case drop-in files are supported
+	if [[ ${use_containerd_drop_in_conf_file} == "true" ]]; then
+		configuration_file="/host${containerd_drop_in_conf_file}"
+	fi
+
+	local containerd_root_conf_file="${containerd_conf_file}"
+	if [[ "${runtime}" =~ ^(k0s-worker|k0s-controller)$ ]]; then
+		containerd_root_conf_file="/etc/containerd/containerd.toml"
+	fi
+
+	if grep -q "version = 3\>" ${containerd_root_conf_file}; then
+		pluginid=\"io.containerd.cri.v1.images\"
+	fi
+
+	case "${snapshotter}" in
+		nydus)
+			configure_nydus_snapshotter "${configuration_file}" "${pluginid}"
+			host_systemctl restart nydus-snapshotter
+			;;
+	esac
+}
+
+function install_nydus_snapshotter() {
+	info "Deploying nydus-snapshotter"
+
+	install -D -m 775 /opt/kata-artifacts/nydus-snapshotter/containerd-nydus-grpc /host/usr/local/bin/containerd-nydus-grpc
+	install -D -m 775 /opt/kata-artifacts/nydus-snapshotter/nydus-overlayfs /host/usr/local/bin/nydus-overlayfs
+
+	mkdir -p /host/etc/nydus-snapshotter/
+	install -D -m 644 /opt/kata-artifacts/nydus-snapshotter/config-guest-pulling.toml /host/etc/nydus-snapshotter/config-guest-pulling.toml
+	install -D -m 644 /opt/kata-artifacts/nydus-snapshotter/nydus-snapshotter.service /host/etc/systemd/system/nydus-snapshotter.service
+
+	host_systemctl daemon-reload
+	host_systemctl enable nydus-snapshotter.service
+}
+
+function uninstall_nydus_snapshotter() {
+	info "Removing deployed nydus-snapshotter"
+	host_systemctl disable --now nydus-snapshotter.service
+
+	rm -f /host/etc/systemd/system/nydus-snapshotter.service
+	rm -f /host/etc/nydus-snapshotter/config-guest-pulling.toml
+
+	rm -f /host/usr/local/bin/nydus-overlayfs
+	rm -f /host/usr/local/bin/containerd-nydus-grpc
+
+	host_systemctl daemon-reload
+}
+
+function install_snapshotter() {
+	snapshotter="${1}"
+
+	case "${snapshotter}" in
+		nydus) install_nydus_snapshotter ;;
+	esac
+}
+
+function uninstall_snapshotter() {
+	snapshotter="${1}"
+
+	case "${snapshotter}" in
+		nydus) uninstall_nydus_snapshotter ;;
+	esac
+}
+
 function main() {
 	action=${1:-}
 	if [ -z "$action" ]; then
@@ -824,6 +917,7 @@ function main() {
 	echo "* INSTALLATION_PREFIX: ${INSTALLATION_PREFIX}"
 	echo "* MULTI_INSTALL_SUFFIX: ${MULTI_INSTALL_SUFFIX}"
 	echo "* HELM_POST_DELETE_HOOK: ${HELM_POST_DELETE_HOOK}"
+	echo "* EXPERIMENTAL_SETUP_SNAPSHOTTER: ${EXPERIMENTAL_SETUP_SNAPSHOTTER}"
 
 	# script requires that user is root
 	euid=$(id -u)
@@ -853,7 +947,6 @@ function main() {
 		containerd_conf_file_backup="${containerd_conf_tmpl_file}.bak"
 	fi
 
-
 	# only install / remove / update if we are dealing with CRIO or containerd
 	if [[ "$runtime" =~ ^(crio|containerd|k3s|k3s-agent|rke2-agent|rke2-server|k0s-worker|k0s-controller|microk8s)$ ]]; then
 		if [ "$runtime" != "crio" ]; then
@@ -873,6 +966,23 @@ function main() {
 
 		case "$action" in
 		install)
+			# Let's fail early on this, so we don't need to do a rollback
+			# in case we reach this situation.
+			if [[ -n "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" ]]; then
+				if [[ "${runtime}" == "cri-o" ]]; then
+					warn "EXPERIMENTAL_SETUP_SNAPSHOTTER is being ignored!"
+					warn "Snapshotter is a containerd specific option."
+				else
+					case "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" in
+						nydus)
+							;;
+						*)
+							die "${EXPERIMENTAL_SETUP_SNAPSHOTTER} is not a supported snapshotter by kata-deploy"
+							;;
+					esac
+				fi
+			fi
+
 			if [[ "$runtime" =~ ^(k3s|k3s-agent|rke2-agent|rke2-server)$ ]]; then
 			       if [ ! -f "$containerd_conf_tmpl_file" ] && [ -f "$containerd_conf_file" ]; then
 				       cp "$containerd_conf_file" "$containerd_conf_tmpl_file"
@@ -897,6 +1007,12 @@ function main() {
 
 			install_artifacts
 			configure_cri_runtime "$runtime"
+			if [[ -n "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" ]]; then
+				install_snapshotter "${EXPERIMENTAL_SETUP_SNAPSHOTTER}"
+				configure_snapshotter "${EXPERIMENTAL_SETUP_SNAPSHOTTER}"
+			fi
+
+			restart_runtime "${runtime}"
 			kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=true
 			;;
 		cleanup)
@@ -918,6 +1034,13 @@ function main() {
 				fi
 			fi
 
+			if [[ -n "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" ]]; then
+				# Here we don't need to do any cleanup on the config, as kata-deploy
+				# will revert the configuration to the state it was before the deployment,
+				# which is also before the snapshotter configuration. :-)
+				uninstall_snapshotter "${EXPERIMENTAL_SETUP_SNAPSHOTTER}"
+			fi
+
 			cleanup_cri_runtime "$runtime"
 			if [ "${HELM_POST_DELETE_HOOK}" == "false" ]; then
 				# If we still have any other installation here, it means we'll break them

versions.yaml

@@ -344,6 +344,8 @@ externals:
     url: "https://github.com/dragonflyoss/image-service"
     version: "v2.2.3"
 
+  # Keep the version here aligned with the NYDUS_SNAPSHOTTER_VERSION
+  # on tools/packaging/kata-deploy/Dockerfile
   nydus-snapshotter:
     description: "Snapshotter for Nydus image acceleration service"
     url: "https://github.com/containerd/nydus-snapshotter"