diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile index eaf94005ad..ab4b459384 100644 --- a/tools/packaging/kata-deploy/Dockerfile +++ b/tools/packaging/kata-deploy/Dockerfile @@ -4,11 +4,37 @@ ARG BASE_IMAGE_NAME=alpine ARG BASE_IMAGE_TAG=3.22 -FROM $BASE_IMAGE_NAME:$BASE_IMAGE_TAG +FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} AS base + +#### Nydus snapshotter & nydus image + +FROM golang:1.24-alpine AS nydus-binary-downloader + +# Keep the version here aligned with "ndyus-snapshotter.version" +# in versions.yaml +ARG NYDUS_SNAPSHOTTER_VERSION=v0.15.2 +ARG NYDUS_SNAPSHOTTER_REPO=https://github.com/containerd/nydus-snapshotter + +RUN \ + mkdir -p /opt/nydus-snapshotter && \ + ARCH=$(uname -m) && \ + if [[ "${ARCH}" == "x86_64" ]]; then ARCH=amd64 ; fi && \ + if [[ "${ARCH}" == "aarch64" ]]; then ARCH=arm64; fi && \ + apk add --no-cache curl && \ + curl -fOL --progress-bar ${NYDUS_SNAPSHOTTER_REPO}/releases/download/${NYDUS_SNAPSHOTTER_VERSION}/nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz && \ + tar xvzpf nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz -C /opt/nydus-snapshotter && \ + rm nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz + + +#### kata-deploy main image + +# kata-deploy args +FROM base + ARG KATA_ARTIFACTS=./kata-static.tar.zst ARG DESTINATION=/opt/kata-artifacts -COPY ${KATA_ARTIFACTS} ${WORKDIR} +COPY ${KATA_ARTIFACTS} / # I understand that in order to be on the safer side, it'd # be good to have the alpine packages pointing to a very @@ -37,4 +63,7 @@ RUN \ pip install --no-cache-dir yq==3.2.3 --break-system-packages COPY scripts ${DESTINATION}/scripts +COPY nydus-snapshotter ${DESTINATION}/nydus-snapshotter +COPY --from=nydus-binary-downloader /opt/nydus-snapshotter/bin/containerd-nydus-grpc ${DESTINATION}/nydus-snapshotter/ +COPY --from=nydus-binary-downloader /opt/nydus-snapshotter/bin/nydus-overlayfs ${DESTINATION}/nydus-snapshotter/ COPY runtimeclasses ${DESTINATION}/runtimeclasses diff --git a/tools/packaging/kata-deploy/nydus-snapshotter/config-guest-pulling.toml b/tools/packaging/kata-deploy/nydus-snapshotter/config-guest-pulling.toml new file mode 100644 index 0000000000..0cc77b0bd4 --- /dev/null +++ b/tools/packaging/kata-deploy/nydus-snapshotter/config-guest-pulling.toml @@ -0,0 +1,15 @@ +version = 1 + +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" + +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" + +[daemon] +# Enable proxy mode +fs_driver = "proxy" + +[snapshot] +# Insert Kata volume information to `Mount.Options` +enable_kata_volume = true diff --git a/tools/packaging/kata-deploy/nydus-snapshotter/nydus-snapshotter.service b/tools/packaging/kata-deploy/nydus-snapshotter/nydus-snapshotter.service new file mode 100644 index 0000000000..a1f4d8c9c8 --- /dev/null +++ b/tools/packaging/kata-deploy/nydus-snapshotter/nydus-snapshotter.service @@ -0,0 +1,10 @@ +[Unit] +Description=Nydus snapshotter +After=network.target local-fs.target +Before=containerd.service + +[Service] +ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus-snapshotter/config-guest-pulling.toml --log-to-stdout + +[Install] +RequiredBy=containerd.service diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index b86266f634..fa5540193b 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -62,6 +62,8 @@ AGENT_NO_PROXY="${AGENT_NO_PROXY:-}" PULL_TYPE_MAPPING="${PULL_TYPE_MAPPING:-}" IFS=',' read -a pull_types <<< "$PULL_TYPE_MAPPING" +EXPERIMENTAL_SETUP_SNAPSHOTTER="${EXPERIMENTAL_SETUP_SNAPSHOTTER:-}" + INSTALLATION_PREFIX="${INSTALLATION_PREFIX:-}" default_dest_dir="/opt/kata" dest_dir="${default_dest_dir}" @@ -497,26 +499,33 @@ function wait_till_node_is_ready() { done } +function restart_runtime() { + local runtime="${1}" + + if [ "${runtime}" == "k0s-worker" ] || [ "${runtime}" == "k0s-controller" ]; then + # do nothing, k0s will automatically load the config on the fly + : + elif [ "${runtime}" == "microk8s" ]; then + host_systemctl restart snap.microk8s.daemon-containerd.service + else + host_systemctl daemon-reload + host_systemctl restart "${runtime}" + fi + + wait_till_node_is_ready +} + function configure_cri_runtime() { - case $1 in + local runtime="${1}" + + case "${runtime}" in crio) configure_crio ;; containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker | microk8s) - configure_containerd "$1" + configure_containerd "${runtime}" ;; esac - if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then - # do nothing, k0s will automatically load the config on the fly - : - elif [ "$1" == "microk8s" ]; then - host_systemctl restart snap.microk8s.daemon-containerd.service - else - host_systemctl daemon-reload - host_systemctl restart "$1" - fi - - wait_till_node_is_ready } function configure_crio_runtime() { @@ -799,6 +808,90 @@ function snapshotter_handler_mapping_validation_check() { done } +function configure_nydus_snapshotter() { + info "Configuring nydus-snapshotter" + + configuration_file="${1}" + pluginid="${2}" + + tomlq -i -t $(printf '.plugins.%s.disable_snapshot_annotations=false' ${pluginid}) ${configuration_file} + + tomlq -i -t $(printf '.proxy_plugins.nydus.type="snapshot"') ${configuration_file} + tomlq -i -t $(printf '.proxy_plugins.nydus.address="/run/containerd-nydus/containerd-nydus-grpc.sock"') ${configuration_file} +} + +function configure_snapshotter() { + snapshotter="${1}" + + local runtime="$(get_container_runtime)" + local pluginid="\"io.containerd.grpc.v1.cri\".containerd" # version = 2 + local configuration_file="${containerd_conf_file}" + + # Properly set the configuration file in case drop-in files are supported + if [[ ${use_containerd_drop_in_conf_file} == "true" ]]; then + configuration_file="/host${containerd_drop_in_conf_file}" + fi + + local containerd_root_conf_file="${containerd_conf_file}" + if [[ "${runtime}" =~ ^(k0s-worker|k0s-controller)$ ]]; then + containerd_root_conf_file="/etc/containerd/containerd.toml" + fi + + if grep -q "version = 3\>" ${containerd_root_conf_file}; then + pluginid=\"io.containerd.cri.v1.images\" + fi + + case "${snapshotter}" in + nydus) + configure_nydus_snapshotter "${configuration_file}" "${pluginid}" + host_systemctl restart nydus-snapshotter + ;; + esac +} + +function install_nydus_snapshotter() { + info "Deploying nydus-snapshotter" + + install -D -m 775 /opt/kata-artifacts/nydus-snapshotter/containerd-nydus-grpc /host/usr/local/bin/containerd-nydus-grpc + install -D -m 775 /opt/kata-artifacts/nydus-snapshotter/nydus-overlayfs /host/usr/local/bin/nydus-overlayfs + + mkdir -p /host/etc/nydus-snapshotter/ + install -D -m 644 /opt/kata-artifacts/nydus-snapshotter/config-guest-pulling.toml /host/etc/nydus-snapshotter/config-guest-pulling.toml + install -D -m 644 /opt/kata-artifacts/nydus-snapshotter/nydus-snapshotter.service /host/etc/systemd/system/nydus-snapshotter.service + + host_systemctl daemon-reload + host_systemctl enable nydus-snapshotter.service +} + +function uninstall_nydus_snapshotter() { + info "Removing deployed nydus-snapshotter" + host_systemctl disable --now nydus-snapshotter.service + + rm -f /host/etc/systemd/system/nydus-snapshotter.service + rm -f /host/etc/nydus-snapshotter/config-guest-pulling.toml + + rm -f /host/usr/local/bin/nydus-overlayfs + rm -f /host/usr/local/bin/containerd-nydus-grpc + + host_systemctl daemon-reload +} + +function install_snapshotter() { + snapshotter="${1}" + + case "${snapshotter}" in + nydus) install_nydus_snapshotter ;; + esac +} + +function uninstall_snapshotter() { + snapshotter="${1}" + + case "${snapshotter}" in + nydus) uninstall_nydus_snapshotter ;; + esac +} + function main() { action=${1:-} if [ -z "$action" ]; then @@ -824,6 +917,7 @@ function main() { echo "* INSTALLATION_PREFIX: ${INSTALLATION_PREFIX}" echo "* MULTI_INSTALL_SUFFIX: ${MULTI_INSTALL_SUFFIX}" echo "* HELM_POST_DELETE_HOOK: ${HELM_POST_DELETE_HOOK}" + echo "* EXPERIMENTAL_SETUP_SNAPSHOTTER: ${EXPERIMENTAL_SETUP_SNAPSHOTTER}" # script requires that user is root euid=$(id -u) @@ -853,7 +947,6 @@ function main() { containerd_conf_file_backup="${containerd_conf_tmpl_file}.bak" fi - # only install / remove / update if we are dealing with CRIO or containerd if [[ "$runtime" =~ ^(crio|containerd|k3s|k3s-agent|rke2-agent|rke2-server|k0s-worker|k0s-controller|microk8s)$ ]]; then if [ "$runtime" != "crio" ]; then @@ -873,6 +966,23 @@ function main() { case "$action" in install) + # Let's fail early on this, so we don't need to do a rollback + # in case we reach this situation. + if [[ -n "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" ]]; then + if [[ "${runtime}" == "cri-o" ]]; then + warn "EXPERIMENTAL_SETUP_SNAPSHOTTER is being ignored!" + warn "Snapshotter is a containerd specific option." + else + case "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" in + nydus) + ;; + *) + die "${EXPERIMENTAL_SETUP_SNAPSHOTTER} is not a supported snapshotter by kata-deploy" + ;; + esac + fi + fi + if [[ "$runtime" =~ ^(k3s|k3s-agent|rke2-agent|rke2-server)$ ]]; then if [ ! -f "$containerd_conf_tmpl_file" ] && [ -f "$containerd_conf_file" ]; then cp "$containerd_conf_file" "$containerd_conf_tmpl_file" @@ -897,6 +1007,12 @@ function main() { install_artifacts configure_cri_runtime "$runtime" + if [[ -n "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" ]]; then + install_snapshotter "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" + configure_snapshotter "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" + fi + + restart_runtime "${runtime}" kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=true ;; cleanup) @@ -918,6 +1034,13 @@ function main() { fi fi + if [[ -n "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" ]]; then + # Here we don't need to do any cleanup on the config, as kata-deploy + # will revert the configuration to the state it was before the deployment, + # which is also before the snapshotter configuration. :-) + uninstall_snapshotter "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" + fi + cleanup_cri_runtime "$runtime" if [ "${HELM_POST_DELETE_HOOK}" == "false" ]; then # If we still have any other installation here, it means we'll break them diff --git a/versions.yaml b/versions.yaml index bc2b4b3b3a..ea8f21257a 100644 --- a/versions.yaml +++ b/versions.yaml @@ -344,6 +344,8 @@ externals: url: "https://github.com/dragonflyoss/image-service" version: "v2.2.3" + # Keep the version here aligned with the NYDUS_SNAPSHOTTER_VERSION + # on tools/packaging/kata-deploy/Dockerfile nydus-snapshotter: description: "Snapshotter for Nydus image acceleration service" url: "https://github.com/containerd/nydus-snapshotter"