From f68c25de6abaac23c9406aabc7d46e035e1b7a44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 15 Jan 2026 17:11:56 +0100 Subject: [PATCH] kata-deploy: Switch to the rust version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's remove the script and rely only on the rust version from now on. Signed-off-by: Fabiano FidĂȘncio --- tools/packaging/kata-deploy/Dockerfile | 245 ++- tools/packaging/kata-deploy/Dockerfile.rust | 232 --- .../kata-deploy/templates/_helpers.tpl | 15 - .../kata-deploy/templates/kata-deploy.yaml | 4 - .../templates/post-delete-job.yaml | 4 - .../kata-deploy/scripts/kata-deploy.sh | 1501 ----------------- 6 files changed, 204 insertions(+), 1797 deletions(-) delete mode 100644 tools/packaging/kata-deploy/Dockerfile.rust delete mode 100755 tools/packaging/kata-deploy/scripts/kata-deploy.sh diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile index 50f455f39d..0ff119796e 100644 --- a/tools/packaging/kata-deploy/Dockerfile +++ b/tools/packaging/kata-deploy/Dockerfile @@ -1,11 +1,8 @@ # Copyright Intel Corporation, 2022 IBM Corp. +# Copyright (c) 2025 NVIDIA Corporation # # SPDX-License-Identifier: Apache-2.0 -ARG BASE_IMAGE_NAME=alpine -ARG BASE_IMAGE_TAG=3.22 -FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} AS base - #### Nydus snapshotter & nydus image FROM golang:1.24-alpine AS nydus-binary-downloader @@ -17,53 +14,219 @@ ARG NYDUS_SNAPSHOTTER_REPO=https://github.com/containerd/nydus-snapshotter RUN \ mkdir -p /opt/nydus-snapshotter && \ - ARCH=$(uname -m) && \ - if [[ "${ARCH}" == "x86_64" ]]; then ARCH=amd64 ; fi && \ - if [[ "${ARCH}" == "aarch64" ]]; then ARCH=arm64; fi && \ + ARCH="$(uname -m)" && \ + if [ "${ARCH}" = "x86_64" ]; then ARCH=amd64 ; fi && \ + if [ "${ARCH}" = "aarch64" ]; then ARCH=arm64; fi && \ apk add --no-cache curl && \ - curl -fOL --progress-bar ${NYDUS_SNAPSHOTTER_REPO}/releases/download/${NYDUS_SNAPSHOTTER_VERSION}/nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz && \ - tar xvzpf nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz -C /opt/nydus-snapshotter && \ - rm nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz + curl -fOL --progress-bar "${NYDUS_SNAPSHOTTER_REPO}/releases/download/${NYDUS_SNAPSHOTTER_VERSION}/nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz" && \ + tar xvzpf "nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz" -C /opt/nydus-snapshotter && \ + rm "nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz" +#### Build binary package +FROM ubuntu:22.04 AS rust-builder -#### kata-deploy main image +# Default to Rust 1.90.0 +ARG RUST_TOOLCHAIN=1.90.0 +ENV DEBIAN_FRONTEND=noninteractive +ENV RUSTUP_HOME="/opt/rustup" +ENV CARGO_HOME="/opt/cargo" +ENV PATH="/opt/cargo/bin/:${PATH}" -# kata-deploy args -FROM base +SHELL ["/bin/bash", "-o", "pipefail", "-c"] -ARG KATA_ARTIFACTS=./kata-static.tar.zst +RUN \ + mkdir ${RUSTUP_HOME} ${CARGO_HOME} && \ + chmod -R a+rwX ${RUSTUP_HOME} ${CARGO_HOME} + +RUN \ + apt-get update && \ + apt-get --no-install-recommends -y install \ + ca-certificates \ + curl \ + gcc \ + libc6-dev \ + musl-tools && \ + apt-get clean && rm -rf /var/lib/apt/lists/ && \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${RUST_TOOLCHAIN} + +WORKDIR /kata-deploy + +# Copy standalone binary project +COPY binary /kata-deploy + +# Install target and run tests based on architecture +# - AMD64/arm64: use musl for fully static binaries +# - PPC64le/s390x: use glibc (musl has issues on these platforms) +RUN \ + HOST_ARCH="$(uname -m)"; \ + rust_arch=""; \ + rust_target=""; \ + case "${HOST_ARCH}" in \ + "x86_64") \ + rust_arch="x86_64"; \ + rust_target="${rust_arch}-unknown-linux-musl"; \ + echo "Installing musl target for ${rust_target}"; \ + rustup target add "${rust_target}"; \ + ;; \ + "aarch64") \ + rust_arch="aarch64"; \ + rust_target="${rust_arch}-unknown-linux-musl"; \ + echo "Installing musl target for ${rust_target}"; \ + rustup target add "${rust_target}"; \ + ;; \ + "ppc64le") \ + rust_arch="powerpc64le"; \ + rust_target="${rust_arch}-unknown-linux-gnu"; \ + echo "Using glibc target for ${rust_target} (musl is not well supported on ppc64le)"; \ + ;; \ + "s390x") \ + rust_arch="s390x"; \ + rust_target="${rust_arch}-unknown-linux-gnu"; \ + echo "Using glibc target for ${rust_target} (musl is not well supported on s390x)"; \ + ;; \ + *) echo "Unsupported architecture: ${HOST_ARCH}" && exit 1 ;; \ + esac; \ + echo "${rust_target}" > /tmp/rust_target + +# Run tests using --test-threads=1 to prevent environment variable pollution between tests, +# and this is fine as we'll never ever have multiple binaries running at the same time. +RUN \ + rust_target="$(cat /tmp/rust_target)"; \ + echo "Running binary tests with target ${rust_target}..." && \ + RUSTFLAGS="-D warnings" cargo test --target "${rust_target}" -- --test-threads=1 && \ + echo "All tests passed!" + +RUN \ + rust_target="$(cat /tmp/rust_target)"; \ + echo "Building kata-deploy binary for ${rust_target}..." && \ + RUSTFLAGS="-D warnings" cargo build --release --target "${rust_target}" && \ + mkdir -p /kata-deploy/bin && \ + cp "/kata-deploy/target/${rust_target}/release/kata-deploy" /kata-deploy/bin/kata-deploy && \ + echo "Cleaning up build artifacts to save disk space..." && \ + rm -rf /kata-deploy/target && \ + cargo clean + +#### Extract kata artifacts +FROM alpine:3.22 AS artifact-extractor + +ARG KATA_ARTIFACTS=kata-static.tar.zst ARG DESTINATION=/opt/kata-artifacts -COPY ${KATA_ARTIFACTS} / - -# I understand that in order to be on the safer side, it'd -# be good to have the alpine packages pointing to a very -# specific version, but this may break anyone else trying -# to use a different version of alpine for one reason or -# another. With this in mind, let's ignore DL3018. -# SC2086 is about using double quotes to prevent globbing and -# word splitting, which can also be ignored for now. -# hadolint ignore=DL3018,SC2086 +COPY ${KATA_ARTIFACTS} /tmp/ RUN \ - apk --no-cache add bash curl tar zstd && \ - ARCH=$(uname -m) && \ - if [ "${ARCH}" = "x86_64" ]; then ARCH=amd64; fi && \ - if [ "${ARCH}" = "aarch64" ]; then ARCH=arm64; fi && \ - DEBIAN_ARCH=${ARCH} && \ - if [ "${DEBIAN_ARCH}" = "ppc64le" ]; then DEBIAN_ARCH=ppc64el; fi && \ - curl -fL --progress-bar -o /usr/bin/kubectl https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl && \ - chmod +x /usr/bin/kubectl && \ - curl -fL --progress-bar -o /usr/bin/jq https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${DEBIAN_ARCH} && \ - chmod +x /usr/bin/jq && \ - mkdir -p ${DESTINATION} && \ - tar --zstd -xvf ${WORKDIR}/${KATA_ARTIFACTS} -C ${DESTINATION} && \ - rm -f ${WORKDIR}/${KATA_ARTIFACTS} && \ - apk del curl tar zstd && \ - apk --no-cache add py3-pip && \ - pip install --no-cache-dir yq==3.2.3 --break-system-packages + apk add --no-cache tar zstd util-linux-misc && \ + mkdir -p "${DESTINATION}" && \ + tar --zstd -xf "/tmp/$(basename "${KATA_ARTIFACTS}")" -C "${DESTINATION}" && \ + rm -f "/tmp/$(basename "${KATA_ARTIFACTS}")" -COPY scripts ${DESTINATION}/scripts +#### Prepare runtime dependencies (nsenter and required libraries) +# This stage assembles all runtime dependencies based on architecture +# using ldd to find exact library dependencies +FROM debian:bookworm-slim AS runtime-assembler + +ARG DESTINATION=/opt/kata-artifacts + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN \ + apt-get update && \ + apt-get --no-install-recommends -y install \ + util-linux && \ + apt-get clean && rm -rf /var/lib/apt/lists/ + +# Copy the built binary to analyze its dependencies +COPY --from=rust-builder /kata-deploy/bin/kata-deploy /tmp/kata-deploy + +# Create output directories +RUN mkdir -p /output/lib /output/lib64 /output/usr/bin + +# Use ldd to find and copy all required libraries for the kata-deploy binary and nsenter +RUN \ + HOST_ARCH="$(uname -m)"; \ + echo "Preparing runtime dependencies for ${HOST_ARCH}"; \ + case "${HOST_ARCH}" in \ + "ppc64le"|"s390x") \ + echo "Using glibc - copying libraries based on ldd output"; \ + \ + # Copy nsenter \ + cp /usr/bin/nsenter /output/usr/bin/nsenter; \ + \ + # Show what the binaries need \ + echo "Libraries needed by kata-deploy:"; \ + ldd /tmp/kata-deploy || echo "ldd failed"; \ + echo "Libraries needed by nsenter:"; \ + ldd /usr/bin/nsenter || echo "ldd failed"; \ + \ + # Extract and copy all library paths from both binaries \ + for binary in /tmp/kata-deploy /usr/bin/nsenter; do \ + echo "Processing ${binary}..."; \ + # Get libraries with "=>" (shared libs) \ + ldd "${binary}" 2>/dev/null | grep "=>" | awk '{print $3}' | sort -u | while read -r lib; do \ + if [ -n "${lib}" ] && [ -f "${lib}" ]; then \ + dest_dir="/output$(dirname "${lib}")"; \ + mkdir -p "${dest_dir}"; \ + cp -Ln "${lib}" "${dest_dir}/" 2>/dev/null || true; \ + echo " Copied lib: ${lib}"; \ + fi; \ + done; \ + done; \ + \ + # Copy the dynamic linker - it's at /lib/ld64.so.1 (not /lib64/) \ + echo "Copying dynamic linker:"; \ + mkdir -p /output/lib; \ + cp -Ln /lib/ld64.so* /output/lib/ 2>/dev/null || true; \ + cp -Ln /lib64/ld64.so* /output/lib64/ 2>/dev/null || true; \ + \ + echo "glibc" > /output/.libc-type; \ + ;; \ + *) \ + echo "amd64/arm64: will use musl-based static binaries"; \ + echo "musl" > /output/.libc-type; \ + # Create placeholder so COPY doesn't fail \ + touch /output/lib/.placeholder; \ + touch /output/lib64/.placeholder; \ + touch /output/usr/bin/.placeholder; \ + ;; \ + esac + +# Copy musl nsenter from alpine for amd64/arm64 +COPY --from=artifact-extractor /usr/bin/nsenter /output/usr/bin/nsenter-musl +COPY --from=artifact-extractor /lib/ld-musl-*.so.1 /output/lib/ + +# For amd64/arm64, use the musl nsenter; for ppc64le/s390x, keep the glibc one +RUN \ + HOST_ARCH="$(uname -m)"; \ + case "${HOST_ARCH}" in \ + "x86_64"|"aarch64") \ + mv /output/usr/bin/nsenter-musl /output/usr/bin/nsenter; \ + ;; \ + *) \ + rm -f /output/usr/bin/nsenter-musl; \ + ;; \ + esac + +#### kata-deploy main image +FROM gcr.io/distroless/static-debian12@sha256:87bce11be0af225e4ca761c40babb06d6d559f5767fbf7dc3c47f0f1a466b92c + +ARG DESTINATION=/opt/kata-artifacts + +# Copy extracted kata artifacts +COPY --from=artifact-extractor ${DESTINATION} ${DESTINATION} + +# Copy Rust binary +COPY --from=rust-builder /kata-deploy/bin/kata-deploy /usr/bin/kata-deploy + +# Copy nsenter and required libraries (assembled based on architecture) +COPY --from=runtime-assembler /output/usr/bin/nsenter /usr/bin/nsenter +COPY --from=runtime-assembler /output/lib/ /lib/ +COPY --from=runtime-assembler /output/lib64/ /lib64/ + +# Copy nydus snapshotter COPY nydus-snapshotter ${DESTINATION}/nydus-snapshotter COPY --from=nydus-binary-downloader /opt/nydus-snapshotter/bin/containerd-nydus-grpc ${DESTINATION}/nydus-snapshotter/ COPY --from=nydus-binary-downloader /opt/nydus-snapshotter/bin/nydus-overlayfs ${DESTINATION}/nydus-snapshotter/ + +# Copy runtimeclasses and node-feature-rules COPY node-feature-rules ${DESTINATION}/node-feature-rules + +ENTRYPOINT ["/usr/bin/kata-deploy"] diff --git a/tools/packaging/kata-deploy/Dockerfile.rust b/tools/packaging/kata-deploy/Dockerfile.rust deleted file mode 100644 index 0ff119796e..0000000000 --- a/tools/packaging/kata-deploy/Dockerfile.rust +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright Intel Corporation, 2022 IBM Corp. -# Copyright (c) 2025 NVIDIA Corporation -# -# SPDX-License-Identifier: Apache-2.0 - -#### Nydus snapshotter & nydus image - -FROM golang:1.24-alpine AS nydus-binary-downloader - -# Keep the version here aligned with "ndyus-snapshotter.version" -# in versions.yaml -ARG NYDUS_SNAPSHOTTER_VERSION=v0.15.10 -ARG NYDUS_SNAPSHOTTER_REPO=https://github.com/containerd/nydus-snapshotter - -RUN \ - mkdir -p /opt/nydus-snapshotter && \ - ARCH="$(uname -m)" && \ - if [ "${ARCH}" = "x86_64" ]; then ARCH=amd64 ; fi && \ - if [ "${ARCH}" = "aarch64" ]; then ARCH=arm64; fi && \ - apk add --no-cache curl && \ - curl -fOL --progress-bar "${NYDUS_SNAPSHOTTER_REPO}/releases/download/${NYDUS_SNAPSHOTTER_VERSION}/nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz" && \ - tar xvzpf "nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz" -C /opt/nydus-snapshotter && \ - rm "nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-linux-${ARCH}.tar.gz" - -#### Build binary package -FROM ubuntu:22.04 AS rust-builder - -# Default to Rust 1.90.0 -ARG RUST_TOOLCHAIN=1.90.0 -ENV DEBIAN_FRONTEND=noninteractive -ENV RUSTUP_HOME="/opt/rustup" -ENV CARGO_HOME="/opt/cargo" -ENV PATH="/opt/cargo/bin/:${PATH}" - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN \ - mkdir ${RUSTUP_HOME} ${CARGO_HOME} && \ - chmod -R a+rwX ${RUSTUP_HOME} ${CARGO_HOME} - -RUN \ - apt-get update && \ - apt-get --no-install-recommends -y install \ - ca-certificates \ - curl \ - gcc \ - libc6-dev \ - musl-tools && \ - apt-get clean && rm -rf /var/lib/apt/lists/ && \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${RUST_TOOLCHAIN} - -WORKDIR /kata-deploy - -# Copy standalone binary project -COPY binary /kata-deploy - -# Install target and run tests based on architecture -# - AMD64/arm64: use musl for fully static binaries -# - PPC64le/s390x: use glibc (musl has issues on these platforms) -RUN \ - HOST_ARCH="$(uname -m)"; \ - rust_arch=""; \ - rust_target=""; \ - case "${HOST_ARCH}" in \ - "x86_64") \ - rust_arch="x86_64"; \ - rust_target="${rust_arch}-unknown-linux-musl"; \ - echo "Installing musl target for ${rust_target}"; \ - rustup target add "${rust_target}"; \ - ;; \ - "aarch64") \ - rust_arch="aarch64"; \ - rust_target="${rust_arch}-unknown-linux-musl"; \ - echo "Installing musl target for ${rust_target}"; \ - rustup target add "${rust_target}"; \ - ;; \ - "ppc64le") \ - rust_arch="powerpc64le"; \ - rust_target="${rust_arch}-unknown-linux-gnu"; \ - echo "Using glibc target for ${rust_target} (musl is not well supported on ppc64le)"; \ - ;; \ - "s390x") \ - rust_arch="s390x"; \ - rust_target="${rust_arch}-unknown-linux-gnu"; \ - echo "Using glibc target for ${rust_target} (musl is not well supported on s390x)"; \ - ;; \ - *) echo "Unsupported architecture: ${HOST_ARCH}" && exit 1 ;; \ - esac; \ - echo "${rust_target}" > /tmp/rust_target - -# Run tests using --test-threads=1 to prevent environment variable pollution between tests, -# and this is fine as we'll never ever have multiple binaries running at the same time. -RUN \ - rust_target="$(cat /tmp/rust_target)"; \ - echo "Running binary tests with target ${rust_target}..." && \ - RUSTFLAGS="-D warnings" cargo test --target "${rust_target}" -- --test-threads=1 && \ - echo "All tests passed!" - -RUN \ - rust_target="$(cat /tmp/rust_target)"; \ - echo "Building kata-deploy binary for ${rust_target}..." && \ - RUSTFLAGS="-D warnings" cargo build --release --target "${rust_target}" && \ - mkdir -p /kata-deploy/bin && \ - cp "/kata-deploy/target/${rust_target}/release/kata-deploy" /kata-deploy/bin/kata-deploy && \ - echo "Cleaning up build artifacts to save disk space..." && \ - rm -rf /kata-deploy/target && \ - cargo clean - -#### Extract kata artifacts -FROM alpine:3.22 AS artifact-extractor - -ARG KATA_ARTIFACTS=kata-static.tar.zst -ARG DESTINATION=/opt/kata-artifacts - -COPY ${KATA_ARTIFACTS} /tmp/ -RUN \ - apk add --no-cache tar zstd util-linux-misc && \ - mkdir -p "${DESTINATION}" && \ - tar --zstd -xf "/tmp/$(basename "${KATA_ARTIFACTS}")" -C "${DESTINATION}" && \ - rm -f "/tmp/$(basename "${KATA_ARTIFACTS}")" - -#### Prepare runtime dependencies (nsenter and required libraries) -# This stage assembles all runtime dependencies based on architecture -# using ldd to find exact library dependencies -FROM debian:bookworm-slim AS runtime-assembler - -ARG DESTINATION=/opt/kata-artifacts - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN \ - apt-get update && \ - apt-get --no-install-recommends -y install \ - util-linux && \ - apt-get clean && rm -rf /var/lib/apt/lists/ - -# Copy the built binary to analyze its dependencies -COPY --from=rust-builder /kata-deploy/bin/kata-deploy /tmp/kata-deploy - -# Create output directories -RUN mkdir -p /output/lib /output/lib64 /output/usr/bin - -# Use ldd to find and copy all required libraries for the kata-deploy binary and nsenter -RUN \ - HOST_ARCH="$(uname -m)"; \ - echo "Preparing runtime dependencies for ${HOST_ARCH}"; \ - case "${HOST_ARCH}" in \ - "ppc64le"|"s390x") \ - echo "Using glibc - copying libraries based on ldd output"; \ - \ - # Copy nsenter \ - cp /usr/bin/nsenter /output/usr/bin/nsenter; \ - \ - # Show what the binaries need \ - echo "Libraries needed by kata-deploy:"; \ - ldd /tmp/kata-deploy || echo "ldd failed"; \ - echo "Libraries needed by nsenter:"; \ - ldd /usr/bin/nsenter || echo "ldd failed"; \ - \ - # Extract and copy all library paths from both binaries \ - for binary in /tmp/kata-deploy /usr/bin/nsenter; do \ - echo "Processing ${binary}..."; \ - # Get libraries with "=>" (shared libs) \ - ldd "${binary}" 2>/dev/null | grep "=>" | awk '{print $3}' | sort -u | while read -r lib; do \ - if [ -n "${lib}" ] && [ -f "${lib}" ]; then \ - dest_dir="/output$(dirname "${lib}")"; \ - mkdir -p "${dest_dir}"; \ - cp -Ln "${lib}" "${dest_dir}/" 2>/dev/null || true; \ - echo " Copied lib: ${lib}"; \ - fi; \ - done; \ - done; \ - \ - # Copy the dynamic linker - it's at /lib/ld64.so.1 (not /lib64/) \ - echo "Copying dynamic linker:"; \ - mkdir -p /output/lib; \ - cp -Ln /lib/ld64.so* /output/lib/ 2>/dev/null || true; \ - cp -Ln /lib64/ld64.so* /output/lib64/ 2>/dev/null || true; \ - \ - echo "glibc" > /output/.libc-type; \ - ;; \ - *) \ - echo "amd64/arm64: will use musl-based static binaries"; \ - echo "musl" > /output/.libc-type; \ - # Create placeholder so COPY doesn't fail \ - touch /output/lib/.placeholder; \ - touch /output/lib64/.placeholder; \ - touch /output/usr/bin/.placeholder; \ - ;; \ - esac - -# Copy musl nsenter from alpine for amd64/arm64 -COPY --from=artifact-extractor /usr/bin/nsenter /output/usr/bin/nsenter-musl -COPY --from=artifact-extractor /lib/ld-musl-*.so.1 /output/lib/ - -# For amd64/arm64, use the musl nsenter; for ppc64le/s390x, keep the glibc one -RUN \ - HOST_ARCH="$(uname -m)"; \ - case "${HOST_ARCH}" in \ - "x86_64"|"aarch64") \ - mv /output/usr/bin/nsenter-musl /output/usr/bin/nsenter; \ - ;; \ - *) \ - rm -f /output/usr/bin/nsenter-musl; \ - ;; \ - esac - -#### kata-deploy main image -FROM gcr.io/distroless/static-debian12@sha256:87bce11be0af225e4ca761c40babb06d6d559f5767fbf7dc3c47f0f1a466b92c - -ARG DESTINATION=/opt/kata-artifacts - -# Copy extracted kata artifacts -COPY --from=artifact-extractor ${DESTINATION} ${DESTINATION} - -# Copy Rust binary -COPY --from=rust-builder /kata-deploy/bin/kata-deploy /usr/bin/kata-deploy - -# Copy nsenter and required libraries (assembled based on architecture) -COPY --from=runtime-assembler /output/usr/bin/nsenter /usr/bin/nsenter -COPY --from=runtime-assembler /output/lib/ /lib/ -COPY --from=runtime-assembler /output/lib64/ /lib64/ - -# Copy nydus snapshotter -COPY nydus-snapshotter ${DESTINATION}/nydus-snapshotter -COPY --from=nydus-binary-downloader /opt/nydus-snapshotter/bin/containerd-nydus-grpc ${DESTINATION}/nydus-snapshotter/ -COPY --from=nydus-binary-downloader /opt/nydus-snapshotter/bin/nydus-overlayfs ${DESTINATION}/nydus-snapshotter/ - -# Copy runtimeclasses and node-feature-rules -COPY node-feature-rules ${DESTINATION}/node-feature-rules - -ENTRYPOINT ["/usr/bin/kata-deploy"] diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl index 28f2e92355..02580672cc 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl @@ -290,18 +290,3 @@ Note: EXPERIMENTAL_FORCE_GUEST_PULL only checks containerd.forceGuestPull, not c {{- end -}} {{- join "," $shimNames -}} {{- end -}} - -{{/* -Detect if this is a Rust-based build by checking the image tag -Returns "true" if the tag contains "-rust", otherwise returns "false" -This is a temporary helper for dual script/rust support -*/}} -{{- define "kata-deploy.isRustBuild" -}} -{{- $tag := default .Chart.AppVersion .Values.image.tag -}} -{{- if or (contains "-rust" $tag) (contains "nightly-rust" $tag) -}} -true -{{- else -}} -false -{{- end -}} -{{- end -}} - diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml index de7ecde14a..f262237502 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml @@ -133,11 +133,7 @@ spec: - name: kube-kata image: {{ .Values.image.reference }}:{{ default .Chart.AppVersion .Values.image.tag }} imagePullPolicy: {{ .Values.imagePullPolicy }} -{{- if eq (include "kata-deploy.isRustBuild" .) "true" }} command: ["/usr/bin/kata-deploy", "install"] -{{- else }} - command: ["/opt/kata-artifacts/scripts/kata-deploy.sh", "install"] -{{- end }} env: - name: NODE_NAME valueFrom: diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml index c56608f431..09a9ff05c6 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml @@ -104,11 +104,7 @@ spec: - name: kube-kata-cleanup image: {{ .Values.image.reference }}:{{ default .Chart.AppVersion .Values.image.tag }} imagePullPolicy: {{ .Values.imagePullPolicy }} -{{- if eq (include "kata-deploy.isRustBuild" .) "true" }} command: ["/usr/bin/kata-deploy", "cleanup"] -{{- else }} - command: ["/opt/kata-artifacts/scripts/kata-deploy.sh", "cleanup"] -{{- end }} env: - name: NODE_NAME valueFrom: diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh deleted file mode 100755 index c300605a12..0000000000 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ /dev/null @@ -1,1501 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2019 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -set -o errexit -set -o pipefail -set -o nounset - -crio_drop_in_conf_dir="/etc/crio/crio.conf.d/" -crio_drop_in_conf_file="${crio_drop_in_conf_dir}/99-kata-deploy" -crio_drop_in_conf_file_debug="${crio_drop_in_conf_dir}/100-debug" -containerd_conf_file="/etc/containerd/config.toml" -containerd_conf_file_backup="${containerd_conf_file}.bak" -containerd_conf_tmpl_file="" -use_containerd_drop_in_conf_file="false" - -# If we fail for any reason a message will be displayed -die() { - msg="$*" - echo "ERROR: $msg" >&2 - exit 1 -} - -warn() { - msg="$*" - echo "WARN: $msg" >&2 -} - -info() { - msg="$*" - echo "INFO: $msg" >&2 -} - -# Get existing values from a TOML array field and return them as a comma-separated string -# * get_field_array_values "${config}" "enable_annotations" "${shim}" -get_field_array_values() { - local config_file="$1" - local field="$2" - local shim="${3:-}" - - # Determine hypervisor name if shim is provided - local hypervisor_name="" - if [[ -n "${shim}" ]]; then - hypervisor_name=$(get_hypervisor_name "${shim}") - fi - - # Get array values using tomlq - output each element on a new line, then convert to comma-separated - local array_values="" - if [[ -n "${hypervisor_name}" ]]; then - array_values=$(tomlq -r '.hypervisor.'"${hypervisor_name}"'.'"${field}"' // [] | .[]' "${config_file}" 2>/dev/null || echo "") - fi - - # Fallback: try without hypervisor prefix (for top-level fields) - if [[ -z "${array_values}" ]] || [[ "${array_values}" == "null" ]]; then - array_values=$(tomlq -r '."'"${field}"'" // [] | .[]' "${config_file}" 2>/dev/null || echo "") - fi - - # Convert newline-separated values to comma-separated string - if [[ -n "${array_values}" ]]; then - echo "${array_values}" | tr '\n' ',' | sed 's/,$//' - else - echo "" - fi -} - -DEBUG="${DEBUG:-"false"}" - -ARCH=$(uname -m) - -SHIMS_X86_64="${SHIMS_X86_64:-"clh cloud-hypervisor dragonball fc qemu qemu-coco-dev qemu-coco-dev-runtime-rs qemu-runtime-rs qemu-snp qemu-tdx qemu-nvidia-gpu qemu-nvidia-gpu-snp qemu-nvidia-gpu-tdx"}" -SHIMS_AARCH64="${SHIMS_AARCH64:-"clh cloud-hypervisor dragonball fc qemu qemu-nvidia-gpu qemu-cca"}" -SHIMS_S390X="${SHIMS_S390X:-"qemu qemu-runtime-rs qemu-se qemu-se-runtime-rs qemu-coco-dev qemu-coco-dev-runtime-rs"}" -SHIMS_PPC64LE="${SHIMS_PPC64LE:-"qemu"}" - -DEFAULT_SHIM_X86_64="${DEFAULT_SHIM_X86_64:-"qemu"}" -DEFAULT_SHIM_AARCH64="${DEFAULT_SHIM_AARCH64:-"qemu"}" -DEFAULT_SHIM_S390X="${DEFAULT_SHIM_S390X:-"qemu"}" -DEFAULT_SHIM_PPC64LE="${DEFAULT_SHIM_PPC64LE:-"qemu"}" - -SNAPSHOTTER_HANDLER_MAPPING_X86_64="${SNAPSHOTTER_HANDLER_MAPPING_X86_64:-}" -SNAPSHOTTER_HANDLER_MAPPING_AARCH64="${SNAPSHOTTER_HANDLER_MAPPING_AARCH64:-}" -SNAPSHOTTER_HANDLER_MAPPING_S390X="${SNAPSHOTTER_HANDLER_MAPPING_S390X:-}" -SNAPSHOTTER_HANDLER_MAPPING_PPC64LE="${SNAPSHOTTER_HANDLER_MAPPING_PPC64LE:-}" - -ALLOWED_HYPERVISOR_ANNOTATIONS_X86_64="${ALLOWED_HYPERVISOR_ANNOTATIONS_X86_64:-}" -ALLOWED_HYPERVISOR_ANNOTATIONS_AARCH64="${ALLOWED_HYPERVISOR_ANNOTATIONS_AARCH64:-}" -ALLOWED_HYPERVISOR_ANNOTATIONS_S390X="${ALLOWED_HYPERVISOR_ANNOTATIONS_S390X:-}" -ALLOWED_HYPERVISOR_ANNOTATIONS_PPC64LE="${ALLOWED_HYPERVISOR_ANNOTATIONS_PPC64LE:-}" - -PULL_TYPE_MAPPING_X86_64="${PULL_TYPE_MAPPING_X86_64:-}" -PULL_TYPE_MAPPING_AARCH64="${PULL_TYPE_MAPPING_AARCH64:-}" -PULL_TYPE_MAPPING_S390X="${PULL_TYPE_MAPPING_S390X:-}" -PULL_TYPE_MAPPING_PPC64LE="${PULL_TYPE_MAPPING_PPC64LE:-}" - -EXPERIMENTAL_FORCE_GUEST_PULL_X86_64="${EXPERIMENTAL_FORCE_GUEST_PULL_X86_64:-}" -EXPERIMENTAL_FORCE_GUEST_PULL_AARCH64="${EXPERIMENTAL_FORCE_GUEST_PULL_AARCH64:-}" -EXPERIMENTAL_FORCE_GUEST_PULL_S390X="${EXPERIMENTAL_FORCE_GUEST_PULL_S390X:-}" -EXPERIMENTAL_FORCE_GUEST_PULL_PPC64LE="${EXPERIMENTAL_FORCE_GUEST_PULL_PPC64LE:-}" - -SHIMS_FOR_ARCH="" -DEFAULT_SHIM_FOR_ARCH="" -SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH="" -ALLOWED_HYPERVISOR_ANNOTATIONS_FOR_ARCH="" -PULL_TYPE_MAPPING_FOR_ARCH="" -EXPERIMENTAL_FORCE_GUEST_PULL_FOR_ARCH="" -case ${ARCH} in - x86_64) - SHIMS_FOR_ARCH="${SHIMS_X86_64}" - DEFAULT_SHIM_FOR_ARCH="${DEFAULT_SHIM_X86_64}" - SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH="${SNAPSHOTTER_HANDLER_MAPPING_X86_64}" - ALLOWED_HYPERVISOR_ANNOTATIONS_FOR_ARCH="${ALLOWED_HYPERVISOR_ANNOTATIONS_X86_64}" - PULL_TYPE_MAPPING_FOR_ARCH="${PULL_TYPE_MAPPING_X86_64}" - EXPERIMENTAL_FORCE_GUEST_PULL_FOR_ARCH="${EXPERIMENTAL_FORCE_GUEST_PULL_X86_64}" - ;; - aarch64) - SHIMS_FOR_ARCH="${SHIMS_AARCH64}" - DEFAULT_SHIM_FOR_ARCH="${DEFAULT_SHIM_AARCH64}" - SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH="${SNAPSHOTTER_HANDLER_MAPPING_AARCH64}" - ALLOWED_HYPERVISOR_ANNOTATIONS_FOR_ARCH="${ALLOWED_HYPERVISOR_ANNOTATIONS_AARCH64}" - PULL_TYPE_MAPPING_FOR_ARCH="${PULL_TYPE_MAPPING_AARCH64}" - EXPERIMENTAL_FORCE_GUEST_PULL_FOR_ARCH="${EXPERIMENTAL_FORCE_GUEST_PULL_AARCH64}" - ;; - s390x) - SHIMS_FOR_ARCH="${SHIMS_S390X}" - DEFAULT_SHIM_FOR_ARCH="${DEFAULT_SHIM_S390X}" - SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH="${SNAPSHOTTER_HANDLER_MAPPING_S390X}" - ALLOWED_HYPERVISOR_ANNOTATIONS_FOR_ARCH="${ALLOWED_HYPERVISOR_ANNOTATIONS_S390X}" - PULL_TYPE_MAPPING_FOR_ARCH="${PULL_TYPE_MAPPING_S390X}" - EXPERIMENTAL_FORCE_GUEST_PULL_FOR_ARCH="${EXPERIMENTAL_FORCE_GUEST_PULL_S390X}" - ;; - ppc64le) - SHIMS_FOR_ARCH="${SHIMS_PPC64LE}" - DEFAULT_SHIM_FOR_ARCH="${DEFAULT_SHIM_PPC64LE}" - SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH="${SNAPSHOTTER_HANDLER_MAPPING_PPC64LE}" - ALLOWED_HYPERVISOR_ANNOTATIONS_FOR_ARCH="${ALLOWED_HYPERVISOR_ANNOTATIONS_PPC64LE}" - PULL_TYPE_MAPPING_FOR_ARCH="${PULL_TYPE_MAPPING_PPC64LE}" - EXPERIMENTAL_FORCE_GUEST_PULL_FOR_ARCH="${EXPERIMENTAL_FORCE_GUEST_PULL_PPC64LE}" - ;; - *) - die "Unsupported architecture: ${ARCH}" - ;; -esac - -IFS=' ' read -a shims <<< "${SHIMS_FOR_ARCH}" -default_shim="${DEFAULT_SHIM_FOR_ARCH}" - -IFS=',' read -a snapshotters <<< "${SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH}" -snapshotters_delimiter=':' - -IFS=' ' read -a hypervisor_annotations <<< "${ALLOWED_HYPERVISOR_ANNOTATIONS_FOR_ARCH}" - -IFS=',' read -a pull_types <<< "${PULL_TYPE_MAPPING_FOR_ARCH}" - -IFS="," read -a experimental_force_guest_pull <<< "${EXPERIMENTAL_FORCE_GUEST_PULL_FOR_ARCH}" - -AGENT_HTTPS_PROXY="${AGENT_HTTPS_PROXY:-}" -AGENT_NO_PROXY="${AGENT_NO_PROXY:-}" - -EXPERIMENTAL_SETUP_SNAPSHOTTER="${EXPERIMENTAL_SETUP_SNAPSHOTTER:-}" -IFS=',' read -a experimental_setup_snapshotter <<< "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" - -INSTALLATION_PREFIX="${INSTALLATION_PREFIX:-}" -default_dest_dir="/opt/kata" -dest_dir="${default_dest_dir}" -if [ -n "${INSTALLATION_PREFIX}" ]; then - if [[ "${INSTALLATION_PREFIX:0:1}" != "/" ]]; then - die 'INSTALLATION_PREFIX must begin with a "/"(ex. /hoge/fuga)' - fi - # There's no `/` in between ${INSTALLATION_PREFIX} and ${default_dest_dir} - # as, otherwise, we'd have it doubled there, as: `/foo/bar//opt/kata` - dest_dir="${INSTALLATION_PREFIX}${default_dest_dir}" -fi - -MULTI_INSTALL_SUFFIX="${MULTI_INSTALL_SUFFIX:-}" -if [ -n "${MULTI_INSTALL_SUFFIX}" ]; then - dest_dir="${dest_dir}-${MULTI_INSTALL_SUFFIX}" - crio_drop_in_conf_file="${crio_drop_in_conf_file}-${MULTI_INSTALL_SUFFIX}" -fi -containerd_drop_in_conf_file="${dest_dir}/containerd/config.d/kata-deploy.toml" - -# Here, again, there's no `/` between /host and ${dest_dir}, otherwise we'd have it -# doubled here as well, as: `/host//opt/kata` -host_install_dir="/host${dest_dir}" - -HELM_POST_DELETE_HOOK="${HELM_POST_DELETE_HOOK:-"false"}" - -function host_systemctl() { - nsenter --target 1 --mount systemctl "${@}" -} - -function host_exec() { - nsenter --target 1 --mount bash -c "$*" -} - -function print_usage() { - echo "Usage: $0 [install/cleanup/reset]" -} - -function patch_runtimeclasses_for_nfd() { - info "Patching existing runtime classes for NFD" - - for shim in "${shims[@]}"; do - local runtime_class_name="kata-${shim}" - if [[ -n "${MULTI_INSTALL_SUFFIX}" ]]; then - runtime_class_name="kata-${shim}-${MULTI_INSTALL_SUFFIX}" - fi - - # Check if runtime class exists - if ! kubectl get runtimeclass "${runtime_class_name}" &>/dev/null; then - continue - fi - - case "${shim}" in - *tdx*) - info "Patching runtime class ${runtime_class_name} for TDX NFD support" - # Use merge patch strategy to add/update tdx.intel.com/keys in overhead.podFixed - # This preserves existing overhead.podFixed values and creates structure if needed - kubectl patch runtimeclass "${runtime_class_name}" --type=merge \ - -p='{"overhead":{"podFixed":{"tdx.intel.com/keys":1}}}' - ;; - *snp*) - info "Patching runtime class ${runtime_class_name} for SNP NFD support" - # Use merge patch strategy to add/update sev-snp.amd.com/esids in overhead.podFixed - # This preserves existing overhead.podFixed values and creates structure if needed - kubectl patch runtimeclass "${runtime_class_name}" --type=merge \ - -p='{"overhead":{"podFixed":{"sev-snp.amd.com/esids":1}}}' - ;; - *) - ;; - esac - done -} - -function get_container_runtime() { - - local runtime=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}') - local microk8s=$(kubectl get node $NODE_NAME -o jsonpath='{.metadata.labels.microk8s\.io\/cluster}') - if [ "$?" -ne 0 ]; then - die "invalid node name" - fi - - if echo "$runtime" | grep -qE "cri-o"; then - echo "cri-o" - elif [ "$microk8s" == "true" ]; then - echo "microk8s" - elif echo "$runtime" | grep -qE 'containerd.*-k3s'; then - if host_systemctl is-active --quiet rke2-agent; then - echo "rke2-agent" - elif host_systemctl is-active --quiet rke2-server; then - echo "rke2-server" - elif host_systemctl is-active --quiet k3s-agent; then - echo "k3s-agent" - else - echo "k3s" - fi - # Note: we assumed you used a conventional k0s setup and k0s will generate a systemd entry k0scontroller.service and k0sworker.service respectively - # and it is impossible to run this script without a kubelet, so this k0s controller must also have worker mode enabled - elif host_systemctl is-active --quiet k0scontroller; then - echo "k0s-controller" - elif host_systemctl is-active --quiet k0sworker; then - echo "k0s-worker" - else - echo "$runtime" | awk -F '[:]' '{print $1}' - fi -} - -function is_containerd_capable_of_using_drop_in_files() { - local runtime="$1" - - if [ "$runtime" == "crio" ]; then - # This should never happen but better be safe than sorry - echo "false" - return - fi - - if [[ "$runtime" =~ ^(k0s-worker|k0s-controller)$ ]]; then - # k0s does the work of using drop-in files better than any other "k8s distro", so - # we don't mess up with what's being correctly done. - echo "false" - return - fi - - if [ "$runtime" == "microk8s" ]; then - # microk8s use snap containerd - echo "false" - return - fi - - local major_version=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}' | grep -oE '[0-9]+\.[0-9]+' | head -n1 | cut -d'.' -f1) - if [ $major_version -lt 2 ]; then - # Only containerd 2.0 does the merge of the plugins section from different snippets, - # instead of overwritting the whole section, which makes things considerably more - # complicated for us to deal with. - # - # It's been discussed with containerd community, and the patch needed will **NOT** be - # backported to the release 1.7, as that breaks the behaviour from an existing release. - echo "false" - return - fi - - echo "true" -} - -function get_kata_containers_config_path() { - local shim="$1" - - # Directory holding pristine configuration files for the current default golang runtime. - local golang_config_path="${dest_dir}/share/defaults/kata-containers" - - # Directory holding pristine configuration files for the new rust runtime. - # - # These are put into a separate directory since: - # - # - In some cases, the rust runtime configuration syntax is - # slightly different to the golang runtime configuration files - # so some hypervisors need two different configuration files, - # one for reach runtime type (for example Cloud Hypervisor which - # uses 'clh' for the golang runtime and 'cloud-hypervisor' for - # the rust runtime. - # - # - Some hypervisors only currently work with the golang runtime. - # - # - Some hypervisors only work with the rust runtime (dragonball). - # - # See: https://github.com/kata-containers/kata-containers/issues/6020 - local rust_config_path="${golang_config_path}/runtime-rs" - - local config_path - - # Map the runtime shim name to the appropriate configuration - # file directory. - case "$shim" in - cloud-hypervisor | dragonball | qemu-runtime-rs | qemu-coco-dev-runtime-rs | qemu-se-runtime-rs) config_path="$rust_config_path" ;; - *) config_path="$golang_config_path" ;; - esac - - echo "$config_path" -} - -function get_kata_containers_runtime_path() { - local shim="$1" - - local runtime_path - case "$shim" in - cloud-hypervisor | dragonball | qemu-runtime-rs | qemu-coco-dev-runtime-rs | qemu-se-runtime-rs) - runtime_path="${dest_dir}/runtime-rs/bin/containerd-shim-kata-v2" - ;; - *) - runtime_path="${dest_dir}/bin/containerd-shim-kata-v2" - ;; - esac - - echo "$runtime_path" -} - -function tdx_not_supported() { - distro="${1}" - version="${2}" - - warn "Distro ${distro} ${version} does not support TDX and the TDX related runtime classes will not work in your cluster!" -} - -function tdx_supported() { - distro="${1}" - version="${2}" - config="${3}" - - local qemu_path=$(get_tdx_qemu_path_from_distro ${distro}) - local ovmf_path=$(get_tdx_ovmf_path_from_distro ${distro}) - - tomlq -i -t '.hypervisor.qemu.path = "'"${qemu_path}"'"' "${config}" 2>/dev/null || true - tomlq -i -t '.hypervisor.qemu.firmware = "'"${ovmf_path}"'"' "${config}" 2>/dev/null || true - - info "In order to use the tdx related runtime classes, ensure TDX is properly configured for ${distro} ${version} by following the instructions provided at: $(get_tdx_distro_instructions ${distro})" -} - -function get_tdx_distro_instructions() { - distro="${1}" - - case ${distro} in - ubuntu) - echo "https://github.com/canonical/tdx/tree/3.3" - ;; - centos) - echo "https://sigs.centos.org/virt/tdx" - ;; - esac -} - -function get_tdx_qemu_path_from_distro() { - distro="${1}" - - case ${distro} in - ubuntu) - echo "/usr/bin/qemu-system-x86_64" - ;; - centos) - echo "/usr/libexec/qemu-kvm" - ;; - esac -} - -function get_tdx_ovmf_path_from_distro() { - distro="${1}" - - case ${distro} in - ubuntu) - echo "/usr/share/ovmf/OVMF.fd" - ;; - centos) - echo "/usr/share/edk2/ovmf/OVMF.inteltdx.fd" - ;; - esac -} - -function adjust_qemu_cmdline() { - shim="${1}" - config_path="${2}" - qemu_share="${shim}" - - # The paths on the kata-containers tarball side look like: - # ${dest_dir}/opt/kata/share/kata-qemu/qemu - # ${dest_dir}/opt/kata/share/kata-qemu-snp-experimnental/qemu - # ${dest_dir}/opt/kata/share/kata-qemu-cca-experimental/qemu - [[ "${shim}" == "qemu-nvidia-gpu-snp" ]] && qemu_share=qemu-snp-experimental - [[ "${shim}" == "qemu-nvidia-gpu-tdx" ]] && qemu_share=qemu-tdx-experimental - [[ "${shim}" == "qemu-cca" ]] && qemu_share=qemu-cca-experimental - - # Both qemu and qemu-coco-dev use exactly the same QEMU, so we can adjust - # the shim on the qemu-coco-dev case to qemu - [[ "${shim}" =~ ^(qemu|qemu-runtime-rs|qemu-snp|qemu-se|qemu-se-runtime-rs|qemu-coco-dev|qemu-coco-dev-runtime-rs|qemu-nvidia-gpu)$ ]] && qemu_share="qemu" - - qemu_binary=$(tomlq '.hypervisor.qemu.path' ${config_path} | tr -d \") - qemu_binary_script="${qemu_binary}-installation-prefix" - qemu_binary_script_host_path="/host/${qemu_binary_script}" - - if [[ ! -f ${qemu_binary_script_host_path} ]]; then - # From the QEMU man page: - # ``` - # -L path - # Set the directory for the BIOS, VGA BIOS and keymaps. - # To list all the data directories, use -L help. - # ``` - # - # The reason we have to do this here, is because otherwise QEMU - # will only look for those files in specific paths, which are - # tied to the location of the PREFIX used during build time - # (/opt/kata, in our case). - cat <${qemu_binary_script_host_path} -#!/usr/bin/env bash - -exec ${qemu_binary} "\$@" -L ${dest_dir}/share/kata-${qemu_share}/qemu/ -EOF - chmod +x ${qemu_binary_script_host_path} - fi - - tomlq -i -t '.hypervisor.qemu.path = "'"${qemu_binary_script}"'"' "${config_path}" 2>/dev/null || true -} - -function get_hypervisor_name() { - local shim="${1}" - case "${shim}" in - qemu-runtime-rs | qemu-coco-dev-runtime-rs | qemu-se-runtime-rs | qemu | qemu-tdx | qemu-snp | qemu-se | qemu-coco-dev | qemu-cca | qemu-nvidia-gpu | qemu-nvidia-gpu-tdx | qemu-nvidia-gpu-snp) - echo "qemu" - ;; - clh) - echo "clh" - ;; - cloud-hypervisor) - echo "cloud-hypervisor" - ;; - dragonball) - echo "dragonball" - ;; - fc | firecracker) - echo "firecracker" - ;; - stratovirt) - echo "stratovirt" - ;; - remote) - echo "remote" - ;; - *) - # Default to the shim name itself if no specific mapping - echo "${shim}" - ;; - esac -} - -function install_artifacts() { - echo "copying kata artifacts onto host" - - mkdir -p ${host_install_dir} - cp -au /opt/kata-artifacts/opt/kata/* ${host_install_dir}/ - chmod +x ${host_install_dir}/bin/* - [ -d ${host_install_dir}/runtime-rs/bin ] && \ - chmod +x ${host_install_dir}/runtime-rs/bin/* - - local config_path - - for shim in "${shims[@]}"; do - config_path="/host/$(get_kata_containers_config_path "${shim}")" - mkdir -p "$config_path" - - local kata_config_file="${config_path}/configuration-${shim}.toml" - - # Till deprecation period is over, we need to support: - # * "http://proxy:8080" (applies to all shims) - # * per-shim format: "qemu-tdx=http://proxy:8080;qemu-snp=http://proxy2:8080" - if [ -n "${AGENT_HTTPS_PROXY}" ]; then - local https_proxy_value="" - - # Parse AGENT_HTTPS_PROXY - check if it contains "=" for per-shim format - if [[ "${AGENT_HTTPS_PROXY}" == *=* ]]; then - # Per-shim format: parse semicolon-separated "shim=proxy" mappings - IFS=';' read -ra proxy_mappings <<< "${AGENT_HTTPS_PROXY}" - for mapping in "${proxy_mappings[@]}"; do - local key="${mapping%%=*}" - local value="${mapping#*=}" - if [[ "${key}" == "${shim}" ]]; then - https_proxy_value="${value}" - break - fi - done - else - # Simple format: apply to all shims - https_proxy_value="${AGENT_HTTPS_PROXY}" - fi - - if [[ -n "${https_proxy_value}" ]]; then - local hypervisor_name=$(get_hypervisor_name "${shim}") - local current_params=$(tomlq -r '.hypervisor.'"${hypervisor_name}"'.kernel_params // ""' "${kata_config_file}" 2>/dev/null || echo "") - # Only add if not already present - if [[ "${current_params}" != *"agent.https_proxy"* ]]; then - local new_params="${current_params}" - [[ -n "${new_params}" ]] && new_params+=" " - new_params+="agent.https_proxy=${https_proxy_value}" - tomlq -i -t '.hypervisor.'"${hypervisor_name}"'.kernel_params = "'"${new_params}"'"' "${kata_config_file}" 2>/dev/null || true - fi - fi - fi - - # Till deprecation period is over, need to support: - # * "localhost,127.0.0.1" (applies to all shims) - # * per-shim format: "qemu-tdx=localhost,127.0.0.1;qemu-snp=192.168.0.0/16" - if [ -n "${AGENT_NO_PROXY}" ]; then - local no_proxy_value="" - - # Parse AGENT_NO_PROXY - check if it contains "=" for per-shim format - if [[ "${AGENT_NO_PROXY}" == *=* ]]; then - # Per-shim format: parse semicolon-separated "shim=no_proxy" mappings - IFS=';' read -ra noproxy_mappings <<< "${AGENT_NO_PROXY}" - for mapping in "${noproxy_mappings[@]}"; do - local key="${mapping%%=*}" - local value="${mapping#*=}" - if [[ "${key}" == "${shim}" ]]; then - no_proxy_value="${value}" - break - fi - done - else - # Simple format: apply to all shims - no_proxy_value="${AGENT_NO_PROXY}" - fi - - if [[ -n "${no_proxy_value}" ]]; then - local hypervisor_name=$(get_hypervisor_name "${shim}") - local current_params=$(tomlq -r '.hypervisor.'"${hypervisor_name}"'.kernel_params // ""' "${kata_config_file}" 2>/dev/null || echo "") - # Only add if not already present - if [[ "${current_params}" != *"agent.no_proxy"* ]]; then - local new_params="${current_params}" - [[ -n "${new_params}" ]] && new_params+=" " - new_params+="agent.no_proxy=${no_proxy_value}" - tomlq -i -t '.hypervisor.'"${hypervisor_name}"'.kernel_params = "'"${new_params}"'"' "${kata_config_file}" 2>/dev/null || true - fi - fi - fi - - # Allow enabling debug for Kata Containers - if [[ "${DEBUG}" == "true" ]]; then - local hypervisor_name=$(get_hypervisor_name "${shim}") - - # Only set these boolean flags if not already set to true - local current_enable_debug=$(tomlq -r '.hypervisor.'"${hypervisor_name}"'.enable_debug // false' "${kata_config_file}" 2>/dev/null || echo "false") - if [[ "${current_enable_debug}" != "true" ]]; then - tomlq -i -t '.hypervisor.'"${hypervisor_name}"'.enable_debug = true' "${kata_config_file}" 2>/dev/null || true - fi - - local current_runtime_debug=$(tomlq -r '.runtime.enable_debug // false' "${kata_config_file}" 2>/dev/null || echo "false") - if [[ "${current_runtime_debug}" != "true" ]]; then - tomlq -i -t '.runtime.enable_debug = true' "${kata_config_file}" 2>/dev/null || true - fi - - local current_debug_console=$(tomlq -r '.agent.kata.debug_console_enabled // false' "${kata_config_file}" 2>/dev/null || echo "false") - if [[ "${current_debug_console}" != "true" ]]; then - tomlq -i -t '.agent.kata.debug_console_enabled = true' "${kata_config_file}" 2>/dev/null || true - fi - - local current_agent_debug=$(tomlq -r '.agent.kata.enable_debug // false' "${kata_config_file}" 2>/dev/null || echo "false") - if [[ "${current_agent_debug}" != "true" ]]; then - tomlq -i -t '.agent.kata.enable_debug = true' "${kata_config_file}" 2>/dev/null || true - fi - - # Add debug kernel params if not already present - local current_params=$(tomlq -r '.hypervisor.'"${hypervisor_name}"'.kernel_params // ""' "${kata_config_file}" 2>/dev/null || echo "") - local debug_params="" - if [[ "${current_params}" != *"agent.log=debug"* ]]; then - debug_params+=" agent.log=debug" - fi - if [[ "${current_params}" != *"initcall_debug"* ]]; then - debug_params+=" initcall_debug" - fi - if [[ -n "${debug_params}" ]]; then - local new_params="${current_params}${debug_params}" - tomlq -i -t '.hypervisor.'"${hypervisor_name}"'.kernel_params = "'"${new_params}"'"' "${kata_config_file}" 2>/dev/null || true - fi - fi - - # Apply allowed_hypervisor_annotations: - # Here we need to support both cases of: - # * A list of annotations, which will be blindly applied to all shims - # * A per-shim list of annotations, which will only be applied to the specific shim - if [[ ${#hypervisor_annotations[@]} -gt 0 ]]; then - local shim_specific_annotations="" - local global_annotations="" - - for m in "${hypervisor_annotations[@]}"; do - # Check if this mapping has a colon (shim-specific) or not - if [[ "${m}" == *:* ]]; then - # Shim-specific mapping like "qemu:foo,bar" - local key="${m%:*}" - local value="${m#*:}" - - if [[ "${key}" != "${shim}" ]]; then - continue - fi - - if [[ -n "${shim_specific_annotations}" ]]; then - shim_specific_annotations+="," - fi - shim_specific_annotations+="${value}" - else - # All shims annotations like "foo bar" - if [[ -n "${global_annotations}" ]]; then - global_annotations+="," - fi - global_annotations+="$(echo "${m}" | sed 's/ /,/g')" - fi - done - - # Combine shim-specific and non-shim-specific annotations - local all_annotations="${global_annotations}" - if [[ -n "${shim_specific_annotations}" ]]; then - if [[ -n "${all_annotations}" ]]; then - all_annotations+="," - fi - all_annotations+="${shim_specific_annotations}" - fi - - if [[ -n "${all_annotations}" ]]; then - local hypervisor_name=$(get_hypervisor_name "${shim}") - local existing_annotations=$(get_field_array_values "${kata_config_file}" "enable_annotations" "${shim}") - - # Combine existing and new annotations - local combined_annotations="${existing_annotations}" - if [[ -n "${combined_annotations}" ]] && [[ -n "${all_annotations}" ]]; then - combined_annotations+=",${all_annotations}" - elif [[ -n "${all_annotations}" ]]; then - combined_annotations="${all_annotations}" - fi - - # Deduplicate all annotations - IFS=',' read -a annotations <<< "${combined_annotations}" - local -A seen_annotations - local unique_annotations=() - - for annotation in "${annotations[@]}"; do - # Trim whitespace - annotation=$(echo "${annotation}" | sed 's/^[[:space:]]//;s/[[:space:]]$//') - if [[ -n "${annotation}" ]] && [[ -z "${seen_annotations[${annotation}]+_}" ]]; then - seen_annotations["${annotation}"]=1 - unique_annotations+=("${annotation}") - fi - done - - if [[ ${#unique_annotations[@]} -gt 0 ]]; then - local formatted_annotations=() - for ann in "${unique_annotations[@]}"; do - formatted_annotations+=("\"${ann}\"") - done - local final_annotations=$(IFS=','; echo "${formatted_annotations[*]}") - tomlq -i -t '.hypervisor.'"${hypervisor_name}"'.enable_annotations = ['"${final_annotations}"']' "${kata_config_file}" 2>/dev/null || true - fi - fi - fi - - if printf '%s\n' "${experimental_force_guest_pull[@]}" | grep -Fxq "${shim}"; then - tomlq -i -t '.runtime.experimental_force_guest_pull = true' "${kata_config_file}" 2>/dev/null || true - fi - - if grep -q "tdx" <<< "$shim"; then - VERSION_ID=version_unset # VERSION_ID may be unset, see https://www.freedesktop.org/software/systemd/man/latest/os-release.html#Notes - source /host/etc/os-release || source /host/usr/lib/os-release - case ${ID} in - ubuntu) - case ${VERSION_ID} in - 24.04|25.04|25.10) - tdx_supported ${ID} ${VERSION_ID} ${kata_config_file} - ;; - *) - tdx_not_supported ${ID} ${VERSION_ID} - ;; - esac - ;; - centos) - case ${VERSION_ID} in - 9) - tdx_supported ${ID} ${VERSION_ID} ${kata_config_file} - ;; - *) - tdx_not_supported ${ID} ${VERSION_ID} - ;; - esac - ;; - *) - tdx_not_supported ${ID} ${VERSION_ID} - ;; - esac - fi - - if [ "${dest_dir}" != "${default_dest_dir}" ]; then - hypervisor="${shim}" - [[ "${shim}" == "qemu"* ]] && hypervisor="qemu" - - kernel_path=$(tomlq ".hypervisor.${hypervisor}.path" ${kata_config_file} | tr -d \") - if echo $kernel_path | grep -q "${dest_dir}"; then - # If we got to this point here, it means that we're dealing with - # a kata containers configuration file that has already been changed - # to support multi-install suffix, and we're here most likely due to - # and update or container restart, and we simply should not try to - # do anything else, thus just leave the conditional. - break - else - # We could always do this sed, regardless, but I have a strong preference - # on not touching the configuration files unless extremelly needed - sed -i -e "s|${default_dest_dir}|${dest_dir}|g" "${kata_config_file}" - - # Let's only adjust qemu_cmdline for the QEMUs that we build and ship ourselves - [[ "${shim}" =~ ^(qemu|qemu-runtime-rs|qemu-snp|qemu-nvidia-gpu|qemu-nvidia-gpu-snp|qemu-nvidia-gpu-tdx|qemu-se|qemu-se-runtime-rs|qemu-coco-dev|qemu-coco-dev-runtime-rs|qemu-cca)$ ]] && \ - adjust_qemu_cmdline "${shim}" "${kata_config_file}" - fi - fi - done - - # Allow Mariner to use custom configuration. - if [ "${HOST_OS:-}" == "cbl-mariner" ]; then - config_path="${host_install_dir}/share/defaults/kata-containers/configuration-clh.toml" - clh_path="${dest_dir}/bin/cloud-hypervisor-glibc" - local mariner_hypervisor_name="clh" - - tomlq -i -t '.hypervisor.'"${mariner_hypervisor_name}"'.static_sandbox_resource_mgmt = true' "${config_path}" 2>/dev/null || true - - # Append to valid_hypervisor_paths if not already present - local existing_paths=$(tomlq -r '.hypervisor.'"${mariner_hypervisor_name}"'.valid_hypervisor_paths // [] | .[]' "${config_path}" 2>/dev/null || echo "") - local path_exists=false - if [[ -n "${existing_paths}" ]]; then - while IFS= read -r path; do - path=$(echo "${path}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - if [[ "${path}" == "${clh_path}" ]]; then - path_exists=true - break - fi - done <<< "${existing_paths}" - fi - - if [[ "${path_exists}" == "false" ]]; then - local formatted_paths=() - if [[ -n "${existing_paths}" ]]; then - while IFS= read -r path; do - path=$(echo "${path}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - formatted_paths+=("\"${path}\"") - done <<< "${existing_paths}" - fi - formatted_paths+=("\"${clh_path}\"") - local final_paths=$(IFS=','; echo "${formatted_paths[*]}") - tomlq -i -t '.hypervisor.'"${mariner_hypervisor_name}"'.valid_hypervisor_paths = ['"${final_paths}"']' "${config_path}" 2>/dev/null || true - fi - - tomlq -i -t '.hypervisor.'"${mariner_hypervisor_name}"'.path = "'"${clh_path}"'"' "${config_path}" 2>/dev/null || true - fi - - if kubectl get crds nodefeaturerules.nfd.k8s-sigs.io &>/dev/null; then - arch="$(uname -m)" - if [[ ${arch} == "x86_64" ]]; then - node_feature_rule_file="/opt/kata-artifacts/node-feature-rules/${arch}-tee-keys.yaml" - - kubectl apply -f "${node_feature_rule_file}" - - info "As NFD is deployed on the node, rules for ${arch} TEEs have been created" - - # Patch existing runtime classes for TDX/SNP when NFD is detected - patch_runtimeclasses_for_nfd - fi - fi -} - -function wait_till_node_is_ready() { - local ready="False" - - while ! [[ "${ready}" == "True" ]]; do - sleep 2s - ready=$(kubectl get node $NODE_NAME -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') - done -} - -function restart_runtime() { - local runtime="${1}" - - if [ "${runtime}" == "k0s-worker" ] || [ "${runtime}" == "k0s-controller" ]; then - # do nothing, k0s will automatically load the config on the fly - : - elif [ "${runtime}" == "microk8s" ]; then - host_systemctl restart snap.microk8s.daemon-containerd.service - else - host_systemctl daemon-reload - host_systemctl restart "${runtime}" - fi - - wait_till_node_is_ready -} - -function configure_cri_runtime() { - local runtime="${1}" - - case "${runtime}" in - crio) - configure_crio - ;; - containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker | microk8s) - configure_containerd "${runtime}" - ;; - esac -} - -function configure_crio_runtime() { - local shim="${1}" - local adjusted_shim_to_multi_install="${shim}" - if [ -n "${MULTI_INSTALL_SUFFIX}" ]; then - adjusted_shim_to_multi_install="${shim}-${MULTI_INSTALL_SUFFIX}" - fi - local runtime="kata-${adjusted_shim_to_multi_install}" - local configuration="configuration-${shim}" - - local config_path=$(get_kata_containers_config_path "${shim}") - - local kata_path=$(get_kata_containers_runtime_path "${shim}") - local kata_conf="crio.runtime.runtimes.${runtime}" - local kata_config_path="${config_path}/${configuration}.toml" - - cat <" $containerd_root_conf_file; then - pluginid=\"io.containerd.grpc.v1.cri\" - fi - - if grep -q "version = 3\>" $containerd_root_conf_file; then - pluginid=\"io.containerd.cri.v1.runtime\" - fi - - local runtime_table=".plugins.${pluginid}.containerd.runtimes.\"${runtime}\"" - local runtime_options_table="${runtime_table}.options" - local runtime_type=\"io.containerd."${runtime}".v2\" - local runtime_config_path=\"$(get_kata_containers_config_path "${shim}")/${configuration}.toml\" - local runtime_path=\"$(get_kata_containers_runtime_path "${shim}")\" - - tomlq -i -t $(printf '%s.runtime_type=%s' ${runtime_table} ${runtime_type}) ${configuration_file} - tomlq -i -t $(printf '%s.runtime_path=%s' ${runtime_table} ${runtime_path}) ${configuration_file} - tomlq -i -t $(printf '%s.privileged_without_host_devices=true' ${runtime_table}) ${configuration_file} - if [[ "${shim}" == *"nvidia-gpu-"* ]]; then - tomlq -i -t $(printf '%s.pod_annotations=["io.katacontainers.*","cdi.k8s.io/*"]' ${runtime_table}) ${configuration_file} - else - tomlq -i -t $(printf '%s.pod_annotations=["io.katacontainers.*"]' ${runtime_table}) ${configuration_file} - fi - - tomlq -i -t $(printf '%s.ConfigPath=%s' ${runtime_options_table} ${runtime_config_path}) ${configuration_file} - - if [ "${DEBUG}" == "true" ]; then - tomlq -i -t '.debug.level = "debug"' ${configuration_file} - fi - - if [[ -n "${SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH}" ]]; then - for m in "${snapshotters[@]}"; do - key="${m%$snapshotters_delimiter*}" - - if [ "${key}" != "${shim}" ]; then - continue - fi - - value="${m#*$snapshotters_delimiter}" - if [[ "${value}" == "nydus" ]] && [[ -n "${MULTI_INSTALL_SUFFIX}" ]]; then - value="${value}-${MULTI_INSTALL_SUFFIX}" - fi - - tomlq -i -t $(printf '%s.snapshotter="%s"' ${runtime_table} ${value}) ${configuration_file} - break - done - fi -} - -function configure_containerd() { - # Configure containerd to use Kata: - echo "Add Kata Containers as a supported runtime for containerd" - - mkdir -p /etc/containerd/ - - if [ $use_containerd_drop_in_conf_file = "false" ] && [ -f "$containerd_conf_file" ]; then - # only backup in case drop-in files are not supported, and when doing the backup - # only do it if a backup doesn't already exist (don't override original) - cp -n "$containerd_conf_file" "$containerd_conf_file_backup" - fi - - if [ $use_containerd_drop_in_conf_file = "true" ]; then - if ! grep -q "${containerd_drop_in_conf_file}" ${containerd_conf_file}; then - tomlq -i -t $(printf '.imports|=.+["%s"]' ${containerd_drop_in_conf_file}) ${containerd_conf_file} - fi - fi - - for shim in "${shims[@]}"; do - configure_containerd_runtime "$1" $shim - done -} - -function remove_artifacts() { - echo "deleting kata artifacts" - - rm -rf ${host_install_dir} - - if kubectl get crds nodefeaturerules.nfd.k8s-sigs.io &>/dev/null; then - arch="$(uname -m)" - if [[ ${arch} == "x86_64" ]]; then - node_feature_rule_file="/opt/kata-artifacts/node-feature-rules/${arch}-tee-keys.yaml" - - kubectl delete --ignore-not-found -f "${node_feature_rule_file}" - - info "As NFD is deployed on the node, rules for ${arch} TEEs have been deleted" - fi - fi -} - -function restart_cri_runtime() { - local runtime="${1}" - - if [ "${runtime}" == "k0s-worker" ] || [ "${runtime}" == "k0s-controller" ]; then - # do nothing, k0s will automatically unload the config on the fly - : - elif [ "$1" == "microk8s" ]; then - host_systemctl restart snap.microk8s.daemon-containerd.service - else - host_systemctl daemon-reload - host_systemctl restart "${runtime}" - fi -} - -function cleanup_cri_runtime() { - case $1 in - crio) - cleanup_crio - ;; - containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker | microk8s) - cleanup_containerd - ;; - esac - - [ "${HELM_POST_DELETE_HOOK}" == "false" ] && return - - # Only run this code in the HELM_POST_DELETE_HOOK - restart_cri_runtime "$1" -} - -function cleanup_crio() { - rm -f $crio_drop_in_conf_file - if [[ "${DEBUG}" == "true" ]]; then - rm -f $crio_drop_in_conf_file_debug - fi -} - -function cleanup_containerd() { - if [ $use_containerd_drop_in_conf_file = "true" ]; then - # There's no need to remove the drop-in file, as it'll be removed as - # part of the artefacts removal. Thus, simply remove the file from - # the imports line of the containerd configuration and return. - tomlq -i -t $(printf '.imports|=.-["%s"]' ${containerd_drop_in_conf_file}) ${containerd_conf_file} - return - fi - - rm -f $containerd_conf_file - if [ -f "$containerd_conf_file_backup" ]; then - mv "$containerd_conf_file_backup" "$containerd_conf_file" - fi -} - -function reset_runtime() { - kubectl label node "$NODE_NAME" katacontainers.io/kata-runtime- - restart_cri_runtime "$1" - - if [ "$1" == "crio" ] || [ "$1" == "containerd" ]; then - host_systemctl restart kubelet - fi - - wait_till_node_is_ready -} - -function containerd_snapshotter_version_check() { - local container_runtime_version=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}') - local containerd_prefix="containerd://" - local containerd_version_to_avoid="1.6" - local containerd_version=${container_runtime_version#$containerd_prefix} - - if grep -q ^${containerd_version_to_avoid} <<< ${containerd_version}; then - if [[ -n "${SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH}" ]]; then - die "kata-deploy only supports snapshotter configuration with containerd 1.7 or newer" - fi - fi -} - -function containerd_erofs_snapshotter_version_check() { - local container_runtime_version=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}') - local containerd_prefix="containerd://" - local containerd_version=${container_runtime_version#$containerd_prefix} - local min_version_major="2" - local min_version_minor="2" - - # Extract major.minor (strip patch and prerelease stuff) - local major=${containerd_version%%.*} - local rest=${containerd_version#*.} - local minor=${rest%%[^0-9]*} - - if [ "${min_version_major}" -gt "${major}" ] || { [ "${min_version_major}" -eq "${major}" ] && [ "${min_version_minor}" -gt "${minor}" ]; }; then - die "In order to use erofs-snapshotter containerd must be 2.2.0 or newer" - fi -} - -function snapshotter_handler_mapping_validation_check() { - echo "Validating the snapshotter-handler mapping: \"${SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH}\"" - if [[ -z "${SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH}" ]]; then - echo "No snapshotter has been requested, using the default value from containerd" - return - fi - - for m in "${snapshotters[@]}"; do - shim="${m%$snapshotters_delimiter*}" - snapshotter="${m#*$snapshotters_delimiter}" - - if [ -z "${shim}" ]; then - die "The snapshotter must follow the \"shim:snapshotter,shim:snapshotter,...\" format, but at least one shim is empty" - fi - - if [ -z "${snapshotter}" ]; then - die "The snapshotter must follow the \"shim:snapshotter,shim:snapshotter,...\" format, but at least one snapshotter is empty" - fi - - if ! grep -q " ${shim} " <<< " ${SHIMS_FOR_ARCH} "; then - die "\"${shim}\" is not part of \"${SHIMS_FOR_ARCH}\"" - fi - - matches=$(grep -o "${shim}${snapshotters_delimiter}" <<< "${SNAPSHOTTER_HANDLER_MAPPING_FOR_ARCH}" | wc -l) - if [[ ${matches} -ne 1 ]]; then - die "One, and only one, entry per shim is required" - fi - done -} - -function configure_erofs_snapshotter() { - info "Configuring erofs-snapshotter" - - # As it's only supported with containerd 2.2.0 or newer - # we don't even care about the config file format, as - # it'll always be 3 (at least till version 4 is out). - # - # Also, drop-in is always supported on containerd 2.x - configuration_file="${1}" - - tomlq -i -t $(printf '.plugins."io.containerd.cri.v1.images".discard_unpacked_layers=false') ${configuration_file} - - tomlq -i -t $(printf '.plugins."io.containerd.service.v1.diff-service".default=["erofs","walking"]') ${configuration_file} - - tomlq -i -t $(printf '.plugins."io.containerd.snapshotter.v1.erofs".enable_fsverity=true') ${configuration_file} - tomlq -i -t $(printf '.plugins."io.containerd.snapshotter.v1.erofs".set_immutable=true') ${configuration_file} -} - -function configure_nydus_snapshotter() { - info "Configuring nydus-snapshotter" - - local nydus="nydus" - local containerd_nydus="nydus-snapshotter" - if [[ -n "${MULTI_INSTALL_SUFFIX}" ]]; then - nydus="${nydus}-${MULTI_INSTALL_SUFFIX}" - containerd_nydus="${containerd_nydus}-${MULTI_INSTALL_SUFFIX}" - fi - - configuration_file="${1}" - pluginid="${2}" - - tomlq -i -t $(printf '.plugins.%s.disable_snapshot_annotations=false' ${pluginid}) ${configuration_file} - - tomlq -i -t $(printf '.proxy_plugins."%s".type="snapshot"' ${nydus} ) ${configuration_file} - tomlq -i -t $(printf '.proxy_plugins."%s".address="/run/%s/containerd-nydus-grpc.sock"' ${nydus} ${containerd_nydus}) ${configuration_file} -} - -function configure_snapshotter() { - snapshotter="${1}" - - local runtime="$(get_container_runtime)" - local pluginid="\"io.containerd.grpc.v1.cri\".containerd" # version = 2 - local configuration_file="${containerd_conf_file}" - - # Properly set the configuration file in case drop-in files are supported - if [[ ${use_containerd_drop_in_conf_file} == "true" ]]; then - configuration_file="/host${containerd_drop_in_conf_file}" - fi - - local containerd_root_conf_file="${containerd_conf_file}" - if [[ "${runtime}" =~ ^(k0s-worker|k0s-controller)$ ]]; then - containerd_root_conf_file="/etc/containerd/containerd.toml" - fi - - if grep -q "version = 3\>" ${containerd_root_conf_file}; then - pluginid=\"io.containerd.cri.v1.images\" - fi - - case "${snapshotter}" in - nydus) - configure_nydus_snapshotter "${configuration_file}" "${pluginid}" - - nydus_snapshotter="nydus-snapshotter" - if [[ -n "${MULTI_INSTALL_SUFFIX}" ]]; then - nydus_snapshotter="${nydus_snapshotter}-${MULTI_INSTALL_SUFFIX}" - fi - host_systemctl restart "${nydus_snapshotter}" - ;; - erofs) - configure_erofs_snapshotter "${configuration_file}" - ;; - esac -} - -function install_nydus_snapshotter() { - info "Deploying nydus-snapshotter" - - local nydus_snapshotter="nydus-snapshotter" - if [[ -n "${MULTI_INSTALL_SUFFIX}" ]]; then - nydus_snapshotter="${nydus_snapshotter}-${MULTI_INSTALL_SUFFIX}" - fi - - local config_guest_pulling="/opt/kata-artifacts/nydus-snapshotter/config-guest-pulling.toml" - local nydus_snapshotter_service="/opt/kata-artifacts/nydus-snapshotter/nydus-snapshotter.service" - - # Adjust the paths for the config-guest-pulling.toml and nydus-snapshotter.service - sed -i -e "s|@SNAPSHOTTER_ROOT_DIR@|/var/lib/${nydus_snapshotter}|g" "${config_guest_pulling}" - sed -i -e "s|@SNAPSHOTTER_GRPC_SOCKET_ADDRESS@|/run/${nydus_snapshotter}/containerd-nydus-grpc.sock|g" "${config_guest_pulling}" - sed -i -e "s|@NYDUS_OVERLAYFS_PATH@|${host_install_dir#/host}/nydus-snapshotter/nydus-overlayfs|g" "${config_guest_pulling}" - - sed -i -e "s|@CONTAINERD_NYDUS_GRPC_BINARY@|${host_install_dir#/host}/nydus-snapshotter/containerd-nydus-grpc|g" "${nydus_snapshotter_service}" - sed -i -e "s|@CONFIG_GUEST_PULLING@|${host_install_dir#/host}/nydus-snapshotter/config-guest-pulling.toml|g" "${nydus_snapshotter_service}" - - mkdir -p "${host_install_dir}/nydus-snapshotter" - install -D -m 775 /opt/kata-artifacts/nydus-snapshotter/containerd-nydus-grpc "${host_install_dir}/nydus-snapshotter/containerd-nydus-grpc" - install -D -m 775 /opt/kata-artifacts/nydus-snapshotter/nydus-overlayfs "${host_install_dir}/nydus-snapshotter/nydus-overlayfs" - - install -D -m 644 "${config_guest_pulling}" "${host_install_dir}/nydus-snapshotter/config-guest-pulling.toml" - install -D -m 644 "${nydus_snapshotter_service}" "/host/etc/systemd/system/${nydus_snapshotter}.service" - - host_systemctl daemon-reload - host_systemctl enable "${nydus_snapshotter}.service" -} - -function uninstall_nydus_snapshotter() { - info "Removing deployed nydus-snapshotter" - - local nydus_snapshotter="nydus-snapshotter" - if [[ -n "${MULTI_INSTALL_SUFFIX}" ]]; then - nydus_snapshotter="${nydus_snapshotter}-${MULTI_INSTALL_SUFFIX}" - fi - - host_systemctl disable --now "${nydus_snapshotter}.service" - - rm -f "/host/etc/systemd/system/${nydus_snapshotter}.service" - rm -rf "${host_install_dir}/nydus-snapshotter" - - host_systemctl daemon-reload -} - -function install_snapshotter() { - snapshotter="${1}" - - case "${snapshotter}" in - erofs) ;; # it's a containerd's built-in snapshotter - nydus) install_nydus_snapshotter ;; - esac -} - -function uninstall_snapshotter() { - snapshotter="${1}" - - case "${snapshotter}" in - nydus) uninstall_nydus_snapshotter ;; - esac -} - -function main() { - action=${1:-} - if [ -z "$action" ]; then - print_usage - die "invalid arguments" - fi - - echo "Action:" - echo "* $action" - echo "" - echo "Environment variables passed to this script" - echo "* NODE_NAME: ${NODE_NAME}" - echo "* DEBUG: ${DEBUG}" - echo "* SHIMS_X86_64: ${SHIMS_X86_64}" - echo "* SHIMS_AARCH64: ${SHIMS_AARCH64}" - echo "* SHIMS_S390X: ${SHIMS_S390X}" - echo "* SHIMS_PPC64LE: ${SHIMS_PPC64LE}" - echo "* DEFAULT_SHIM_X86_64: ${DEFAULT_SHIM_X86_64}" - echo "* DEFAULT_SHIM_AARCH64: ${DEFAULT_SHIM_AARCH64}" - echo "* DEFAULT_SHIM_S390X: ${DEFAULT_SHIM_S390X}" - echo "* DEFAULT_SHIM_PPC64LE: ${DEFAULT_SHIM_PPC64LE}" - echo "* ALLOWED_HYPERVISOR_ANNOTATIONS_X86_64: ${ALLOWED_HYPERVISOR_ANNOTATIONS_X86_64}" - echo "* ALLOWED_HYPERVISOR_ANNOTATIONS_AARCH64: ${ALLOWED_HYPERVISOR_ANNOTATIONS_AARCH64}" - echo "* ALLOWED_HYPERVISOR_ANNOTATIONS_S390X: ${ALLOWED_HYPERVISOR_ANNOTATIONS_S390X}" - echo "* ALLOWED_HYPERVISOR_ANNOTATIONS_PPC64LE: ${ALLOWED_HYPERVISOR_ANNOTATIONS_PPC64LE}" - echo "* SNAPSHOTTER_HANDLER_MAPPING_X86_64: ${SNAPSHOTTER_HANDLER_MAPPING_X86_64}" - echo "* SNAPSHOTTER_HANDLER_MAPPING_AARCH64: ${SNAPSHOTTER_HANDLER_MAPPING_AARCH64}" - echo "* SNAPSHOTTER_HANDLER_MAPPING_S390X: ${SNAPSHOTTER_HANDLER_MAPPING_S390X}" - echo "* SNAPSHOTTER_HANDLER_MAPPING_PPC64LE: ${SNAPSHOTTER_HANDLER_MAPPING_PPC64LE}" - echo "* AGENT_HTTPS_PROXY: ${AGENT_HTTPS_PROXY}" - echo "* AGENT_NO_PROXY: ${AGENT_NO_PROXY}" - echo "* PULL_TYPE_MAPPING_X86_64: ${PULL_TYPE_MAPPING_X86_64}" - echo "* PULL_TYPE_MAPPING_AARCH64: ${PULL_TYPE_MAPPING_AARCH64}" - echo "* PULL_TYPE_MAPPING_S390X: ${PULL_TYPE_MAPPING_S390X}" - echo "* PULL_TYPE_MAPPING_PPC64LE: ${PULL_TYPE_MAPPING_PPC64LE}" - echo "* INSTALLATION_PREFIX: ${INSTALLATION_PREFIX}" - echo "* MULTI_INSTALL_SUFFIX: ${MULTI_INSTALL_SUFFIX}" - echo "* HELM_POST_DELETE_HOOK: ${HELM_POST_DELETE_HOOK}" - echo "* EXPERIMENTAL_SETUP_SNAPSHOTTER: ${EXPERIMENTAL_SETUP_SNAPSHOTTER}" - echo "* EXPERIMENTAL_FORCE_GUEST_PULL_X86_64: ${EXPERIMENTAL_FORCE_GUEST_PULL_X86_64}" - echo "* EXPERIMENTAL_FORCE_GUEST_PULL_AARCH64: ${EXPERIMENTAL_FORCE_GUEST_PULL_AARCH64}" - echo "* EXPERIMENTAL_FORCE_GUEST_PULL_S390X: ${EXPERIMENTAL_FORCE_GUEST_PULL_S390X}" - echo "* EXPERIMENTAL_FORCE_GUEST_PULL_PPC64LE: ${EXPERIMENTAL_FORCE_GUEST_PULL_PPC64LE}" - - # script requires that user is root - euid=$(id -u) - if [[ $euid -ne 0 ]]; then - die "This script must be run as root" - fi - - runtime=$(get_container_runtime) - - # CRI-O isn't consistent with the naming -- let's use crio to match the service file - if [ "$runtime" == "cri-o" ]; then - runtime="crio" - elif [ "$runtime" == "microk8s" ]; then - containerd_conf_file="/etc/containerd/containerd-template.toml" - containerd_conf_file_backup="${containerd_conf_file}.bak" - elif [[ "$runtime" =~ ^(k3s|k3s-agent|rke2-agent|rke2-server)$ ]]; then - containerd_conf_tmpl_file="${containerd_conf_file}.tmpl" - containerd_conf_file_backup="${containerd_conf_tmpl_file}.bak" - elif [[ "$runtime" =~ ^(k0s-worker|k0s-controller)$ ]]; then - # From 1.27.1 onwards k0s enables dynamic configuration on containerd CRI runtimes. - # This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets. - # k0s will automatically pick up these files and adds these in containerd configuration imports list. - containerd_conf_file="/etc/containerd/containerd.d/kata-containers.toml" - if [ -n "$MULTI_INSTALL_SUFFIX" ]; then - containerd_conf_file="/etc/containerd/containerd.d/kata-containers-$MULTI_INSTALL_SUFFIX.toml" - fi - containerd_conf_file_backup="${containerd_conf_tmpl_file}.bak" - fi - - # only install / remove / update if we are dealing with CRIO or containerd - if [[ "$runtime" =~ ^(crio|containerd|k3s|k3s-agent|rke2-agent|rke2-server|k0s-worker|k0s-controller|microk8s)$ ]]; then - if [ "$runtime" != "crio" ]; then - containerd_snapshotter_version_check - snapshotter_handler_mapping_validation_check - - use_containerd_drop_in_conf_file=$(is_containerd_capable_of_using_drop_in_files "$runtime") - echo "Using containerd drop-in files: $use_containerd_drop_in_conf_file" - - if [[ ! "$runtime" =~ ^(k0s-worker|k0s-controller)$ ]]; then - # We skip this check for k0s, as they handle things differently on their side - if [ -n "$MULTI_INSTALL_SUFFIX" ] && [ $use_containerd_drop_in_conf_file = "false" ]; then - die "Multi installation can only be done if $runtime supports drop-in configuration files" - fi - fi - fi - - case "$action" in - install) - # Let's fail early on this, so we don't need to do a rollback - # in case we reach this situation. - if [[ -n "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" ]]; then - if [[ "${runtime}" == "cri-o" ]]; then - warn "EXPERIMENTAL_SETUP_SNAPSHOTTER is being ignored!" - warn "Snapshotter is a containerd specific option." - else - for snapshotter in "${experimental_setup_snapshotter[@]}"; do - case "${snapshotter}" in - erofs) - containerd_erofs_snapshotter_version_check - ;; - nydus) - ;; - *) - die "${EXPERIMENTAL_SETUP_SNAPSHOTTER} is not a supported snapshotter by kata-deploy" - ;; - esac - done - fi - fi - - if [[ "$runtime" =~ ^(k3s|k3s-agent|rke2-agent|rke2-server)$ ]]; then - if [ ! -f "$containerd_conf_tmpl_file" ] && [ -f "$containerd_conf_file" ]; then - cp "$containerd_conf_file" "$containerd_conf_tmpl_file" - fi - # Only set the containerd_conf_file to its new value after - # copying the file to the template location - containerd_conf_file="${containerd_conf_tmpl_file}" - containerd_conf_file_backup="${containerd_conf_tmpl_file}.bak" - elif [[ "$runtime" =~ ^(k0s-worker|k0s-controller)$ ]]; then - mkdir -p $(dirname "$containerd_conf_file") - touch "$containerd_conf_file" - elif [[ "$runtime" == "containerd" ]]; then - if [ ! -f "$containerd_conf_file" ] && [ -d $(dirname "$containerd_conf_file") ]; then - host_exec containerd config default > "$containerd_conf_file" - fi - fi - - if [ $use_containerd_drop_in_conf_file = "true" ]; then - mkdir -p $(dirname "/host$containerd_drop_in_conf_file") - touch "/host$containerd_drop_in_conf_file" - fi - - install_artifacts - configure_cri_runtime "$runtime" - - for snapshotter in "${experimental_setup_snapshotter[@]}"; do - install_snapshotter "${snapshotter}" - configure_snapshotter "${snapshotter}" - done - restart_runtime "${runtime}" - kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=true - ;; - cleanup) - if [[ "$runtime" =~ ^(k3s|k3s-agent|rke2-agent|rke2-server)$ ]]; then - containerd_conf_file_backup="${containerd_conf_tmpl_file}.bak" - containerd_conf_file="${containerd_conf_tmpl_file}" - fi - - local kata_deploy_installations=$(kubectl -n kube-system get ds | grep kata-deploy | wc -l) - - if [ "${HELM_POST_DELETE_HOOK}" == "true" ]; then - # Remove the label as the first thing, so we ensure no more kata-containers - # pods would be scheduled here. - # - # If we still have any other installation here, it means we'll break them - # removing the label, so we just don't do it. - if [ $kata_deploy_installations -eq 0 ]; then - kubectl label node "$NODE_NAME" katacontainers.io/kata-runtime- - fi - fi - - for snapshotter in "${experimental_setup_snapshotter[@]}"; do - # Here we don't need to do any cleanup on the config, as kata-deploy - # will revert the configuration to the state it was before the deployment, - # which is also before the snapshotter configuration. :-) - uninstall_snapshotter "${EXPERIMENTAL_SETUP_SNAPSHOTTER}" - done - - cleanup_cri_runtime "$runtime" - if [ "${HELM_POST_DELETE_HOOK}" == "false" ]; then - # If we still have any other installation here, it means we'll break them - # removing the label, so we just don't do it. - if [ $kata_deploy_installations -eq 0 ]; then - # The Confidential Containers operator relies on this label - kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup - fi - fi - remove_artifacts - - if [ "${HELM_POST_DELETE_HOOK}" == "true" ]; then - # After everything was cleaned up, there's no reason to continue - # and sleep forever. Let's just return success.. - exit 0 - fi - ;; - reset) - reset_runtime $runtime - ;; - *) - print_usage - die "invalid arguments" - ;; - esac - fi - - #It is assumed this script will be called as a daemonset. As a result, do - # not return, otherwise the daemon will restart and rexecute the script - sleep infinity -} - -main "$@"