From 9b816aa0acff6fc94e4cf44b0da322de6412fcd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 23 Jul 2026 11:47:41 +0200 Subject: [PATCH] packaging: add generic devkit debug guest extension image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a self-contained "devkit" guest extension: a minimal Alpine (musl + busybox + apk) rootfs with common debug tools prebaked (strace, ltrace, iproute2, procps, lsof, tcpdump, pciutils, util-linux, ...), built as a measured erofs+dm-verity image mounted at /run/kata-extensions/devkit. The production guest rootfs is minimal (and, for some bases, shell-less), so the agent debug console has no rich interactive shell. Rather than rebuilding the whole rootfs with debug tooling, this optional extension can be cold-plugged alongside any base image. At runtime the guest helper scripts overlay a writable tmpfs on the read-only extension and chroot in, so apk and every tool run natively against a normal root filesystem; `devkit-apk add ` (or plain `apk add` inside the debug shell) installs anything else into the overlay. Alpine's own busybox-static (/bin/busybox.static) bootstraps the overlay/chroot from the shell-less base (the dynamic musl loader is not present there yet); it is installed at a dedicated path so it never clobbers Alpine's /bin/busybox or /bin/sh, which are needed unclobbered inside the chroot. The image is assembled directly and handed to image_builder.sh (mirroring the CoCo extension), never going through rootfs.sh: it ships no kata-agent, kernel or driver userspace. No runtime wiring is added here; that follows in later commits. Nothing consumes the image yet. Signed-off-by: Fabiano Fidêncio Assisted-by: Cursor --- .../build-kata-static-tarball-amd64.yaml | 1 + .../build-kata-static-tarball-arm64.yaml | 1 + .../rootfs-builder/devkit/devkit-apk.sh | 14 ++ .../rootfs-builder/devkit/devkit-enter.sh | 31 ++++ .../rootfs-builder/devkit/devkit-init.sh | 117 +++++++++++++++ .../kata-deploy/local-build/Makefile | 8 ++ .../local-build/kata-deploy-binaries.sh | 133 ++++++++++++++++++ versions.yaml | 6 + 8 files changed, 311 insertions(+) create mode 100644 tools/osbuilder/rootfs-builder/devkit/devkit-apk.sh create mode 100644 tools/osbuilder/rootfs-builder/devkit/devkit-enter.sh create mode 100644 tools/osbuilder/rootfs-builder/devkit/devkit-init.sh diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index e57ce85670..357a698b2a 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -210,6 +210,7 @@ jobs: - rootfs-image - rootfs-image-confidential - rootfs-image-coco-extension + - rootfs-image-devkit-extension - rootfs-image-mariner - rootfs-image-nvidia-gpu - rootfs-image-nvidia-gpu-confidential diff --git a/.github/workflows/build-kata-static-tarball-arm64.yaml b/.github/workflows/build-kata-static-tarball-arm64.yaml index e153413e51..da18f47199 100644 --- a/.github/workflows/build-kata-static-tarball-arm64.yaml +++ b/.github/workflows/build-kata-static-tarball-arm64.yaml @@ -203,6 +203,7 @@ jobs: - rootfs-image - rootfs-image-confidential - rootfs-image-coco-extension + - rootfs-image-devkit-extension - rootfs-image-nvidia-gpu - rootfs-image-nvidia - rootfs-image-nvidia-gpu-extension diff --git a/tools/osbuilder/rootfs-builder/devkit/devkit-apk.sh b/tools/osbuilder/rootfs-builder/devkit/devkit-apk.sh new file mode 100644 index 0000000000..0a796512a7 --- /dev/null +++ b/tools/osbuilder/rootfs-builder/devkit/devkit-apk.sh @@ -0,0 +1,14 @@ +#!/run/kata-extensions/devkit/bin/busybox.static sh +# shellcheck shell=dash +# +# Copyright (c) Kata Containers Community +# +# SPDX-License-Identifier: Apache-2.0 +# +# Thin apk wrapper: runs Alpine's apk inside the devkit overlay/chroot, so +# `devkit-apk add htop` installs into the writable overlay at runtime. +DEVKIT=/run/kata-extensions/devkit +# shellcheck source=tools/osbuilder/rootfs-builder/devkit/devkit-init.sh +. "${DEVKIT}/usr/bin/devkit-init" + +devkit_chroot_exec /sbin/apk "$@" diff --git a/tools/osbuilder/rootfs-builder/devkit/devkit-enter.sh b/tools/osbuilder/rootfs-builder/devkit/devkit-enter.sh new file mode 100644 index 0000000000..7ba0efb534 --- /dev/null +++ b/tools/osbuilder/rootfs-builder/devkit/devkit-enter.sh @@ -0,0 +1,31 @@ +#!/run/kata-extensions/devkit/bin/busybox.static sh +# shellcheck shell=dash +# +# Copyright (c) Kata Containers Community +# +# SPDX-License-Identifier: Apache-2.0 +# +# Interactive devkit debug shell: the debug_console_shell target, reached via +# ${DEVKIT}/bin/devkit-sh -> this. +# +# NOTE: the bootstrap busybox ash has no builtin/PATH `[`, so use "${BB}" test. +DEVKIT=/run/kata-extensions/devkit +# BB and MERGED are exported by the sourced devkit-init, resolved at runtime. +# shellcheck source=tools/osbuilder/rootfs-builder/devkit/devkit-init.sh +. "${DEVKIT}/usr/bin/devkit-init" + +devkit_setup_chroot || { + "${BB}" echo "devkit: failed to set up chroot environment" >&2 + exit 1 +} + +# Prefer the prebaked bash, falling back to busybox ash (/bin/sh) if bash is +# missing or not executable in the merged tree. +shell=/bin/bash +"${BB}" test -x "${MERGED}${shell}" || shell=/bin/sh + +# The agent execs the shell with no arguments, so default to a login shell; any +# arguments (-i, -c "cmd", ...) pass straight through. +"${BB}" test "$#" -eq 0 && set -- -l + +exec "${BB}" chroot "${MERGED}" "${shell}" "$@" diff --git a/tools/osbuilder/rootfs-builder/devkit/devkit-init.sh b/tools/osbuilder/rootfs-builder/devkit/devkit-init.sh new file mode 100644 index 0000000000..2a945fb33f --- /dev/null +++ b/tools/osbuilder/rootfs-builder/devkit/devkit-init.sh @@ -0,0 +1,117 @@ +#!/run/kata-extensions/devkit/bin/busybox.static sh +# shellcheck shell=dash +# +# Copyright (c) Kata Containers Community +# +# SPDX-License-Identifier: Apache-2.0 +# +# Shared devkit guest runtime library, sourced by devkit-enter and devkit-apk. +# +# The devkit extension (read-only at ${DEVKIT}) is a full minimal Alpine rootfs. +# We overlay a writable, exec-enabled tmpfs on it and chroot in, so apk and the +# prebaked tools run natively against a normal root filesystem. +# +# BB is Alpine's statically-linked busybox (busybox-static, /bin/busybox.static), +# the ONLY binary we can exec on the shell-less guest base before the musl loader +# exists. We deliberately do NOT reuse /bin/busybox: that is Alpine's dynamic +# busybox providing the chroot's coreutils applets, and clobbering it breaks +# ls/cat/ps and apk's busybox trigger. + +DEVKIT_INIT_VERSION=1 + +DEVKIT="${DEVKIT:-/run/kata-extensions/devkit}" +WRITABLE="${WRITABLE:-/run/kata-devkit-writable}" +BB="${BB:-${DEVKIT}/bin/busybox.static}" + +UPPER="${WRITABLE}/upper" +WORK="${WRITABLE}/work" +MERGED="${WRITABLE}/merged" + +devkit_is_mounted() { + "${BB}" grep -q "[[:space:]]${1}[[:space:]]" /proc/mounts 2>/dev/null +} + +# Mount an exec-enabled tmpfs at ${WRITABLE}: /run is typically noexec, so the +# overlay upper/work (and the fallback rootfs copy) must live on our own tmpfs +# for the prebaked and apk-installed binaries to exec. +devkit_mount_writable() { + "${BB}" mkdir -p "${WRITABLE}" + devkit_is_mounted "${WRITABLE}" && return 0 + "${BB}" mount -t tmpfs -o mode=755,size=2G tmpfs "${WRITABLE}" +} + +# Overlay (devkit ro lower + tmpfs rw upper), falling back to a plain copy of the +# rootfs into the tmpfs if this kernel refuses an overlay mount. +devkit_mount_root() { + "${BB}" mkdir -p "${UPPER}" "${WORK}" "${MERGED}" + devkit_is_mounted "${MERGED}" && return 0 + "${BB}" test -f "${WRITABLE}/.copied" && return 0 + + local ovl_err + if ovl_err="$("${BB}" mount -t overlay overlay "${MERGED}" \ + -o "lowerdir=${DEVKIT},upperdir=${UPPER},workdir=${WORK}" 2>&1)" \ + && devkit_is_mounted "${MERGED}"; then + return 0 + fi + + # Fallback: overlay unavailable on this kernel - copy the rootfs into tmpfs. + "${BB}" echo "devkit: overlay mount unavailable (${ovl_err:-unknown}), copying rootfs into tmpfs" >&2 + "${BB}" cp -a "${DEVKIT}/." "${MERGED}/" + "${BB}" touch "${WRITABLE}/.copied" +} + +# Bind the kernel virtual filesystems so apk and the debug tools behave; /dev is +# rbind'd to bring in pts/shm for interactive shells. +devkit_bind_mounts() { + "${BB}" mkdir -p "${MERGED}/proc" "${MERGED}/sys" "${MERGED}/dev" + + devkit_is_mounted "${MERGED}/proc" || "${BB}" mount -t proc proc "${MERGED}/proc" + devkit_is_mounted "${MERGED}/sys" || "${BB}" mount -t sysfs sysfs "${MERGED}/sys" + + if ! devkit_is_mounted "${MERGED}/dev"; then + if ! "${BB}" mount -o rbind /dev "${MERGED}/dev" 2>/dev/null; then + "${BB}" mount -t devtmpfs devtmpfs "${MERGED}/dev" 2>/dev/null \ + || "${BB}" mount -t tmpfs tmpfs "${MERGED}/dev" + "${BB}" mkdir -p "${MERGED}/dev/pts" + "${BB}" mount -t devpts devpts "${MERGED}/dev/pts" 2>/dev/null || true + fi + fi +} + +# Give apk/curl working DNS by importing the guest resolver config. +devkit_seed_resolv_conf() { + "${BB}" test -e /etc/resolv.conf || return 0 + "${BB}" mkdir -p "${MERGED}/etc" + "${BB}" cp -L /etc/resolv.conf "${MERGED}/etc/resolv.conf" 2>/dev/null || true +} + +# Expose the guest's real root as /real_root so container rootfses and other +# guest state are reachable from the debug shell (e.g. +# /real_root/run/kata-containers//rootfs). +# +# A symlink to /proc/1/root, not a bind mount: proc is mounted in the chroot, so +# the kernel resolves it to PID 1's root regardless of the chroot, with no +# bind-mount recursion or teardown ordering to worry about. +devkit_link_real_root() { + "${BB}" test -e "${MERGED}/real_root" && return 0 + "${BB}" ln -s /proc/1/root "${MERGED}/real_root" 2>/dev/null || true +} + +# Idempotent: safe to call from every devkit-* invocation. +devkit_setup_chroot() { + devkit_mount_writable || return 1 + devkit_mount_root || return 1 + devkit_bind_mounts || return 1 + devkit_seed_resolv_conf + devkit_link_real_root + "${BB}" echo "${DEVKIT_INIT_VERSION}" > "${WRITABLE}/.initialized" 2>/dev/null || true + return 0 +} + +devkit_chroot_exec() { + devkit_setup_chroot || { + "${BB}" echo "devkit: failed to set up chroot environment" >&2 + return 1 + } + exec "${BB}" chroot "${MERGED}" "$@" +} diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index 3680e16a7a..d055a60263 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -32,6 +32,7 @@ BASE_TARBALLS = serial-targets \ BASE_SERIAL_TARBALLS = rootfs-image-tarball \ rootfs-image-confidential-tarball \ rootfs-image-coco-extension-tarball \ + rootfs-image-devkit-extension-tarball \ rootfs-image-mariner-tarball \ rootfs-initrd-confidential-tarball \ rootfs-initrd-tarball \ @@ -58,6 +59,7 @@ BASE_TARBALLS = serial-targets \ BASE_SERIAL_TARBALLS = rootfs-image-tarball \ rootfs-image-confidential-tarball \ rootfs-image-coco-extension-tarball \ + rootfs-image-devkit-extension-tarball \ rootfs-initrd-tarball endif @@ -262,6 +264,12 @@ DEPS := pause-image-tarball coco-guest-components-tarball rootfs-image-coco-extension-tarball: $(DEPS) ${MAKE} $@-build +# Self-contained debug rootfs (no kata-agent, kernel or driver tree), so it has +# no build-time dependency on any other target. +DEPS := +rootfs-image-devkit-extension-tarball: $(DEPS) + ${MAKE} $@-build + DEPS := agent-tarball rootfs-image-mariner-tarball: $(DEPS) ${MAKE} $@-build diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index f212554ad5..792853b0a0 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -81,6 +81,7 @@ readonly MEASURED_ROOTFS_VARIANTS=( base confidential coco-extension + devkit-extension nvidia-gpu nvidia-gpu-confidential nvidia @@ -740,6 +741,136 @@ EOF rm -rf "${extension_rootfs}" } +# Debug tools prebaked so they work offline; `apk add ` covers anything else +# inside the chroot. busybox-static is the static /bin/busybox.static that +# bootstraps the overlay/chroot from the shell-less guest base (see +# devkit-init.sh); pciutils and util-linux let the guest's devices and namespaces +# be inspected from the chroot. Kept lean: heavier tools are pulled on demand. +readonly devkit_debug_packages="busybox-static bash ca-certificates strace ltrace iproute2 procps psmisc lsof tcpdump curl wget file less netcat-openbsd bind-tools pciutils util-linux" + +# Install the generic devkit debug extension image (erofs+verity): a self-contained +# minimal Alpine rootfs with debug tools prebaked. It ships no kata-agent, kernel +# or driver userspace, so - unlike the monolithic images - it is assembled here +# and handed to image_builder.sh (like install_image_coco_extension), never via +# rootfs.sh. +install_image_devkit_extension() { + local component="rootfs-image-devkit-extension" + + local branch ver mirror + branch="$(get_from_kata_deps ".externals.alpine.branch")" + ver="$(get_from_kata_deps ".externals.alpine.version")" + mirror="$(get_from_kata_deps ".externals.alpine.mirror")" + [[ -n "${branch}" ]] || die "externals.alpine.branch must be set in versions.yaml" + [[ -n "${ver}" ]] || die "externals.alpine.version must be set in versions.yaml" + [[ -n "${mirror}" ]] || mirror="https://dl-cdn.alpinelinux.org/alpine" + + # Self-contained (Alpine release + guest scripts), so key the cache on the + # Alpine version and the devkit source directory. + local devkit_last_commit + devkit_last_commit="$(git -C "${repo_root_dir}" log -1 --abbrev=9 --pretty=format:"%h" \ + -- tools/osbuilder/rootfs-builder/devkit 2>/dev/null || echo "unknown")" + latest_artefact="$(get_kata_version)-devkit-extension-${ver}-${devkit_last_commit}" + latest_builder_image="" + + install_cached_tarball_component \ + "${component}" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + + info "Create devkit extension image" + + # Temp dir under the repo root so the path is valid both in the outer build + # container and the nested image-builder (Docker-in-Docker uses host paths). + local extension_rootfs + extension_rootfs="$(mktemp -d "${repo_root_dir}/.devkit-extension-rootfs.XXXX")" + + # Alpine's arch naming matches ${ARCH} (x86_64/aarch64). + local tarball url + tarball="alpine-minirootfs-${ver}-${ARCH}.tar.gz" + url="${mirror}/${branch}/releases/${ARCH}/${tarball}" + info "Fetching Alpine minirootfs ${url}" + curl -fsSL -o "${extension_rootfs}/../${tarball}" "${url}" \ + || die "failed to download Alpine minirootfs ${url}" + tar -xzf "${extension_rootfs}/../${tarball}" -C "${extension_rootfs}" + rm -f "${extension_rootfs}/../${tarball}" + + # Pin repositories and seed a resolver so apk can fetch indexes and content. + mkdir -p "${extension_rootfs}/etc/apk" + cat > "${extension_rootfs}/etc/apk/repositories" <<-EOF + ${mirror}/${branch}/main + ${mirror}/${branch}/community + EOF + cp -L /etc/resolv.conf "${extension_rootfs}/etc/resolv.conf" 2>/dev/null || true + + # The build container is unprivileged, so a direct chroot is denied. Run apk + # in a privileged sibling container instead (Docker-in-Docker uses host paths, + # hence the repo-root temp dir): apk installs into --root but still runs + # package scripts chrooted, which needs the sibling's privileges. apk writes + # as root, so chown the tree back afterwards or the trim/cleanup and + # image_builder steps below hit "Permission denied". + info "Installing devkit debug toolset with apk" + # shellcheck disable=SC2086 + docker run --rm --privileged \ + -v "${extension_rootfs}:${extension_rootfs}" \ + "docker.io/library/alpine:${ver}" \ + sh -c "/sbin/apk add --no-cache --root '${extension_rootfs}' ${devkit_debug_packages} \ + && chown -R $(id -u):$(id -g) '${extension_rootfs}'" \ + || die "apk failed to install the devkit debug toolset" + + # Install the guest-side helper scripts and expose the debug console entry as + # ${DEVKIT}/bin/devkit-sh. Do NOT reuse ${DEVKIT}/bin/sh: that is Alpine's own + # /bin/sh -> busybox symlink, needed unclobbered inside the chroot. + local devkit_src="${repo_root_dir}/tools/osbuilder/rootfs-builder/devkit" + local script dest + for script in devkit-init.sh devkit-enter.sh devkit-apk.sh; do + [[ -f "${devkit_src}/${script}" ]] || die "missing ${devkit_src}/${script}" + dest="${script%.sh}" + install -D -m0755 "${devkit_src}/${script}" "${extension_rootfs}/usr/bin/${dest}" + done + mkdir -p "${extension_rootfs}/bin" + ln -sf ../usr/bin/devkit-enter "${extension_rootfs}/bin/devkit-sh" + + # Trim what a debug rootfs does not need (man/doc/info, apk cache) and the + # seeded resolver (devkit-init re-seeds one at runtime). Root inode must be + # 0755 (mktemp makes it 0700), else the mount point is unsearchable non-root. + rm -rf \ + "${extension_rootfs}/usr/share/man"/* \ + "${extension_rootfs}/usr/share/doc"/* \ + "${extension_rootfs}/usr/share/info"/* \ + "${extension_rootfs}/var/cache/apk"/* \ + "${extension_rootfs}/etc/resolv.conf" \ + 2>/dev/null || true + chmod 0755 "${extension_rootfs}" + + local install_dir="${destdir}/${prefix}/share/kata-containers/" + mkdir -p "${install_dir}" + + local image_builder="${repo_root_dir}/tools/osbuilder/image-builder/image_builder.sh" + + export USE_DOCKER="1" + export BUILD_VARIANT="devkit-extension" + export FS_TYPE="erofs" + # s390x does not use a measured rootfs, so no dm-verity hash is produced. + if [[ "${ARCH}" == "s390x" ]]; then + export MEASURED_ROOTFS="no" + else + export MEASURED_ROOTFS="yes" + fi + export SKIP_DAX_HEADER="yes" + export SKIP_ROOTFS_CHECK="yes" + + "${image_builder}" -o "${install_dir}/kata-containers-devkit-extension.img" "${extension_rootfs}" + + if [[ -e "${install_dir}/root_hash_devkit-extension.txt" ]]; then + info "Root hash file: ${install_dir}/root_hash_devkit-extension.txt" + fi + + rm -rf "${extension_rootfs}" +} + #Install cbl-mariner guest image install_image_mariner() { export IMAGE_SIZE_ALIGNMENT_MB=2 @@ -1718,6 +1849,8 @@ handle_build() { rootfs-image-coco-extension) install_image_coco_extension ;; + rootfs-image-devkit-extension) install_image_devkit_extension ;; + rootfs-image-mariner) install_image_mariner ;; rootfs-initrd) install_initrd ;; diff --git a/versions.yaml b/versions.yaml index 0b893ee72a..6f0a3bba39 100644 --- a/versions.yaml +++ b/versions.yaml @@ -211,6 +211,12 @@ assets: externals: description: "Third-party projects used by the system" + alpine: + desc: "Alpine Linux minirootfs, base for the devkit debug extension" + branch: "v3.21" + version: "3.21.3" + mirror: "https://dl-cdn.alpinelinux.org/alpine" + nvrc: # yamllint disable-line rule:line-length desc: "The NVRC project provides a Rust binary that implements a simple init system for microVMs"