packaging: add generic devkit debug guest extension image

Add a self-contained "devkit" guest extension: a minimal Alpine (musl +
busybox + apk) rootfs with common debug tools prebaked (strace, ltrace,
iproute2, procps, lsof, tcpdump, pciutils, util-linux, ...), built as a
measured erofs+dm-verity image mounted at /run/kata-extensions/devkit.

The production guest rootfs is minimal (and, for some bases, shell-less), so
the agent debug console has no rich interactive shell. Rather than rebuilding
the whole rootfs with debug tooling, this optional extension can be cold-plugged
alongside any base image. At runtime the guest helper scripts overlay a
writable tmpfs on the read-only extension and chroot in, so apk and every tool
run natively against a normal root filesystem; `devkit-apk add <pkg>` (or plain
`apk add` inside the debug shell) installs anything else into the overlay.

Alpine's own busybox-static (/bin/busybox.static) bootstraps the overlay/chroot
from the shell-less base (the dynamic musl loader is not present there yet); it
is installed at a dedicated path so it never clobbers Alpine's /bin/busybox or
/bin/sh, which are needed unclobbered inside the chroot.

The image is assembled directly and handed to image_builder.sh (mirroring the
CoCo extension), never going through rootfs.sh: it ships no kata-agent, kernel
or driver userspace.

No runtime wiring is added here; that follows in later commits. Nothing
consumes the image yet.

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
Assisted-by: Cursor <cursoragent@cursor.com>

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -210,6 +210,7 @@ jobs:
           - rootfs-image
           - rootfs-image-confidential
           - rootfs-image-coco-extension
+          - rootfs-image-devkit-extension
           - rootfs-image-mariner
           - rootfs-image-nvidia-gpu
           - rootfs-image-nvidia-gpu-confidential

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -203,6 +203,7 @@ jobs:
           - rootfs-image
           - rootfs-image-confidential
           - rootfs-image-coco-extension
+          - rootfs-image-devkit-extension
           - rootfs-image-nvidia-gpu
           - rootfs-image-nvidia
           - rootfs-image-nvidia-gpu-extension

tools/osbuilder/rootfs-builder/devkit/devkit-apk.sh

@@ -0,0 +1,14 @@
+#!/run/kata-extensions/devkit/bin/busybox.static sh
+# shellcheck shell=dash
+#
+# Copyright (c) Kata Containers Community
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Thin apk wrapper: runs Alpine's apk inside the devkit overlay/chroot, so
+# `devkit-apk add htop` installs into the writable overlay at runtime.
+DEVKIT=/run/kata-extensions/devkit
+# shellcheck source=tools/osbuilder/rootfs-builder/devkit/devkit-init.sh
+. "${DEVKIT}/usr/bin/devkit-init"
+
+devkit_chroot_exec /sbin/apk "$@"

tools/osbuilder/rootfs-builder/devkit/devkit-enter.sh

@@ -0,0 +1,31 @@
+#!/run/kata-extensions/devkit/bin/busybox.static sh
+# shellcheck shell=dash
+#
+# Copyright (c) Kata Containers Community
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Interactive devkit debug shell: the debug_console_shell target, reached via
+# ${DEVKIT}/bin/devkit-sh -> this.
+#
+# NOTE: the bootstrap busybox ash has no builtin/PATH `[`, so use "${BB}" test.
+DEVKIT=/run/kata-extensions/devkit
+# BB and MERGED are exported by the sourced devkit-init, resolved at runtime.
+# shellcheck source=tools/osbuilder/rootfs-builder/devkit/devkit-init.sh
+. "${DEVKIT}/usr/bin/devkit-init"
+
+devkit_setup_chroot || {
+	"${BB}" echo "devkit: failed to set up chroot environment" >&2
+	exit 1
+}
+
+# Prefer the prebaked bash, falling back to busybox ash (/bin/sh) if bash is
+# missing or not executable in the merged tree.
+shell=/bin/bash
+"${BB}" test -x "${MERGED}${shell}" || shell=/bin/sh
+
+# The agent execs the shell with no arguments, so default to a login shell; any
+# arguments (-i, -c "cmd", ...) pass straight through.
+"${BB}" test "$#" -eq 0 && set -- -l
+
+exec "${BB}" chroot "${MERGED}" "${shell}" "$@"

tools/osbuilder/rootfs-builder/devkit/devkit-init.sh

@@ -0,0 +1,117 @@
+#!/run/kata-extensions/devkit/bin/busybox.static sh
+# shellcheck shell=dash
+#
+# Copyright (c) Kata Containers Community
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Shared devkit guest runtime library, sourced by devkit-enter and devkit-apk.
+#
+# The devkit extension (read-only at ${DEVKIT}) is a full minimal Alpine rootfs.
+# We overlay a writable, exec-enabled tmpfs on it and chroot in, so apk and the
+# prebaked tools run natively against a normal root filesystem.
+#
+# BB is Alpine's statically-linked busybox (busybox-static, /bin/busybox.static),
+# the ONLY binary we can exec on the shell-less guest base before the musl loader
+# exists. We deliberately do NOT reuse /bin/busybox: that is Alpine's dynamic
+# busybox providing the chroot's coreutils applets, and clobbering it breaks
+# ls/cat/ps and apk's busybox trigger.
+
+DEVKIT_INIT_VERSION=1
+
+DEVKIT="${DEVKIT:-/run/kata-extensions/devkit}"
+WRITABLE="${WRITABLE:-/run/kata-devkit-writable}"
+BB="${BB:-${DEVKIT}/bin/busybox.static}"
+
+UPPER="${WRITABLE}/upper"
+WORK="${WRITABLE}/work"
+MERGED="${WRITABLE}/merged"
+
+devkit_is_mounted() {
+	"${BB}" grep -q "[[:space:]]${1}[[:space:]]" /proc/mounts 2>/dev/null
+}
+
+# Mount an exec-enabled tmpfs at ${WRITABLE}: /run is typically noexec, so the
+# overlay upper/work (and the fallback rootfs copy) must live on our own tmpfs
+# for the prebaked and apk-installed binaries to exec.
+devkit_mount_writable() {
+	"${BB}" mkdir -p "${WRITABLE}"
+	devkit_is_mounted "${WRITABLE}" && return 0
+	"${BB}" mount -t tmpfs -o mode=755,size=2G tmpfs "${WRITABLE}"
+}
+
+# Overlay (devkit ro lower + tmpfs rw upper), falling back to a plain copy of the
+# rootfs into the tmpfs if this kernel refuses an overlay mount.
+devkit_mount_root() {
+	"${BB}" mkdir -p "${UPPER}" "${WORK}" "${MERGED}"
+	devkit_is_mounted "${MERGED}" && return 0
+	"${BB}" test -f "${WRITABLE}/.copied" && return 0
+
+	local ovl_err
+	if ovl_err="$("${BB}" mount -t overlay overlay "${MERGED}" \
+		-o "lowerdir=${DEVKIT},upperdir=${UPPER},workdir=${WORK}" 2>&1)" \
+		&& devkit_is_mounted "${MERGED}"; then
+		return 0
+	fi
+
+	# Fallback: overlay unavailable on this kernel - copy the rootfs into tmpfs.
+	"${BB}" echo "devkit: overlay mount unavailable (${ovl_err:-unknown}), copying rootfs into tmpfs" >&2
+	"${BB}" cp -a "${DEVKIT}/." "${MERGED}/"
+	"${BB}" touch "${WRITABLE}/.copied"
+}
+
+# Bind the kernel virtual filesystems so apk and the debug tools behave; /dev is
+# rbind'd to bring in pts/shm for interactive shells.
+devkit_bind_mounts() {
+	"${BB}" mkdir -p "${MERGED}/proc" "${MERGED}/sys" "${MERGED}/dev"
+
+	devkit_is_mounted "${MERGED}/proc" || "${BB}" mount -t proc proc "${MERGED}/proc"
+	devkit_is_mounted "${MERGED}/sys"  || "${BB}" mount -t sysfs sysfs "${MERGED}/sys"
+
+	if ! devkit_is_mounted "${MERGED}/dev"; then
+		if ! "${BB}" mount -o rbind /dev "${MERGED}/dev" 2>/dev/null; then
+			"${BB}" mount -t devtmpfs devtmpfs "${MERGED}/dev" 2>/dev/null \
+				|| "${BB}" mount -t tmpfs tmpfs "${MERGED}/dev"
+			"${BB}" mkdir -p "${MERGED}/dev/pts"
+			"${BB}" mount -t devpts devpts "${MERGED}/dev/pts" 2>/dev/null || true
+		fi
+	fi
+}
+
+# Give apk/curl working DNS by importing the guest resolver config.
+devkit_seed_resolv_conf() {
+	"${BB}" test -e /etc/resolv.conf || return 0
+	"${BB}" mkdir -p "${MERGED}/etc"
+	"${BB}" cp -L /etc/resolv.conf "${MERGED}/etc/resolv.conf" 2>/dev/null || true
+}
+
+# Expose the guest's real root as /real_root so container rootfses and other
+# guest state are reachable from the debug shell (e.g.
+# /real_root/run/kata-containers/<id>/rootfs).
+#
+# A symlink to /proc/1/root, not a bind mount: proc is mounted in the chroot, so
+# the kernel resolves it to PID 1's root regardless of the chroot, with no
+# bind-mount recursion or teardown ordering to worry about.
+devkit_link_real_root() {
+	"${BB}" test -e "${MERGED}/real_root" && return 0
+	"${BB}" ln -s /proc/1/root "${MERGED}/real_root" 2>/dev/null || true
+}
+
+# Idempotent: safe to call from every devkit-* invocation.
+devkit_setup_chroot() {
+	devkit_mount_writable || return 1
+	devkit_mount_root || return 1
+	devkit_bind_mounts || return 1
+	devkit_seed_resolv_conf
+	devkit_link_real_root
+	"${BB}" echo "${DEVKIT_INIT_VERSION}" > "${WRITABLE}/.initialized" 2>/dev/null || true
+	return 0
+}
+
+devkit_chroot_exec() {
+	devkit_setup_chroot || {
+		"${BB}" echo "devkit: failed to set up chroot environment" >&2
+		return 1
+	}
+	exec "${BB}" chroot "${MERGED}" "$@"
+}

tools/packaging/kata-deploy/local-build/Makefile

@@ -32,6 +32,7 @@ BASE_TARBALLS = serial-targets \
 BASE_SERIAL_TARBALLS = rootfs-image-tarball \
 	rootfs-image-confidential-tarball \
 	rootfs-image-coco-extension-tarball \
+	rootfs-image-devkit-extension-tarball \
 	rootfs-image-mariner-tarball \
 	rootfs-initrd-confidential-tarball \
 	rootfs-initrd-tarball \
@@ -58,6 +59,7 @@ BASE_TARBALLS = serial-targets \
 BASE_SERIAL_TARBALLS = rootfs-image-tarball \
 	rootfs-image-confidential-tarball \
 	rootfs-image-coco-extension-tarball \
+	rootfs-image-devkit-extension-tarball \
 	rootfs-initrd-tarball
 endif
 
@@ -262,6 +264,12 @@ DEPS := pause-image-tarball coco-guest-components-tarball
 rootfs-image-coco-extension-tarball: $(DEPS)
 	${MAKE} $@-build
 
+# Self-contained debug rootfs (no kata-agent, kernel or driver tree), so it has
+# no build-time dependency on any other target.
+DEPS :=
+rootfs-image-devkit-extension-tarball: $(DEPS)
+	${MAKE} $@-build
+
 DEPS := agent-tarball
 rootfs-image-mariner-tarball: $(DEPS)
 	${MAKE} $@-build

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -81,6 +81,7 @@ readonly MEASURED_ROOTFS_VARIANTS=(
 	base
 	confidential
 	coco-extension
+	devkit-extension
 	nvidia-gpu
 	nvidia-gpu-confidential
 	nvidia
@@ -740,6 +741,136 @@ EOF
 	rm -rf "${extension_rootfs}"
 }
 
+# Debug tools prebaked so they work offline; `apk add <pkg>` covers anything else
+# inside the chroot. busybox-static is the static /bin/busybox.static that
+# bootstraps the overlay/chroot from the shell-less guest base (see
+# devkit-init.sh); pciutils and util-linux let the guest's devices and namespaces
+# be inspected from the chroot. Kept lean: heavier tools are pulled on demand.
+readonly devkit_debug_packages="busybox-static bash ca-certificates strace ltrace iproute2 procps psmisc lsof tcpdump curl wget file less netcat-openbsd bind-tools pciutils util-linux"
+
+# Install the generic devkit debug extension image (erofs+verity): a self-contained
+# minimal Alpine rootfs with debug tools prebaked. It ships no kata-agent, kernel
+# or driver userspace, so - unlike the monolithic images - it is assembled here
+# and handed to image_builder.sh (like install_image_coco_extension), never via
+# rootfs.sh.
+install_image_devkit_extension() {
+	local component="rootfs-image-devkit-extension"
+
+	local branch ver mirror
+	branch="$(get_from_kata_deps ".externals.alpine.branch")"
+	ver="$(get_from_kata_deps ".externals.alpine.version")"
+	mirror="$(get_from_kata_deps ".externals.alpine.mirror")"
+	[[ -n "${branch}" ]] || die "externals.alpine.branch must be set in versions.yaml"
+	[[ -n "${ver}" ]] || die "externals.alpine.version must be set in versions.yaml"
+	[[ -n "${mirror}" ]] || mirror="https://dl-cdn.alpinelinux.org/alpine"
+
+	# Self-contained (Alpine release + guest scripts), so key the cache on the
+	# Alpine version and the devkit source directory.
+	local devkit_last_commit
+	devkit_last_commit="$(git -C "${repo_root_dir}" log -1 --abbrev=9 --pretty=format:"%h" \
+		-- tools/osbuilder/rootfs-builder/devkit 2>/dev/null || echo "unknown")"
+	latest_artefact="$(get_kata_version)-devkit-extension-${ver}-${devkit_last_commit}"
+	latest_builder_image=""
+
+	install_cached_tarball_component \
+		"${component}" \
+		"${latest_artefact}" \
+		"${latest_builder_image}" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
+	info "Create devkit extension image"
+
+	# Temp dir under the repo root so the path is valid both in the outer build
+	# container and the nested image-builder (Docker-in-Docker uses host paths).
+	local extension_rootfs
+	extension_rootfs="$(mktemp -d "${repo_root_dir}/.devkit-extension-rootfs.XXXX")"
+
+	# Alpine's arch naming matches ${ARCH} (x86_64/aarch64).
+	local tarball url
+	tarball="alpine-minirootfs-${ver}-${ARCH}.tar.gz"
+	url="${mirror}/${branch}/releases/${ARCH}/${tarball}"
+	info "Fetching Alpine minirootfs ${url}"
+	curl -fsSL -o "${extension_rootfs}/../${tarball}" "${url}" \
+		|| die "failed to download Alpine minirootfs ${url}"
+	tar -xzf "${extension_rootfs}/../${tarball}" -C "${extension_rootfs}"
+	rm -f "${extension_rootfs}/../${tarball}"
+
+	# Pin repositories and seed a resolver so apk can fetch indexes and content.
+	mkdir -p "${extension_rootfs}/etc/apk"
+	cat > "${extension_rootfs}/etc/apk/repositories" <<-EOF
+		${mirror}/${branch}/main
+		${mirror}/${branch}/community
+	EOF
+	cp -L /etc/resolv.conf "${extension_rootfs}/etc/resolv.conf" 2>/dev/null || true
+
+	# The build container is unprivileged, so a direct chroot is denied. Run apk
+	# in a privileged sibling container instead (Docker-in-Docker uses host paths,
+	# hence the repo-root temp dir): apk installs into --root but still runs
+	# package scripts chrooted, which needs the sibling's privileges. apk writes
+	# as root, so chown the tree back afterwards or the trim/cleanup and
+	# image_builder steps below hit "Permission denied".
+	info "Installing devkit debug toolset with apk"
+	# shellcheck disable=SC2086
+	docker run --rm --privileged \
+		-v "${extension_rootfs}:${extension_rootfs}" \
+		"docker.io/library/alpine:${ver}" \
+		sh -c "/sbin/apk add --no-cache --root '${extension_rootfs}' ${devkit_debug_packages} \
+			&& chown -R $(id -u):$(id -g) '${extension_rootfs}'" \
+		|| die "apk failed to install the devkit debug toolset"
+
+	# Install the guest-side helper scripts and expose the debug console entry as
+	# ${DEVKIT}/bin/devkit-sh. Do NOT reuse ${DEVKIT}/bin/sh: that is Alpine's own
+	# /bin/sh -> busybox symlink, needed unclobbered inside the chroot.
+	local devkit_src="${repo_root_dir}/tools/osbuilder/rootfs-builder/devkit"
+	local script dest
+	for script in devkit-init.sh devkit-enter.sh devkit-apk.sh; do
+		[[ -f "${devkit_src}/${script}" ]] || die "missing ${devkit_src}/${script}"
+		dest="${script%.sh}"
+		install -D -m0755 "${devkit_src}/${script}" "${extension_rootfs}/usr/bin/${dest}"
+	done
+	mkdir -p "${extension_rootfs}/bin"
+	ln -sf ../usr/bin/devkit-enter "${extension_rootfs}/bin/devkit-sh"
+
+	# Trim what a debug rootfs does not need (man/doc/info, apk cache) and the
+	# seeded resolver (devkit-init re-seeds one at runtime). Root inode must be
+	# 0755 (mktemp makes it 0700), else the mount point is unsearchable non-root.
+	rm -rf \
+		"${extension_rootfs}/usr/share/man"/* \
+		"${extension_rootfs}/usr/share/doc"/* \
+		"${extension_rootfs}/usr/share/info"/* \
+		"${extension_rootfs}/var/cache/apk"/* \
+		"${extension_rootfs}/etc/resolv.conf" \
+		2>/dev/null || true
+	chmod 0755 "${extension_rootfs}"
+
+	local install_dir="${destdir}/${prefix}/share/kata-containers/"
+	mkdir -p "${install_dir}"
+
+	local image_builder="${repo_root_dir}/tools/osbuilder/image-builder/image_builder.sh"
+
+	export USE_DOCKER="1"
+	export BUILD_VARIANT="devkit-extension"
+	export FS_TYPE="erofs"
+	# s390x does not use a measured rootfs, so no dm-verity hash is produced.
+	if [[ "${ARCH}" == "s390x" ]]; then
+		export MEASURED_ROOTFS="no"
+	else
+		export MEASURED_ROOTFS="yes"
+	fi
+	export SKIP_DAX_HEADER="yes"
+	export SKIP_ROOTFS_CHECK="yes"
+
+	"${image_builder}" -o "${install_dir}/kata-containers-devkit-extension.img" "${extension_rootfs}"
+
+	if [[ -e "${install_dir}/root_hash_devkit-extension.txt" ]]; then
+		info "Root hash file: ${install_dir}/root_hash_devkit-extension.txt"
+	fi
+
+	rm -rf "${extension_rootfs}"
+}
+
 #Install cbl-mariner guest image
 install_image_mariner() {
 	export IMAGE_SIZE_ALIGNMENT_MB=2
@@ -1718,6 +1849,8 @@ handle_build() {
 
 	rootfs-image-coco-extension) install_image_coco_extension ;;
 
+	rootfs-image-devkit-extension) install_image_devkit_extension ;;
+
 	rootfs-image-mariner) install_image_mariner ;;
 
 	rootfs-initrd) install_initrd ;;

versions.yaml

@@ -211,6 +211,12 @@ assets:
 externals:
   description: "Third-party projects used by the system"
 
+  alpine:
+    desc: "Alpine Linux minirootfs, base for the devkit debug extension"
+    branch: "v3.21"
+    version: "3.21.3"
+    mirror: "https://dl-cdn.alpinelinux.org/alpine"
+
   nvrc:
     # yamllint disable-line rule:line-length
     desc: "The NVRC project provides a Rust binary that implements a simple init system for microVMs"