From 51ffafa0ac2da02286b08cc08afac813f7343507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 24 Apr 2026 14:07:56 +0200 Subject: [PATCH] kernel: Add MLX5 modules image build infrastructure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add config fragment, build script, and CI integration for building Mellanox MLX5/InfiniBand kernel modules as a standalone disk image. The orchestrator script (build-kernel-modules-images.sh) builds the kernel with extra module config fragments, runs modules_install, filters modules by subsystem into per-set staging trees, and packages each into its own disk image using build-modules-volume.sh. Since these modules are built within the Kata CI using the same KBUILD_SIGN_PIN, they are signed and loadable on the official released Kata kernel. Signed-off-by: Fabiano Fidêncio --- .../build-kata-static-tarball-amd64.yaml | 3 +- .../kernel/build-kernel-modules-images.sh | 206 ++++++++++++++++++ tools/packaging/kernel/build-kernel.sh | 151 +++++++++++++ .../configs/fragments/modules/mlx5.conf | 10 + .../kernel-modules-images/build.sh | 44 ++++ .../packaging/static-build/kernel/Dockerfile | 1 + tools/packaging/static-build/kernel/build.sh | 9 + 7 files changed, 423 insertions(+), 1 deletion(-) create mode 100755 tools/packaging/kernel/build-kernel-modules-images.sh create mode 100644 tools/packaging/kernel/configs/fragments/modules/mlx5.conf create mode 100755 tools/packaging/static-build/kernel-modules-images/build.sh diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index 18c3228d7c..a54f0082d2 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -49,6 +49,7 @@ jobs: - kernel - kernel-debug - kernel-dragonball-experimental + - kernel-modules-images - kernel-nvidia-gpu - nydus - ovmf @@ -103,7 +104,7 @@ jobs: ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} TARGET_BRANCH: ${{ inputs.target-branch }} RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }} - KBUILD_SIGN_PIN: ${{ contains(matrix.asset, 'nvidia') && secrets.KBUILD_SIGN_PIN || '' }} + KBUILD_SIGN_PIN: ${{ (contains(matrix.asset, 'nvidia') || matrix.asset == 'kernel-modules-images') && secrets.KBUILD_SIGN_PIN || '' }} - name: Parse OCI image name and digest id: parse-oci-segments diff --git a/tools/packaging/kernel/build-kernel-modules-images.sh b/tools/packaging/kernel/build-kernel-modules-images.sh new file mode 100755 index 0000000000..373368f0a7 --- /dev/null +++ b/tools/packaging/kernel/build-kernel-modules-images.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026 Kata Contributors +# +# SPDX-License-Identifier: Apache-2.0 +# +# Build kernel module disk images for kata guest VMs. +# +# This script: +# 1. Invokes build-kernel.sh to compile the kernel with extra module +# config fragments (e.g., MLNX, NTFS3). +# 2. Runs modules_install to collect all built modules. +# 3. Filters modules by subsystem into per-set staging directories. +# 4. Packages each set into a disk image using build-modules-volume.sh. +# +# The kernel binary itself is discarded; only the module images are +# the desired output. + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +packaging_scripts_dir="${script_dir}/../scripts" +# shellcheck source=tools/packaging/scripts/lib.sh +source "${packaging_scripts_dir}/lib.sh" + +readonly build_kernel="${script_dir}/build-kernel.sh" +readonly build_modules_volume="${script_dir}/build-modules-volume.sh" + +usage() { + cat < Target architecture (default: host arch). + -o Output directory for module images (default: PWD). + -V Enable dm-verity on module images. + -h Display this help. + +The script builds the kernel with all module-set config fragments, +then splits the resulting modules into per-set disk images: + + - mlx5: Mellanox MLX5 + InfiniBand drivers + +EOF + exit "${1:-0}" +} + +# Module sets: name -> array of source paths under lib/modules//kernel/ +# that belong to this set. +declare -A MODULE_SETS +MODULE_SETS[mlx5]="drivers/net/ethernet/mellanox drivers/infiniband" + +# Module set config fragments (relative to fragments/modules/) +declare -A MODULE_FRAGMENTS +MODULE_FRAGMENTS[mlx5]="mlx5.conf" + +output_dir="${PWD}" +enable_verity="false" +arch_target="" + +while getopts "a:o:Vh" opt; do + case "${opt}" in + a) arch_target="${OPTARG}" ;; + o) output_dir="${OPTARG}" ;; + V) enable_verity="true" ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done + +[[ -n "${arch_target}" ]] || arch_target="$(uname -m)" + +fragments_dir="${script_dir}/configs/fragments/modules" +for name in "${!MODULE_FRAGMENTS[@]}"; do + frag="${fragments_dir}/${MODULE_FRAGMENTS[${name}]}" + [[ -f "${frag}" ]] || die "Config fragment not found: ${frag}" +done + +kernel_version=$(get_from_kata_deps ".assets.kernel.version") +kernel_version="${kernel_version#v}" +config_version=$(cat "${script_dir}/kata_config_version") +kernel_path="${PWD}/kata-linux-modules-${kernel_version}-${config_version}" + +info "Building kernel ${kernel_version} with module configs" + +# Collect all module fragment paths as extra config files for build-kernel.sh +extra_configs="" +for name in "${!MODULE_FRAGMENTS[@]}"; do + extra_configs="${extra_configs} ${fragments_dir}/${MODULE_FRAGMENTS[${name}]}" +done + +# Build the kernel with module support + module signing + extra module fragments. +# We use -x (confidential) so that modules.conf and module_signing.conf are +# included, and KBUILD_SIGN_PIN (if set) is used to sign the modules. +# The -s flag skips redundant config checks since our module fragments may +# overlap with configs already set by the confidential build. +"${build_kernel}" \ + -a "${arch_target}" \ + -v "${kernel_version}" \ + -k "${kernel_path}" \ + -x \ + -s \ + -f \ + setup + +# Append module fragment configs to the generated .config and re-run olddefconfig +arch_kernel=$(case "${arch_target}" in + x86_64) echo "x86_64" ;; + aarch64) echo "arm64" ;; + s390x) echo "s390" ;; + ppc64le|powerpc64) echo "powerpc" ;; + *) echo "${arch_target}" ;; +esac) + +arch_frag_dir="${script_dir}/configs/fragments/${arch_kernel}" +config_path="${arch_frag_dir}/.config" + +info "Merging module config fragments into kernel config" +for name in "${!MODULE_FRAGMENTS[@]}"; do + frag="${fragments_dir}/${MODULE_FRAGMENTS[${name}]}" + info " Appending: ${frag}" + cat "${frag}" >> "${config_path}" +done + +cp "${config_path}" "${kernel_path}/.config" +make -C "${kernel_path}" ARCH="${arch_kernel}" olddefconfig + +info "Building kernel" +make -C "${kernel_path}" -j "$(nproc)" ARCH="${arch_kernel}" + +info "Installing modules" +make -C "${kernel_path}" -j "$(nproc)" \ + INSTALL_MOD_STRIP=1 \ + INSTALL_MOD_PATH="${kernel_path}" \ + modules_install + +# Find the installed modules version directory +modules_base="${kernel_path}/lib/modules" +mod_version=$(ls "${modules_base}" | head -1) +modules_tree="${modules_base}/${mod_version}" + +[[ -d "${modules_tree}/kernel" ]] || die "No modules installed at ${modules_tree}/kernel" + +info "Modules installed at ${modules_tree}" + +mkdir -p "${output_dir}" + +for name in "${!MODULE_SETS[@]}"; do + paths="${MODULE_SETS[${name}]}" + staging=$(mktemp -d) + staging_modules="${staging}/lib/modules/${mod_version}" + mkdir -p "${staging_modules}/kernel" + + has_modules="false" + for subpath in ${paths}; do + src="${modules_tree}/kernel/${subpath}" + if [[ -d "${src}" ]]; then + dst="${staging_modules}/kernel/${subpath}" + mkdir -p "$(dirname "${dst}")" + cp -a "${src}" "${dst}" + has_modules="true" + fi + done + + if [[ "${has_modules}" == "false" ]]; then + info "No modules found for set '${name}', skipping" + rm -rf "${staging}" + continue + fi + + # Copy metadata files needed by depmod + for f in modules.order modules.builtin; do + [[ -f "${modules_tree}/${f}" ]] && cp "${modules_tree}/${f}" "${staging_modules}/" + done + + # Generate modules.dep and related files for this subset + depmod -b "${staging}" "${mod_version}" + + info "Creating intermediate tarball for module set '${name}'" + tarball=$(mktemp --suffix=".tar.gz") + tar -czf "${tarball}" -C "${staging}" . + + info "Building disk image for module set '${name}'" + verity_flag="" + [[ "${enable_verity}" == "true" ]] && verity_flag="-V" + # shellcheck disable=SC2086 + "${build_modules_volume}" -m "${tarball}" -o "${output_dir}" ${verity_flag} + rm -f "${tarball}" + + # Rename generic output to per-set name + mv "${output_dir}/kata-modules-volume.img" "${output_dir}/kata-modules-${name}.img" + if [[ "${enable_verity}" == "true" ]] && [[ -f "${output_dir}/modules_verity_params.txt" ]]; then + mv "${output_dir}/modules_verity_params.txt" "${output_dir}/kata-modules-${name}-verity-params.txt" + fi + + info "Module image created: ${output_dir}/kata-modules-${name}.img" + rm -rf "${staging}" +done + +info "All module images built successfully" diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 55a1131387..1f3b033699 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -588,6 +588,154 @@ build_kernel_headers() { popd >>/dev/null } +build_modules_images() { + local kernel_path=${1:-} + [[ -n "${kernel_path}" ]] || die "kernel_path not provided" + [[ -d "${kernel_path}" ]] || die "path to kernel does not exist, use ${script_name} setup" + + if [[ "${gpu_vendor}" != "" ]]; then + info "Skipping module images for gpu-vendor kernel (modules are built in-tree)" + return 0 + fi + + if ! grep -q '^CONFIG_MODULES=y' "${kernel_path}/.config" 2>/dev/null; then + info "Skipping module images: CONFIG_MODULES not enabled for this kernel" + return 0 + fi + + [[ -n "${arch_target}" ]] || arch_target="$(uname -m)" + arch_target=$(arch_to_kernel "${arch_target}") + + local verity_flag="" + if [[ "${measured_rootfs}" == "true" ]]; then + verity_flag="-V" + fi + + local modules_frag_dir="${default_config_frags_dir}/modules" + [[ -d "${modules_frag_dir}" ]] || die "No modules fragment directory at ${modules_frag_dir}" + + local install_path + install_path=$(readlink -m "${DESTDIR}/${PREFIX}/share/${project_name}") + mkdir -p "${install_path}" + + local build_modules_volume="${script_dir}/build-modules-volume.sh" + [[ -x "${build_modules_volume}" ]] || die "build-modules-volume.sh not found at ${build_modules_volume}" + + declare -A module_sets + module_sets[mlx5]="drivers/net/ethernet/mellanox drivers/infiniband" + module_sets[ntfs]="fs/ntfs3" + + declare -A module_frags + module_frags[mlx5]="mlx5.conf" + module_frags[ntfs]="ntfs.conf" + + local frags_to_apply="" + for name in "${!module_frags[@]}"; do + local frag="${modules_frag_dir}/${module_frags[${name}]}" + [[ -f "${frag}" ]] || die "Config fragment not found: ${frag}" + frags_to_apply="${frags_to_apply} ${frag}" + done + + pushd "${kernel_path}" >>/dev/null + + for frag in ${frags_to_apply}; do + info " Appending: ${frag}" + cat "${frag}" >> .config + done + # shellcheck disable=SC2086 + make ARCH="${arch_target}" olddefconfig + + info "Rebuilding kernel with module fragment dependencies" + # shellcheck disable=SC2086 + make -j "$(nproc)" ARCH="${arch_target}" ${CROSS_BUILD_ARG} + + info "Installing modules" + make -j "$(nproc)" INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH="${kernel_path}" modules_install + + local modules_base="${kernel_path}/lib/modules" + local mod_version + mod_version=$(ls "${modules_base}" | head -1) + local modules_tree="${modules_base}/${mod_version}" + + [[ -d "${modules_tree}/kernel" ]] || die "No modules installed at ${modules_tree}/kernel" + info "Modules installed at ${modules_tree}" + + for name in "${!module_sets[@]}"; do + local paths="${module_sets[${name}]}" + local staging + staging=$(mktemp -d) + local staging_modules="${staging}/lib/modules/${mod_version}" + mkdir -p "${staging_modules}/kernel" + + local has_modules="false" + for subpath in ${paths}; do + local src="${modules_tree}/kernel/${subpath}" + if [[ -d "${src}" ]]; then + local dst="${staging_modules}/kernel/${subpath}" + mkdir -p "$(dirname "${dst}")" + cp -a "${src}" "${dst}" + has_modules="true" + fi + done + + if [[ "${has_modules}" == "false" ]]; then + info "No modules found for set '${name}', skipping" + rm -rf "${staging}" + continue + fi + + for f in modules.order modules.builtin; do + [[ -f "${modules_tree}/${f}" ]] && cp "${modules_tree}/${f}" "${staging_modules}/" + done + + depmod -b "${staging}" "${mod_version}" + + local tarball + tarball=$(mktemp --suffix=".tar.gz") + tar -czf "${tarball}" -C "${staging}" . + + # shellcheck disable=SC2086 + "${build_modules_volume}" -m "${tarball}" -o "${install_path}" ${verity_flag} + rm -f "${tarball}" + + mv "${install_path}/kata-modules-volume.img" "${install_path}/kata-modules-${name}.img" + if [[ -f "${install_path}/modules_verity_params.txt" ]]; then + mv "${install_path}/modules_verity_params.txt" "${install_path}/kata-modules-${name}-verity-params.txt" + info "Verity params: ${install_path}/kata-modules-${name}-verity-params.txt" + fi + info "Module image created: ${install_path}/kata-modules-${name}.img" + rm -rf "${staging}" + done + + if [[ "${has_any_modules}" == "true" ]]; then + info "Building combined module image (all module sets)" + for f in modules.order modules.builtin; do + [[ -f "${modules_tree}/${f}" ]] && cp "${modules_tree}/${f}" "${combined_modules}/" + done + + depmod -b "${combined_staging}" "${mod_version}" + + local combined_tarball + combined_tarball=$(mktemp --suffix=".tar.gz") + tar -czf "${combined_tarball}" -C "${combined_staging}" . + + # shellcheck disable=SC2086 + "${build_modules_volume}" -m "${combined_tarball}" -o "${install_path}" ${verity_flag} + rm -f "${combined_tarball}" + + mv "${install_path}/kata-modules-volume.img" "${install_path}/kata-modules-all.img" + if [[ -f "${install_path}/modules_verity_params.txt" ]]; then + mv "${install_path}/modules_verity_params.txt" "${install_path}/kata-modules-all-verity-params.txt" + info "Verity params: ${install_path}/kata-modules-all-verity-params.txt" + fi + info "Combined module image created: ${install_path}/kata-modules-all.img" + fi + rm -rf "${combined_staging}" + + popd >>/dev/null + info "All module images built successfully" +} + install_kata() { local kernel_path=${1:-} [[ -n "${kernel_path}" ]] || die "kernel_path not provided" @@ -795,6 +943,9 @@ main() { build-headers) build_kernel_headers "${kernel_path}" ;; + build-modules-images) + build_modules_images "${kernel_path}" + ;; install) install_kata "${kernel_path}" ;; diff --git a/tools/packaging/kernel/configs/fragments/modules/mlx5.conf b/tools/packaging/kernel/configs/fragments/modules/mlx5.conf new file mode 100644 index 0000000000..187abe6340 --- /dev/null +++ b/tools/packaging/kernel/configs/fragments/modules/mlx5.conf @@ -0,0 +1,10 @@ +# Mellanox ConnectX networking and InfiniBand drivers (as modules). +# Provides mlx5_core, mlx5_ib, and supporting InfiniBand stack. +CONFIG_NET_VENDOR_MELLANOX=y +CONFIG_MLX5_CORE=m +CONFIG_MLX5_CORE_EN=m +CONFIG_MLX5_EN_ARFS=y +CONFIG_MLX5_EN_RXNFC=y +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_MLX5_INFINIBAND=m diff --git a/tools/packaging/static-build/kernel-modules-images/build.sh b/tools/packaging/static-build/kernel-modules-images/build.sh new file mode 100755 index 0000000000..25a2f48bae --- /dev/null +++ b/tools/packaging/static-build/kernel-modules-images/build.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026 Kata Contributors +# +# SPDX-License-Identifier: Apache-2.0 +# +# Build kernel module disk images inside the kernel builder container. + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +source "${script_dir}/../../scripts/lib.sh" + +repo_root_dir="${repo_root_dir:-}" +[[ -n "${repo_root_dir}" ]] || die "repo_root_dir is not set" + +readonly modules_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel-modules-images.sh" + +DESTDIR=${DESTDIR:-${PWD}} +PREFIX=${PREFIX:-/opt/kata} +KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN:-}" + +output_dir="${DESTDIR}/${PREFIX}/share/kata-containers" +mkdir -p "${output_dir}" + +container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}" +container_engine="${CONTAINER_ENGINE:-docker}" + +"${container_engine}" pull "${container_image}" || \ + "${container_engine}" build \ + --build-arg "ARCH=${ARCH:-}" \ + -t "${container_image}" \ + "${script_dir}/../kernel" + +"${container_engine}" run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + --privileged \ + -w "${PWD}" \ + --env KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN}" \ + "${container_image}" \ + bash -c "${modules_builder} -a ${ARCH:-$(uname -m)} -o ${output_dir}" diff --git a/tools/packaging/static-build/kernel/Dockerfile b/tools/packaging/static-build/kernel/Dockerfile index bebbeb2d2d..c2360bc8cf 100644 --- a/tools/packaging/static-build/kernel/Dockerfile +++ b/tools/packaging/static-build/kernel/Dockerfile @@ -30,6 +30,7 @@ RUN apt-get update && \ gettext \ rsync \ cpio \ + cryptsetup-bin \ patch \ python3 && \ if [ "${ARCH}" != "$(uname -m)" ]; then apt-get install --no-install-recommends -y gcc-"${ARCH}"-linux-gnu binutils-"${ARCH}"-linux-gnu; fi && \ diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index 3e2297ecfe..7753593fd1 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -82,6 +82,15 @@ container_build+=" --build-arg ARCH=${ARCH:-}" "${container_image}" \ bash -c "${kernel_builder} ${kernel_builder_args} build" +"${container_engine}" run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + -w "${PWD}" \ + --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ + --env KERNEL_DEBUG_ENABLED="${KERNEL_DEBUG_ENABLED}" \ + --env KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN}" \ + --user "$(id -u)":"$(id -g)" \ + "${container_image}" \ + bash -c "${kernel_builder} ${kernel_builder_args} build-modules-images" + "${container_engine}" run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \