mirror of
https://github.com/kata-containers/kata-containers.git
synced 2026-05-04 12:31:27 +00:00
kernel: Add MLX5 modules image build infrastructure
Add config fragment, build script, and CI integration for building Mellanox MLX5/InfiniBand kernel modules as a standalone disk image. The orchestrator script (build-kernel-modules-images.sh) builds the kernel with extra module config fragments, runs modules_install, filters modules by subsystem into per-set staging trees, and packages each into its own disk image using build-modules-volume.sh. Since these modules are built within the Kata CI using the same KBUILD_SIGN_PIN, they are signed and loadable on the official released Kata kernel. Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
This commit is contained in:
Fabiano Fidêncio
@@ -49,6 +49,7 @@ jobs:
|
||||
- kernel
|
||||
- kernel-debug
|
||||
- kernel-dragonball-experimental
|
||||
- kernel-modules-images
|
||||
- kernel-nvidia-gpu
|
||||
- nydus
|
||||
- ovmf
|
||||
@@ -103,7 +104,7 @@ jobs:
|
||||
ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
|
||||
TARGET_BRANCH: ${{ inputs.target-branch }}
|
||||
RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
|
||||
KBUILD_SIGN_PIN: ${{ contains(matrix.asset, 'nvidia') && secrets.KBUILD_SIGN_PIN || '' }}
|
||||
KBUILD_SIGN_PIN: ${{ (contains(matrix.asset, 'nvidia') || matrix.asset == 'kernel-modules-images') && secrets.KBUILD_SIGN_PIN || '' }}
|
||||
|
||||
- name: Parse OCI image name and digest
|
||||
id: parse-oci-segments
|
||||
|
||||
206
tools/packaging/kernel/build-kernel-modules-images.sh
Executable file
206
tools/packaging/kernel/build-kernel-modules-images.sh
Executable file
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Copyright (c) 2026 Kata Contributors
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Build kernel module disk images for kata guest VMs.
|
||||
#
|
||||
# This script:
|
||||
# 1. Invokes build-kernel.sh to compile the kernel with extra module
|
||||
# config fragments (e.g., MLNX, NTFS3).
|
||||
# 2. Runs modules_install to collect all built modules.
|
||||
# 3. Filters modules by subsystem into per-set staging directories.
|
||||
# 4. Packages each set into a disk image using build-modules-volume.sh.
|
||||
#
|
||||
# The kernel binary itself is discarded; only the module images are
|
||||
# the desired output.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
packaging_scripts_dir="${script_dir}/../scripts"
|
||||
# shellcheck source=tools/packaging/scripts/lib.sh
|
||||
source "${packaging_scripts_dir}/lib.sh"
|
||||
|
||||
readonly build_kernel="${script_dir}/build-kernel.sh"
|
||||
readonly build_modules_volume="${script_dir}/build-modules-volume.sh"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Build kernel module disk images for kata guest VMs.
|
||||
|
||||
Usage:
|
||||
$(basename "$0") [options]
|
||||
|
||||
Options:
|
||||
-a <arch> Target architecture (default: host arch).
|
||||
-o <dir> Output directory for module images (default: PWD).
|
||||
-V Enable dm-verity on module images.
|
||||
-h Display this help.
|
||||
|
||||
The script builds the kernel with all module-set config fragments,
|
||||
then splits the resulting modules into per-set disk images:
|
||||
|
||||
- mlx5: Mellanox MLX5 + InfiniBand drivers
|
||||
|
||||
EOF
|
||||
exit "${1:-0}"
|
||||
}
|
||||
|
||||
# Module sets: name -> array of source paths under lib/modules/<ver>/kernel/
|
||||
# that belong to this set.
|
||||
declare -A MODULE_SETS
|
||||
MODULE_SETS[mlx5]="drivers/net/ethernet/mellanox drivers/infiniband"
|
||||
|
||||
# Module set config fragments (relative to fragments/modules/)
|
||||
declare -A MODULE_FRAGMENTS
|
||||
MODULE_FRAGMENTS[mlx5]="mlx5.conf"
|
||||
|
||||
output_dir="${PWD}"
|
||||
enable_verity="false"
|
||||
arch_target=""
|
||||
|
||||
while getopts "a:o:Vh" opt; do
|
||||
case "${opt}" in
|
||||
a) arch_target="${OPTARG}" ;;
|
||||
o) output_dir="${OPTARG}" ;;
|
||||
V) enable_verity="true" ;;
|
||||
h) usage 0 ;;
|
||||
*) usage 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "${arch_target}" ]] || arch_target="$(uname -m)"
|
||||
|
||||
fragments_dir="${script_dir}/configs/fragments/modules"
|
||||
for name in "${!MODULE_FRAGMENTS[@]}"; do
|
||||
frag="${fragments_dir}/${MODULE_FRAGMENTS[${name}]}"
|
||||
[[ -f "${frag}" ]] || die "Config fragment not found: ${frag}"
|
||||
done
|
||||
|
||||
kernel_version=$(get_from_kata_deps ".assets.kernel.version")
|
||||
kernel_version="${kernel_version#v}"
|
||||
config_version=$(cat "${script_dir}/kata_config_version")
|
||||
kernel_path="${PWD}/kata-linux-modules-${kernel_version}-${config_version}"
|
||||
|
||||
info "Building kernel ${kernel_version} with module configs"
|
||||
|
||||
# Collect all module fragment paths as extra config files for build-kernel.sh
|
||||
extra_configs=""
|
||||
for name in "${!MODULE_FRAGMENTS[@]}"; do
|
||||
extra_configs="${extra_configs} ${fragments_dir}/${MODULE_FRAGMENTS[${name}]}"
|
||||
done
|
||||
|
||||
# Build the kernel with module support + module signing + extra module fragments.
|
||||
# We use -x (confidential) so that modules.conf and module_signing.conf are
|
||||
# included, and KBUILD_SIGN_PIN (if set) is used to sign the modules.
|
||||
# The -s flag skips redundant config checks since our module fragments may
|
||||
# overlap with configs already set by the confidential build.
|
||||
"${build_kernel}" \
|
||||
-a "${arch_target}" \
|
||||
-v "${kernel_version}" \
|
||||
-k "${kernel_path}" \
|
||||
-x \
|
||||
-s \
|
||||
-f \
|
||||
setup
|
||||
|
||||
# Append module fragment configs to the generated .config and re-run olddefconfig
|
||||
arch_kernel=$(case "${arch_target}" in
|
||||
x86_64) echo "x86_64" ;;
|
||||
aarch64) echo "arm64" ;;
|
||||
s390x) echo "s390" ;;
|
||||
ppc64le|powerpc64) echo "powerpc" ;;
|
||||
*) echo "${arch_target}" ;;
|
||||
esac)
|
||||
|
||||
arch_frag_dir="${script_dir}/configs/fragments/${arch_kernel}"
|
||||
config_path="${arch_frag_dir}/.config"
|
||||
|
||||
info "Merging module config fragments into kernel config"
|
||||
for name in "${!MODULE_FRAGMENTS[@]}"; do
|
||||
frag="${fragments_dir}/${MODULE_FRAGMENTS[${name}]}"
|
||||
info " Appending: ${frag}"
|
||||
cat "${frag}" >> "${config_path}"
|
||||
done
|
||||
|
||||
cp "${config_path}" "${kernel_path}/.config"
|
||||
make -C "${kernel_path}" ARCH="${arch_kernel}" olddefconfig
|
||||
|
||||
info "Building kernel"
|
||||
make -C "${kernel_path}" -j "$(nproc)" ARCH="${arch_kernel}"
|
||||
|
||||
info "Installing modules"
|
||||
make -C "${kernel_path}" -j "$(nproc)" \
|
||||
INSTALL_MOD_STRIP=1 \
|
||||
INSTALL_MOD_PATH="${kernel_path}" \
|
||||
modules_install
|
||||
|
||||
# Find the installed modules version directory
|
||||
modules_base="${kernel_path}/lib/modules"
|
||||
mod_version=$(ls "${modules_base}" | head -1)
|
||||
modules_tree="${modules_base}/${mod_version}"
|
||||
|
||||
[[ -d "${modules_tree}/kernel" ]] || die "No modules installed at ${modules_tree}/kernel"
|
||||
|
||||
info "Modules installed at ${modules_tree}"
|
||||
|
||||
mkdir -p "${output_dir}"
|
||||
|
||||
for name in "${!MODULE_SETS[@]}"; do
|
||||
paths="${MODULE_SETS[${name}]}"
|
||||
staging=$(mktemp -d)
|
||||
staging_modules="${staging}/lib/modules/${mod_version}"
|
||||
mkdir -p "${staging_modules}/kernel"
|
||||
|
||||
has_modules="false"
|
||||
for subpath in ${paths}; do
|
||||
src="${modules_tree}/kernel/${subpath}"
|
||||
if [[ -d "${src}" ]]; then
|
||||
dst="${staging_modules}/kernel/${subpath}"
|
||||
mkdir -p "$(dirname "${dst}")"
|
||||
cp -a "${src}" "${dst}"
|
||||
has_modules="true"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${has_modules}" == "false" ]]; then
|
||||
info "No modules found for set '${name}', skipping"
|
||||
rm -rf "${staging}"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Copy metadata files needed by depmod
|
||||
for f in modules.order modules.builtin; do
|
||||
[[ -f "${modules_tree}/${f}" ]] && cp "${modules_tree}/${f}" "${staging_modules}/"
|
||||
done
|
||||
|
||||
# Generate modules.dep and related files for this subset
|
||||
depmod -b "${staging}" "${mod_version}"
|
||||
|
||||
info "Creating intermediate tarball for module set '${name}'"
|
||||
tarball=$(mktemp --suffix=".tar.gz")
|
||||
tar -czf "${tarball}" -C "${staging}" .
|
||||
|
||||
info "Building disk image for module set '${name}'"
|
||||
verity_flag=""
|
||||
[[ "${enable_verity}" == "true" ]] && verity_flag="-V"
|
||||
# shellcheck disable=SC2086
|
||||
"${build_modules_volume}" -m "${tarball}" -o "${output_dir}" ${verity_flag}
|
||||
rm -f "${tarball}"
|
||||
|
||||
# Rename generic output to per-set name
|
||||
mv "${output_dir}/kata-modules-volume.img" "${output_dir}/kata-modules-${name}.img"
|
||||
if [[ "${enable_verity}" == "true" ]] && [[ -f "${output_dir}/modules_verity_params.txt" ]]; then
|
||||
mv "${output_dir}/modules_verity_params.txt" "${output_dir}/kata-modules-${name}-verity-params.txt"
|
||||
fi
|
||||
|
||||
info "Module image created: ${output_dir}/kata-modules-${name}.img"
|
||||
rm -rf "${staging}"
|
||||
done
|
||||
|
||||
info "All module images built successfully"
|
||||
@@ -588,6 +588,154 @@ build_kernel_headers() {
|
||||
popd >>/dev/null
|
||||
}
|
||||
|
||||
build_modules_images() {
|
||||
local kernel_path=${1:-}
|
||||
[[ -n "${kernel_path}" ]] || die "kernel_path not provided"
|
||||
[[ -d "${kernel_path}" ]] || die "path to kernel does not exist, use ${script_name} setup"
|
||||
|
||||
if [[ "${gpu_vendor}" != "" ]]; then
|
||||
info "Skipping module images for gpu-vendor kernel (modules are built in-tree)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! grep -q '^CONFIG_MODULES=y' "${kernel_path}/.config" 2>/dev/null; then
|
||||
info "Skipping module images: CONFIG_MODULES not enabled for this kernel"
|
||||
return 0
|
||||
fi
|
||||
|
||||
[[ -n "${arch_target}" ]] || arch_target="$(uname -m)"
|
||||
arch_target=$(arch_to_kernel "${arch_target}")
|
||||
|
||||
local verity_flag=""
|
||||
if [[ "${measured_rootfs}" == "true" ]]; then
|
||||
verity_flag="-V"
|
||||
fi
|
||||
|
||||
local modules_frag_dir="${default_config_frags_dir}/modules"
|
||||
[[ -d "${modules_frag_dir}" ]] || die "No modules fragment directory at ${modules_frag_dir}"
|
||||
|
||||
local install_path
|
||||
install_path=$(readlink -m "${DESTDIR}/${PREFIX}/share/${project_name}")
|
||||
mkdir -p "${install_path}"
|
||||
|
||||
local build_modules_volume="${script_dir}/build-modules-volume.sh"
|
||||
[[ -x "${build_modules_volume}" ]] || die "build-modules-volume.sh not found at ${build_modules_volume}"
|
||||
|
||||
declare -A module_sets
|
||||
module_sets[mlx5]="drivers/net/ethernet/mellanox drivers/infiniband"
|
||||
module_sets[ntfs]="fs/ntfs3"
|
||||
|
||||
declare -A module_frags
|
||||
module_frags[mlx5]="mlx5.conf"
|
||||
module_frags[ntfs]="ntfs.conf"
|
||||
|
||||
local frags_to_apply=""
|
||||
for name in "${!module_frags[@]}"; do
|
||||
local frag="${modules_frag_dir}/${module_frags[${name}]}"
|
||||
[[ -f "${frag}" ]] || die "Config fragment not found: ${frag}"
|
||||
frags_to_apply="${frags_to_apply} ${frag}"
|
||||
done
|
||||
|
||||
pushd "${kernel_path}" >>/dev/null
|
||||
|
||||
for frag in ${frags_to_apply}; do
|
||||
info " Appending: ${frag}"
|
||||
cat "${frag}" >> .config
|
||||
done
|
||||
# shellcheck disable=SC2086
|
||||
make ARCH="${arch_target}" olddefconfig
|
||||
|
||||
info "Rebuilding kernel with module fragment dependencies"
|
||||
# shellcheck disable=SC2086
|
||||
make -j "$(nproc)" ARCH="${arch_target}" ${CROSS_BUILD_ARG}
|
||||
|
||||
info "Installing modules"
|
||||
make -j "$(nproc)" INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH="${kernel_path}" modules_install
|
||||
|
||||
local modules_base="${kernel_path}/lib/modules"
|
||||
local mod_version
|
||||
mod_version=$(ls "${modules_base}" | head -1)
|
||||
local modules_tree="${modules_base}/${mod_version}"
|
||||
|
||||
[[ -d "${modules_tree}/kernel" ]] || die "No modules installed at ${modules_tree}/kernel"
|
||||
info "Modules installed at ${modules_tree}"
|
||||
|
||||
for name in "${!module_sets[@]}"; do
|
||||
local paths="${module_sets[${name}]}"
|
||||
local staging
|
||||
staging=$(mktemp -d)
|
||||
local staging_modules="${staging}/lib/modules/${mod_version}"
|
||||
mkdir -p "${staging_modules}/kernel"
|
||||
|
||||
local has_modules="false"
|
||||
for subpath in ${paths}; do
|
||||
local src="${modules_tree}/kernel/${subpath}"
|
||||
if [[ -d "${src}" ]]; then
|
||||
local dst="${staging_modules}/kernel/${subpath}"
|
||||
mkdir -p "$(dirname "${dst}")"
|
||||
cp -a "${src}" "${dst}"
|
||||
has_modules="true"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${has_modules}" == "false" ]]; then
|
||||
info "No modules found for set '${name}', skipping"
|
||||
rm -rf "${staging}"
|
||||
continue
|
||||
fi
|
||||
|
||||
for f in modules.order modules.builtin; do
|
||||
[[ -f "${modules_tree}/${f}" ]] && cp "${modules_tree}/${f}" "${staging_modules}/"
|
||||
done
|
||||
|
||||
depmod -b "${staging}" "${mod_version}"
|
||||
|
||||
local tarball
|
||||
tarball=$(mktemp --suffix=".tar.gz")
|
||||
tar -czf "${tarball}" -C "${staging}" .
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
"${build_modules_volume}" -m "${tarball}" -o "${install_path}" ${verity_flag}
|
||||
rm -f "${tarball}"
|
||||
|
||||
mv "${install_path}/kata-modules-volume.img" "${install_path}/kata-modules-${name}.img"
|
||||
if [[ -f "${install_path}/modules_verity_params.txt" ]]; then
|
||||
mv "${install_path}/modules_verity_params.txt" "${install_path}/kata-modules-${name}-verity-params.txt"
|
||||
info "Verity params: ${install_path}/kata-modules-${name}-verity-params.txt"
|
||||
fi
|
||||
info "Module image created: ${install_path}/kata-modules-${name}.img"
|
||||
rm -rf "${staging}"
|
||||
done
|
||||
|
||||
if [[ "${has_any_modules}" == "true" ]]; then
|
||||
info "Building combined module image (all module sets)"
|
||||
for f in modules.order modules.builtin; do
|
||||
[[ -f "${modules_tree}/${f}" ]] && cp "${modules_tree}/${f}" "${combined_modules}/"
|
||||
done
|
||||
|
||||
depmod -b "${combined_staging}" "${mod_version}"
|
||||
|
||||
local combined_tarball
|
||||
combined_tarball=$(mktemp --suffix=".tar.gz")
|
||||
tar -czf "${combined_tarball}" -C "${combined_staging}" .
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
"${build_modules_volume}" -m "${combined_tarball}" -o "${install_path}" ${verity_flag}
|
||||
rm -f "${combined_tarball}"
|
||||
|
||||
mv "${install_path}/kata-modules-volume.img" "${install_path}/kata-modules-all.img"
|
||||
if [[ -f "${install_path}/modules_verity_params.txt" ]]; then
|
||||
mv "${install_path}/modules_verity_params.txt" "${install_path}/kata-modules-all-verity-params.txt"
|
||||
info "Verity params: ${install_path}/kata-modules-all-verity-params.txt"
|
||||
fi
|
||||
info "Combined module image created: ${install_path}/kata-modules-all.img"
|
||||
fi
|
||||
rm -rf "${combined_staging}"
|
||||
|
||||
popd >>/dev/null
|
||||
info "All module images built successfully"
|
||||
}
|
||||
|
||||
install_kata() {
|
||||
local kernel_path=${1:-}
|
||||
[[ -n "${kernel_path}" ]] || die "kernel_path not provided"
|
||||
@@ -795,6 +943,9 @@ main() {
|
||||
build-headers)
|
||||
build_kernel_headers "${kernel_path}"
|
||||
;;
|
||||
build-modules-images)
|
||||
build_modules_images "${kernel_path}"
|
||||
;;
|
||||
install)
|
||||
install_kata "${kernel_path}"
|
||||
;;
|
||||
|
||||
10
tools/packaging/kernel/configs/fragments/modules/mlx5.conf
Normal file
10
tools/packaging/kernel/configs/fragments/modules/mlx5.conf
Normal file
@@ -0,0 +1,10 @@
|
||||
# Mellanox ConnectX networking and InfiniBand drivers (as modules).
|
||||
# Provides mlx5_core, mlx5_ib, and supporting InfiniBand stack.
|
||||
CONFIG_NET_VENDOR_MELLANOX=y
|
||||
CONFIG_MLX5_CORE=m
|
||||
CONFIG_MLX5_CORE_EN=m
|
||||
CONFIG_MLX5_EN_ARFS=y
|
||||
CONFIG_MLX5_EN_RXNFC=y
|
||||
CONFIG_INFINIBAND=m
|
||||
CONFIG_INFINIBAND_USER_MAD=m
|
||||
CONFIG_MLX5_INFINIBAND=m
|
||||
44
tools/packaging/static-build/kernel-modules-images/build.sh
Executable file
44
tools/packaging/static-build/kernel-modules-images/build.sh
Executable file
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Copyright (c) 2026 Kata Contributors
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Build kernel module disk images inside the kernel builder container.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source "${script_dir}/../../scripts/lib.sh"
|
||||
|
||||
repo_root_dir="${repo_root_dir:-}"
|
||||
[[ -n "${repo_root_dir}" ]] || die "repo_root_dir is not set"
|
||||
|
||||
readonly modules_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel-modules-images.sh"
|
||||
|
||||
DESTDIR=${DESTDIR:-${PWD}}
|
||||
PREFIX=${PREFIX:-/opt/kata}
|
||||
KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN:-}"
|
||||
|
||||
output_dir="${DESTDIR}/${PREFIX}/share/kata-containers"
|
||||
mkdir -p "${output_dir}"
|
||||
|
||||
container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}"
|
||||
container_engine="${CONTAINER_ENGINE:-docker}"
|
||||
|
||||
"${container_engine}" pull "${container_image}" || \
|
||||
"${container_engine}" build \
|
||||
--build-arg "ARCH=${ARCH:-}" \
|
||||
-t "${container_image}" \
|
||||
"${script_dir}/../kernel"
|
||||
|
||||
"${container_engine}" run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
|
||||
--privileged \
|
||||
-w "${PWD}" \
|
||||
--env KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN}" \
|
||||
"${container_image}" \
|
||||
bash -c "${modules_builder} -a ${ARCH:-$(uname -m)} -o ${output_dir}"
|
||||
@@ -30,6 +30,7 @@ RUN apt-get update && \
|
||||
gettext \
|
||||
rsync \
|
||||
cpio \
|
||||
cryptsetup-bin \
|
||||
patch \
|
||||
python3 && \
|
||||
if [ "${ARCH}" != "$(uname -m)" ]; then apt-get install --no-install-recommends -y gcc-"${ARCH}"-linux-gnu binutils-"${ARCH}"-linux-gnu; fi && \
|
||||
|
||||
@@ -82,6 +82,15 @@ container_build+=" --build-arg ARCH=${ARCH:-}"
|
||||
"${container_image}" \
|
||||
bash -c "${kernel_builder} ${kernel_builder_args} build"
|
||||
|
||||
"${container_engine}" run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
|
||||
-w "${PWD}" \
|
||||
--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
|
||||
--env KERNEL_DEBUG_ENABLED="${KERNEL_DEBUG_ENABLED}" \
|
||||
--env KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN}" \
|
||||
--user "$(id -u)":"$(id -g)" \
|
||||
"${container_image}" \
|
||||
bash -c "${kernel_builder} ${kernel_builder_args} build-modules-images"
|
||||
|
||||
"${container_engine}" run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
|
||||
-w "${PWD}" \
|
||||
--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
|
||||
|
||||
Reference in New Issue
Block a user