diff --git a/tools/osbuilder/README.md b/tools/osbuilder/README.md index b68ae4e8b5..f402411cff 100644 --- a/tools/osbuilder/README.md +++ b/tools/osbuilder/README.md @@ -3,24 +3,26 @@ # osbuilder * [osbuilder](#osbuilder) - * [Introduction](#introduction) - * [Terms](#terms) - * [Building](#building) - * [Rootfs creation](#rootfs-creation) - * [Rootfs with systemd as init](#rootfs-with-systemd-as-init) - * [Rootfs with the agent as init](#rootfs-with-the-agent-as-init) - * [dracut based rootfs](#dracut-based-rootfs) - * [Image creation](#image-creation) - * [Image with systemd as init](#image-with-systemd-as-init) - * [Image with the agent as init](#image-with-the-agent-as-init) - * [dracut based image](#dracut-based-image) - * [Initrd creation](#initrd-creation) - * [Rootfs based initrd](#rootfs-based-initrd) - * [dracut based initrd](#dracut-based-initrd) - * [dracut options](#dracut-options) - * [Add kernel modules](#add-kernel-modules) - * [Testing](#testing) - * [Platform-Distro Compatibility Matrix](#platform-distro-compatibility-matrix) + * [Introduction](#introduction) + * [Terms](#terms) + * [Building](#building) + * [Rootfs creation](#rootfs-creation) + * [Rootfs with systemd as init](#rootfs-with-systemd-as-init) + * [Rootfs with the agent as init](#rootfs-with-the-agent-as-init) + * [dracut based rootfs](#dracut-based-rootfs) + * [Image creation](#image-creation) + * [Image with systemd as init](#image-with-systemd-as-init) + * [Image with the agent as init](#image-with-the-agent-as-init) + * [dracut based image](#dracut-based-image) + * [Initrd creation](#initrd-creation) + * [Rootfs based initrd](#rootfs-based-initrd) + * [dracut based initrd](#dracut-based-initrd) + * [dracut options](#dracut-options) + * [Add kernel modules](#add-kernel-modules) + * [Custom images](#custom-images) + * [Intel® QuickAssist Technology (QAT) customized kernel and rootfs](#intel-quickassist-technology-qat-customized-kernel-and-rootfs) + * [Testing](#testing) + * [Platform-Distro Compatibility Matrix](#platform-distro-compatibility-matrix) ## Introduction @@ -198,6 +200,23 @@ is paired with the built image or initrd, using the `uname -r` format. For examp $ make BUILD_METHOD=dracut DRACUT_KVERSION=5.2.1-23-kata AGENT_INIT=yes initrd ``` +### Custom images + +The Kata Containers kernel and rootfs images are by design "minimal". If advanced, +site specific, or customized features are required, then building a customized +kernel and/or rootfs may be required. + +The below are some examples which may help or be useful for generating a +customized system. + +#### Intel® QuickAssist Technology (QAT) customized kernel and rootfs + +As documented in the +[Intel® QAT Kata use-case documentation](../../docs/use-cases/using-Intel-QAT-and-kata.md), +enabling this hardware requires a customized kernel and rootfs to work with Kata. +To ease building of the kernel and rootfs, a [Dockerfile](./dockerfiles/QAT) is +supplied, that when run, generates the required kernel and rootfs binaries. + ## Testing ``` diff --git a/tools/osbuilder/dockerfiles/QAT/Dockerfile b/tools/osbuilder/dockerfiles/QAT/Dockerfile new file mode 100644 index 0000000000..5437d43a38 --- /dev/null +++ b/tools/osbuilder/dockerfiles/QAT/Dockerfile @@ -0,0 +1,58 @@ +# Copyright (c) 2021 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# Kata osbuilder 'works best' on Fedora +FROM fedora:latest + +# Version of the Dockerfile - update if you change this file to avoid 'stale' +# images being pulled from the registry. +# Set AGENT_VERSION as an env variable to specify a specific version of Kata Agent to install + +LABEL DOCKERFILE_VERSION="2.0" + +ENV QAT_DRIVER_VER "qat1.7.l.4.12.0-00011.tar.gz" +ENV QAT_DRIVER_URL "https://downloadmirror.intel.com/30178/eng/${QAT_DRIVER_VER}" +ENV QAT_CONFIGURE_OPTIONS "--enable-icp-sriov=guest" +ENV KATA_REPO_VERSION "main" +ENV AGENT_VERSION "" +ENV ROOTFS_OS "debian" +ENV OUTPUT_DIR "/output" + +RUN dnf install -y \ + bc \ + bison \ + curl \ + debootstrap \ + diffutils \ + e2fsprogs \ + elfutils-libelf-devel \ + findutils \ + flex \ + gcc \ + gcc-c++ \ + git \ + kiwi \ + kmod \ + openssl \ + openssl-devel \ + make \ + parted \ + patch \ + qemu-img \ + systemd-devel \ + sudo \ + xz + +# Pull in our local files +COPY ./run.sh /input/ +COPY ./qat.conf /input/ + +# Output is placed in the /output directory. +# We could make this a VOLUME to force it to be attached to the host, but let's +# just leave it as a container dir that can then be over-ridden from a host commandline +# volume setup. +# VOLUME /output + +# By default build everything +CMD ["/input/run.sh"] diff --git a/tools/osbuilder/dockerfiles/QAT/README.md b/tools/osbuilder/dockerfiles/QAT/README.md new file mode 100644 index 0000000000..1a43fce9c6 --- /dev/null +++ b/tools/osbuilder/dockerfiles/QAT/README.md @@ -0,0 +1,89 @@ + + * [Introduction](#introduction) + * [Building](#building) + * [Options](#options) + +## Introduction + +The files in this directory can be used to build a modified Kata Containers rootfs +and kernel with modifications to support Intel® QuickAssist Technology (QAT) +hardware. It is designed to work with Kata Container versions 2.0 and higher. + +To properly load the driver modules, systemd init must be used. It is not adequate +to use the agent as the init. Because of this, alpine is not a valid base OS image +to use. The following rootfs OS's have been tested with this Dockerfile. + +* CentOS +* Clear Linux +* Debian +* Fedora +* SUSE +* Ubuntu + +The generated files will need to be copied and configured into your Kata Containers +setup. + +Please see the +[Using Intel® QuickAssist Technology and Kata](../../../../docs/use-cases/using-Intel-QAT-and-kata.md) +documentation for more specific details on how to configure a host system and +enable acceleration of workloads. + +## Building + +The image build and run are executed using Docker, from within this `QAT` folder. +It is required to use **all** the files in this directory to build the Docker +image: + +```sh +$ docker build --label kataqat --tag kataqat:latest . +$ mkdir ./output +$ docker run -ti --rm --privileged -v /dev:/dev -v $(pwd)/output:/output kataqat +``` + +> **Note:** The use of the `--privileged` and `-v /dev:/dev` arguments to the `docker run` are +> necessary, to enable the scripts within the container to generate a roofs file system. + +When complete, the generated files will be placed into the output directory. +Sample config files that have been modified with a `[SHIM`] section are also +placed into the `config` subdirectory as a reference that can be used with +Kata Containers. + +```sh +# ls -lR output +output: +total 136656 +drwxr-xr-x 2 root root 4096 Feb 11 23:59 configs +-rw-r--r-- 1 root root 134217728 Feb 11 23:59 kata-containers.img +-rw-r--r-- 1 root root 5710336 Feb 11 23:59 vmlinuz-kata-linux-5.4.71-84_qat + +output/configs: +total 20 +-rw-r--r-- 1 root root 4082 Feb 11 23:59 200xxvf_dev0.conf +-rw-r--r-- 1 root root 4082 Feb 11 23:59 c3xxxvf_dev0.conf +-rw-r--r-- 1 root root 4082 Feb 11 23:59 c6xxvf_dev0.conf +-rw-r--r-- 1 root root 4082 Feb 11 23:59 d15xxvf_dev0.conf +-rw-r--r-- 1 root root 4082 Feb 11 23:59 dh895xccvf_dev0.conf +``` + +## Options + +A number of parameters to the scripts are configured in the `Dockerfile`, and thus can be modified +on the commandline. The `AGENT_VERSION` is not set and by default will use the +latest stable version of Kata Containers. + + +| Variable | Definition | Default value | +| -------- | ---------- | ------------- | +| AGENT_VERSION | Kata agent that is installed into the rootfs | | +| KATA_REPO_VERSION | Kata Branch or Tag to build from | `main` | +| OUTPUT_DIR | Directory inside container where results are stored | `/output` | +| QAT_CONFIGURE_OPTIONS | `configure` options for QAT driver | `--enable-icp-sriov=guest` | +| QAT_DRIVER_URL | URL to curl QAT driver from | `https://01.org/sites/default/files/downloads/${QAT_DRIVER_VER}` | +| QAT_DRIVER_VER | QAT driver version to use | `qat1.7.l.4.9.0-00008.tar.gz` | +| ROOTFS_OS | Operating system to use for the rootfs | `debian` | + +Variables can be set on the `docker run` commandline, for example: + +```sh +$ docker run -ti --rm --privileged -e "AGENT_VERSION=2.0.0" -v /dev:/dev -v ${PWD}/output:/output kataqat +``` diff --git a/tools/osbuilder/dockerfiles/QAT/qat.conf b/tools/osbuilder/dockerfiles/QAT/qat.conf new file mode 100644 index 0000000000..dd9c9e9187 --- /dev/null +++ b/tools/osbuilder/dockerfiles/QAT/qat.conf @@ -0,0 +1,17 @@ +# +# Copyright (c) 2021 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +CONFIG_PCIEAER=y +CONFIG_UIO=y +CONFIG_CRYPTO_HW=y +# This is a hack. By setting this QAT device as a module, we get the kernel +# to configure/build all the other parts required for QAT - and then later we +# build and load the out-of-tree QAT kernel modules instead of this one. +CONFIG_CRYPTO_DEV_QAT_C62XVF=m +CONFIG_CRYPTO_CBC=y +CONFIG_MODULES=y +CONFIG_MODULE_SIG=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_DH=y diff --git a/tools/osbuilder/dockerfiles/QAT/run.sh b/tools/osbuilder/dockerfiles/QAT/run.sh new file mode 100644 index 0000000000..c3cafb00f2 --- /dev/null +++ b/tools/osbuilder/dockerfiles/QAT/run.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# +# Copyright (c) 2021 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -e +set -u + +# NOTE: Some env variables are set in the Dockerfile - those that are +# intended to be over-rideable. +export QAT_SRC=~/src/QAT +export ROOTFS_DIR=~/src/rootfs +export GOPATH=~/src/go +export PATH=${PATH}:/usr/local/go/bin:${GOPATH}/bin + +katarepo=github.com/kata-containers/kata-containers +katarepopath=${GOPATH}/src/${katarepo} + +testsrepo=github.com/kata-containers/tests +testsrepopath=${GOPATH}/src/${testsrepo} + +grab_kata_repos() +{ + # Check out all the repos we will use now, so we can try and ensure they use the specified branch + # Only check out the branch needed, and make it shallow and thus space/bandwidth efficient + # Use a green prompt with white text for easy viewing + bin/echo -e "\n\e[1;42mClone and checkout Kata repos\e[0m" + git clone --single-branch --branch $KATA_REPO_VERSION --depth=1 https://${katarepo} ${katarepopath} + git clone --single-branch --branch $KATA_REPO_VERSION --depth=1 https://${testsrepo} ${testsrepopath} +} + +configure_kernel() +{ + cp /input/qat.conf ${katarepopath}/tools/packaging/kernel/configs/fragments/common/qat.conf + # We need yq and go to grab kernel versions etc. + ${testsrepopath}/.ci/install_yq.sh + ${testsrepopath}/.ci/install_go.sh -p + cd ${katarepopath} + /bin/echo -e "\n\e[1;42mDownload and configure Kata kernel with CRYPTO support\e[0m" + ./tools/packaging/kernel/build-kernel.sh setup +} + +build_kernel() +{ + cd ${katarepopath} + LINUX_VER=$(ls -d kata-linux-*) + sed -i 's/EXTRAVERSION =/EXTRAVERSION = .qat.container/' $LINUX_VER/Makefile + /bin/echo -e "\n\e[1;42mBuild Kata kernel with CRYPTO support\e[0m" + ./tools/packaging/kernel/build-kernel.sh build +} + +build_rootfs() +{ + # Due to an issue with debootstrap unmounting /proc when running in a + # --privileged container, change into /proc to keep it from being umounted. + # This should only be done for Ubuntu and Debian based OS's. Other OS + # distributions had issues if building the rootfs from /proc + + if [ "${ROOTFS_OS}" == "debian" ] || [ "${ROOTFS_OS}" == "ubuntu" ]; then + cd /proc + fi + /bin/echo -e "\n\e[1;42mDownload ${ROOTFS_OS} based rootfs\e[0m" + SECCOMP=no EXTRA_PKGS='kmod' ${katarepopath}/tools/osbuilder/rootfs-builder/rootfs.sh $ROOTFS_OS +} + +grab_qat_drivers() +{ + /bin/echo -e "\n\e[1;42mDownload and extract the drivers\e[0m" + mkdir -p $QAT_SRC + cd $QAT_SRC + curl -L $QAT_DRIVER_URL | tar zx +} + +build_qat_drivers() +{ + /bin/echo -e "\n\e[1;42mCompile driver modules\e[0m" + cd ${katarepopath} + linux_kernel_path=${katarepopath}/${LINUX_VER} + KERNEL_MAJOR_VERSION=$(awk '/^VERSION =/{print $NF}' ${linux_kernel_path}/Makefile) + KERNEL_PATHLEVEL=$(awk '/^PATCHLEVEL =/{print $NF}' ${linux_kernel_path}/Makefile) + KERNEL_SUBLEVEL=$(awk '/^SUBLEVEL =/{print $NF}' ${linux_kernel_path}/Makefile) + KERNEL_EXTRAVERSION=$(awk '/^EXTRAVERSION =/{print $NF}' ${linux_kernel_path}/Makefile) + KERNEL_ROOTFS_DIR=${KERNEL_MAJOR_VERSION}.${KERNEL_PATHLEVEL}.${KERNEL_SUBLEVEL}${KERNEL_EXTRAVERSION} + cd $QAT_SRC + KERNEL_SOURCE_ROOT=${linux_kernel_path} ./configure ${QAT_CONFIGURE_OPTIONS} + make all -j$(nproc) +} + +add_qat_to_rootfs() +{ + /bin/echo -e "\n\e[1;42mCopy driver modules to rootfs\e[0m" + cd $QAT_SRC + make INSTALL_MOD_PATH=${ROOTFS_DIR} qat-driver-install -j$(nproc) + cp $QAT_SRC/build/usdm_drv.ko ${ROOTFS_DIR}/lib/modules/${KERNEL_ROOTFS_DIR}/updates/drivers + depmod -a -b ${ROOTFS_DIR} ${KERNEL_ROOTFS_DIR} + cd ${katarepopath}/tools/osbuilder/image-builder + /bin/echo -e "\n\e[1;42mBuild rootfs image\e[0m" + ./image_builder.sh ${ROOTFS_DIR} +} + +copy_outputs() +{ + /bin/echo -e "\n\e[1;42mCopy kernel and rootfs to the output directory and provide sample configuration files\e[0m" + mkdir -p ${OUTPUT_DIR} || true + cp ${linux_kernel_path}/arch/x86/boot/bzImage $OUTPUT_DIR/vmlinuz-${LINUX_VER}_qat + cp ${linux_kernel_path}/vmlinux $OUTPUT_DIR/vmlinux-${LINUX_VER}_qat + cp ${katarepopath}/tools/osbuilder/image-builder/kata-containers.img $OUTPUT_DIR + mkdir -p ${OUTPUT_DIR}/configs || true + # Change extension from .conf.vm to just .conf and change the SSL section to + # SHIM so it works with Kata containers + for f in $QAT_SRC/quickassist/utilities/adf_ctl/conf_files/*.conf.vm; do + output_conf_file=$(basename -- "$f" .conf.vm).conf + cp -- "$f" "${OUTPUT_DIR}/configs/${output_conf_file}" + sed -i 's/\[SSL\]/\[SHIM\]/g' ${OUTPUT_DIR}/configs/${output_conf_file} + done +} + +help() { +cat << EOF +Usage: $0 [-h] [options] + Description: + This script builds kernel and rootfs artifacts for Kata Containers, + configured and built to support QAT hardware. + Options: + -d, Enable debug mode + -h, Show this help +EOF +} + +main() +{ + local check_in_container=${OUTPUT_DIR:-} + if [ -z "${check_in_container}" ]; then + echo "Error: 'OUTPUT_DIR' not set" >&2 + echo "$0 should be run using the Dockerfile supplied." >&2 + exit -1 + fi + + local OPTIND + while getopts "dh" opt;do + case ${opt} in + d) + set -x + ;; + h) + help + exit 0; + ;; + ?) + # parse failure + help + echo "ERROR: Failed to parse arguments" + exit -1 + ;; + esac + done + shift $((OPTIND-1)) + + grab_kata_repos + configure_kernel + build_kernel + build_rootfs + grab_qat_drivers + build_qat_drivers + add_qat_to_rootfs + copy_outputs +} + +main "$@"