From 2fc5f0e2e0ab18e2d39f6a2a43378a87eb4d98db Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Sun, 2 Apr 2023 09:09:13 +0000 Subject: [PATCH 1/7] kata-depoly: prepare env for cross build in lib.sh We leverage three env, TARGET_ARCH means the buid target tuple; ARCH nearly the same meaning with TARGET_ARCH but has been widely used in kata; CROSS_BUILD means if you want to do cross compile. Signed-off-by: Jianyong Wu --- tools/packaging/scripts/lib.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index 1f0173e473..e7a6fc987d 100644 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -19,6 +19,16 @@ short_commit_length=10 hub_bin="hub-bin" +#for cross build +CROSS_BUILD=${CROSS_BUILD-:} +BUILDX="" +PLATFORM="" +TARGET_ARCH=${TARGET_ARCH:-$(uname -m)} +ARCH=${ARCH:-$(uname -m)} +[ "${TARGET_ARCH}" == "aarch64" ] && TARGET_ARCH=arm64 +TARGET_OS=${TARGET_OS:-linux} +[ "${CROSS_BUILD}" == "true" ] && BUILDX=buildx && PLATFORM="--platform=${TARGET_OS}/${TARGET_ARCH}" + clone_tests_repo() { # KATA_CI_NO_NETWORK is (has to be) ignored if there is # no existing clone. @@ -189,7 +199,7 @@ get_ovmf_image_name() { } get_virtiofsd_image_name() { - ARCH=$(uname -m) + ARCH=${ARCH:-$(uname -m)} case ${ARCH} in "aarch64") libc="musl" From e2c31fce235ffdc42ac3efd6405848d788c7ff8e Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Mon, 24 Jul 2023 16:06:22 +0800 Subject: [PATCH 2/7] kata-deploy: enable cross build for kata deploy script kata-deploy-binaries-in-docker.sh is the entry to build kata components. set some environment to facilitate the following cross build work. Fixes: #6557 Signed-off-by: Jianyong Wu --- .../kata-deploy-binaries-in-docker.sh | 26 +++++++++++++++++++ .../local-build/kata-deploy-binaries.sh | 14 +++++----- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index fc82082c4d..ebcc80c814 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -19,6 +19,29 @@ gid=$(id -g ${USER}) http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" +ARCH=${ARCH:-$(uname -m)} +CROSS_BUILD= +BUILDX="" +PLATFORM="" +TARGET_ARCH=${TARGET_ARCH:-$(uname -m)} +[ "$(uname -m)" != "${TARGET_ARCH}" ] && CROSS_BUILD=true + +[ "${TARGET_ARCH}" == "aarch64" ] && TARGET_ARCH=arm64 + +# used for cross build +TARGET_OS=${TARGET_OS:-linux} +TARGET_ARCH=${TARGET_ARCH:-$ARCH} + +[ "${CROSS_BUILD}" == "true" ] && BUILDX="buildx" && PLATFORM="--platform=${TARGET_OS}/${TARGET_ARCH}" +if [ "${CROSS_BUILD}" == "true" ]; then + # check if the current docker support docker buildx + docker buildx ls > /dev/null 2>&1 || true + [ $? != 0 ] && echo "no docker buildx support, please upgrad your docker" && exit 1 + # check if docker buildx support target_arch, if not install it + r=$(docker buildx ls | grep "${TARGET_ARCH}" || true) + [ -z "$r" ] && sudo docker run --privileged --rm tonistiigi/binfmt --install ${TARGET_ARCH} +fi + if [ "${script_dir}" != "${PWD}" ]; then ln -sf "${script_dir}/build" "${PWD}/build" fi @@ -66,6 +89,9 @@ docker run \ --env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" \ --env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \ --env USE_CACHE="${USE_CACHE:-}" \ + --env CROSS_BUILD="${CROSS_BUILD}" \ + --env TARGET_ARCH="${TARGET_ARCH}" \ + --env ARCH="${ARCH}" \ --rm \ -w ${script_dir} \ build-kata-deploy "${kata_deploy_create}" $@ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 2af5cd36b8..8f819166af 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -38,7 +38,7 @@ readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_imag readonly jenkins_url="http://jenkins.katacontainers.io" readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts" -ARCH=$(uname -m) +ARCH=${ARCH:-$(uname -m)} MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} USE_CACHE="${USE_CACHE:-"yes"}" @@ -150,7 +150,7 @@ install_image() { image_type+="-${variant}" fi - local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${image_type}-$(uname -m)/${cached_artifacts_path}" + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${image_type}-${ARCH}/${cached_artifacts_path}" local component="rootfs-${image_type}" local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" @@ -197,7 +197,7 @@ install_initrd() { initrd_type+="-${variant}" fi - local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-$(uname -m)/${cached_artifacts_path}" + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-${ARCH}/${cached_artifacts_path}" local component="rootfs-${initrd_type}" local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" @@ -247,7 +247,7 @@ install_cached_kernel_tarball_component() { install_cached_tarball_component \ "${kernel_name}" \ - "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ + "${jenkins_url}/job/kata-containers-main-${kernel_name}-${ARCH}/${cached_artifacts_path}" \ "${kernel_version}-${kernel_kata_config_version}-$(get_last_modification $(dirname $kernel_builder))" \ "$(get_kernel_image_name)" \ "${final_tarball_name}" \ @@ -384,7 +384,7 @@ install_qemu_helper() { install_cached_tarball_component \ "${qemu_name}" \ - "${jenkins_url}/job/kata-containers-main-${qemu_name}-$(uname -m)/${cached_artifacts_path}" \ + "${jenkins_url}/job/kata-containers-main-${qemu_name}-${ARCH}/${cached_artifacts_path}" \ "${qemu_version}-$(calc_qemu_files_sha256sum)" \ "$(get_qemu_image_name)" \ "${final_tarball_name}" \ @@ -495,7 +495,7 @@ install_clh_glibc() { install_virtiofsd() { install_cached_tarball_component \ "virtiofsd" \ - "${jenkins_url}/job/kata-containers-main-virtiofsd-$(uname -m)/${cached_artifacts_path}" \ + "${jenkins_url}/job/kata-containers-main-virtiofsd-${ARCH}/${cached_artifacts_path}" \ "$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" \ "$(get_virtiofsd_image_name)" \ "${final_tarball_name}" \ @@ -542,7 +542,7 @@ install_shimv2() { install_cached_tarball_component \ "shim-v2" \ - "${jenkins_url}/job/kata-containers-main-shim-v2-$(uname -m)/${cached_artifacts_path}" \ + "${jenkins_url}/job/kata-containers-main-shim-v2-${ARCH}/${cached_artifacts_path}" \ "${shim_v2_version}" \ "$(get_shim_v2_image_name)" \ "${final_tarball_name}" \ From 7923de8999de055c7f55c568d3354d4c36f830e5 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Wed, 19 Jul 2023 17:31:58 +0800 Subject: [PATCH 3/7] static-build: cross build kernel Prepare cross build environment based on current Dockerfile. Fixes: #6557 Signed-off-by: Jianyong Wu --- tools/packaging/kernel/build-kernel.sh | 8 +++++-- tools/packaging/kernel/kata_config_version | 2 +- .../packaging/static-build/kernel/Dockerfile | 5 +++- tools/packaging/static-build/kernel/build.sh | 24 +++++++++++++++---- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 18725c1b87..8119be7047 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -65,6 +65,8 @@ kernel_url="" #Linux headers for GPU guest fs module building linux_headers="" +CROSS_BUILD_ARG="" + MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} packaging_scripts_dir="${script_dir}/../scripts" @@ -436,7 +438,7 @@ setup_kernel() { info "Copying config file from: ${kernel_config_path}" cp "${kernel_config_path}" ./.config - make oldconfig + ARCH=${arch_target} make oldconfig ${CROSS_BUILD_ARG} ) } @@ -447,7 +449,7 @@ build_kernel() { [ -n "${arch_target}" ] || arch_target="$(uname -m)" arch_target=$(arch_to_kernel "${arch_target}") pushd "${kernel_path}" >>/dev/null - make -j $(nproc ${CI:+--ignore 1}) ARCH="${arch_target}" + make -j $(nproc ${CI:+--ignore 1}) ARCH="${arch_target}" ${CROSS_BUILD_ARG} if [ "${conf_guest}" == "sev" ]; then make -j $(nproc ${CI:+--ignore 1}) INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=${kernel_path} modules_install fi @@ -658,6 +660,8 @@ main() { info "Kernel version: ${kernel_version}" + [ "${arch_target}" != "" -a "${arch_target}" != $(uname -m) ] && CROSS_BUILD_ARG="CROSS_COMPILE=${arch_target}-linux-gnu-" + case "${subcmd}" in build) build_kernel "${kernel_path}" diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index 58c9bdf9d0..194b81caae 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -111 +112 diff --git a/tools/packaging/static-build/kernel/Dockerfile b/tools/packaging/static-build/kernel/Dockerfile index 183f8a47e2..c806e35743 100644 --- a/tools/packaging/static-build/kernel/Dockerfile +++ b/tools/packaging/static-build/kernel/Dockerfile @@ -5,6 +5,8 @@ FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive +ARG ARCH + # kernel deps RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -23,4 +25,5 @@ RUN apt-get update && \ rsync \ cpio \ patch && \ - apt-get clean && apt-get autoclean + if [ "${ARCH}" != "$(uname -m)" ]; then apt-get install --no-install-recommends -y gcc-"${ARCH}"-linux-gnu binutils-"${ARCH}"-linux-gnu; fi && \ + apt-get clean && apt-get autoclean && rm -rf /var/lib/apt/lists/* diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index b36f06cff8..b8deea4f09 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -14,12 +14,26 @@ source "${script_dir}/../../scripts/lib.sh" readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh" +BUILDX= +PLATFORM= + DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}" +if [ "${CROSS_BUILD}" == "true" ]; then + container_image="${container_image}-${ARCH}-cross-build" + # Need to build a s390x image due to an issue at + # https://github.com/kata-containers/kata-containers/pull/6586#issuecomment-1603189242 + if [ ${ARCH} == "s390x" ]; then + BUILDX="buildx" + PLATFORM="--platform=linux/s390x" + fi +fi + sudo docker pull ${container_image} || \ - (sudo docker build -t "${container_image}" "${script_dir}" && \ + (sudo docker ${BUILDX} build ${PLATFORM} \ + --build-arg ARCH=${ARCH} -t "${container_image}" "${script_dir}" && \ # No-op unless PUSH_TO_REGISTRY is exported as "yes" push_to_registry "${container_image}") @@ -27,21 +41,21 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ --env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \ "${container_image}" \ - bash -c "${kernel_builder} $* setup" + bash -c "${kernel_builder} -a ${ARCH} $* setup" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ "${container_image}" \ - bash -c "${kernel_builder} $* build" + bash -c "${kernel_builder} -a ${ARCH} $* build" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ "${container_image}" \ - bash -c "${kernel_builder} $* install" + bash -c "${kernel_builder} -a ${ARCH} $* install" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ "${container_image}" \ - bash -c "${kernel_builder} $* build-headers" + bash -c "${kernel_builder} -a ${ARCH} $* build-headers" From 11631c681ae8f4fafc4a1001a33eba6fcf3ae0bf Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Tue, 20 Jun 2023 22:30:37 +0800 Subject: [PATCH 4/7] static-build: enable cross build for shim-v2 shim-v2 has go and rust code. For rust code, we use messense/rust-musl-cross to build for speed up as it doesn't depends on qemu emulation. Build go code based on docker buildx as it doesn't support cross build now. Fixes: #6557 Signed-off-by: Jianyong Wu --- tools/packaging/static-build/shim-v2/build.sh | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index 9330445646..aeb48f2da8 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -16,6 +16,7 @@ VMM_CONFIGS="qemu fc" GO_VERSION=${GO_VERSION} RUST_VERSION=${RUST_VERSION} +CC="" DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} @@ -23,29 +24,43 @@ container_image="${SHIM_V2_CONTAINER_BUILDER:-$(get_shim_v2_image_name)}" EXTRA_OPTS="${EXTRA_OPTS:-""}" +[ "${CROSS_BUILD}" == "true" ] && container_image_bk="${container_image}" && container_image="${container_image}-cross-build" sudo docker pull ${container_image} || \ - (sudo docker build \ + (sudo docker ${BUILDX} build ${PLATFORM} \ --build-arg GO_VERSION="${GO_VERSION}" \ --build-arg RUST_VERSION="${RUST_VERSION}" \ -t "${container_image}" \ "${script_dir}" && \ push_to_registry "${container_image}") -arch=$(uname -m) +arch=${ARCH:-$(uname -m)} +GCC_ARCH=${arch} if [ ${arch} = "ppc64le" ]; then + GCC_ARCH="powerpc64le" arch="ppc64" fi +#Build rust project using cross build musl image to speed up +[[ "${CROSS_BUILD}" == "true" && ${ARCH} != "s390x" ]] && container_image="messense/rust-musl-cross:${GCC_ARCH}-musl" && CC=${GCC_ARCH}-unknown-linux-musl-gcc + sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + --env CROSS_BUILD=${CROSS_BUILD} \ + --env ARCH=${ARCH} \ + --env CC="${CC}" \ -w "${repo_root_dir}/src/runtime-rs" \ "${container_image}" \ bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch}" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + --env CROSS_BUILD=${CROSS_BUILD} \ + --env ARCH=${ARCH} \ + --env CC="${CC}" \ -w "${repo_root_dir}/src/runtime-rs" \ "${container_image}" \ bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" install" - + +[ "${CROSS_BUILD}" == "true" ] && container_image="${container_image_bk}-cross-build" + sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${repo_root_dir}/src/runtime" \ "${container_image}" \ From 2205fb9d050ad2bdaf77edd3a16f8a543d490333 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Sun, 2 Apr 2023 10:58:29 +0000 Subject: [PATCH 5/7] static-build: enable cross build for virtiofsd Based on messense/rust-musl-cross which offer cross build musl lib environment to cross compile virtiofsd. Fixes: #6557 Signed-off-by: Jianyong Wu --- .../static-build/virtiofsd/build-static-virtiofsd.sh | 2 +- tools/packaging/static-build/virtiofsd/build.sh | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh b/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh index 6bad74ffb2..a6370f8944 100755 --- a/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh +++ b/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh @@ -8,7 +8,7 @@ set -o errexit set -o nounset set -o pipefail -ARCH=$(uname -m) +ARCH=${ARCH:-$(uname -m)} ARCH_LIBC="" LIBC="" diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh index 6eb5ad51bf..7f85c5c0d4 100755 --- a/tools/packaging/static-build/virtiofsd/build.sh +++ b/tools/packaging/static-build/virtiofsd/build.sh @@ -13,6 +13,7 @@ readonly virtiofsd_builder="${script_dir}/build-static-virtiofsd.sh" source "${script_dir}/../../scripts/lib.sh" +ARCH=${ARCH:-$(uname -m)} DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} kata_version="${kata_version:-}" @@ -32,7 +33,6 @@ package_output_dir="${package_output_dir:-}" [ -n "${virtiofsd_toolchain}" ] || die "Failed to get the rust toolchain to build virtiofsd" [ -n "${virtiofsd_zip}" ] || die "Failed to get virtiofsd binary URL" -ARCH=$(uname -m) case ${ARCH} in "aarch64") libc="musl" @@ -49,9 +49,10 @@ case ${ARCH} in esac container_image="${VIRTIOFSD_CONTAINER_BUILDER:-$(get_virtiofsd_image_name)}" +[ "${CROSS_BUILD}" == "true" ] && container_image="${container_image}-cross-build" sudo docker pull ${container_image} || \ - (sudo docker build \ + (sudo docker $BUILDX build $PLATFORM \ --build-arg RUST_TOOLCHAIN="${virtiofsd_toolchain}" \ -t "${container_image}" "${script_dir}/${libc}" && \ # No-op unless PUSH_TO_REGISTRY is exported as "yes" @@ -64,5 +65,6 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ --env virtiofsd_repo="${virtiofsd_repo}" \ --env virtiofsd_version="${virtiofsd_version}" \ --env virtiofsd_zip="${virtiofsd_zip}" \ + --env ARCH="${ARCH}" \ "${container_image}" \ bash -c "${virtiofsd_builder}" From 35d6d86ab57664b805b8acb0aed5bab94e23055f Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Wed, 19 Jul 2023 17:41:42 +0800 Subject: [PATCH 6/7] static-build: enable cross-build for image build It's too long a time to cross build agent based on docker buildx, thus we cross build rootfs based on a container with cross compile toolchain of gcc and rust with musl libc. Then we get fast build just like native build. rootfs initrd cross build is disabled as no cross compile tolchain for rust with musl lib if found for alpine and based on docker buildx takes too long a time. Fixes: #6557 Signed-off-by: Jianyong Wu --- ci/install_libseccomp.sh | 3 ++- tools/osbuilder/image-builder/image_builder.sh | 14 +++++++++++++- tools/osbuilder/rootfs-builder/rootfs.sh | 10 ++++++++++ .../osbuilder/rootfs-builder/ubuntu/Dockerfile.in | 2 ++ tools/packaging/guest-image/build_image.sh | 8 +++++++- .../local-build/kata-deploy-binaries.sh | 2 ++ utils.mk | 4 ---- 7 files changed, 36 insertions(+), 7 deletions(-) diff --git a/ci/install_libseccomp.sh b/ci/install_libseccomp.sh index 3b1f8aa86f..683d0f65b1 100755 --- a/ci/install_libseccomp.sh +++ b/ci/install_libseccomp.sh @@ -87,7 +87,8 @@ build_and_install_libseccomp() { curl -sLO "${libseccomp_tarball_url}" tar -xf "${libseccomp_tarball}" pushd "libseccomp-${libseccomp_version}" - ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}" + [ "${arch}" == $(uname -m) ] && cc_name="" || cc_name="${arch}-linux-gnu-gcc" + CC=${cc_name} ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}" make make install popd diff --git a/tools/osbuilder/image-builder/image_builder.sh b/tools/osbuilder/image-builder/image_builder.sh index 3e7f0babc0..26f37d122a 100755 --- a/tools/osbuilder/image-builder/image_builder.sh +++ b/tools/osbuilder/image-builder/image_builder.sh @@ -13,6 +13,16 @@ set -o pipefail DOCKER_RUNTIME=${DOCKER_RUNTIME:-runc} MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} +#For cross build +CROSS_BUILD=${CROSS_BUILD:-false} +BUILDX="" +PLATFORM="" +TARGET_ARCH=${TARGET_ARCH:-$(uname -m)} +ARCH=${ARCH:-$(uname -m)} +[ "${TARGET_ARCH}" == "aarch64" ] && TARGET_ARCH=arm64 +TARGET_OS=${TARGET_OS:-linux} +[ "${CROSS_BUILD}" == "true" ] && BUILDX=buildx && PLATFORM="--platform=${TARGET_OS}/${TARGET_ARCH}" + readonly script_name="${0##*/}" readonly script_dir=$(dirname "$(readlink -f "$0")") readonly lib_file="${script_dir}/../scripts/lib.sh" @@ -154,7 +164,7 @@ build_with_container() { engine_build_args+=" --runtime ${DOCKER_RUNTIME}" fi - "${container_engine}" build \ + "${container_engine}" ${BUILDX} build ${PLATFORM} \ ${engine_build_args} \ --build-arg http_proxy="${http_proxy}" \ --build-arg https_proxy="${https_proxy}" \ @@ -189,6 +199,8 @@ build_with_container() { --env MEASURED_ROOTFS="${MEASURED_ROOTFS}" \ --env SELINUX="${SELINUX}" \ --env DEBUG="${DEBUG}" \ + --env ARCH="${ARCH}" \ + --env TARGET_ARCH="${TARGET_ARCH}" \ -v /dev:/dev \ -v "${script_dir}":"/osbuilder" \ -v "${script_dir}/../scripts":"/scripts" \ diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index 22940ee994..89efd388b5 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -31,6 +31,16 @@ SELINUX=${SELINUX:-"no"} lib_file="${script_dir}/../scripts/lib.sh" source "$lib_file" +#For cross build +CROSS_BUILD=${CROSS_BUILD:-false} +BUILDX="" +PLATFORM="" +TARGET_ARCH=${TARGET_ARCH:-$(uname -m)} +ARCH=${ARCH:-$(uname -m)} +[ "${TARGET_ARCH}" == "aarch64" ] && TARGET_ARCH=arm64 +TARGET_OS=${TARGET_OS:-linux} +[ "${CROSS_BUILD}" == "true" ] && BUILDX=buildx && PLATFORM="--platform=${TARGET_OS}/${TARGET_ARCH}" + handle_error() { local exit_code="${?}" local line_number="${1:-}" diff --git a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in index 937ea89f43..61307e9560 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in +++ b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in @@ -8,6 +8,7 @@ FROM ${IMAGE_REGISTRY}/ubuntu:@OS_VERSION@ # makedev tries to mknod from postinst RUN [ -x /usr/bin/systemd-detect-virt ] || ( echo "echo docker" >/usr/bin/systemd-detect-virt && chmod +x /usr/bin/systemd-detect-virt ) +# hadolint ignore=DL3009,SC2046 RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive \ apt-get --no-install-recommends -y install \ @@ -18,6 +19,7 @@ RUN apt-get update && \ libc_arch="$gcc_arch" && \ [ "$gcc_arch" = aarch64 ] && libc_arch=arm64; \ [ "$gcc_arch" = ppc64le ] && gcc_arch=powerpc64le && libc_arch=ppc64el; \ + [ "$gcc_arch" = s390x ] && gcc_arch=s390x && libc_arch=s390x; \ [ "$gcc_arch" = x86_64 ] && gcc_arch=x86-64 && libc_arch=amd64; \ echo "gcc-$gcc_arch-linux-gnu libc6-dev-$libc_arch-cross")) \ git \ diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index fad6646517..372ddf3a46 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -21,7 +21,13 @@ readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)" export GOPATH=${GOPATH:-${HOME}/go} -arch_target="$(uname -m)" +ARCH=${ARCH:-$(uname -m)} +if [ $(uname -m) == "${ARCH}" ]; then + arch_target="$(uname -m)" +else + arch_target="${ARCH}" +fi + final_artifact_name="kata-containers" image_initrd_extension=".img" diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 8f819166af..a13f9c51f1 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -208,6 +208,8 @@ install_initrd() { local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + [[ "${ARCH}" == "aarch64" && "${CROSS_BUILD}" == "true" ]] && echo "warning: Don't cross build initrd for aarch64 as it's too slow" && exit 0 + install_cached_tarball_component \ "${component}" \ "${jenkins}" \ diff --git a/utils.mk b/utils.mk index 3c809cf544..1a153e26cc 100644 --- a/utils.mk +++ b/utils.mk @@ -157,10 +157,6 @@ endif EXTRA_RUSTFLAGS := -ifeq ($(ARCH), aarch64) - override EXTRA_RUSTFLAGS = -C link-arg=-lgcc - $(warning "WARNING: aarch64-musl needs extra symbols from libgcc") -endif ifneq ($(HOST_ARCH),$(ARCH)) ifeq ($(CC),) From 7ffc0c1225c3972efd2c227504e5194f7866ddf6 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Sun, 2 Apr 2023 09:25:22 +0000 Subject: [PATCH 7/7] static-build: enable cross build for qemu Depends on mutiarch feature of ubuntu, we can set up cross build environment easily and achive as good build performance as native build. Fixes: #6557 Signed-off-by: Jianyong Wu --- tools/packaging/static-build/qemu/Dockerfile | 61 +++++++++++++------ .../static-build/qemu/build-base-qemu.sh | 14 ++++- .../packaging/static-build/qemu/build-qemu.sh | 12 +++- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/tools/packaging/static-build/qemu/Dockerfile b/tools/packaging/static-build/qemu/Dockerfile index 930a907817..f5e812c915 100644 --- a/tools/packaging/static-build/qemu/Dockerfile +++ b/tools/packaging/static-build/qemu/Dockerfile @@ -8,8 +8,23 @@ from ubuntu:20.04 # This is required to keep build dependencies with security fixes. ARG CACHE_TIMEOUT ARG DEBIAN_FRONTEND=noninteractive +ARG DPKG_ARCH +ARG ARCH +ARG GCC_ARCH +ARG PREFIX SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN if [ "${ARCH}" != "$(uname -m)" ]; then sed -i 's/^deb/deb [arch=amd64]/g' /etc/apt/sources.list && \ + dpkg --add-architecture "${DPKG_ARCH#:}" && \ + echo "deb [arch=${DPKG_ARCH#:}] http://ports.ubuntu.com/ focal main restricted" >> /etc/apt/sources.list && \ + echo "deb [arch=${DPKG_ARCH#:}] http://ports.ubuntu.com/ focal-updates main restricted" >> /etc/apt/sources.list && \ + echo "deb [arch=${DPKG_ARCH#:}] http://ports.ubuntu.com/ focal universe" >> /etc/apt/sources.list && \ + echo "deb [arch=${DPKG_ARCH#:}] http://ports.ubuntu.com/ focal-updates universe" >> /etc/apt/sources.list && \ + echo "deb [arch=${DPKG_ARCH#:}] http://ports.ubuntu.com/ focal multiverse" >> /etc/apt/sources.list && \ + echo "deb [arch=${DPKG_ARCH#:}] http://ports.ubuntu.com/ focal-updates multiverse" >> /etc/apt/sources.list && \ + echo "deb [arch=${DPKG_ARCH#:}] http://ports.ubuntu.com/ focal-backports main restricted universe multiverse" >> /etc/apt/sources.list; fi + RUN apt-get update && apt-get upgrade -y && \ apt-get --no-install-recommends install -y \ apt-utils \ @@ -19,37 +34,43 @@ RUN apt-get update && apt-get upgrade -y && \ bison \ ca-certificates \ cpio \ + dpkg-dev \ flex \ gawk \ - libaudit-dev \ - libblkid-dev \ - libcap-dev \ - libcap-ng-dev \ - libdw-dev \ - libelf-dev \ - libffi-dev \ - libglib2.0-0 \ - libglib2.0-dev \ - libglib2.0-dev git \ - libltdl-dev \ - libmount-dev \ - libpixman-1-dev \ - libselinux1-dev \ - libtool \ + libaudit-dev${DPKG_ARCH} \ + libblkid-dev${DPKG_ARCH} \ + libcap-dev${DPKG_ARCH} \ + libcap-ng-dev${DPKG_ARCH} \ + libdw-dev${DPKG_ARCH} \ + libelf-dev${DPKG_ARCH} \ + libffi-dev${DPKG_ARCH} \ + libglib2.0-0${DPKG_ARCH} \ + libglib2.0-dev${DPKG_ARCH} \ + libglib2.0-dev${DPKG_ARCH} git \ + libltdl-dev${DPKG_ARCH} \ + libmount-dev${DPKG_ARCH} \ + libpixman-1-dev${DPKG_ARCH} \ + libselinux1-dev${DPKG_ARCH} \ + libtool${DPKG_ARCH} \ make \ ninja-build \ - pkg-config \ - libseccomp-dev \ - libseccomp2 \ + pkg-config${DPKG_ARCH} \ + libseccomp-dev${DPKG_ARCH} \ + libseccomp2${DPKG_ARCH} \ patch \ python \ python-dev \ rsync \ - zlib1g-dev && \ - if [ "$(uname -m)" != "s390x" ]; then apt-get install -y --no-install-recommends libpmem-dev; fi && \ + zlib1g-dev${DPKG_ARCH} && \ + if [ "${ARCH}" != s390x ]; then apt-get install -y --no-install-recommends libpmem-dev${DPKG_ARCH}; fi && \ + GCC_ARCH="${ARCH}" && if [ "${ARCH}" = "ppc64le" ]; then GCC_ARCH="powerpc64le"; fi && \ + if [ "${ARCH}" != "$(uname -m)" ]; then apt-get install --no-install-recommends -y gcc-"${GCC_ARCH}"-linux-gnu; fi && \ apt-get clean && rm -rf /var/lib/apt/lists/ RUN git clone https://github.com/axboe/liburing/ ~/liburing && \ cd ~/liburing && \ git checkout tags/liburing-2.1 && \ + GCC_ARCH="${ARCH}" && if [ "${ARCH}" = "ppc64le" ]; then GCC_ARCH="powerpc64le"; fi && \ + if [ "${ARCH}" != "$(uname -m)" ]; then PREFIX="${GCC_ARCH}-linux-gnu"; fi && \ + ./configure --cc=${GCC_ARCH}-linux-gnu-gcc --cxx=${GCC_ARCH}-linux-gnu-cpp --prefix=/usr/${PREFIX}/ && \ make && make install && ldconfig diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh index 9767e5d548..1514419d78 100755 --- a/tools/packaging/static-build/qemu/build-base-qemu.sh +++ b/tools/packaging/static-build/qemu/build-base-qemu.sh @@ -14,6 +14,12 @@ readonly qemu_builder="${script_dir}/build-qemu.sh" source "${script_dir}/../../scripts/lib.sh" source "${script_dir}/../qemu.blacklist" +ARCH=${ARCH:-$(uname -m)} +dpkg_arch=":${ARCH}" +[ ${dpkg_arch} == ":aarch64" ] && dpkg_arch=":arm64" +[ ${dpkg_arch} == ":x86_64" ] && dpkg_arch="" +[ "${dpkg_arch}" == ":ppc64le" ] && dpkg_arch=":ppc64el" + packaging_dir="${script_dir}/../.." qemu_destdir="/tmp/qemu-static/" container_engine="${USE_PODMAN:+podman}" @@ -39,11 +45,14 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d") [ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static" container_image="${QEMU_CONTAINER_BUILDER:-$(get_qemu_image_name)}" +[ "${CROSS_BUILD}" == "true" ] && container_image="${container_image}-cross-build" sudo docker pull ${container_image} || (sudo "${container_engine}" build \ --build-arg CACHE_TIMEOUT="${CACHE_TIMEOUT}" \ --build-arg http_proxy="${http_proxy}" \ --build-arg https_proxy="${https_proxy}" \ + --build-arg DPKG_ARCH="${dpkg_arch}" \ + --build-arg ARCH="${ARCH}" \ "${packaging_dir}" \ -f "${script_dir}/Dockerfile" \ -t "${container_image}" && \ @@ -54,13 +63,14 @@ sudo "${container_engine}" run \ --rm \ -i \ --env BUILD_SUFFIX="${build_suffix}" \ - --env HYPERVISOR_NAME="${HYPERVISOR_NAME}" \ --env PKGVERSION="${PKGVERSION}" \ --env QEMU_DESTDIR="${qemu_destdir}" \ --env QEMU_REPO="${qemu_repo}" \ - --env QEMU_VERSION="${qemu_version}" \ --env QEMU_TARBALL="${qemu_tar}" \ --env PREFIX="${prefix}" \ + --env HYPERVISOR_NAME="${HYPERVISOR_NAME}" \ + --env QEMU_VERSION_NUM="${qemu_version}" \ + --env ARCH="${ARCH}" \ -v "${repo_root_dir}:/root/kata-containers" \ -v "${PWD}":/share "${container_image}" \ bash -c "/root/kata-containers/tools/packaging/static-build/qemu/build-qemu.sh" diff --git a/tools/packaging/static-build/qemu/build-qemu.sh b/tools/packaging/static-build/qemu/build-qemu.sh index edab348910..cab5f251b1 100755 --- a/tools/packaging/static-build/qemu/build-qemu.sh +++ b/tools/packaging/static-build/qemu/build-qemu.sh @@ -14,13 +14,19 @@ kata_packaging_scripts="${kata_packaging_dir}/scripts" kata_static_build_dir="${kata_packaging_dir}/static-build" kata_static_build_scripts="${kata_static_build_dir}/scripts" +ARCH=${ARCH:-$(uname -m)} + git clone --depth=1 "${QEMU_REPO}" qemu pushd qemu -git fetch --depth=1 origin "${QEMU_VERSION}" +git fetch --depth=1 origin "${QEMU_VERSION_NUM}" git checkout FETCH_HEAD scripts/git-submodule.sh update meson capstone -${kata_packaging_scripts}/patch_qemu.sh "${QEMU_VERSION}" "${kata_packaging_dir}/qemu/patches" -PREFIX="${PREFIX}" ${kata_packaging_scripts}/configure-hypervisor.sh -s "${HYPERVISOR_NAME}" | xargs ./configure --with-pkgversion="${PKGVERSION}" +${kata_packaging_scripts}/patch_qemu.sh "${QEMU_VERSION_NUM}" "${kata_packaging_dir}/qemu/patches" +if [ "$(uname -m)" != "${ARCH}" ] && [ "${ARCH}" == "s390x" ]; then + PREFIX="${PREFIX}" ${kata_packaging_scripts}/configure-hypervisor.sh -s "${HYPERVISOR_NAME}" "${ARCH}" | xargs ./configure --with-pkgversion="${PKGVERSION}" --cc=s390x-linux-gnu-gcc --cross-prefix=s390x-linux-gnu- --prefix="${PREFIX}" --target-list=s390x-softmmu +else + PREFIX="${PREFIX}" ${kata_packaging_scripts}/configure-hypervisor.sh -s "${HYPERVISOR_NAME}" "${ARCH}" | xargs ./configure --with-pkgversion="${PKGVERSION}" +fi make -j"$(nproc +--ignore 1)" make install DESTDIR="${QEMU_DESTDIR}" popd