From 207e325a0d3baacb7ea51026e6cf758957b550f4 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 7 Mar 2022 11:56:45 +0100 Subject: [PATCH 1/6] osbuilder: apk add --no-cache Hadolint DL3019. If you're wondering why this is in this PR, that's because I touch the file later, and we're only triggering the lints for changed files. Signed-off-by: Jakob Naucke --- tools/osbuilder/rootfs-builder/alpine/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/osbuilder/rootfs-builder/alpine/Dockerfile.in b/tools/osbuilder/rootfs-builder/alpine/Dockerfile.in index b6b77f1214..1e34148a21 100644 --- a/tools/osbuilder/rootfs-builder/alpine/Dockerfile.in +++ b/tools/osbuilder/rootfs-builder/alpine/Dockerfile.in @@ -6,7 +6,7 @@ ARG IMAGE_REGISTRY=docker.io FROM ${IMAGE_REGISTRY}/alpine:3.15 -RUN apk update && apk add \ +RUN apk update && apk add --no-cache \ apk-tools-static \ autoconf \ automake \ From 77ea087ae7573e77d627fbf6ea6890109e04beba Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 4 Mar 2022 18:22:19 +0100 Subject: [PATCH 2/6] osbuilder: Remove musl installations Remove a lot of cruft of musl installations -- we needed those for the Go agent, but Rustup just takes care of everything. aarch64 on Debian-based & Alpine is an exception -- create a symlink `aarch64-linux-musl-gcc` to `musl-tools`'s `musl-gcc` or `gcc` on Alpine. This is unified -- arch-specific Dockerfiles are removed. Furthermore, we should keep it in Ubuntu for supporting the offline SEV KBC. We also keep it in Clear Linux, as that runs our internal checks, but it is e.g. not shipped in CentOS Stream 9. Signed-off-by: Jakob Naucke --- ci/install_musl.sh | 24 ---- tools/osbuilder/dracut/Dockerfile.in | 3 - .../rootfs-builder/alpine/Dockerfile.in | 2 + .../rootfs-builder/clearlinux/Dockerfile.in | 2 +- .../debian/Dockerfile-aarch64.in | 34 ------ .../rootfs-builder/debian/Dockerfile.in | 4 +- tools/osbuilder/rootfs-builder/rootfs.sh | 9 -- .../template/Dockerfile.template | 1 - .../ubuntu/Dockerfile-aarch64.in | 48 -------- .../rootfs-builder/ubuntu/Dockerfile.in | 4 +- tools/osbuilder/scripts/lib.sh | 103 ++---------------- tools/osbuilder/tests/test_images.sh | 2 - versions.yaml | 13 --- 13 files changed, 15 insertions(+), 234 deletions(-) delete mode 100755 ci/install_musl.sh delete mode 100644 tools/osbuilder/rootfs-builder/debian/Dockerfile-aarch64.in delete mode 100644 tools/osbuilder/rootfs-builder/ubuntu/Dockerfile-aarch64.in diff --git a/ci/install_musl.sh b/ci/install_musl.sh deleted file mode 100755 index 4beec29113..0000000000 --- a/ci/install_musl.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2020 Ant Group -# -# SPDX-License-Identifier: Apache-2.0 -# - -set -e - -install_aarch64_musl() { - local arch=$(uname -m) - if [ "${arch}" == "aarch64" ]; then - local musl_tar="${arch}-linux-musl-native.tgz" - local musl_dir="${arch}-linux-musl-native" - pushd /tmp - if curl -sLO --fail https://musl.cc/${musl_tar}; then - tar -zxf ${musl_tar} - mkdir -p /usr/local/musl/ - cp -r ${musl_dir}/* /usr/local/musl/ - fi - popd - fi -} - -install_aarch64_musl diff --git a/tools/osbuilder/dracut/Dockerfile.in b/tools/osbuilder/dracut/Dockerfile.in index f84838bc3d..e80fa374a3 100644 --- a/tools/osbuilder/dracut/Dockerfile.in +++ b/tools/osbuilder/dracut/Dockerfile.in @@ -36,7 +36,4 @@ RUN zypper --non-interactive refresh; \ zypper --non-interactive clean --all; -# This will install the proper golang to build Kata components -@INSTALL_MUSL@ -@INSTALL_GO@ @INSTALL_RUST@ diff --git a/tools/osbuilder/rootfs-builder/alpine/Dockerfile.in b/tools/osbuilder/rootfs-builder/alpine/Dockerfile.in index 1e34148a21..7e05704223 100644 --- a/tools/osbuilder/rootfs-builder/alpine/Dockerfile.in +++ b/tools/osbuilder/rootfs-builder/alpine/Dockerfile.in @@ -30,3 +30,5 @@ RUN apk update && apk add --no-cache \ pkgconfig \ protoc \ tar +# aarch64 requires this name -- link for all +RUN ln -s /usr/bin/gcc "/usr/bin/$(uname -m)-linux-musl-gcc" diff --git a/tools/osbuilder/rootfs-builder/clearlinux/Dockerfile.in b/tools/osbuilder/rootfs-builder/clearlinux/Dockerfile.in index 1206185ad4..ceb67c0f44 100644 --- a/tools/osbuilder/rootfs-builder/clearlinux/Dockerfile.in +++ b/tools/osbuilder/rootfs-builder/clearlinux/Dockerfile.in @@ -28,6 +28,7 @@ RUN dnf -y update && dnf install -y \ libstdc++-static \ m4 \ make \ + musl-gcc \ openssl-devel \ perl \ perl-IPC-Cmd \ @@ -41,5 +42,4 @@ RUN dnf -y update && dnf install -y \ dnf clean all # This will install the proper packages to build Kata components -@INSTALL_MUSL@ @INSTALL_RUST@ diff --git a/tools/osbuilder/rootfs-builder/debian/Dockerfile-aarch64.in b/tools/osbuilder/rootfs-builder/debian/Dockerfile-aarch64.in deleted file mode 100644 index 727506f47a..0000000000 --- a/tools/osbuilder/rootfs-builder/debian/Dockerfile-aarch64.in +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (c) 2020 ARM Limited -# -# SPDX-License-Identifier: Apache-2.0 - -ARG IMAGE_REGISTRY=docker.io -# NOTE: OS_VERSION is set according to config.sh -FROM ${IMAGE_REGISTRY}/debian:@OS_VERSION@ - -# RUN commands -RUN apt-get update && apt-get install -y \ - autoconf \ - automake \ - binutils \ - build-essential \ - chrony \ - coreutils \ - curl \ - debianutils \ - debootstrap \ - g++ \ - gcc \ - git \ - libc-dev \ - libstdc++-8-dev \ - m4 \ - make \ - sed \ - systemd \ - tar \ - vim -# This will install the proper packages to build Kata components -@INSTALL_MUSL@ -@INSTALL_RUST@ diff --git a/tools/osbuilder/rootfs-builder/debian/Dockerfile.in b/tools/osbuilder/rootfs-builder/debian/Dockerfile.in index 685dd0f4d5..0220598570 100644 --- a/tools/osbuilder/rootfs-builder/debian/Dockerfile.in +++ b/tools/osbuilder/rootfs-builder/debian/Dockerfile.in @@ -27,14 +27,14 @@ RUN apt-get update && apt-get --no-install-recommends install -y \ libstdc++-8-dev \ m4 \ make \ - musl \ - musl-dev \ musl-tools \ sed \ systemd \ tar \ vim \ wget +# aarch64 requires this name -- link for all +RUN ln -s /usr/bin/musl-gcc "/usr/bin/$(uname -m)-linux-musl-gcc" # This will install the proper packages to build Kata components @INSTALL_RUST@ diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index b7b6798a5a..831aba78e2 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -14,7 +14,6 @@ script_name="${0##*/}" script_dir="$(dirname $(readlink -f $0))" AGENT_VERSION=${AGENT_VERSION:-} RUST_VERSION="null" -MUSL_VERSION=${MUSL_VERSION:-"null"} AGENT_BIN=${AGENT_BIN:-kata-agent} AGENT_INIT=${AGENT_INIT:-no} KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""} @@ -355,11 +354,6 @@ build_rootfs_distro() echo "Required rust version: $RUST_VERSION" - detect_musl_version || - die "Could not detect the required musl version for AGENT_VERSION='${AGENT_VERSION:-main}'." - - echo "Required musl version: $MUSL_VERSION" - if [ -z "${USE_DOCKER}" ] && [ -z "${USE_PODMAN}" ]; then info "build directly" build_rootfs ${ROOTFS_DIR} @@ -569,7 +563,6 @@ EOF fi if [ -z "${AGENT_SOURCE_BIN}" ] ; then - [ "$LIBC" == "musl" ] && bash ${script_dir}/../../../ci/install_musl.sh test -r "${HOME}/.cargo/env" && source "${HOME}/.cargo/env" # rust agent needs ${arch}-unknown-linux-${LIBC} if ! (rustup show | grep -v linux-${LIBC} > /dev/null); then @@ -580,7 +573,6 @@ EOF bash ${script_dir}/../../../ci/install_rust.sh ${RUST_VERSION} fi test -r "${HOME}/.cargo/env" && source "${HOME}/.cargo/env" - [ "$ARCH" == "aarch64" ] && OLD_PATH=$PATH && export PATH=$PATH:/usr/local/musl/bin agent_dir="${script_dir}/../../../src/agent/" @@ -602,7 +594,6 @@ EOF make clean make LIBC=${LIBC} INIT=${AGENT_INIT} SECCOMP=${SECCOMP} make install DESTDIR="${ROOTFS_DIR}" LIBC=${LIBC} INIT=${AGENT_INIT} - [ "$ARCH" == "aarch64" ] && export PATH=$OLD_PATH && rm -rf /usr/local/musl if [ "${SECCOMP}" == "yes" ]; then rm -rf "${libseccomp_install_dir}" "${gperf_install_dir}" fi diff --git a/tools/osbuilder/rootfs-builder/template/Dockerfile.template b/tools/osbuilder/rootfs-builder/template/Dockerfile.template index b881dac439..863cab3e57 100644 --- a/tools/osbuilder/rootfs-builder/template/Dockerfile.template +++ b/tools/osbuilder/rootfs-builder/template/Dockerfile.template @@ -14,5 +14,4 @@ FROM ${IMAGE_REGISTRY}/@distro@:@OS_VERSION@ # RUN commands # This will install the proper packages to build Kata components -@INSTALL_MUSL@ @INSTALL_RUST@ diff --git a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile-aarch64.in b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile-aarch64.in deleted file mode 100644 index ef4374ec43..0000000000 --- a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile-aarch64.in +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright (c) 2020 ARM Limited -# -# SPDX-License-Identifier: Apache-2.0 - -ARG IMAGE_REGISTRY=docker.io -#ubuntu: docker image to be used to create a rootfs -#@OS_VERSION@: Docker image version to build this dockerfile -FROM ${IMAGE_REGISTRY}/ubuntu:@OS_VERSION@ - -# This dockerfile needs to provide all the componets need to build a rootfs -# Install any package need to create a rootfs (package manager, extra tools) - -# Avoid tzdata setup -ENV DEBIAN_FRONTEND noninteractive - -# RUN commands -RUN apt-get update && apt-get install -y \ - autoconf \ - automake \ - binutils \ - build-essential \ - chrony \ - coreutils \ - curl \ - debianutils \ - debootstrap \ - g++ \ - gcc \ - git \ - golang-go \ - libdevmapper-dev \ - libc6-dev \ - libgpgme-dev \ - libssl-dev \ - libstdc++-8-dev \ - m4 \ - make \ - pkg-config \ - sed \ - systemd \ - tar \ - vim && \ - apt-get clean && rm -rf /var/lib/apt/lists/ - -# This will install the proper packages to build Kata components -@INSTALL_MUSL@ -@INSTALL_RUST@ diff --git a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in index f5acbac4e3..c0fffd07eb 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in +++ b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in @@ -35,8 +35,6 @@ RUN apt-get update && apt-get --no-install-recommends install -y \ libstdc++-8-dev \ m4 \ make \ - musl \ - musl-dev \ musl-tools \ pkg-config \ protobuf-compiler \ @@ -45,6 +43,8 @@ RUN apt-get update && apt-get --no-install-recommends install -y \ tar \ vim \ wget +# aarch64 requires this name -- link for all +RUN ln -s /usr/bin/musl-gcc "/usr/bin/$(uname -m)-linux-musl-gcc" # This will install the proper packages to build Kata components @INSTALL_RUST@ diff --git a/tools/osbuilder/scripts/lib.sh b/tools/osbuilder/scripts/lib.sh index 96d65d375f..7254c92dbc 100644 --- a/tools/osbuilder/scripts/lib.sh +++ b/tools/osbuilder/scripts/lib.sh @@ -7,7 +7,6 @@ set -e KATA_REPO=${KATA_REPO:-github.com/kata-containers/kata-containers} -MUSL_VERSION=${MUSL_VERSION:-"null"} # Give preference to variable set by CI yq_file="${script_dir}/../../../ci/install_yq.sh" kata_versions_file="${script_dir}/../../../versions.yaml" @@ -228,68 +227,12 @@ generate_dockerfile() dir="$1" [ -d "${dir}" ] || die "${dir}: not a directory" - local architecture=$(uname -m) - local rustarch=${architecture} - local muslarch=${architecture} - local libc=musl - case "$(uname -m)" in - "ppc64le") - rustarch=powerpc64le - muslarch=powerpc64 - libc=gnu - ;; - "s390x") - libc=gnu - ;; - - *) - ;; - esac + local rustarch=$(uname -m) + [ "$rustarch" = ppc64le ] && rustarch=powerpc64le [ -n "${http_proxy:-}" ] && readonly set_proxy="RUN sed -i '$ a proxy="${http_proxy:-}"' /etc/dnf/dnf.conf /etc/yum.conf; true" # Rust agent - # rust installer should set path apropiately, just in case - # install musl for compiling rust-agent - local musl_source_url="https://git.zv.io/toolchains/musl-cross-make.git" - local musl_source_dir="musl-cross-make" - install_musl= - if [ "${muslarch}" == "aarch64" ]; then - local musl_tar="${muslarch}-linux-musl-native.tgz" - local musl_dir="${muslarch}-linux-musl-native" - local aarch64_musl_target="aarch64-linux-musl" - install_musl=" -RUN cd /tmp; \ - mkdir -p /usr/local/musl/; \ - if curl -sLO --fail https://musl.cc/${musl_tar}; then \ - tar -zxf ${musl_tar}; \ - cp -r ${musl_dir}/* /usr/local/musl/; \ - else \ - git clone ${musl_source_url}; \ - TARGET=${aarch64_musl_target} make -j$(nproc) -C ${musl_source_dir} install; \ - cp -r ${musl_source_dir}/output/* /usr/local/musl/; \ - cp /usr/local/musl/bin/aarch64-linux-musl-g++ /usr/local/musl/bin/g++; \ - fi -ENV PATH=\$PATH:/usr/local/musl/bin -RUN ln -sf /usr/local/musl/bin/g++ /usr/bin/g++ -" - else - local musl_tar="musl-${MUSL_VERSION}.tar.gz" - local musl_dir="musl-${MUSL_VERSION}" - install_musl=" -RUN pushd /root; \ - curl -sLO https://www.musl-libc.org/releases/${musl_tar}; tar -zxf ${musl_tar}; \ - cd ${musl_dir}; \ - sed -i \"s/^ARCH = .*/ARCH = ${muslarch}/g\" dist/config.mak; \ - ./configure > /dev/null 2>\&1; \ - make > /dev/null 2>\&1; \ - make install > /dev/null 2>\&1; \ - echo \"/usr/local/musl/lib\" > /etc/ld-musl-${muslarch}.path; \ - popd -ENV PATH=\$PATH:/usr/local/musl/bin -" - fi - readonly install_rust=" RUN curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf --output /tmp/rust-init; \ chmod a+x /tmp/rust-init; \ @@ -304,32 +247,13 @@ RUN . /root/.cargo/env; \ RUN ln -sf /usr/bin/g++ /bin/musl-g++ " pushd "${dir}" - dockerfile_template="Dockerfile.in" - dockerfile_arch_template="Dockerfile-${architecture}.in" - # if arch-specific docker file exists, swap the univesal one with it. - if [ -f "${dockerfile_arch_template}" ]; then - dockerfile_template="${dockerfile_arch_template}" - else - [ -f "${dockerfile_template}" ] || die "${dockerfile_template}: file not found" - fi - # ppc64le and s390x have no musl target - if [ "${architecture}" == "ppc64le" ] || [ "${architecture}" == "s390x" ]; then - sed \ - -e "s|@OS_VERSION@|${OS_VERSION:-}|g" \ - -e "s|@INSTALL_MUSL@||g" \ - -e "s|@INSTALL_RUST@|${install_rust//$'\n'/\\n}|g" \ - -e "s|@SET_PROXY@|${set_proxy:-}|g" \ - "${dockerfile_template}" > Dockerfile - else - sed \ - -e "s|@OS_VERSION@|${OS_VERSION:-}|g" \ - -e "s|@INSTALL_MUSL@|${install_musl//$'\n'/\\n}|g" \ - -e "s|@INSTALL_RUST@|${install_rust//$'\n'/\\n}|g" \ - -e "s|@SET_PROXY@|${set_proxy:-}|g" \ - -e "s|@INSTALL_AA_KBC@|${AA_KBC_EXTRAS//$'\n'/\\n}|g" \ - "${dockerfile_template}" > Dockerfile - fi + sed \ + -e "s#@OS_VERSION@#${OS_VERSION:-}#g" \ + -e "s#@INSTALL_RUST@#${install_rust//$'\n'/\\n}#g" \ + -e "s#@SET_PROXY@#${set_proxy:-}#g" \ + -e "s#@INSTALL_AA_KBC@#${AA_KBC_EXTRAS//$'\n'/\\n}#g" \ + Dockerfile.in > Dockerfile popd } @@ -370,17 +294,6 @@ detect_rust_version() [ -n "$RUST_VERSION" ] } -detect_musl_version() -{ - info "Detecting musl version" - local yq_path="externals.musl.version" - - info "Get musl version from ${kata_versions_file}" - MUSL_VERSION="$(get_package_version_from_kata_yaml "$yq_path")" - - [ -n "$MUSL_VERSION" ] -} - before_starting_container() { return 0 } diff --git a/tools/osbuilder/tests/test_images.sh b/tools/osbuilder/tests/test_images.sh index 5def439b7c..338e5d3ada 100755 --- a/tools/osbuilder/tests/test_images.sh +++ b/tools/osbuilder/tests/test_images.sh @@ -640,8 +640,6 @@ test_dracut() die "Could not detect the required Go version for AGENT_VERSION='${AGENT_VERSION:-master}'." detect_rust_version || die "Could not detect the required rust version for AGENT_VERSION='${AGENT_VERSION:-master}'." - detect_musl_version || - die "Could not detect the required musl version for AGENT_VERSION='${AGENT_VERSION:-master}'." generate_dockerfile ${dracut_dir} info "Creating container for dracut" diff --git a/versions.yaml b/versions.yaml index 69540d54fb..8291d6d0da 100644 --- a/versions.yaml +++ b/versions.yaml @@ -246,19 +246,6 @@ externals: url: "https://github.com/opencontainers/umoci" tag: "v0.4.7" - musl: - description: | - The musl library is used to build the rust agent. - url: "https://www.musl-libc.org/" - uscan-url: >- - https://www.musl-libc.org/releases/ - musl-([\d\.]+)\.tar\.gz - version: "1.1.23" - meta: - description: | - 'newest-version' is the latest version known to work. - newest-version: "1.1.23" - nydus: description: "Nydus image acceleration service" url: "https://github.com/dragonflyoss/image-service" From e167237b13732307751af0a55001a5da53b472c6 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 4 Mar 2022 18:23:19 +0100 Subject: [PATCH 3/6] osbuilder: Simplify Rust installation no double export, direct target Signed-off-by: Jakob Naucke --- tools/osbuilder/scripts/lib.sh | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tools/osbuilder/scripts/lib.sh b/tools/osbuilder/scripts/lib.sh index 7254c92dbc..5601a5b2b6 100644 --- a/tools/osbuilder/scripts/lib.sh +++ b/tools/osbuilder/scripts/lib.sh @@ -234,17 +234,11 @@ generate_dockerfile() # Rust agent readonly install_rust=" -RUN curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf --output /tmp/rust-init; \ - chmod a+x /tmp/rust-init; \ - export http_proxy=${http_proxy:-}; \ - export https_proxy=${http_proxy:-}; \ - /tmp/rust-init -y --default-toolchain ${RUST_VERSION} -RUN . /root/.cargo/env; \ - export http_proxy=${http_proxy:-}; \ - export https_proxy=${http_proxy:-}; \ - cargo install cargo-when; \ - rustup target install ${rustarch}-unknown-linux-${libc} -RUN ln -sf /usr/bin/g++ /bin/musl-g++ +ENV http_proxy=${http_proxy:-} +ENV https_proxy=${http_proxy:-} +RUN curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf | \ + sh -s -- -y --default-toolchain ${RUST_VERSION} -t ${rustarch}-unknown-linux-${LIBC} +RUN . /root/.cargo/env; cargo install cargo-when " pushd "${dir}" From 527d741c07a3b2ca4237d402fcbd0c5006c3580e Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 1 Mar 2022 15:20:35 +0100 Subject: [PATCH 4/6] osbuilder: Fix use of LIBC in rootfs.sh - Add a doc comment - Pass to build container, e.g. to build x86_64 with glibc (would always use musl) Signed-off-by: Jakob Naucke --- tools/osbuilder/rootfs-builder/rootfs.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index 831aba78e2..323c541450 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -124,6 +124,9 @@ KERNEL_MODULES_DIR Path to a directory containing kernel modules to include in the rootfs. Default value: +LIBC libc the agent is built against (gnu or musl). + Default value: ${LIBC} (varies with architecture) + ROOTFS_DIR Path to the directory that is populated with the rootfs. Default value: <${script_name} path>/rootfs- @@ -427,11 +430,11 @@ build_rootfs_distro() --env AGENT_INIT="${AGENT_INIT}" \ --env CI="${CI}" \ --env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \ + --env LIBC="${LIBC}" \ --env EXTRA_PKGS="${EXTRA_PKGS}" \ --env OSBUILDER_VERSION="${OSBUILDER_VERSION}" \ --env OS_VERSION="${OS_VERSION}" \ --env INSIDE_CONTAINER=1 \ - --env LIBC="${LIBC}" \ --env SKOPEO="${SKOPEO}" \ --env UMOCI="${UMOCI}" \ --env AA_KBC="${AA_KBC}" \ From 578678e0510b6b5f422dc16975a07ad7b15274a4 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 15 Feb 2022 19:12:41 +0100 Subject: [PATCH 5/6] packaging: Enable cross-building agent Requires setting ARCH and CC. - Add CC linker option for building agent. - Set host for building libseccomp. Fixes: #3681 Signed-off-by: Jakob Naucke --- ci/install_libseccomp.sh | 8 +++++--- utils.mk | 11 ++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ci/install_libseccomp.sh b/ci/install_libseccomp.sh index 8933438860..5cbf8f0798 100755 --- a/ci/install_libseccomp.sh +++ b/ci/install_libseccomp.sh @@ -19,7 +19,7 @@ source "${tests_repo_dir}/.ci/lib.sh" # fail. So let's ensure they are unset here. unset PREFIX DESTDIR -arch=$(uname -m) +arch=${ARCH:-$(uname -m)} workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)" # Variables for libseccomp @@ -70,7 +70,9 @@ build_and_install_gperf() { curl -sLO "${gperf_tarball_url}" tar -xf "${gperf_tarball}" pushd "gperf-${gperf_version}" - ./configure --prefix="${gperf_install_dir}" + # gperf is a build time dependency of libseccomp and not to be used in the target. + # Unset $CC since that might point to a cross compiler. + CC= ./configure --prefix="${gperf_install_dir}" make make install export PATH=$PATH:"${gperf_install_dir}"/bin @@ -84,7 +86,7 @@ build_and_install_libseccomp() { curl -sLO "${libseccomp_tarball_url}" tar -xf "${libseccomp_tarball}" pushd "libseccomp-${libseccomp_version}" - ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static + ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}" make make install popd diff --git a/utils.mk b/utils.mk index e833b40d7a..3816fac2a1 100644 --- a/utils.mk +++ b/utils.mk @@ -112,8 +112,9 @@ endef ##VAR BUILD_TYPE=release|debug type of rust build BUILD_TYPE = release +HOST_ARCH = $(shell uname -m) ##VAR ARCH=arch target to build (format: uname -m) -ARCH = $(shell uname -m) +ARCH ?= $(HOST_ARCH) ##VAR LIBC=musl|gnu LIBC ?= musl ifneq ($(LIBC),musl) @@ -142,6 +143,14 @@ ifeq ($(ARCH), aarch64) $(warning "WARNING: aarch64-musl needs extra symbols from libgcc") endif +ifneq ($(HOST_ARCH),$(ARCH)) + ifeq ($(CC),) + CC = gcc + $(warning "WARNING: A foreign ARCH was passed, but no CC alternative. Using $(CC).") + endif + override EXTRA_RUSTFLAGS += -C linker=$(CC) +endif + TRIPLE = $(ARCH)-unknown-linux-$(LIBC) CWD := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) From 8fbf6c4e14cca7a7080b658f1e4fb25bdf1ee04f Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 21 Mar 2022 20:20:24 +0100 Subject: [PATCH 6/6] osbuilder: Multistrap Ubuntu Use `multistrap` for building Ubuntu rootfs. Adds support for building for foreign architectures using the `ARCH` environment variable (including umoci). In the process, the Ubuntu rootfs workflow is vastly simplified. Signed-off-by: Jakob Naucke --- tools/osbuilder/rootfs-builder/rootfs.sh | 37 +++++-- .../rootfs-builder/ubuntu/Dockerfile.in | 43 +++----- .../osbuilder/rootfs-builder/ubuntu/config.sh | 59 +++++----- .../rootfs-builder/ubuntu/rootfs_lib.sh | 103 +++++------------- tools/osbuilder/scripts/lib.sh | 5 +- 5 files changed, 100 insertions(+), 147 deletions(-) diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index 323c541450..fa2dfad496 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -39,7 +39,11 @@ handle_error() { trap 'handle_error $LINENO' ERR # Default architecture -ARCH=$(uname -m) +export ARCH=${ARCH:-$(uname -m)} +if [ "$ARCH" == "ppc64le" ] || [ "$ARCH" == "s390x" ]; then + LIBC=gnu + echo "WARNING: Forcing LIBC=gnu because $ARCH has no musl Rust target" +fi # distro-specific config file typeset -r CONFIG_SH="config.sh" @@ -103,6 +107,11 @@ AGENT_SOURCE_BIN Path to the directory of agent binary. AGENT_VERSION Version of the agent to include in the rootfs. Default value: ${AGENT_VERSION:-} +ARCH Target architecture (according to \`uname -m\`). + Foreign bootstraps are currently only supported for Ubuntu + and glibc agents. + Default value: $(uname -m) + DISTRO_REPO Use host repositories to install guest packages. Default value: @@ -428,6 +437,7 @@ build_rootfs_distro() --env ROOTFS_DIR="/rootfs" \ --env AGENT_BIN="${AGENT_BIN}" \ --env AGENT_INIT="${AGENT_INIT}" \ + --env ARCH="${ARCH}" \ --env CI="${CI}" \ --env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \ --env LIBC="${LIBC}" \ @@ -560,11 +570,6 @@ EOF AGENT_DIR="${ROOTFS_DIR}/usr/bin" AGENT_DEST="${AGENT_DIR}/${AGENT_BIN}" - if [ "$ARCH" == "ppc64le" ] || [ "$ARCH" == "s390x" ]; then - LIBC=gnu - warning "Forcing LIBC=gnu because $ARCH has no musl Rust target" - fi - if [ -z "${AGENT_SOURCE_BIN}" ] ; then test -r "${HOME}/.cargo/env" && source "${HOME}/.cargo/env" # rust agent needs ${arch}-unknown-linux-${LIBC} @@ -583,7 +588,7 @@ EOF info "Set up libseccomp" libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX) gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX) - bash ${script_dir}/../../../ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}" + ${script_dir}/../../../ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}" echo "Set environment variables for the libseccomp crate to link the libseccomp library statically" export LIBSECCOMP_LINK_TYPE=static export LIBSECCOMP_LIB_PATH="${libseccomp_install_dir}/lib" @@ -667,16 +672,28 @@ EOF source "${HOME}/.cargo/env" target="${ARCH}-unknown-linux-${LIBC}" if [ "${AA_KBC}" == "eaa_kbc" ] && [ "${ARCH}" == "x86_64" ]; then - AA_RUSTFLAG="-C link-args=-Wl,-rpath,/usr/local/lib/rats-tls" + RUSTFLAGS="-C link-args=-Wl,-rpath,/usr/local/lib/rats-tls" # Currently eaa_kbc module only support this specific platform target="x86_64-unknown-linux-gnu" fi - RUSTFLAGS=${AA_RUSTFLAG} cargo build --release --target "${target}" --no-default-features --features "${AA_KBC}" - install -o root -g root -m 0755 "target/${target}/release/attestation-agent" "${ROOTFS_DIR}/usr/local/bin/" + if [ "$(uname -m)" != "$ARCH" ]; then + RUSTFLAGS+=" -C linker=$CC" + fi + export RUSTFLAGS + # Foreign CC is incompatible with libgit2 -- CC is still handled by `-C linker=...` flag + CC= cargo build --release --target "${target}" --no-default-features --features "${AA_KBC}" + install -D -o root -g root -m 0755 "target/${target}/release/attestation-agent" -t "${ROOTFS_DIR}/usr/local/bin/" popd fi if [ "${UMOCI}" = "yes" ]; then + case "$ARCH" in + aarch64) GOARCH=arm64;; + x86_64) GOARCH=amd64;; + *) GOARCH="$ARCH" + esac + export GOARCH + umoci_url="$(get_package_version_from_kata_yaml externals.umoci.url)" umoci_tag="$(get_package_version_from_kata_yaml externals.umoci.tag)" info "Install umoci" diff --git a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in index c0fffd07eb..25d1907a6f 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in +++ b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in @@ -1,51 +1,36 @@ -# -# Copyright (c) 2018 Yash Jain +# Copyright (c) 2018 Yash Jain, 2022 IBM Corp. # # SPDX-License-Identifier: Apache-2.0 ARG IMAGE_REGISTRY=docker.io -#ubuntu: docker image to be used to create a rootfs -#@OS_VERSION@: Docker image version to build this dockerfile FROM ${IMAGE_REGISTRY}/ubuntu:@OS_VERSION@ +@SET_PROXY@ -# This dockerfile needs to provide all the componets need to build a rootfs -# Install any package need to create a rootfs (package manager, extra tools) - -# RUN commands -RUN apt-get update && apt-get --no-install-recommends install -y \ - apt-utils \ - autoconf \ - automake \ - binutils \ - build-essential \ +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive \ + apt-get --no-install-recommends -y install \ ca-certificates \ - chrony \ - coreutils \ curl \ - debianutils \ - debootstrap \ g++ \ - gcc \ + $(gcc_arch="@ARCH@" && [ "$(uname -m)" != "$gcc_arch" ] && ( \ + libc_arch="$gcc_arch" && \ + [ "$gcc_arch" = aarch64 ] && libc_arch=arm64; \ + [ "$gcc_arch" = ppc64le ] && gcc_arch=powerpc64le && libc_arch=ppc64el; \ + [ "$gcc_arch" = x86_64 ] && gcc_arch=x86-64 && libc_arch=amd64; \ + echo "gcc-$gcc_arch-linux-gnu libc6-dev-$libc_arch-cross")) \ git \ golang-go \ libdevmapper-dev \ - libc6-dev \ libgpgme-dev \ libssl-dev \ - libstdc++-8-dev \ - m4 \ make \ + multistrap \ musl-tools \ pkg-config \ - protobuf-compiler \ - sed \ - systemd \ - tar \ - vim \ - wget + protobuf-compiler + # aarch64 requires this name -- link for all RUN ln -s /usr/bin/musl-gcc "/usr/bin/$(uname -m)-linux-musl-gcc" -# This will install the proper packages to build Kata components @INSTALL_RUST@ @INSTALL_AA_KBC@ diff --git a/tools/osbuilder/rootfs-builder/ubuntu/config.sh b/tools/osbuilder/rootfs-builder/ubuntu/config.sh index 3c4dbb9319..d9e249f8d0 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/config.sh +++ b/tools/osbuilder/rootfs-builder/ubuntu/config.sh @@ -1,46 +1,39 @@ -# This is a configuration file add extra variables to -# -# Copyright (c) 2018 Yash Jain +# Copyright (c) 2018 Yash Jain, 2022 IBM Corp. # # SPDX-License-Identifier: Apache-2.0 -# be used by build_rootfs() from rootfs_lib.sh the variables will be -# loaded just before call the function. For more information see the -# rootfs-builder/README.md file. -OS_VERSION=${OS_VERSION:-20.04} +OS_NAME=ubuntu # This should be Ubuntu's code name, e.g. "focal" (Focal Fossa) for 20.04 -OS_NAME=${OS_NAME:-"focal"} +OS_VERSION=${OS_VERSION:-focal} +PACKAGES=chrony +[ "$AGENT_INIT" = no ] && PACKAGES+=" init" +[ "$SECCOMP" = yes ] && PACKAGES+=" libseccomp2" +[ "$SKOPEO" = yes ] && PACKAGES+=" libgpgme11" +REPO_URL=http://ports.ubuntu.com -# packages to be installed by default -# Note: ca-certificates is required for confidential containers -# to pull the container image on the guest -PACKAGES="systemd coreutils init kmod ca-certificates" -EXTRA_PKGS+=" chrony" - -DEBOOTSTRAP=${PACKAGE_MANAGER:-"debootstrap"} - -case $(uname -m) in - x86_64) ARCHITECTURE="amd64";; - ppc64le) ARCHITECTURE="ppc64el";; - aarch64) ARCHITECTURE="arm64";; - s390x) ARCHITECTURE="s390x";; - (*) die "$(uname -m) not supported " +case "$ARCH" in + aarch64) DEB_ARCH=arm64;; + ppc64le) DEB_ARCH=ppc64el;; + s390x) DEB_ARCH="$ARCH";; + x86_64) DEB_ARCH=amd64; REPO_URL=http://archive.ubuntu.com/ubuntu;; + *) die "$ARCH not supported" esac -# Init process must be one of {systemd,kata-agent} -INIT_PROCESS=systemd -# List of zero or more architectures to exclude from build, -# as reported by `uname -m` -ARCH_EXCLUDE_LIST=() - -[ "$SECCOMP" = "yes" ] && PACKAGES+=" libseccomp2" || true -[ "$SKOPEO" = "yes" ] && PACKAGES+=" libgpgme11" || true - if [ "${AA_KBC}" == "eaa_kbc" ] && [ "${ARCH}" == "x86_64" ]; then - AA_KBC_EXTRAS=" + PACKAGES+=" apt gnupg" + AA_KBC_EXTRAS=" RUN echo 'deb [arch=amd64] http://mirrors.openanolis.cn/inclavare-containers/ubuntu20.04 bionic main' \| tee /etc/apt/sources.list.d/inclavare-containers.list; \ - wget -qO - http://mirrors.openanolis.cn/inclavare-containers/ubuntu20.04/DEB-GPG-KEY.key \| apt-key add -; \ + curl -L http://mirrors.openanolis.cn/inclavare-containers/ubuntu20.04/DEB-GPG-KEY.key \| apt-key add -; \ apt-get update; \ apt-get install -y rats-tls " fi + +if [ "$(uname -m)" != "$ARCH" ]; then + case "$ARCH" in + ppc64le) cc_arch=powerpc64le;; + x86_64) cc_arch=x86-64;; + *) cc_arch="$ARCH" + esac + export CC="$cc_arch-linux-gnu-gcc" +fi diff --git a/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh b/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh index 4e048ca6a0..90c13dd0a0 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh +++ b/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh @@ -1,84 +1,41 @@ -# - Arguments -# -# Copyright (c) 2018 Yash Jain +# Copyright (c) 2018 Yash Jain, 2022 IBM Corp. # # SPDX-License-Identifier: Apache-2.0 -# -# -# rootfs_dir=$1 -# -# - Optional environment variables -# -# EXTRA_PKGS: Variable to add extra PKGS provided by the user -# -# BIN_AGENT: Name of the Kata-Agent binary -# -# REPO_URL: URL to distribution repository ( should be configured in -# config.sh file) -# -# Any other configuration variable for a specific distro must be added -# and documented on its own config.sh -# -# - Expected result -# -# rootfs_dir populated with rootfs pkgs -# It must provide a binary in /sbin/init -# + build_rootfs() { - # Mandatory - local ROOTFS_DIR=$1 + local rootfs_dir=$1 + local multistrap_conf=multistrap.conf - # Name of the Kata-Agent binary - local BIN_AGENT=${BIN_AGENT} + [ -z "$rootfs_dir" ] && die "need rootfs" + [ "$rootfs_dir" = "/" ] && die "rootfs cannot be slash" - # In case of support EXTRA packages, use it to allow - # users to add more packages to the base rootfs - local EXTRA_PKGS=${EXTRA_PKGS:-} + # For simplicity's sake, use multistrap for foreign and native bootstraps. + cat > "$multistrap_conf" << EOF +[General] +cleanup=true +aptsources=Ubuntu +bootstrap=Ubuntu - # In case rootfs is created using repositories allow user to modify - # the default URL - local REPO_URL=${REPO_URL:-YOUR_REPO} +[Ubuntu] +source=$REPO_URL +keyring=ubuntu-keyring +suite=focal +packages=$PACKAGES $EXTRA_PKGS +EOF + multistrap -a "$DEB_ARCH" -d "$rootfs_dir" -f "$multistrap_conf" + rm -rf "$rootfs_dir/var/run" + ln -s /run "$rootfs_dir/var/run" + for file in /etc/{resolv.conf,ssl/certs/ca-certificates.crt}; do + mkdir -p "$rootfs_dir$(dirname $file)" + cp --remove-destination "$file" "$rootfs_dir$file" + done - # PATH where files this script is placed - # Use it to refer to files in the same directory - # Example: ${CONFIG_DIR}/foo - local CONFIG_DIR=${CONFIG_DIR} + # Reduce image size and memory footprint by removing unnecessary files and directories. + rm -rf $rootfs_dir/usr/share/{bash-completion,bug,doc,info,lintian,locale,man,menu,misc,pixmaps,terminfo,zsh} - - # Populate ROOTFS_DIR - # Must provide /sbin/init and /bin/${BIN_AGENT} - DEBOOTSTRAP="debootstrap" - check_root - mkdir -p "${ROOTFS_DIR}" - if [ -n "${PKG_MANAGER}" ]; then - info "debootstrap path provided by user: ${PKG_MANAGER}" - elif check_program $DEBOOTSTRAP ; then - PKG_MANAGER=$DEBOOTSTRAP - else - die "$DEBOOTSTRAP is not installed" - fi - # trim whitespace - PACKAGES=$(echo $PACKAGES |xargs ) - # add comma as debootstrap needs , separated package names. - # Don't change $PACKAGES in config.sh to include ',' - # This is done to maintain consistency - PACKAGES=$(echo $PACKAGES | sed -e 's/ /,/g' ) - - ${PKG_MANAGER} --variant=minbase \ - --arch=${ARCHITECTURE}\ - --include="$PACKAGES" \ - ${OS_NAME} \ - ${ROOTFS_DIR} - - [ -n "${EXTRA_PKGS}" ] && chroot $ROOTFS_DIR apt-get install -y ${EXTRA_PKGS} - - # Reduce image size and memory footprint - # removing not needed files and directories. - chroot $ROOTFS_DIR rm -rf /usr/share/{bash-completion,bug,doc,info,lintian,locale,man,menu,misc,pixmaps,terminfo,zoneinfo,zsh} - - if [ "${AA_KBC}" == "eaa_kbc" ] && [ "${ARCH}" == "x86_64" ]; then - wget -qO - http://mirrors.openanolis.cn/inclavare-containers/ubuntu20.04/DEB-GPG-KEY.key | chroot $ROOTFS_DIR apt-key add - - cat << EOF | chroot $ROOTFS_DIR + if [ "${AA_KBC}" == "eaa_kbc" ] && [ "${ARCH}" == "x86_64" ]; then + curl -L http://mirrors.openanolis.cn/inclavare-containers/ubuntu20.04/DEB-GPG-KEY.key | chroot "$rootfs_dir" apt-key add - + cat << EOF | chroot "$rootfs_dir" echo 'deb [arch=amd64] http://mirrors.openanolis.cn/inclavare-containers/ubuntu20.04 bionic main' | tee /etc/apt/sources.list.d/inclavare-containers.list apt-get update apt-get install -y rats-tls diff --git a/tools/osbuilder/scripts/lib.sh b/tools/osbuilder/scripts/lib.sh index 5601a5b2b6..7ee6b79986 100644 --- a/tools/osbuilder/scripts/lib.sh +++ b/tools/osbuilder/scripts/lib.sh @@ -227,8 +227,8 @@ generate_dockerfile() dir="$1" [ -d "${dir}" ] || die "${dir}: not a directory" - local rustarch=$(uname -m) - [ "$rustarch" = ppc64le ] && rustarch=powerpc64le + local rustarch="$ARCH" + [ "$ARCH" = ppc64le ] && rustarch=powerpc64le [ -n "${http_proxy:-}" ] && readonly set_proxy="RUN sed -i '$ a proxy="${http_proxy:-}"' /etc/dnf/dnf.conf /etc/yum.conf; true" @@ -244,6 +244,7 @@ RUN . /root/.cargo/env; cargo install cargo-when sed \ -e "s#@OS_VERSION@#${OS_VERSION:-}#g" \ + -e "s#@ARCH@#$ARCH#g" \ -e "s#@INSTALL_RUST@#${install_rust//$'\n'/\\n}#g" \ -e "s#@SET_PROXY@#${set_proxy:-}#g" \ -e "s#@INSTALL_AA_KBC@#${AA_KBC_EXTRAS//$'\n'/\\n}#g" \