From 2c6b3d114c94d99cb09eead9a1295c9420c1c2a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 28 Nov 2022 19:44:38 +0100 Subject: [PATCH 1/7] initramfs: get_from_kata_deps does't require a "kata_version" arg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's been dropped for a long time now, as part of a09e58fa80ae81c0d336d9a782da4f38f9492da1. Signed-off-by: Fabiano Fidêncio --- tools/packaging/static-build/initramfs/build.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/packaging/static-build/initramfs/build.sh b/tools/packaging/static-build/initramfs/build.sh index 72fb459325..cbf02e2a60 100755 --- a/tools/packaging/static-build/initramfs/build.sh +++ b/tools/packaging/static-build/initramfs/build.sh @@ -15,17 +15,16 @@ readonly default_install_dir="$(cd "${script_dir}/../../kernel" && pwd)" source "${script_dir}/../../scripts/lib.sh" -kata_version="${kata_version:-}" cryptsetup_repo="${cryptsetup_repo:-}" cryptsetup_version="${cryptsetup_version:-}" lvm2_repo="${lvm2_repo:-}" lvm2_version="${lvm2_version:-}" package_output_dir="${package_output_dir:-}" -[ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps "externals.cryptsetup.url" "${kata_version}") -[ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps "externals.cryptsetup.version" "${kata_version}") -[ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps "externals.lvm2.url" "${kata_version}") -[ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps "externals.lvm2.version" "${kata_version}") +[ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps "externals.cryptsetup.url") +[ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps "externals.cryptsetup.version") +[ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps "externals.lvm2.url") +[ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps "externals.lvm2.version") [ -n "${cryptsetup_repo}" ] || die "Failed to get cryptsetup repo" [ -n "${cryptsetup_version}" ] || die "Failed to get cryptsetup version" From 5022a0d2c2863304a9e97d42001dafdce840d92d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 28 Nov 2022 21:16:51 +0100 Subject: [PATCH 2/7] =?UTF-8?q?packaging:=20Don=C2=B4t=20print=20sha256sum?= =?UTF-8?q?=20as=20part=20of=20sha256sum=5Ffrom=5Ffiles()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If you're directly using the output of this function, the info message will show up as part of the string, and that's not what we want. Signed-off-by: Fabiano Fidêncio --- tools/packaging/scripts/lib.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index b119b6c9cd..49520c6616 100755 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -173,7 +173,6 @@ sha256sum_from_files() { files="$(echo $files | tr ' ' '\n' | LC_ALL=C sort -u)" # Concate the files and calculate a hash. shasum="$(cat $files | sha256sum -b)" || true - info "shasum of files $shasum" if [ -n "$shasum" ];then # Return only the SHA field. echo $(awk '{ print $1 }' <<< $shasum) From 58480aac342edab57b96378eb7d19a638909d2d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 28 Nov 2022 19:41:45 +0100 Subject: [PATCH 3/7] packaging: Move calc_qemu_files_sha256sum() to lib.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're going to use this function from different places, so we better move it to lib.sh and avoid rewriting it. Signed-off-by: Fabiano Fidêncio --- tools/packaging/scripts/lib.sh | 8 ++++++++ .../packaging/static-build/qemu/build-static-qemu-cc.sh | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index 49520c6616..4ad7d0b14f 100755 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -178,3 +178,11 @@ sha256sum_from_files() { echo $(awk '{ print $1 }' <<< $shasum) fi } + +calc_qemu_files_sha256sum() { + local files="${this_script_dir}/../qemu \ + ${this_script_dir}/../static-build/qemu.blacklist \ + ${this_script_dir}/../static-build/scripts" + + sha256sum_from_files "$files" +} diff --git a/tools/packaging/static-build/qemu/build-static-qemu-cc.sh b/tools/packaging/static-build/qemu/build-static-qemu-cc.sh index 47ffe6cd03..724e606b08 100755 --- a/tools/packaging/static-build/qemu/build-static-qemu-cc.sh +++ b/tools/packaging/static-build/qemu/build-static-qemu-cc.sh @@ -36,15 +36,6 @@ get_qemu_information() { [ -n "${qemu_version}" ] || die "failed to get qemu version" } -calc_qemu_files_sha256sum() { - info "pkg directory is at ${pkg_dir}" - local files="${pkg_dir}/qemu \ - ${pkg_dir}/static-build/qemu.blacklist \ - ${pkg_dir}/static-build/scripts" - - sha256sum_from_files "$files" -} - cached_or_build_qemu_tar() { # Check latest qemu cc tar version sha256sum local latest=$(curl -sfL "${qemu_latest_build_url}/latest") || latest="none" From 02f6af9e1e787e567c9694141aa367a636fe42fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 28 Nov 2022 21:10:52 +0100 Subject: [PATCH 4/7] packaging: Move repo_root_dir to lib.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is used in several parts of the code, and can have a single declaration as part of the `lib.sh` file, which is already imported by all the places where it's used. Signed-off-by: Fabiano Fidêncio --- tools/packaging/guest-image/build_image.sh | 6 +++--- .../kata-deploy/local-build/kata-deploy-binaries.sh | 3 ++- tools/packaging/scripts/lib.sh | 2 ++ tools/packaging/static-build/initramfs/build.sh | 1 - tools/packaging/static-build/kernel/build.sh | 4 ++-- tools/packaging/static-build/ovmf/build.sh | 1 - tools/packaging/static-build/qemu/build-base-qemu.sh | 1 - tools/packaging/static-build/shim-v2/build.sh | 4 ++-- tools/packaging/static-build/td-shim/build.sh | 1 - tools/packaging/static-build/virtiofsd/build.sh | 1 - 10 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index ae8bf8ef4b..c254778399 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -14,7 +14,9 @@ set -o pipefail readonly script_name="$(basename "${BASH_SOURCE[0]}")" readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly packaging_root_dir="$(cd "${script_dir}/../" && pwd)" -readonly repo_root_dir="$(cd "${script_dir}/../../../" && pwd)" + +source "${packaging_root_dir}/scripts/lib.sh" + readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)" patches_path="" @@ -26,8 +28,6 @@ final_image_name="kata-containers" final_initrd_name="kata-containers-initrd" image_initrd_extension=".img" -source "${packaging_root_dir}/scripts/lib.sh" - arch_target="$(uname -m)" build_initrd() { diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 1a5b89197d..d6577f9219 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -15,8 +15,9 @@ readonly project="kata-containers" readonly script_name="$(basename "${BASH_SOURCE[0]}")" readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${script_dir}/../../scripts/lib.sh" + readonly prefix="/opt/kata" -readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)" readonly static_build_dir="${repo_root_dir}/tools/packaging/static-build" readonly version_file="${repo_root_dir}/VERSION" readonly versions_yaml="${repo_root_dir}/versions.yaml" diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index 4ad7d0b14f..886131ba7f 100755 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -13,6 +13,8 @@ export PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}" this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export repo_root_dir="$(cd "${this_script_dir}/../../../" && pwd)" + short_commit_length=10 hub_bin="hub-bin" diff --git a/tools/packaging/static-build/initramfs/build.sh b/tools/packaging/static-build/initramfs/build.sh index cbf02e2a60..00ea30f95e 100755 --- a/tools/packaging/static-build/initramfs/build.sh +++ b/tools/packaging/static-build/initramfs/build.sh @@ -9,7 +9,6 @@ set -o nounset set -o pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)" readonly initramfs_builder="${script_dir}/build-initramfs.sh" readonly default_install_dir="$(cd "${script_dir}/../../kernel" && pwd)" diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index 0ffd191938..53839e2e48 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -9,11 +9,11 @@ set -o nounset set -o pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)" -readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh" source "${script_dir}/../../scripts/lib.sh" +readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh" + DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} container_image="${KERNEL_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" diff --git a/tools/packaging/static-build/ovmf/build.sh b/tools/packaging/static-build/ovmf/build.sh index 50ef9a73e0..a85fb9839e 100755 --- a/tools/packaging/static-build/ovmf/build.sh +++ b/tools/packaging/static-build/ovmf/build.sh @@ -9,7 +9,6 @@ set -o nounset set -o pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)" readonly ovmf_builder="${script_dir}/build-ovmf.sh" source "${script_dir}/../../scripts/lib.sh" diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh index 5cdb7e654a..0c858b302e 100755 --- a/tools/packaging/static-build/qemu/build-base-qemu.sh +++ b/tools/packaging/static-build/qemu/build-base-qemu.sh @@ -9,7 +9,6 @@ set -o nounset set -o pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)" readonly qemu_builder="${script_dir}/build-qemu.sh" source "${script_dir}/../../scripts/lib.sh" diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index 10a6e332a2..77e4ae12e4 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -9,11 +9,11 @@ set -o nounset set -o pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)" -readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh" source "${script_dir}/../../scripts/lib.sh" +readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh" + GO_VERSION=${GO_VERSION} RUST_VERSION=${RUST_VERSION:-} diff --git a/tools/packaging/static-build/td-shim/build.sh b/tools/packaging/static-build/td-shim/build.sh index fd55d3148b..a19b692b22 100755 --- a/tools/packaging/static-build/td-shim/build.sh +++ b/tools/packaging/static-build/td-shim/build.sh @@ -9,7 +9,6 @@ set -o nounset set -o pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)" readonly tdshim_builder="${script_dir}/build-td-shim.sh" source "${script_dir}/../../scripts/lib.sh" diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh index dbf0fac2b7..7d8b1a253c 100755 --- a/tools/packaging/static-build/virtiofsd/build.sh +++ b/tools/packaging/static-build/virtiofsd/build.sh @@ -9,7 +9,6 @@ set -o nounset set -o pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)" readonly virtiofsd_builder="${script_dir}/build-static-virtiofsd.sh" source "${script_dir}/../../scripts/lib.sh" From 8a4e77114465d4d35a964edadea832b3469a669a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 28 Nov 2022 19:47:33 +0100 Subject: [PATCH 5/7] packaging: Add functions to generate component's image name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's do this as the component name will be re-used later on, when we start checking whether a cached component needs to be rebuilt or not. Signed-off-by: Fabiano Fidêncio --- tools/packaging/scripts/lib.sh | 59 +++++++++++++++++-- .../packaging/static-build/initramfs/build.sh | 2 +- tools/packaging/static-build/kernel/build.sh | 2 +- tools/packaging/static-build/ovmf/build.sh | 2 +- .../static-build/qemu/build-base-qemu.sh | 2 +- tools/packaging/static-build/shim-v2/build.sh | 2 +- tools/packaging/static-build/td-shim/build.sh | 2 +- .../packaging/static-build/virtiofsd/build.sh | 2 +- 8 files changed, 62 insertions(+), 11 deletions(-) diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index 886131ba7f..b14689ae1e 100755 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -123,12 +123,11 @@ get_config_version() { fi } -# $1 - Repo's root dir -# $2 - The file we're looking for the last modification +# $1 - The file we're looking for the last modification get_last_modification() { - local repo_root_dir="${1}" - local file="${2}" + local file="${1}" + pushd ${repo_root_dir} &> /dev/null # This is a workaround needed for when running this code on Jenkins git config --global --add safe.directory ${repo_root_dir} &> /dev/null @@ -136,6 +135,7 @@ get_last_modification() { [ $(git status --porcelain | grep "${file#${repo_root_dir}/}" | wc -l) -gt 0 ] && dirty="-dirty" echo "$(git log -1 --pretty=format:"%H" ${file})${dirty}" + popd &> /dev/null } # $1 - The tag to be pushed to the registry @@ -188,3 +188,54 @@ calc_qemu_files_sha256sum() { sha256sum_from_files "$files" } + +get_initramfs_image_name() { + initramfs_script_dir="${this_script_dir}/../static-build/initramfs" + echo "${CC_BUILDER_REGISTRY}:initramfs-cryptosetup$(get_from_kata_deps "externals.cryptsetup.version")-lvm2-$(get_from_kata_deps "externals.lvm2.version")-$(get_last_modification ${initramfs_script_dir})-$(uname -m)" +} + +get_kernel_image_name() { + kernel_script_dir="${this_script_dir}/../static-build/kernel" + echo "${CC_BUILDER_REGISTRY}:kernel-$(get_last_modification ${kernel_script_dir})-$(uname -m)" +} + +get_ovmf_image_name() { + ovmf_script_dir="${this_script_dir}/../static-build/ovmf" + echo "${CC_BUILDER_REGISTRY}:ovmf-$(get_last_modification ${ovmf_script_dir})-$(uname -m)" +} + +get_qemu_image_name() { + qemu_script_dir="${this_script_dir}/../static-build/qemu" + echo "${CC_BUILDER_REGISTRY}:qemu-$(get_last_modification ${qemu_script_dir})-$(uname -m)" +} + +get_shim_v2_image_name() { + shim_v2_script_dir="${this_script_dir}/../static-build/shim-v2" + echo "${CC_BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps "languages.golang.meta.newest-version")-rust-$(get_from_kata_deps "languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)" +} + +get_td_shim_image_name() { + td_shim_script_dir="${this_script_dir}/../static-build/td-shim" + echo "${CC_BUILDER_REGISTRY}:td-shim-$(get_last_modification ${td_shim_script_dir})-$(uname -m)" +} + +get_virtiofsd_image_name() { + ARCH=$(uname -m) + case ${ARCH} in + "aarch64") + libc="musl" + ;; + "ppc64le") + libc="gnu" + ;; + "s390x") + libc="gnu" + ;; + "x86_64") + libc="musl" + ;; + esac + + virtiofsd_script_dir="${this_script_dir}/../static-build/virtiofsd" + echo "${CC_BUILDER_REGISTRY}:virtiofsd-$(get_from_kata_deps "externals.virtiofsd.toolchain")-${libc}-$(get_last_modification ${virtiofsd_script_dir})-$(uname -m)" +} diff --git a/tools/packaging/static-build/initramfs/build.sh b/tools/packaging/static-build/initramfs/build.sh index 00ea30f95e..5f35624192 100755 --- a/tools/packaging/static-build/initramfs/build.sh +++ b/tools/packaging/static-build/initramfs/build.sh @@ -30,7 +30,7 @@ package_output_dir="${package_output_dir:-}" [ -n "${lvm2_repo}" ] || die "Failed to get lvm2 repo" [ -n "${lvm2_version}" ] || die "Failed to get lvm2 version" -container_image="${INITRAMFS_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:initramfs-cryptsetup-${cryptsetup_version}-lvm2-${lvm2_version}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" +container_image="${INITRAMFS_CONTAINER_BUILDER:-$(get_initramfs_image_name)}" sudo docker pull ${container_image} || (sudo docker build \ --build-arg cryptsetup_repo="${cryptsetup_repo}" \ diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index 53839e2e48..db0e65d77d 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -16,7 +16,7 @@ readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} -container_image="${KERNEL_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" +container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}" kernel_latest_build_url="${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}" current_kernel_version=${kernel_version:-$(get_from_kata_deps "assets.kernel.version")} cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" diff --git a/tools/packaging/static-build/ovmf/build.sh b/tools/packaging/static-build/ovmf/build.sh index a85fb9839e..53444254b1 100755 --- a/tools/packaging/static-build/ovmf/build.sh +++ b/tools/packaging/static-build/ovmf/build.sh @@ -15,7 +15,7 @@ source "${script_dir}/../../scripts/lib.sh" DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} -container_image="${OVMF_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:ovmf-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" +container_image="${OVMF_CONTAINER_BUILDER:-$(get_ovmf_image_name)}" ovmf_build="${ovmf_build:-x86_64}" kata_version="${kata_version:-}" ovmf_repo="${ovmf_repo:-}" diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh index 0c858b302e..ea72898c29 100755 --- a/tools/packaging/static-build/qemu/build-base-qemu.sh +++ b/tools/packaging/static-build/qemu/build-base-qemu.sh @@ -38,7 +38,7 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d") [ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu" [ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static" -container_image="${QEMU_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:qemu-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" +container_image="${QEMU_CONTAINER_BUILDER:-$(get_qemu_image_name)}" sudo docker pull ${container_image} || \ (sudo "${container_engine}" build \ diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index 77e4ae12e4..db78cc14cd 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -19,7 +19,7 @@ RUST_VERSION=${RUST_VERSION:-} DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} -container_image="${SHIM_V2_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" +container_image="${SHIM_V2_CONTAINER_BUILDER:-$(get_shim_v2_image_name)}" EXTRA_OPTS="${EXTRA_OPTS:-""}" VMM_CONFIGS="qemu fc" diff --git a/tools/packaging/static-build/td-shim/build.sh b/tools/packaging/static-build/td-shim/build.sh index a19b692b22..0f898cec88 100755 --- a/tools/packaging/static-build/td-shim/build.sh +++ b/tools/packaging/static-build/td-shim/build.sh @@ -29,7 +29,7 @@ package_output_dir="${package_output_dir:-}" [ -n "${tdshim_version}" ] || die "Failed to get TD-shim version or commit" [ -n "${tdshim_toolchain}" ] || die "Failed to get TD-shim toolchain to be used to build the project" -container_image="${TDSHIM_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:td-shim-${tdshim_toolchain}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" +container_image="${TDSHIM_CONTAINER_BUILDER:-$(get_td_shim_image_name)}" sudo docker pull ${container_image} || \ (sudo docker build \ diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh index 7d8b1a253c..6eb5ad51bf 100755 --- a/tools/packaging/static-build/virtiofsd/build.sh +++ b/tools/packaging/static-build/virtiofsd/build.sh @@ -48,7 +48,7 @@ case ${ARCH} in ;; esac -container_image="${VIRTIOFSD_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" +container_image="${VIRTIOFSD_CONTAINER_BUILDER:-$(get_virtiofsd_image_name)}" sudo docker pull ${container_image} || \ (sudo docker build \ From 3c79af1ee119e61070814a9cced980979a7c66c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 28 Nov 2022 14:09:09 +0100 Subject: [PATCH 6/7] cache_components: Cache the final tarball MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of caching files generated during the component build, let's cache the final tarball generated for each component. Signed-off-by: Fabiano Fidêncio --- .../static-build/cache_components.sh | 66 ++++++++----------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/tools/packaging/static-build/cache_components.sh b/tools/packaging/static-build/cache_components.sh index 6bd295ec1f..678ef4fd09 100755 --- a/tools/packaging/static-build/cache_components.sh +++ b/tools/packaging/static-build/cache_components.sh @@ -13,61 +13,53 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${script_dir}/../scripts/lib.sh" export KATA_BUILD_CC="${KATA_BUILD_CC:-}" -export qemu_cc_tarball_name="kata-static-qemu-cc.tar.gz" +export TEE="${TEE:-}" cache_qemu_artifacts() { - source "${script_dir}/qemu/build-static-qemu-cc.sh" + local qemu_tarball_name="kata-static-cc-qemu.tar.xz" local current_qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version") - create_cache_asset "${qemu_cc_tarball_name}" "${current_qemu_version}" + if [ -n "${TEE}" ]; then + qemu_tarball_name="kata-static-cc-${TEE}-qemu.tar.xz" + [ "${TEE}" == "tdx" ] && current_qemu_version=$(get_from_kata_deps "asserts.hypervisor.qemu.tdx.tag") + fi + local qemu_script_dir="${repo_root_dir}/tools/packaging/static-build/qemu" local qemu_sha=$(calc_qemu_files_sha256sum) - echo "${current_qemu_version} ${qemu_sha}" > "latest" + local current_qemu_image="$(get_qemu_image_name)" + create_cache_asset "${qemu_tarball_name}" "${current_qemu_version}-${qemu_sha}" "${current_qemu_image}" } cache_clh_artifacts() { - local binary="cloud-hypervisor" - local binary_path="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" - echo "binary path $binary_path" - local current_cloud_hypervisor_version=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version") - local clh_binary_path="${binary_path}/tools/packaging/kata-deploy/local-build/build/cc-cloud-hypervisor/builddir/cloud-hypervisor" - if [ -f "${clh_binary_path}/cloud-hypervisor" ]; then - cp "${clh_binary_path}/${binary}" . - else - cloud_hypervisor_build_path="${binary_path}/cloud-hypervisor" - cp "${cloud_hypervisor_build_path}/${binary}" . - fi - create_cache_asset "${binary}" "${current_cloud_hypervisor_version}" - echo "${current_cloud_hypervisor_version}" > "latest" + local clh_tarball_name="kata-static-cc-clh.tar.xz" + [ -n "${TEE}" ] && clh_tarball_name="kata-static-cc-tdx-clh.tar.xz" + local current_clh_version=$(get_from_kata_deps "assets.cloud-hypervisor.version") + create_cache_asset "${clh_tarball_name}" "${current_clh_version}" "" } cache_kernel_artifacts() { - local current_kernel_version=$(get_from_kata_deps "assets.kernel.version" | cut -c2- ) - local gral_path="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" - local kernel_config_file="${gral_path}/tools/packaging/kernel/kata_config_version" - local kernel_config="$(cat ${kernel_config_file})" - echo "${current_kernel_version} ${kernel_config}" > "latest" - local kernel_path="${gral_path}/tools/packaging/kata-deploy/local-build/build/cc-kernel/destdir/opt/confidential-containers/share/kata-containers" - local vmlinux_binary_name="vmlinux-${current_kernel_version}-${kernel_config}" - ls ${kernel_path} - local vmlinux_file="${kernel_path}/${vmlinux_binary_name}" - if [ -f "${vmlinux_file}" ]; then - cp -a "${vmlinux_file}" . - create_cache_asset "${vmlinux_binary_name}" "${current_kernel_version}" - fi - local vmlinuz_binary_name="vmlinuz-${current_kernel_version}-${kernel_config}" - local vmlinuz_file="${kernel_path}/${vmlinuz_binary_name}" - if [ -f "${vmlinuz_file}" ]; then - cp -a "${vmlinuz_file}" . - create_cache_asset "${vmlinuz_binary_name}" "${current_kernel_version}" + local kernel_tarball_name="kata-static-cc-kernel.tar.xz" + local current_kernel_image="$(get_kernel_image_name)" + local current_kernel_version="$(get_from_kata_deps "assets.kernel.version")" + if [ -n "${TEE}" ]; then + kernel_tarball_name="kata-stastic-cc-${TEE}-kernel.tar.xz" + [ "${TEE}" == "tdx" ] && current_kernel_version="$(get_from_kata_deps "assets.kernel.${TEE}.tag")" + [ "${TEE}" == "sev" ] && current_kernel_version="$(get_from_kata_deps "assets.kernel.${TEE}.version")" fi + create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}" "${current_kernel_image}" } create_cache_asset() { - local component_name="$1" - local component_version="$2" + local component_name="${1}" + local component_version="${2}" + local component_image="${3}" + sudo cp "${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/${component_name}" . sudo chown -R "${USER}:${USER}" . sha256sum "${component_name}" > "sha256sum-${component_name}" cat "sha256sum-${component_name}" + echo "${component_version}" > "latest" + cat "latest" + echo "${component_image}" > "latest_image" + cat "latest_image" } help() { From 0eb2b1f58c97b5299bfceb9fedfdbf5c1cb24260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 28 Nov 2022 14:56:38 +0100 Subject: [PATCH 7/7] kata-deploy-binaries: Check for cached version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's check for the cached version of the components as part of the kata-deploy-binaries.sh as here we already have the needed info for checking whether a component is cached or not, and to use it without depending on changes made on each one of the builder scripts. Fixes: #5816 Signed-off-by: Fabiano Fidêncio --- .../local-build/kata-deploy-binaries.sh | 129 ++++++++++++++++-- .../cloud-hypervisor/build-static-clh.sh | 39 +----- tools/packaging/static-build/kernel/build.sh | 93 +++---------- .../static-build/qemu/build-static-qemu-cc.sh | 81 ++--------- 4 files changed, 156 insertions(+), 186 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index d6577f9219..dc1ff041cc 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -38,6 +38,8 @@ readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_imag readonly cc_prefix="/opt/confidential-containers" readonly qemu_cc_builder="${static_build_dir}/qemu/build-static-qemu-cc.sh" +source "${script_dir}/../../scripts/lib.sh" + ARCH=$(uname -m) workdir="${WORKDIR:-$PWD}" @@ -104,8 +106,45 @@ EOF exit "${return_code}" } +cleanup_and_fail() { + rm -f "${component_tarball_path}" + return 1 +} + +install_cached_component() { + local component="${1}" + local jenkins_build_url="${2}" + local current_version="${3}" + local current_image_version="${4}" + local component_tarball_name="${5}" + local component_tarball_path="${6}" + + local cached_version=$(curl -sfL "${jenkins_build_url}/latest" | awk '{print $1}') || cached_version="none" + local cached_image_version=$(curl -sfL "${jenkins_build_url}/latest_image" | awk '{print $1}') || cached_image_version="none" + + [ "${cached_image_version}" != "${current_image_version}" ] && return 1 + [ "${cached_version}" != "${current_version}" ] && return 1 + + info "Using cached tarball of ${component}" + pushd ${workdir} + echo "Downloading tarball from: ${jenkins_build_url}/${component_tarball_name}" + curl -fL --progress-bar "${jenkins_build_url}/${component_tarball_name}" -o "${component_tarball_path}" || return cleanup_and_fail + curl -fsOL "${jenkins_build_url}/sha256sum-${component_tarball_name}" || return cleanup_and_fail + sha256sum -c "sha256sum-${component_tarball_name}" && return cleanup_and_fail + popd +} + # Install static CC cloud-hypervisor asset install_cc_clh() { + install_cached_component \ + "cloud-hypervisor" \ + "${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + if [[ "${ARCH}" == "x86_64" ]]; then export features="tdx" fi @@ -143,11 +182,21 @@ install_cc_tdx_image() { #Install CC kernel asset install_cc_kernel() { - export KATA_BUILD_CC=yes - info "build initramfs for cc kernel" - "${initramfs_builder}" + + export KATA_BUILD_CC=yes export kernel_version="$(yq r $versions_yaml assets.kernel.version)" + + install_cached_component \ + "kernel" \ + "${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + + "${initramfs_builder}" DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -f -v "${kernel_version}" } @@ -156,6 +205,16 @@ install_cc_qemu() { info "build static CC qemu" export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)" export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)" + + install_cached_component \ + "QEMU" \ + "${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}" \ + "${qemu_version}-$(calc_qemu_files_sha256sum)" \ + "$(get_qemu_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + "${qemu_cc_builder}" tar xvf "${builddir}/kata-static-qemu-cc.tar.gz" -C "${destdir}" } @@ -178,6 +237,15 @@ install_cc_shimv2() { # Install static CC virtiofsd asset install_cc_virtiofsd() { + install_cached_component \ + "virtiofsd" \ + "${jenkins_url}/job/kata-containers-2.0-virtiofsd-cc-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "assets.externals.virtiofsd.version")" \ + "$(get_virtiofsd_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "build static CC virtiofsd" "${virtiofsd_builder}" info "Install static CC virtiofsd" @@ -199,6 +267,16 @@ install_cc_tee_kernel() { info "build initramfs for tee kernel" export kernel_version=${kernel_version} + + install_cached_component \ + "kernel" \ + "${jenkins_url}/job/kata-containers-2.0-kernel-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + "${initramfs_builder}" kernel_url="$(yq r $versions_yaml assets.kernel.${tee}.url)" DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -x "${tee}" -v "${kernel_version}" -u "${kernel_url}" @@ -223,6 +301,16 @@ install_cc_tee_qemu() { export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.url)" export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.tag)" export tee="${tee}" + + install_cached_component \ + "QEMU ${tee}" \ + "${jenkins_url}/job/kata-containers-2.0-qemu-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \ + "${qemu_version}-$(calc_qemu_files_sha256sum)" \ + "$(get_qemu_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + "${qemu_cc_builder}" tar xvf "${builddir}/kata-static-${tee}-qemu-cc.tar.gz" -C "${destdir}" } @@ -232,6 +320,15 @@ install_cc_tdx_qemu() { } install_cc_tdx_td_shim() { + install_cached_component \ + "td-shim" \ + "${jenkins_url}/job/kata-containers-2.0-td-shim-cc-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "assets.externals.td-shim.version")" \ + "$(get_td_shim_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${td_shim_builder}" tar xvf "${builddir}/td-shim.tar.gz" -C "${destdir}" } @@ -240,6 +337,18 @@ install_cc_tee_ovmf() { tee="${1}" tarball_name="${2}" + local component_name="ovmf" + local component_version="$(get_from_kata_deps "assets.external.ovmf.${tee}.version")" + [ "${tee}" == "tdx" ] && component_name="tdvf" + install_cached_component \ + "${component_name}" \ + "${jenkins_url}/job/kata-containers-2.0-${component_name}-cc-$(uname -m)/${cached_artifacts_path}" \ + "$(component_version)" \ + "$(get_ovmf_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + DESTDIR="${destdir}" PREFIX="${cc_prefix}" ovmf_build="${tee}" "${ovmf_builder}" tar xvf "${builddir}/${tarball_name}" -C "${destdir}" } @@ -350,6 +459,11 @@ handle_build() { info "DESTDIR ${destdir}" local build_target build_target="$1" + + export final_tarball_path="${workdir}/kata-static-${build_target}.tar.xz" + export final_tarball_name="$(basename ${final_tarball_path})" + rm -f ${final_tarball_name} + case "${build_target}" in all) install_clh @@ -429,12 +543,11 @@ handle_build() { ;; esac - tarball_name="${workdir}/kata-static-${build_target}.tar.xz" - ( + if [ ! -f "${final_tarball_path}" ]; then cd "${destdir}" - sudo tar cvfJ "${tarball_name}" "." - ) - tar tvf "${tarball_name}" + sudo tar cvfJ "${final_tarball_path}" "." + fi + tar tvf "${final_tarball_path}" } silent_mode_error_trap() { diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh index 1fda017402..0bee1ea041 100755 --- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh +++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh @@ -23,7 +23,6 @@ cloud_hypervisor_repo="${cloud_hypervisor_repo:-}" cloud_hypervisor_version="${cloud_hypervisor_version:-}" cloud_hypervisor_pr="${cloud_hypervisor_pr:-}" cloud_hypervisor_pull_ref_branch="${cloud_hypervisor_pull_ref_branch:-main}" -cloud_hypervisor_latest_build_url="${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}" if [ -z "$cloud_hypervisor_repo" ]; then info "Get cloud_hypervisor information from runtime versions.yaml" @@ -83,40 +82,6 @@ build_clh_from_source() { popd } -check_cached_cloud_hypervisor() { - local cached_cloud_hypervisor_version=$(curl -sfL "${cloud_hypervisor_latest_build_url}"/latest) || latest="none" - info "Current cloud hypervisor version: ${cloud_hypervisor_version}" - info "Cached cloud hypervisor version: ${cached_cloud_hypervisor_version}" - if [ "${cloud_hypervisor_version}" == "${cached_cloud_hypervisor_version}" ] && [ "${ARCH}" == "x86_64" ]; then - install_cached_cloud_hypervisor - else - build_clh_from_source - fi -} - -install_cached_cloud_hypervisor() { - local cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" - local clh_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-cloud-hypervisor/builddir/cloud-hypervisor" - mkdir cloud-hypervisor - pushd cloud-hypervisor - local checksum_file="sha256sum-cloud-hypervisor" - info "Downloading the cloud hypervisor binary" - curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/cloud-hypervisor" || return 1 - info "Checking cloud hypervisor binary checksum" - curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/${checksum_file}" || return 1 - info "Verify checksum" - sudo sha256sum -c "${checksum_file}" || return 1 - chmod +x cloud-hypervisor - local clh_binary_path="${cached_path}/cloud-hypervisor" - if [ ! -d "${clh_binary_path}" ]; then - mkdir -p "${clh_binary_path}" - fi - if [ ! -f "${clh_binary_path}/cloud-hypervisor" ]; then - cp cloud-hypervisor "${clh_binary_path}" - fi - popd -} - if [ "${ARCH}" == "aarch64" ]; then info "aarch64 binaries are not distributed as part of the Cloud Hypervisor releases, forcing to build from source" force_build_from_source="true" @@ -129,8 +94,8 @@ fi if [ "${force_build_from_source}" == "true" ]; then info "Build cloud-hypervisor from source as it's been request via the force_build_from_source flag" - check_cached_cloud_hypervisor + build_clh_from_source else pull_clh_released_binary || - (info "Failed to pull cloud-hypervisor released binary, trying to build from source" && check_cached_cloud_hypervisor) + (info "Failed to pull cloud-hypervisor released binary, trying to build from source" && build_clh_from_source) fi diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index db0e65d77d..c9bb71be4c 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -17,82 +17,25 @@ readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}" -kernel_latest_build_url="${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}" -current_kernel_version=${kernel_version:-$(get_from_kata_deps "assets.kernel.version")} -cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" -current_kernel_config_file="${cached_path}/tools/packaging/kernel/kata_config_version" -current_kernel_config="$(cat $current_kernel_config_file)" -kernel_version="$(echo ${current_kernel_version} | cut -c2- )" -build_from_source() { - sudo docker pull ${container_image} || \ - (sudo docker build -t "${container_image}" "${script_dir}" && \ - # No-op unless PUSH_TO_REGISTRY is exported as "yes" - push_to_registry "${container_image}") +sudo docker pull ${container_image} || \ + (sudo docker build -t "${container_image}" "${script_dir}" && \ + # No-op unless PUSH_TO_REGISTRY is exported as "yes" + push_to_registry "${container_image}") - sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ - -w "${PWD}" \ - --env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \ - "${container_image}" \ - bash -c "${kernel_builder} $* setup" +sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + -w "${PWD}" \ + --env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \ + "${container_image}" \ + bash -c "${kernel_builder} $* setup" - sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ - -w "${PWD}" \ - "${container_image}" \ - bash -c "${kernel_builder} $* build" +sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + -w "${PWD}" \ + "${container_image}" \ + bash -c "${kernel_builder} $* build" - sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ - -w "${PWD}" \ - --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ - "${container_image}" \ - bash -c "${kernel_builder} $* install" -} - -check_cached_kernel() { - local latest=$(curl -sfL "${kernel_latest_build_url}"/latest) || latest="none" - local cached_kernel_version="$(echo ${latest} | awk '{print $1}')" - info "Current kernel version: ${kernel_version}" - info "Cached kernel version: ${cached_kernel_version}" - if [ "${kernel_version}" == "${cached_kernel_version}" ] && [ "$(uname -m)" == "x86_64" ]; then - local cached_kernel_config="$(echo ${latest} | awk '{print $2}')" - info "Cached kernel config: ${cached_kernel_config}" - info "Current kernel config: ${current_kernel_config}" - if [ -z "${cached_kernel_config}" ]; then - build_from_source $* - else - install_cached_kernel $* - fi - else - build_from_source $* - fi -} - -install_cached_kernel() { - local kernel_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-kernel/destdir/opt/confidential-containers/share/kata-containers" - local vmlinux_kernel_name="vmlinux-${cached_kernel_version}-${cached_kernel_config}" - local vmlinuz_kernel_name="vmlinuz-${cached_kernel_version}-${cached_kernel_config}" - mkdir -p "${kernel_directory}" - pushd "${kernel_directory}" - ls - local vmlinux_url="${kernel_latest_build_url}/${vmlinux_kernel_name}" - if curl --output /dev/null --silent --head --fail "${vmlinux_url}"; then - info "Installing vmlinux cached kernel" - curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinux_kernel_name}" -o "${vmlinux_kernel_name}" || return 1 - sudo -E ln -sf "${kernel_directory}/${vmlinux_kernel_name}" "${kernel_directory}/vmlinux.container" - fi - - local vmlinuz_url="${kernel_latest_build_url}/${vmlinuz_kernel_name}" - if curl --output /dev/null --silent --head --fail "${vmlinuz_url}"; then - info "Installing vmlinuz cached kernel" - curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinuz_kernel_name}" -o "${vmlinuz_kernel_name}" || return 1 - sudo -E ln -sf "${kernel_directory}/${vmlinuz_kernel_name}" "${kernel_directory}/vmlinuz.container" - fi - popd - -} - -main() { - check_cached_kernel $* -} - -main $* +sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + -w "${PWD}" \ + --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ + "${container_image}" \ + bash -c "${kernel_builder} $* install" diff --git a/tools/packaging/static-build/qemu/build-static-qemu-cc.sh b/tools/packaging/static-build/qemu/build-static-qemu-cc.sh index 724e606b08..4ec3dcfd16 100755 --- a/tools/packaging/static-build/qemu/build-static-qemu-cc.sh +++ b/tools/packaging/static-build/qemu/build-static-qemu-cc.sh @@ -12,74 +12,23 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${script_dir}/../../scripts/lib.sh" -export qemu_repo="${qemu_repo:-}" -export qemu_version="${qemu_version:-}" -export qemu_latest_build_url="${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}" -export katacontainers_repo="${katacontainers_repo:=github.com/kata-containers/kata-containers}" -export qemu_tarball_name="kata-static-qemu-cc.tar.gz" -export pkg_dir="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" -export qemu_tarball_directory="${pkg_dir}/kata-deploy/local-build/build/cc-qemu/builddir" -export tee="${tee:-}" +qemu_repo="${qemu_repo:-}" +qemu_version="${qemu_version:-}" +tee="${tee:-}" export prefix="/opt/confidential-containers/" -get_qemu_information() { - if [ -z "${qemu_repo}" ]; then - info "Get qemu information from runtime versions.yaml" - export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url") - [ -n "${qemu_url}" ] || die "failed to get qemu url" - export qemu_repo="${qemu_url}.git" - fi +if [ -z "${qemu_repo}" ]; then + info "Get qemu information from runtime versions.yaml" + export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url") + [ -n "${qemu_url}" ] || die "failed to get qemu url" + export qemu_repo="${qemu_url}.git" +fi - [ -n "${qemu_repo}" ] || die "failed to get qemu repo" - [ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version") - [ -n "${qemu_version}" ] || die "failed to get qemu version" -} +[ -n "${qemu_repo}" ] || die "failed to get qemu repo" +[ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version") +[ -n "${qemu_version}" ] || die "failed to get qemu version" -cached_or_build_qemu_tar() { - # Check latest qemu cc tar version sha256sum - local latest=$(curl -sfL "${qemu_latest_build_url}/latest") || latest="none" - local cached_qemu_version="$(echo ${latest} | awk '{print $1}')" - info "Current qemu version: ${qemu_version}" - info "Cached qemu version: ${cached_qemu_version}" - if [ "${qemu_version}" == "${cached_qemu_version}" ]; then - info "Get latest cached information ${latest}" - local cached_sha256sum="$(echo ${latest} | awk '{print $2}')" - info "Cached sha256sum version: ${cached_sha256sum}" - local current_sha256sum="$(calc_qemu_files_sha256sum)" - info "Current sha256sum of the qemu directory ${current_sha256sum}" - if [ -z "${cached_sha256sum}" ]; then - build_qemu_tar - elif [ "${current_sha256sum}" == "${cached_sha256sum}" ]; then - install_cached_qemu_tar - else - build_qemu_tar - fi - else - build_qemu_tar - fi -} - -build_qemu_tar() { - [ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz" - "${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}" -} - -install_cached_qemu_tar() { - info "Using cached tarball of qemu" - curl -fL --progress-bar "${qemu_latest_build_url}/${qemu_tarball_name}" -o "${qemu_tarball_name}" || return 1 - curl -fsOL "${qemu_latest_build_url}/sha256sum-${qemu_tarball_name}" || return 1 - sha256sum -c "sha256sum-${qemu_tarball_name}" || return 1 -} - -main() { - get_qemu_information - # Currently the cached for qemu cc only works in x86_64 - if [ "$(uname -m)" == "x86_64" ]; then - cached_or_build_qemu_tar - else - build_qemu_tar - fi -} - -main $@ +qemu_tarball_name="kata-static-qemu-cc.tar.gz" +[ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz" +"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}"