Merge pull request #5784 from fidencio/topic/cached-components

Re-work the way we cache components for the CCv0 branch

tools/packaging/guest-image/build_image.sh

@@ -14,7 +14,9 @@ set -o pipefail
 readonly script_name="$(basename "${BASH_SOURCE[0]}")"
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly packaging_root_dir="$(cd "${script_dir}/../" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../" && pwd)"
+
+source "${packaging_root_dir}/scripts/lib.sh"
+
 readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)"
 
 patches_path=""
@@ -26,8 +28,6 @@ final_image_name="kata-containers"
 final_initrd_name="kata-containers-initrd"
 image_initrd_extension=".img"
 
-source "${packaging_root_dir}/scripts/lib.sh"
-
 arch_target="$(uname -m)"
 
 build_initrd() {

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -15,8 +15,9 @@ readonly project="kata-containers"
 readonly script_name="$(basename "${BASH_SOURCE[0]}")"
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
+source "${script_dir}/../../scripts/lib.sh"
+
 readonly prefix="/opt/kata"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly static_build_dir="${repo_root_dir}/tools/packaging/static-build"
 readonly version_file="${repo_root_dir}/VERSION"
 readonly versions_yaml="${repo_root_dir}/versions.yaml"
@@ -37,6 +38,8 @@ readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_imag
 readonly cc_prefix="/opt/confidential-containers"
 readonly qemu_cc_builder="${static_build_dir}/qemu/build-static-qemu-cc.sh"
 
+source "${script_dir}/../../scripts/lib.sh"
+
 ARCH=$(uname -m)
 
 workdir="${WORKDIR:-$PWD}"
@@ -103,8 +106,45 @@ EOF
 	exit "${return_code}"
 }
 
+cleanup_and_fail() {
+	rm -f "${component_tarball_path}"
+	return 1
+}
+
+install_cached_component() {
+	local component="${1}"
+	local jenkins_build_url="${2}"
+	local current_version="${3}"
+	local current_image_version="${4}"
+	local component_tarball_name="${5}"
+	local component_tarball_path="${6}"
+
+	local cached_version=$(curl -sfL "${jenkins_build_url}/latest" | awk '{print $1}') || cached_version="none"
+	local cached_image_version=$(curl -sfL "${jenkins_build_url}/latest_image" | awk '{print $1}') || cached_image_version="none"
+
+	[ "${cached_image_version}" != "${current_image_version}" ] && return 1
+	[ "${cached_version}" != "${current_version}" ] && return 1
+
+	info "Using cached tarball of ${component}"
+	pushd ${workdir}
+	echo "Downloading tarball from: ${jenkins_build_url}/${component_tarball_name}"
+	curl -fL --progress-bar "${jenkins_build_url}/${component_tarball_name}" -o "${component_tarball_path}" || return cleanup_and_fail
+	curl -fsOL "${jenkins_build_url}/sha256sum-${component_tarball_name}" || return cleanup_and_fail
+	sha256sum -c "sha256sum-${component_tarball_name}" && return cleanup_and_fail
+	popd
+}
+
 # Install static CC cloud-hypervisor asset
 install_cc_clh() {
+	install_cached_component \
+		"cloud-hypervisor" \
+		"${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}" \
+		"$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \
+		"" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	if [[ "${ARCH}" == "x86_64" ]]; then
 		export features="tdx"
 	fi
@@ -142,11 +182,21 @@ install_cc_tdx_image() {
 
 #Install CC kernel asset
 install_cc_kernel() {
-	export KATA_BUILD_CC=yes
-
 	info "build initramfs for cc kernel"
-	"${initramfs_builder}"
+
+	export KATA_BUILD_CC=yes
 	export kernel_version="$(yq r $versions_yaml assets.kernel.version)"
+
+	install_cached_component \
+		"kernel" \
+		"${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}" \
+		"${kernel_version}" \
+		"$(get_kernel_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
+	"${initramfs_builder}"
 	DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -f -v "${kernel_version}"
 }
 
@@ -155,6 +205,16 @@ install_cc_qemu() {
 	info "build static CC qemu"
 	export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)"
 	export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)"
+
+	install_cached_component \
+		"QEMU" \
+		"${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}" \
+		"${qemu_version}-$(calc_qemu_files_sha256sum)" \
+		"$(get_qemu_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	"${qemu_cc_builder}"
 	tar xvf "${builddir}/kata-static-qemu-cc.tar.gz" -C "${destdir}"
 }
@@ -177,6 +237,15 @@ install_cc_shimv2() {
 
 # Install static CC virtiofsd asset
 install_cc_virtiofsd() {
+	install_cached_component \
+		"virtiofsd" \
+		"${jenkins_url}/job/kata-containers-2.0-virtiofsd-cc-$(uname -m)/${cached_artifacts_path}" \
+		"$(get_from_kata_deps "assets.externals.virtiofsd.version")" \
+		"$(get_virtiofsd_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	info "build static CC virtiofsd"
 	"${virtiofsd_builder}"
 	info "Install static CC virtiofsd"
@@ -198,6 +267,16 @@ install_cc_tee_kernel() {
 
 	info "build initramfs for tee kernel"
 	export kernel_version=${kernel_version}
+
+	install_cached_component \
+		"kernel" \
+		"${jenkins_url}/job/kata-containers-2.0-kernel-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \
+		"${kernel_version}" \
+		"$(get_kernel_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	"${initramfs_builder}"
 	kernel_url="$(yq r $versions_yaml assets.kernel.${tee}.url)"
 	DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -x "${tee}" -v "${kernel_version}" -u "${kernel_url}"
@@ -222,6 +301,16 @@ install_cc_tee_qemu() {
 	export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.url)"
 	export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.tag)"
 	export tee="${tee}"
+
+	install_cached_component \
+		"QEMU ${tee}" \
+		"${jenkins_url}/job/kata-containers-2.0-qemu-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \
+		"${qemu_version}-$(calc_qemu_files_sha256sum)" \
+		"$(get_qemu_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	"${qemu_cc_builder}"
 	tar xvf "${builddir}/kata-static-${tee}-qemu-cc.tar.gz" -C "${destdir}"
 }
@@ -231,6 +320,15 @@ install_cc_tdx_qemu() {
 }
 
 install_cc_tdx_td_shim() {
+	install_cached_component \
+		"td-shim" \
+		"${jenkins_url}/job/kata-containers-2.0-td-shim-cc-$(uname -m)/${cached_artifacts_path}" \
+		"$(get_from_kata_deps "assets.externals.td-shim.version")" \
+		"$(get_td_shim_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${td_shim_builder}"
 	tar xvf "${builddir}/td-shim.tar.gz" -C "${destdir}"
 }
@@ -239,6 +337,18 @@ install_cc_tee_ovmf() {
 	tee="${1}"
 	tarball_name="${2}"
 
+	local component_name="ovmf"
+	local component_version="$(get_from_kata_deps "assets.external.ovmf.${tee}.version")"
+	[ "${tee}" == "tdx" ] && component_name="tdvf"
+	install_cached_component \
+		"${component_name}" \
+		"${jenkins_url}/job/kata-containers-2.0-${component_name}-cc-$(uname -m)/${cached_artifacts_path}" \
+		"$(component_version)" \
+		"$(get_ovmf_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	DESTDIR="${destdir}" PREFIX="${cc_prefix}" ovmf_build="${tee}" "${ovmf_builder}"
 	tar xvf "${builddir}/${tarball_name}" -C "${destdir}"
 }
@@ -349,6 +459,11 @@ handle_build() {
 	info "DESTDIR ${destdir}"
 	local build_target
 	build_target="$1"
+
+	export final_tarball_path="${workdir}/kata-static-${build_target}.tar.xz"
+	export final_tarball_name="$(basename ${final_tarball_path})"
+	rm -f ${final_tarball_name}
+
 	case "${build_target}" in
 	all)
 		install_clh
@@ -428,12 +543,11 @@ handle_build() {
 		;;
 	esac
 
-	tarball_name="${workdir}/kata-static-${build_target}.tar.xz"
+	if [ ! -f "${final_tarball_path}" ]; then
-	(
 		cd "${destdir}"
-		sudo tar cvfJ "${tarball_name}" "."
+		sudo tar cvfJ "${final_tarball_path}" "."
-	)
+	fi
-	tar tvf "${tarball_name}"
+	tar tvf "${final_tarball_path}"
 }
 
 silent_mode_error_trap() {

tools/packaging/scripts/lib.sh

@@ -13,6 +13,8 @@ export PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}"
 
 this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
+export repo_root_dir="$(cd "${this_script_dir}/../../../" && pwd)"
+
 short_commit_length=10
 
 hub_bin="hub-bin"
@@ -121,12 +123,11 @@ get_config_version() {
 	fi
 }
 
-# $1 - Repo's root dir
+# $1 - The file we're looking for the last modification
-# $2 - The file we're looking for the last modification
 get_last_modification() {
-	local repo_root_dir="${1}"
+	local file="${1}"
-	local file="${2}"
 
+	pushd ${repo_root_dir} &> /dev/null
 	# This is a workaround needed for when running this code on Jenkins
 	git config --global --add safe.directory ${repo_root_dir} &> /dev/null
 
@@ -134,6 +135,7 @@ get_last_modification() {
 	[ $(git status --porcelain | grep "${file#${repo_root_dir}/}" | wc -l) -gt 0 ] && dirty="-dirty"
 
 	echo "$(git log -1 --pretty=format:"%H" ${file})${dirty}"
+	popd &> /dev/null
 }
 
 # $1 - The tag to be pushed to the registry
@@ -173,9 +175,67 @@ sha256sum_from_files() {
 	files="$(echo $files | tr ' ' '\n' | LC_ALL=C sort -u)"
 	# Concate the files and calculate a hash.
 	shasum="$(cat $files | sha256sum -b)" || true
-	info "shasum of files $shasum"
 	if [ -n "$shasum" ];then
 		# Return only the SHA field.
 		echo $(awk '{ print $1 }' <<< $shasum)
 	fi
 }
+
+calc_qemu_files_sha256sum() {
+	local files="${this_script_dir}/../qemu \
+		${this_script_dir}/../static-build/qemu.blacklist \
+		${this_script_dir}/../static-build/scripts"
+
+	sha256sum_from_files "$files"
+}
+
+get_initramfs_image_name() {
+	initramfs_script_dir="${this_script_dir}/../static-build/initramfs"
+	echo "${CC_BUILDER_REGISTRY}:initramfs-cryptosetup$(get_from_kata_deps "externals.cryptsetup.version")-lvm2-$(get_from_kata_deps "externals.lvm2.version")-$(get_last_modification ${initramfs_script_dir})-$(uname -m)"
+}
+
+get_kernel_image_name() {
+	kernel_script_dir="${this_script_dir}/../static-build/kernel"
+	echo "${CC_BUILDER_REGISTRY}:kernel-$(get_last_modification ${kernel_script_dir})-$(uname -m)"
+}
+
+get_ovmf_image_name() {
+	ovmf_script_dir="${this_script_dir}/../static-build/ovmf"
+	echo "${CC_BUILDER_REGISTRY}:ovmf-$(get_last_modification ${ovmf_script_dir})-$(uname -m)"
+}
+
+get_qemu_image_name() {
+	qemu_script_dir="${this_script_dir}/../static-build/qemu"
+	echo "${CC_BUILDER_REGISTRY}:qemu-$(get_last_modification ${qemu_script_dir})-$(uname -m)"
+}
+
+get_shim_v2_image_name() {
+	shim_v2_script_dir="${this_script_dir}/../static-build/shim-v2"
+	echo "${CC_BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps "languages.golang.meta.newest-version")-rust-$(get_from_kata_deps "languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)"
+}
+
+get_td_shim_image_name() {
+	td_shim_script_dir="${this_script_dir}/../static-build/td-shim"
+	echo "${CC_BUILDER_REGISTRY}:td-shim-$(get_last_modification ${td_shim_script_dir})-$(uname -m)"
+}
+
+get_virtiofsd_image_name() {
+	ARCH=$(uname -m)
+	case ${ARCH} in
+	        "aarch64")
+	                libc="musl"
+	                ;;
+	        "ppc64le")
+	                libc="gnu"
+	                ;;
+	        "s390x")
+	                libc="gnu"
+	                ;;
+	        "x86_64")
+	                libc="musl"
+	                ;;
+	esac
+
+	virtiofsd_script_dir="${this_script_dir}/../static-build/virtiofsd"
+	echo "${CC_BUILDER_REGISTRY}:virtiofsd-$(get_from_kata_deps "externals.virtiofsd.toolchain")-${libc}-$(get_last_modification ${virtiofsd_script_dir})-$(uname -m)"
+}

tools/packaging/static-build/cache_components.sh

@@ -13,61 +13,53 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${script_dir}/../scripts/lib.sh"
 
 export KATA_BUILD_CC="${KATA_BUILD_CC:-}"
-export qemu_cc_tarball_name="kata-static-qemu-cc.tar.gz"
+export TEE="${TEE:-}"
 
 cache_qemu_artifacts() {
-	source "${script_dir}/qemu/build-static-qemu-cc.sh"
+	local qemu_tarball_name="kata-static-cc-qemu.tar.xz"
 	local current_qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
-	create_cache_asset "${qemu_cc_tarball_name}" "${current_qemu_version}"
+	if [ -n "${TEE}" ]; then
+		qemu_tarball_name="kata-static-cc-${TEE}-qemu.tar.xz"
+		[ "${TEE}" == "tdx" ] && current_qemu_version=$(get_from_kata_deps "asserts.hypervisor.qemu.tdx.tag")
+	fi
+	local qemu_script_dir="${repo_root_dir}/tools/packaging/static-build/qemu"
 	local qemu_sha=$(calc_qemu_files_sha256sum)
-        echo "${current_qemu_version} ${qemu_sha}" > "latest"
+	local current_qemu_image="$(get_qemu_image_name)"
+	create_cache_asset "${qemu_tarball_name}" "${current_qemu_version}-${qemu_sha}" "${current_qemu_image}"
 }
 
 cache_clh_artifacts() {
-	local binary="cloud-hypervisor"
+	local clh_tarball_name="kata-static-cc-clh.tar.xz"
-	local binary_path="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
+	[ -n "${TEE}" ] && clh_tarball_name="kata-static-cc-tdx-clh.tar.xz"
-	echo "binary path $binary_path"
+	local current_clh_version=$(get_from_kata_deps "assets.cloud-hypervisor.version")
-	local current_cloud_hypervisor_version=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")
+	create_cache_asset "${clh_tarball_name}" "${current_clh_version}" ""
-	local clh_binary_path="${binary_path}/tools/packaging/kata-deploy/local-build/build/cc-cloud-hypervisor/builddir/cloud-hypervisor"
-	if [ -f "${clh_binary_path}/cloud-hypervisor" ]; then
-		cp "${clh_binary_path}/${binary}" .
-	else
-		cloud_hypervisor_build_path="${binary_path}/cloud-hypervisor"
-		cp "${cloud_hypervisor_build_path}/${binary}" .
-	fi
-	create_cache_asset "${binary}" "${current_cloud_hypervisor_version}"
-	echo "${current_cloud_hypervisor_version}"  > "latest"
 }
 
 cache_kernel_artifacts() {
-	local current_kernel_version=$(get_from_kata_deps "assets.kernel.version" | cut -c2- )
+	local kernel_tarball_name="kata-static-cc-kernel.tar.xz"
-	local gral_path="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
+	local current_kernel_image="$(get_kernel_image_name)"
-	local kernel_config_file="${gral_path}/tools/packaging/kernel/kata_config_version"
+	local current_kernel_version="$(get_from_kata_deps "assets.kernel.version")"
-	local kernel_config="$(cat ${kernel_config_file})"
+	if [ -n "${TEE}" ]; then 
-	echo "${current_kernel_version} ${kernel_config}" > "latest"
+		kernel_tarball_name="kata-stastic-cc-${TEE}-kernel.tar.xz"
-	local kernel_path="${gral_path}/tools/packaging/kata-deploy/local-build/build/cc-kernel/destdir/opt/confidential-containers/share/kata-containers"
+		[ "${TEE}" == "tdx" ] && current_kernel_version="$(get_from_kata_deps "assets.kernel.${TEE}.tag")"
-	local vmlinux_binary_name="vmlinux-${current_kernel_version}-${kernel_config}"
+		[ "${TEE}" == "sev" ] && current_kernel_version="$(get_from_kata_deps "assets.kernel.${TEE}.version")"
-	ls ${kernel_path}
-	local vmlinux_file="${kernel_path}/${vmlinux_binary_name}"
-	if [ -f "${vmlinux_file}" ]; then
-		cp -a "${vmlinux_file}" .
-		create_cache_asset "${vmlinux_binary_name}" "${current_kernel_version}"
-	fi
-	local vmlinuz_binary_name="vmlinuz-${current_kernel_version}-${kernel_config}"
-	local vmlinuz_file="${kernel_path}/${vmlinuz_binary_name}"
-	if [ -f "${vmlinuz_file}" ]; then
-		cp -a "${vmlinuz_file}" .
-		create_cache_asset "${vmlinuz_binary_name}" "${current_kernel_version}"
 	fi
+	create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}" "${current_kernel_image}"
 }
 
 create_cache_asset() {
-	local component_name="$1"
+	local component_name="${1}"
-	local component_version="$2"
+	local component_version="${2}"
+	local component_image="${3}"
 
+	sudo cp "${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/${component_name}" .
 	sudo chown -R "${USER}:${USER}" .
 	sha256sum "${component_name}" > "sha256sum-${component_name}"
 	cat "sha256sum-${component_name}"
+	echo "${component_version}" > "latest"
+	cat "latest"
+	echo "${component_image}" > "latest_image"
+	cat "latest_image"
 }
 
 help() {

tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh

@@ -23,7 +23,6 @@ cloud_hypervisor_repo="${cloud_hypervisor_repo:-}"
 cloud_hypervisor_version="${cloud_hypervisor_version:-}"
 cloud_hypervisor_pr="${cloud_hypervisor_pr:-}"
 cloud_hypervisor_pull_ref_branch="${cloud_hypervisor_pull_ref_branch:-main}"
-cloud_hypervisor_latest_build_url="${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}"
 
 if [ -z "$cloud_hypervisor_repo" ]; then
 	info "Get cloud_hypervisor information from runtime versions.yaml"
@@ -83,40 +82,6 @@ build_clh_from_source() {
 	popd
 }
 
-check_cached_cloud_hypervisor() {
-	local cached_cloud_hypervisor_version=$(curl -sfL "${cloud_hypervisor_latest_build_url}"/latest) || latest="none"
-	info "Current cloud hypervisor version: ${cloud_hypervisor_version}"
-	info "Cached cloud hypervisor version: ${cached_cloud_hypervisor_version}"
-	if [ "${cloud_hypervisor_version}" == "${cached_cloud_hypervisor_version}" ] && [ "${ARCH}" == "x86_64" ]; then
-		install_cached_cloud_hypervisor
-	else
-		build_clh_from_source
-	fi
-}
-
-install_cached_cloud_hypervisor() {
-	local cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
-	local clh_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-cloud-hypervisor/builddir/cloud-hypervisor"
-	mkdir cloud-hypervisor
-	pushd cloud-hypervisor
-	local checksum_file="sha256sum-cloud-hypervisor"
-	info "Downloading the cloud hypervisor binary"
-	curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/cloud-hypervisor" || return 1
-	info "Checking cloud hypervisor binary checksum"
-	curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/${checksum_file}" || return 1
-	info "Verify checksum"
-	sudo sha256sum -c "${checksum_file}" || return 1
-	chmod +x cloud-hypervisor
-	local clh_binary_path="${cached_path}/cloud-hypervisor"
-	if [ ! -d "${clh_binary_path}" ]; then
-		mkdir -p "${clh_binary_path}"
-	fi
-	if [ ! -f "${clh_binary_path}/cloud-hypervisor" ]; then
-		cp cloud-hypervisor "${clh_binary_path}"
-	fi
-	popd
-}
-
 if [ "${ARCH}" == "aarch64" ]; then
 	info "aarch64 binaries are not distributed as part of the Cloud Hypervisor releases, forcing to build from source"
 	force_build_from_source="true"
@@ -129,8 +94,8 @@ fi
 
 if [ "${force_build_from_source}" == "true" ]; then
 	info "Build cloud-hypervisor from source as it's been request via the force_build_from_source flag"
-	check_cached_cloud_hypervisor
+	build_clh_from_source
 else
 	pull_clh_released_binary ||
-	(info "Failed to pull cloud-hypervisor released binary, trying to build from source" && check_cached_cloud_hypervisor)
+	(info "Failed to pull cloud-hypervisor released binary, trying to build from source" && build_clh_from_source)
 fi

tools/packaging/static-build/initramfs/build.sh

@@ -9,30 +9,28 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly initramfs_builder="${script_dir}/build-initramfs.sh"
 readonly default_install_dir="$(cd "${script_dir}/../../kernel" && pwd)"
 
 source "${script_dir}/../../scripts/lib.sh"
 
-kata_version="${kata_version:-}"
 cryptsetup_repo="${cryptsetup_repo:-}"
 cryptsetup_version="${cryptsetup_version:-}"
 lvm2_repo="${lvm2_repo:-}"
 lvm2_version="${lvm2_version:-}"
 package_output_dir="${package_output_dir:-}"
 
-[ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps "externals.cryptsetup.url" "${kata_version}")
+[ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps "externals.cryptsetup.url")
-[ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps "externals.cryptsetup.version" "${kata_version}")
+[ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps "externals.cryptsetup.version")
-[ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps "externals.lvm2.url" "${kata_version}")
+[ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps "externals.lvm2.url")
-[ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps "externals.lvm2.version" "${kata_version}")
+[ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps "externals.lvm2.version")
 
 [ -n "${cryptsetup_repo}" ] || die "Failed to get cryptsetup repo"
 [ -n "${cryptsetup_version}" ] || die "Failed to get cryptsetup version"
 [ -n "${lvm2_repo}" ] || die "Failed to get lvm2 repo"
 [ -n "${lvm2_version}" ] || die "Failed to get lvm2 version"
 
-container_image="${INITRAMFS_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:initramfs-cryptsetup-${cryptsetup_version}-lvm2-${lvm2_version}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${INITRAMFS_CONTAINER_BUILDER:-$(get_initramfs_image_name)}"
 
 sudo docker pull ${container_image} || (sudo docker build \
 	--build-arg cryptsetup_repo="${cryptsetup_repo}" \

tools/packaging/static-build/kernel/build.sh

@@ -9,90 +9,33 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
-readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
 
 source "${script_dir}/../../scripts/lib.sh"
 
+readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
+
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="${KERNEL_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}"
-kernel_latest_build_url="${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}"
-current_kernel_version=${kernel_version:-$(get_from_kata_deps "assets.kernel.version")}
-cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
-current_kernel_config_file="${cached_path}/tools/packaging/kernel/kata_config_version"
-current_kernel_config="$(cat $current_kernel_config_file)"
-kernel_version="$(echo ${current_kernel_version} | cut -c2- )"
 
-build_from_source() {
+sudo docker pull ${container_image} || \
-	sudo docker pull ${container_image} || \
+	(sudo docker build -t "${container_image}" "${script_dir}" && \
-		(sudo docker build -t "${container_image}" "${script_dir}" && \
+ 	# No-op unless PUSH_TO_REGISTRY is exported as "yes"
-	 	# No-op unless PUSH_TO_REGISTRY is exported as "yes"
+ 	push_to_registry "${container_image}")
-	 	push_to_registry "${container_image}")
 
-	sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
+sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
-		-w "${PWD}" \
+	-w "${PWD}" \
-		--env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \
+	--env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \
-		"${container_image}" \
+	"${container_image}" \
-		bash -c "${kernel_builder} $* setup"
+	bash -c "${kernel_builder} $* setup"
 
-	sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
+sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
-		-w "${PWD}" \
+	-w "${PWD}" \
-		"${container_image}" \
+	"${container_image}" \
-		bash -c "${kernel_builder} $* build"
+	bash -c "${kernel_builder} $* build"
 
-	sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
+sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
-		-w "${PWD}" \
+	-w "${PWD}" \
-		--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
+	--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
-		"${container_image}" \
+	"${container_image}" \
-		bash -c "${kernel_builder} $* install"
+	bash -c "${kernel_builder} $* install"
-}
-
-check_cached_kernel() {
-	local latest=$(curl -sfL "${kernel_latest_build_url}"/latest) || latest="none"
-	local cached_kernel_version="$(echo ${latest} | awk '{print $1}')"
-	info "Current kernel version: ${kernel_version}"
-	info "Cached kernel version: ${cached_kernel_version}"
-	if [ "${kernel_version}" == "${cached_kernel_version}" ] && [ "$(uname -m)" == "x86_64" ]; then
-		local cached_kernel_config="$(echo ${latest} | awk '{print $2}')"
-		info "Cached kernel config: ${cached_kernel_config}"
-		info "Current kernel config: ${current_kernel_config}"
-		if [ -z "${cached_kernel_config}" ]; then
-			build_from_source $*
-		else
-			install_cached_kernel $*
-		fi
-	else
-		build_from_source $*
-	fi
-}
-
-install_cached_kernel() {
-	local kernel_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-kernel/destdir/opt/confidential-containers/share/kata-containers"
-	local vmlinux_kernel_name="vmlinux-${cached_kernel_version}-${cached_kernel_config}"
-	local vmlinuz_kernel_name="vmlinuz-${cached_kernel_version}-${cached_kernel_config}"
-	mkdir -p "${kernel_directory}"
-	pushd "${kernel_directory}"
-	ls
-	local vmlinux_url="${kernel_latest_build_url}/${vmlinux_kernel_name}"
-	if curl --output /dev/null --silent --head --fail "${vmlinux_url}"; then
-		info "Installing vmlinux cached kernel"
-		curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinux_kernel_name}" -o "${vmlinux_kernel_name}" || return 1
-		sudo -E ln -sf "${kernel_directory}/${vmlinux_kernel_name}" "${kernel_directory}/vmlinux.container"
-	fi
-
-	local vmlinuz_url="${kernel_latest_build_url}/${vmlinuz_kernel_name}"
-	if curl --output /dev/null --silent --head --fail "${vmlinuz_url}"; then
-		info "Installing vmlinuz cached kernel"
-		curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinuz_kernel_name}" -o "${vmlinuz_kernel_name}" || return 1
-		sudo -E ln -sf "${kernel_directory}/${vmlinuz_kernel_name}" "${kernel_directory}/vmlinuz.container"
-	fi
-	popd
-
-}
-
-main() {
-	check_cached_kernel $*
-}
-
-main $*

tools/packaging/static-build/ovmf/build.sh

@@ -9,14 +9,13 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly ovmf_builder="${script_dir}/build-ovmf.sh"
 
 source "${script_dir}/../../scripts/lib.sh"
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="${OVMF_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:ovmf-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${OVMF_CONTAINER_BUILDER:-$(get_ovmf_image_name)}"
 ovmf_build="${ovmf_build:-x86_64}"
 kata_version="${kata_version:-}"
 ovmf_repo="${ovmf_repo:-}"

tools/packaging/static-build/qemu/build-base-qemu.sh

@@ -9,7 +9,6 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly qemu_builder="${script_dir}/build-qemu.sh"
 
 source "${script_dir}/../../scripts/lib.sh"
@@ -39,7 +38,7 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d")
 [ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu"
 [ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static"
 
-container_image="${QEMU_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:qemu-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${QEMU_CONTAINER_BUILDER:-$(get_qemu_image_name)}"
 
 sudo docker pull ${container_image} || \
 	(sudo "${container_engine}" build \

tools/packaging/static-build/qemu/build-static-qemu-cc.sh

@@ -12,83 +12,23 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 source "${script_dir}/../../scripts/lib.sh"
 
-export qemu_repo="${qemu_repo:-}"
+qemu_repo="${qemu_repo:-}"
-export qemu_version="${qemu_version:-}"
+qemu_version="${qemu_version:-}"
-export qemu_latest_build_url="${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}"
+tee="${tee:-}"
-export katacontainers_repo="${katacontainers_repo:=github.com/kata-containers/kata-containers}"
-export qemu_tarball_name="kata-static-qemu-cc.tar.gz"
-export pkg_dir="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
-export qemu_tarball_directory="${pkg_dir}/kata-deploy/local-build/build/cc-qemu/builddir"
-export tee="${tee:-}"
 
 export prefix="/opt/confidential-containers/"
 
-get_qemu_information() {
+if [ -z "${qemu_repo}" ]; then
-	if [ -z "${qemu_repo}" ]; then
+	info "Get qemu information from runtime versions.yaml"
-		info "Get qemu information from runtime versions.yaml"
+	export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url")
-		export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url")
+	[ -n "${qemu_url}" ] || die "failed to get qemu url"
-		[ -n "${qemu_url}" ] || die "failed to get qemu url"
+	export qemu_repo="${qemu_url}.git"
-		export qemu_repo="${qemu_url}.git"
+fi
-	fi
 
-	[ -n "${qemu_repo}" ] || die "failed to get qemu repo"
+[ -n "${qemu_repo}" ] || die "failed to get qemu repo"
-	[ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
+[ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
-	[ -n "${qemu_version}" ] || die "failed to get qemu version"
+[ -n "${qemu_version}" ] || die "failed to get qemu version"
-}
 
-calc_qemu_files_sha256sum() {
+qemu_tarball_name="kata-static-qemu-cc.tar.gz"
-	info "pkg directory is at ${pkg_dir}"
+[ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz"
-	local files="${pkg_dir}/qemu \
+"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}"
-		${pkg_dir}/static-build/qemu.blacklist \
-		${pkg_dir}/static-build/scripts"
-
-	sha256sum_from_files "$files"
-}
-
-cached_or_build_qemu_tar() {
-	# Check latest qemu cc tar version sha256sum
-	local latest=$(curl -sfL "${qemu_latest_build_url}/latest") || latest="none"
-	local cached_qemu_version="$(echo ${latest} | awk '{print $1}')"
-	info "Current qemu version: ${qemu_version}"
-	info "Cached qemu version: ${cached_qemu_version}"
-	if [ "${qemu_version}" == "${cached_qemu_version}" ]; then
-		info "Get latest cached information ${latest}"
-		local cached_sha256sum="$(echo ${latest} | awk '{print $2}')"
-		info "Cached sha256sum version: ${cached_sha256sum}"
-		local current_sha256sum="$(calc_qemu_files_sha256sum)"
-		info "Current sha256sum of the qemu directory ${current_sha256sum}"
-		if [ -z "${cached_sha256sum}" ]; then
-			build_qemu_tar
-		elif [ "${current_sha256sum}" == "${cached_sha256sum}" ]; then
-			install_cached_qemu_tar
-		else
-			build_qemu_tar
-		fi
-	else
-		build_qemu_tar
-	fi
-}
-
-build_qemu_tar() {
-	[ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz"
-	"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}"
-}
-
-install_cached_qemu_tar() {
-	info "Using cached tarball of qemu"
-	curl -fL --progress-bar "${qemu_latest_build_url}/${qemu_tarball_name}" -o "${qemu_tarball_name}" || return 1
-	curl -fsOL "${qemu_latest_build_url}/sha256sum-${qemu_tarball_name}" || return 1
-	sha256sum -c "sha256sum-${qemu_tarball_name}" || return 1
-}
-
-main() {
-	get_qemu_information
-	# Currently the cached for qemu cc only works in x86_64
-	if [ "$(uname -m)" == "x86_64" ]; then
-		cached_or_build_qemu_tar
-	else
-		build_qemu_tar
-	fi
-}
-
-main $@

tools/packaging/static-build/shim-v2/build.sh

@@ -9,17 +9,17 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
-readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
 
 source "${script_dir}/../../scripts/lib.sh"
 
+readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
+
 GO_VERSION=${GO_VERSION}
 RUST_VERSION=${RUST_VERSION:-}
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="${SHIM_V2_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${SHIM_V2_CONTAINER_BUILDER:-$(get_shim_v2_image_name)}"
 
 EXTRA_OPTS="${EXTRA_OPTS:-""}"
 VMM_CONFIGS="qemu fc"

tools/packaging/static-build/td-shim/build.sh

@@ -9,7 +9,6 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly tdshim_builder="${script_dir}/build-td-shim.sh"
 
 source "${script_dir}/../../scripts/lib.sh"
@@ -30,7 +29,7 @@ package_output_dir="${package_output_dir:-}"
 [ -n "${tdshim_version}" ] || die "Failed to get TD-shim version or commit"
 [ -n "${tdshim_toolchain}" ] || die "Failed to get TD-shim toolchain to be used to build the project"
 
-container_image="${TDSHIM_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:td-shim-${tdshim_toolchain}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${TDSHIM_CONTAINER_BUILDER:-$(get_td_shim_image_name)}"
 
 sudo docker pull ${container_image} || \
 	(sudo docker build \

tools/packaging/static-build/virtiofsd/build.sh

@@ -9,7 +9,6 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly virtiofsd_builder="${script_dir}/build-static-virtiofsd.sh"
 
 source "${script_dir}/../../scripts/lib.sh"
@@ -49,7 +48,7 @@ case ${ARCH} in
 		;;
 esac
 
-container_image="${VIRTIOFSD_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${VIRTIOFSD_CONTAINER_BUILDER:-$(get_virtiofsd_image_name)}"
 
 sudo docker pull ${container_image} || \
 	(sudo docker build \