kata-deploy-binaries: Check for cached version

Let's check for the cached version of the components as part of the
kata-deploy-binaries.sh as here we already have the needed info for
checking whether a component is cached or not, and to use it without
depending on changes made on each one of the builder scripts.

Fixes: #5816

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -38,6 +38,8 @@ readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_imag
 readonly cc_prefix="/opt/confidential-containers"
 readonly qemu_cc_builder="${static_build_dir}/qemu/build-static-qemu-cc.sh"
 
+source "${script_dir}/../../scripts/lib.sh"
+
 ARCH=$(uname -m)
 
 workdir="${WORKDIR:-$PWD}"
@@ -104,8 +106,45 @@ EOF
 	exit "${return_code}"
 }
 
+cleanup_and_fail() {
+	rm -f "${component_tarball_path}"
+	return 1
+}
+
+install_cached_component() {
+	local component="${1}"
+	local jenkins_build_url="${2}"
+	local current_version="${3}"
+	local current_image_version="${4}"
+	local component_tarball_name="${5}"
+	local component_tarball_path="${6}"
+
+	local cached_version=$(curl -sfL "${jenkins_build_url}/latest" | awk '{print $1}') || cached_version="none"
+	local cached_image_version=$(curl -sfL "${jenkins_build_url}/latest_image" | awk '{print $1}') || cached_image_version="none"
+
+	[ "${cached_image_version}" != "${current_image_version}" ] && return 1
+	[ "${cached_version}" != "${current_version}" ] && return 1
+
+	info "Using cached tarball of ${component}"
+	pushd ${workdir}
+	echo "Downloading tarball from: ${jenkins_build_url}/${component_tarball_name}"
+	curl -fL --progress-bar "${jenkins_build_url}/${component_tarball_name}" -o "${component_tarball_path}" || return cleanup_and_fail
+	curl -fsOL "${jenkins_build_url}/sha256sum-${component_tarball_name}" || return cleanup_and_fail
+	sha256sum -c "sha256sum-${component_tarball_name}" && return cleanup_and_fail
+	popd
+}
+
 # Install static CC cloud-hypervisor asset
 install_cc_clh() {
+	install_cached_component \
+		"cloud-hypervisor" \
+		"${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}" \
+		"$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \
+		"" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	if [[ "${ARCH}" == "x86_64" ]]; then
 		export features="tdx"
 	fi
@@ -143,11 +182,21 @@ install_cc_tdx_image() {
 
 #Install CC kernel asset
 install_cc_kernel() {
-	export KATA_BUILD_CC=yes
-
 	info "build initramfs for cc kernel"
-	"${initramfs_builder}"
+
+	export KATA_BUILD_CC=yes
 	export kernel_version="$(yq r $versions_yaml assets.kernel.version)"
+
+	install_cached_component \
+		"kernel" \
+		"${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}" \
+		"${kernel_version}" \
+		"$(get_kernel_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
+	"${initramfs_builder}"
 	DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -f -v "${kernel_version}"
 }
 
@@ -156,6 +205,16 @@ install_cc_qemu() {
 	info "build static CC qemu"
 	export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)"
 	export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)"
+
+	install_cached_component \
+		"QEMU" \
+		"${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}" \
+		"${qemu_version}-$(calc_qemu_files_sha256sum)" \
+		"$(get_qemu_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	"${qemu_cc_builder}"
 	tar xvf "${builddir}/kata-static-qemu-cc.tar.gz" -C "${destdir}"
 }
@@ -178,6 +237,15 @@ install_cc_shimv2() {
 
 # Install static CC virtiofsd asset
 install_cc_virtiofsd() {
+	install_cached_component \
+		"virtiofsd" \
+		"${jenkins_url}/job/kata-containers-2.0-virtiofsd-cc-$(uname -m)/${cached_artifacts_path}" \
+		"$(get_from_kata_deps "assets.externals.virtiofsd.version")" \
+		"$(get_virtiofsd_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	info "build static CC virtiofsd"
 	"${virtiofsd_builder}"
 	info "Install static CC virtiofsd"
@@ -199,6 +267,16 @@ install_cc_tee_kernel() {
 
 	info "build initramfs for tee kernel"
 	export kernel_version=${kernel_version}
+
+	install_cached_component \
+		"kernel" \
+		"${jenkins_url}/job/kata-containers-2.0-kernel-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \
+		"${kernel_version}" \
+		"$(get_kernel_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	"${initramfs_builder}"
 	kernel_url="$(yq r $versions_yaml assets.kernel.${tee}.url)"
 	DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -x "${tee}" -v "${kernel_version}" -u "${kernel_url}"
@@ -223,6 +301,16 @@ install_cc_tee_qemu() {
 	export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.url)"
 	export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.tag)"
 	export tee="${tee}"
+
+	install_cached_component \
+		"QEMU ${tee}" \
+		"${jenkins_url}/job/kata-containers-2.0-qemu-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \
+		"${qemu_version}-$(calc_qemu_files_sha256sum)" \
+		"$(get_qemu_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	"${qemu_cc_builder}"
 	tar xvf "${builddir}/kata-static-${tee}-qemu-cc.tar.gz" -C "${destdir}"
 }
@@ -232,6 +320,15 @@ install_cc_tdx_qemu() {
 }
 
 install_cc_tdx_td_shim() {
+	install_cached_component \
+		"td-shim" \
+		"${jenkins_url}/job/kata-containers-2.0-td-shim-cc-$(uname -m)/${cached_artifacts_path}" \
+		"$(get_from_kata_deps "assets.externals.td-shim.version")" \
+		"$(get_td_shim_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${td_shim_builder}"
 	tar xvf "${builddir}/td-shim.tar.gz" -C "${destdir}"
 }
@@ -240,6 +337,18 @@ install_cc_tee_ovmf() {
 	tee="${1}"
 	tarball_name="${2}"
 
+	local component_name="ovmf"
+	local component_version="$(get_from_kata_deps "assets.external.ovmf.${tee}.version")"
+	[ "${tee}" == "tdx" ] && component_name="tdvf"
+	install_cached_component \
+		"${component_name}" \
+		"${jenkins_url}/job/kata-containers-2.0-${component_name}-cc-$(uname -m)/${cached_artifacts_path}" \
+		"$(component_version)" \
+		"$(get_ovmf_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
 	DESTDIR="${destdir}" PREFIX="${cc_prefix}" ovmf_build="${tee}" "${ovmf_builder}"
 	tar xvf "${builddir}/${tarball_name}" -C "${destdir}"
 }
@@ -350,6 +459,11 @@ handle_build() {
 	info "DESTDIR ${destdir}"
 	local build_target
 	build_target="$1"
+
+	export final_tarball_path="${workdir}/kata-static-${build_target}.tar.xz"
+	export final_tarball_name="$(basename ${final_tarball_path})"
+	rm -f ${final_tarball_name}
+
 	case "${build_target}" in
 	all)
 		install_clh
@@ -429,12 +543,11 @@ handle_build() {
 		;;
 	esac
 
-	tarball_name="${workdir}/kata-static-${build_target}.tar.xz"
+	if [ ! -f "${final_tarball_path}" ]; then
-	(
 		cd "${destdir}"
-		sudo tar cvfJ "${tarball_name}" "."
+		sudo tar cvfJ "${final_tarball_path}" "."
-	)
+	fi
-	tar tvf "${tarball_name}"
+	tar tvf "${final_tarball_path}"
 }
 
 silent_mode_error_trap() {

tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh

@@ -23,7 +23,6 @@ cloud_hypervisor_repo="${cloud_hypervisor_repo:-}"
 cloud_hypervisor_version="${cloud_hypervisor_version:-}"
 cloud_hypervisor_pr="${cloud_hypervisor_pr:-}"
 cloud_hypervisor_pull_ref_branch="${cloud_hypervisor_pull_ref_branch:-main}"
-cloud_hypervisor_latest_build_url="${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}"
 
 if [ -z "$cloud_hypervisor_repo" ]; then
 	info "Get cloud_hypervisor information from runtime versions.yaml"
@@ -83,40 +82,6 @@ build_clh_from_source() {
 	popd
 }
 
-check_cached_cloud_hypervisor() {
-	local cached_cloud_hypervisor_version=$(curl -sfL "${cloud_hypervisor_latest_build_url}"/latest) || latest="none"
-	info "Current cloud hypervisor version: ${cloud_hypervisor_version}"
-	info "Cached cloud hypervisor version: ${cached_cloud_hypervisor_version}"
-	if [ "${cloud_hypervisor_version}" == "${cached_cloud_hypervisor_version}" ] && [ "${ARCH}" == "x86_64" ]; then
-		install_cached_cloud_hypervisor
-	else
-		build_clh_from_source
-	fi
-}
-
-install_cached_cloud_hypervisor() {
-	local cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
-	local clh_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-cloud-hypervisor/builddir/cloud-hypervisor"
-	mkdir cloud-hypervisor
-	pushd cloud-hypervisor
-	local checksum_file="sha256sum-cloud-hypervisor"
-	info "Downloading the cloud hypervisor binary"
-	curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/cloud-hypervisor" || return 1
-	info "Checking cloud hypervisor binary checksum"
-	curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/${checksum_file}" || return 1
-	info "Verify checksum"
-	sudo sha256sum -c "${checksum_file}" || return 1
-	chmod +x cloud-hypervisor
-	local clh_binary_path="${cached_path}/cloud-hypervisor"
-	if [ ! -d "${clh_binary_path}" ]; then
-		mkdir -p "${clh_binary_path}"
-	fi
-	if [ ! -f "${clh_binary_path}/cloud-hypervisor" ]; then
-		cp cloud-hypervisor "${clh_binary_path}"
-	fi
-	popd
-}
-
 if [ "${ARCH}" == "aarch64" ]; then
 	info "aarch64 binaries are not distributed as part of the Cloud Hypervisor releases, forcing to build from source"
 	force_build_from_source="true"
@@ -129,8 +94,8 @@ fi
 
 if [ "${force_build_from_source}" == "true" ]; then
 	info "Build cloud-hypervisor from source as it's been request via the force_build_from_source flag"
-	check_cached_cloud_hypervisor
+	build_clh_from_source
 else
 	pull_clh_released_binary ||
-	(info "Failed to pull cloud-hypervisor released binary, trying to build from source" && check_cached_cloud_hypervisor)
+	(info "Failed to pull cloud-hypervisor released binary, trying to build from source" && build_clh_from_source)
 fi

tools/packaging/static-build/kernel/build.sh

@@ -17,82 +17,25 @@ readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
 container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}"
-kernel_latest_build_url="${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}"
-current_kernel_version=${kernel_version:-$(get_from_kata_deps "assets.kernel.version")}
-cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
-current_kernel_config_file="${cached_path}/tools/packaging/kernel/kata_config_version"
-current_kernel_config="$(cat $current_kernel_config_file)"
-kernel_version="$(echo ${current_kernel_version} | cut -c2- )"
 
-build_from_source() {
+sudo docker pull ${container_image} || \
-	sudo docker pull ${container_image} || \
 	(sudo docker build -t "${container_image}" "${script_dir}" && \
  	# No-op unless PUSH_TO_REGISTRY is exported as "yes"
  	push_to_registry "${container_image}")
 
-	sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
+sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	--env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \
 	"${container_image}" \
 	bash -c "${kernel_builder} $* setup"
 
-	sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
+sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	"${container_image}" \
 	bash -c "${kernel_builder} $* build"
 
-	sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
+sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
 	"${container_image}" \
 	bash -c "${kernel_builder} $* install"
-}
-
-check_cached_kernel() {
-	local latest=$(curl -sfL "${kernel_latest_build_url}"/latest) || latest="none"
-	local cached_kernel_version="$(echo ${latest} | awk '{print $1}')"
-	info "Current kernel version: ${kernel_version}"
-	info "Cached kernel version: ${cached_kernel_version}"
-	if [ "${kernel_version}" == "${cached_kernel_version}" ] && [ "$(uname -m)" == "x86_64" ]; then
-		local cached_kernel_config="$(echo ${latest} | awk '{print $2}')"
-		info "Cached kernel config: ${cached_kernel_config}"
-		info "Current kernel config: ${current_kernel_config}"
-		if [ -z "${cached_kernel_config}" ]; then
-			build_from_source $*
-		else
-			install_cached_kernel $*
-		fi
-	else
-		build_from_source $*
-	fi
-}
-
-install_cached_kernel() {
-	local kernel_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-kernel/destdir/opt/confidential-containers/share/kata-containers"
-	local vmlinux_kernel_name="vmlinux-${cached_kernel_version}-${cached_kernel_config}"
-	local vmlinuz_kernel_name="vmlinuz-${cached_kernel_version}-${cached_kernel_config}"
-	mkdir -p "${kernel_directory}"
-	pushd "${kernel_directory}"
-	ls
-	local vmlinux_url="${kernel_latest_build_url}/${vmlinux_kernel_name}"
-	if curl --output /dev/null --silent --head --fail "${vmlinux_url}"; then
-		info "Installing vmlinux cached kernel"
-		curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinux_kernel_name}" -o "${vmlinux_kernel_name}" || return 1
-		sudo -E ln -sf "${kernel_directory}/${vmlinux_kernel_name}" "${kernel_directory}/vmlinux.container"
-	fi
-
-	local vmlinuz_url="${kernel_latest_build_url}/${vmlinuz_kernel_name}"
-	if curl --output /dev/null --silent --head --fail "${vmlinuz_url}"; then
-		info "Installing vmlinuz cached kernel"
-		curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinuz_kernel_name}" -o "${vmlinuz_kernel_name}" || return 1
-		sudo -E ln -sf "${kernel_directory}/${vmlinuz_kernel_name}" "${kernel_directory}/vmlinuz.container"
-	fi
-	popd
-
-}
-
-main() {
-	check_cached_kernel $*
-}
-
-main $*

tools/packaging/static-build/qemu/build-static-qemu-cc.sh

@@ -12,74 +12,23 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 source "${script_dir}/../../scripts/lib.sh"
 
-export qemu_repo="${qemu_repo:-}"
+qemu_repo="${qemu_repo:-}"
-export qemu_version="${qemu_version:-}"
+qemu_version="${qemu_version:-}"
-export qemu_latest_build_url="${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}"
+tee="${tee:-}"
-export katacontainers_repo="${katacontainers_repo:=github.com/kata-containers/kata-containers}"
-export qemu_tarball_name="kata-static-qemu-cc.tar.gz"
-export pkg_dir="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
-export qemu_tarball_directory="${pkg_dir}/kata-deploy/local-build/build/cc-qemu/builddir"
-export tee="${tee:-}"
 
 export prefix="/opt/confidential-containers/"
 
-get_qemu_information() {
+if [ -z "${qemu_repo}" ]; then
-	if [ -z "${qemu_repo}" ]; then
 	info "Get qemu information from runtime versions.yaml"
 	export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url")
 	[ -n "${qemu_url}" ] || die "failed to get qemu url"
 	export qemu_repo="${qemu_url}.git"
-	fi
+fi
 
-	[ -n "${qemu_repo}" ] || die "failed to get qemu repo"
+[ -n "${qemu_repo}" ] || die "failed to get qemu repo"
-	[ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
+[ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
-	[ -n "${qemu_version}" ] || die "failed to get qemu version"
+[ -n "${qemu_version}" ] || die "failed to get qemu version"
-}
 
-cached_or_build_qemu_tar() {
+qemu_tarball_name="kata-static-qemu-cc.tar.gz"
-	# Check latest qemu cc tar version sha256sum
+[ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz"
-	local latest=$(curl -sfL "${qemu_latest_build_url}/latest") || latest="none"
+"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}"
-	local cached_qemu_version="$(echo ${latest} | awk '{print $1}')"
-	info "Current qemu version: ${qemu_version}"
-	info "Cached qemu version: ${cached_qemu_version}"
-	if [ "${qemu_version}" == "${cached_qemu_version}" ]; then
-		info "Get latest cached information ${latest}"
-		local cached_sha256sum="$(echo ${latest} | awk '{print $2}')"
-		info "Cached sha256sum version: ${cached_sha256sum}"
-		local current_sha256sum="$(calc_qemu_files_sha256sum)"
-		info "Current sha256sum of the qemu directory ${current_sha256sum}"
-		if [ -z "${cached_sha256sum}" ]; then
-			build_qemu_tar
-		elif [ "${current_sha256sum}" == "${cached_sha256sum}" ]; then
-			install_cached_qemu_tar
-		else
-			build_qemu_tar
-		fi
-	else
-		build_qemu_tar
-	fi
-}
-
-build_qemu_tar() {
-	[ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz"
-	"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}"
-}
-
-install_cached_qemu_tar() {
-	info "Using cached tarball of qemu"
-	curl -fL --progress-bar "${qemu_latest_build_url}/${qemu_tarball_name}" -o "${qemu_tarball_name}" || return 1
-	curl -fsOL "${qemu_latest_build_url}/sha256sum-${qemu_tarball_name}" || return 1
-	sha256sum -c "sha256sum-${qemu_tarball_name}" || return 1
-}
-
-main() {
-	get_qemu_information
-	# Currently the cached for qemu cc only works in x86_64
-	if [ "$(uname -m)" == "x86_64" ]; then
-		cached_or_build_qemu_tar
-	else
-		build_qemu_tar
-	fi
-}
-
-main $@