mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-08-15 14:43:51 +00:00
CCv0: Use cached cc qemu tarball
This PR implements the use of a cached cc qemu tarball to speed up the CI and avoid building the cc qemu tarball when it is not necessary. Fixes #5363 Signed-off-by: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
This commit is contained in:
Gabriela Cervantes
parent
712177a337
commit
72a13f6064
34
tools/packaging/scripts/lib.sh
Normal file → Executable file
34
tools/packaging/scripts/lib.sh
Normal file → Executable file
@ -17,6 +17,11 @@ short_commit_length=10
|
||||
|
||||
hub_bin="hub-bin"
|
||||
|
||||
# Jenkins URL
|
||||
jenkins_url="http://jenkins.katacontainers.io"
|
||||
# Path where cached artifacts are found.
|
||||
cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts"
|
||||
|
||||
clone_tests_repo() {
|
||||
# KATA_CI_NO_NETWORK is (has to be) ignored if there is
|
||||
# no existing clone.
|
||||
@ -145,3 +150,32 @@ push_to_registry() {
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
sha256sum_from_files() {
|
||||
local files_in=${@:-}
|
||||
local files=""
|
||||
local shasum=""
|
||||
|
||||
# Process the input files:
|
||||
# - discard the files/directories that don't exist.
|
||||
# - find the files if it is a directory
|
||||
for f in $files_in; do
|
||||
if [ -d "$f" ]; then
|
||||
files+=" $(find $f -type f)"
|
||||
elif [ -f "$f" ]; then
|
||||
files+=" $f"
|
||||
fi
|
||||
done
|
||||
# Return in case there is none input files.
|
||||
[ -n "$files" ] || return 0
|
||||
|
||||
# Alphabetically sorting the files.
|
||||
files="$(echo $files | tr ' ' '\n' | LC_ALL=C sort -u)"
|
||||
# Concate the files and calculate a hash.
|
||||
shasum="$(cat $files | sha256sum -b)" || true
|
||||
info "shasum of files $shasum"
|
||||
if [ -n "$shasum" ];then
|
||||
# Return only the SHA field.
|
||||
echo $(awk '{ print $1 }' <<< $shasum)
|
||||
fi
|
||||
}
|
||||
|
||||
47
tools/packaging/static-build/cache_components.sh
Executable file
47
tools/packaging/static-build/cache_components.sh
Executable file
@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
# Copyright (c) 2022 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "${script_dir}/../scripts/lib.sh"
|
||||
source "${script_dir}/qemu/build-static-qemu-cc.sh"
|
||||
|
||||
export KATA_BUILD_CC="${KATA_BUILD_CC:-}"
|
||||
export qemu_cc_tarball_name="kata-static-qemu-cc.tar.gz"
|
||||
|
||||
cache_qemu_artifacts() {
|
||||
local current_qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
|
||||
create_qemu_cache_asset "${qemu_cc_tarball_name}" "${current_qemu_version}"
|
||||
local qemu_sha=$(calc_qemu_files_sha256sum)
|
||||
echo "${current_qemu_version} ${qemu_sha}" > "latest"
|
||||
}
|
||||
|
||||
create_qemu_cache_asset() {
|
||||
local component_name="$1"
|
||||
local component_version="$2"
|
||||
local qemu_cc_tarball_path=$(sudo find / -iname "${qemu_cc_tarball_name}")
|
||||
info "qemu cc tarball_path ${qemu_cc_tarball_path}"
|
||||
cp -a "${qemu_cc_tarball_path}" .
|
||||
sudo chown -R "${USER}:${USER}" .
|
||||
sha256sum "${component_name}" > "sha256sum-${component_name}"
|
||||
cat "sha256sum-${component_name}"
|
||||
}
|
||||
|
||||
main() {
|
||||
mkdir -p "${WORKSPACE}/artifacts"
|
||||
pushd "${WORKSPACE}/artifacts"
|
||||
echo "Artifacts:"
|
||||
cache_qemu_artifacts
|
||||
ls -la "${WORKSPACE}/artifacts/"
|
||||
popd
|
||||
sync
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@ -12,24 +12,83 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "${script_dir}/../../scripts/lib.sh"
|
||||
|
||||
qemu_repo="${qemu_repo:-}"
|
||||
qemu_version="${qemu_version:-}"
|
||||
tee="${tee:-}"
|
||||
export qemu_repo="${qemu_repo:-}"
|
||||
export qemu_version="${qemu_version:-}"
|
||||
export qemu_latest_build_url="${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}"
|
||||
export katacontainers_repo="${katacontainers_repo:=github.com/kata-containers/kata-containers}"
|
||||
export qemu_tarball_name="kata-static-qemu-cc.tar.gz"
|
||||
export pkg_dir="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
|
||||
export qemu_tarball_directory="${pkg_dir}/kata-deploy/local-build/build/cc-qemu/builddir"
|
||||
export tee="${tee:-}"
|
||||
|
||||
export prefix="/opt/confidential-containers/"
|
||||
|
||||
if [ -z "$qemu_repo" ]; then
|
||||
info "Get qemu information from runtime versions.yaml"
|
||||
qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url")
|
||||
[ -n "$qemu_url" ] || die "failed to get qemu url"
|
||||
qemu_repo="${qemu_url}.git"
|
||||
fi
|
||||
[ -n "$qemu_repo" ] || die "failed to get qemu repo"
|
||||
get_qemu_information() {
|
||||
if [ -z "${qemu_repo}" ]; then
|
||||
info "Get qemu information from runtime versions.yaml"
|
||||
export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url")
|
||||
[ -n "${qemu_url}" ] || die "failed to get qemu url"
|
||||
export qemu_repo="${qemu_url}.git"
|
||||
fi
|
||||
|
||||
[ -n "$qemu_version" ] || qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
|
||||
[ -n "$qemu_version" ] || die "failed to get qemu version"
|
||||
[ -n "${qemu_repo}" ] || die "failed to get qemu repo"
|
||||
[ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
|
||||
[ -n "${qemu_version}" ] || die "failed to get qemu version"
|
||||
}
|
||||
|
||||
calc_qemu_files_sha256sum() {
|
||||
info "pkg directory is at ${pkg_dir}"
|
||||
local files="${pkg_dir}/qemu \
|
||||
${pkg_dir}/static-build/qemu.blacklist \
|
||||
${pkg_dir}/static-build/scripts"
|
||||
|
||||
tarball_name="kata-static-qemu-cc.tar.gz"
|
||||
[ -n "${tee}" ] && tarball_name="kata-static-${tee}-qemu-cc.tar.gz"
|
||||
"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${tarball_name}"
|
||||
sha256sum_from_files "$files"
|
||||
}
|
||||
|
||||
cached_or_build_qemu_tar() {
|
||||
# Check latest qemu cc tar version sha256sum
|
||||
local latest=$(curl -sfL "${qemu_latest_build_url}/latest") || latest="none"
|
||||
local cached_qemu_version="$(echo ${latest} | awk '{print $1}')"
|
||||
info "Current qemu version: ${qemu_version}"
|
||||
info "Cached qemu version: ${cached_qemu_version}"
|
||||
if [ "${qemu_version}" == "${cached_qemu_version}" ]; then
|
||||
info "Get latest cached information ${latest}"
|
||||
local cached_sha256sum="$(echo ${latest} | awk '{print $2}')"
|
||||
info "Cached sha256sum version: ${cached_sha256sum}"
|
||||
local current_sha256sum="$(calc_qemu_files_sha256sum)"
|
||||
info "Current sha256sum of the qemu directory ${current_sha256sum}"
|
||||
if [ -z "${cached_sha256sum}" ]; then
|
||||
build_qemu_tar
|
||||
elif [ "${current_sha256sum}" == "${cached_sha256sum}" ]; then
|
||||
install_cached_qemu_tar
|
||||
else
|
||||
build_qemu_tar
|
||||
fi
|
||||
else
|
||||
build_qemu_tar
|
||||
fi
|
||||
}
|
||||
|
||||
build_qemu_tar() {
|
||||
[ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz"
|
||||
"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}"
|
||||
}
|
||||
|
||||
install_cached_qemu_tar() {
|
||||
info "Using cached tarball of qemu"
|
||||
curl -fL --progress-bar "${qemu_latest_build_url}/${qemu_tarball_name}" -o "${qemu_tarball_name}" || return 1
|
||||
curl -fsOL "${qemu_latest_build_url}/sha256sum-${qemu_tarball_name}" || return 1
|
||||
sha256sum -c "sha256sum-${qemu_tarball_name}" || return 1
|
||||
}
|
||||
|
||||
main() {
|
||||
get_qemu_information
|
||||
# Currently the cached for qemu cc only works in x86_64
|
||||
if [ "$(uname -m)" == "x86_64" ]; then
|
||||
cached_or_build_qemu_tar
|
||||
else
|
||||
build_qemu_tar
|
||||
fi
|
||||
}
|
||||
|
||||
main $@
|
||||
|
||||
Loading…
Reference in New Issue
Block a user