github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-17 15:38:00 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #5784 from fidencio/topic/cached-components

Browse Source 
Re-work the way we cache components for the CCv0 branch
This commit is contained in:
Fabiano Fidêncio 2022-12-01 23:19:59 +01:00 committed by GitHub
commit 72d2f19c98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 270 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tools/packaging/guest-image/build_image.sh
View File

@ -14,7 +14,9 @@ set -o pipefail
readonly script_name="$(basename "${BASH_SOURCE[0]}")"
readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly packaging_root_dir="$(cd "${script_dir}/../" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../" && pwd)"
source "${packaging_root_dir}/scripts/lib.sh"
readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)"
patches_path=""
@ -26,8 +28,6 @@ final_image_name="kata-containers"
final_initrd_name="kata-containers-initrd"
image_initrd_extension=".img"
source "${packaging_root_dir}/scripts/lib.sh"
arch_target="$(uname -m)"
build_initrd() {

132
tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
View File

@ -15,8 +15,9 @@ readonly project="kata-containers"
readonly script_name="$(basename "${BASH_SOURCE[0]}")"
readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${script_dir}/../../scripts/lib.sh"
readonly prefix="/opt/kata"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly static_build_dir="${repo_root_dir}/tools/packaging/static-build"
readonly version_file="${repo_root_dir}/VERSION"
readonly versions_yaml="${repo_root_dir}/versions.yaml"
@ -37,6 +38,8 @@ readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_imag
readonly cc_prefix="/opt/confidential-containers"
readonly qemu_cc_builder="${static_build_dir}/qemu/build-static-qemu-cc.sh"
source "${script_dir}/../../scripts/lib.sh"
ARCH=$(uname -m)
workdir="${WORKDIR:-$PWD}"
@ -103,8 +106,45 @@ EOF
exit "${return_code}"
}
cleanup_and_fail() {
rm -f "${component_tarball_path}"
return 1
}
install_cached_component() {
local component="${1}"
local jenkins_build_url="${2}"
local current_version="${3}"
local current_image_version="${4}"
local component_tarball_name="${5}"
local component_tarball_path="${6}"
local cached_version=$(curl -sfL "${jenkins_build_url}/latest" | awk '{print $1}') || cached_version="none"
local cached_image_version=$(curl -sfL "${jenkins_build_url}/latest_image" | awk '{print $1}') || cached_image_version="none"
[ "${cached_image_version}" != "${current_image_version}" ] && return 1
[ "${cached_version}" != "${current_version}" ] && return 1
info "Using cached tarball of ${component}"
pushd ${workdir}
echo "Downloading tarball from: ${jenkins_build_url}/${component_tarball_name}"
curl -fL --progress-bar "${jenkins_build_url}/${component_tarball_name}" -o "${component_tarball_path}" || return cleanup_and_fail
curl -fsOL "${jenkins_build_url}/sha256sum-${component_tarball_name}" || return cleanup_and_fail
sha256sum -c "sha256sum-${component_tarball_name}" && return cleanup_and_fail
popd
}
# Install static CC cloud-hypervisor asset
install_cc_clh() {
install_cached_component \
"cloud-hypervisor" \
"${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}" \
"$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \
"" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
if [[ "${ARCH}" == "x86_64" ]]; then
export features="tdx"
fi
@ -142,11 +182,21 @@ install_cc_tdx_image() {
#Install CC kernel asset
install_cc_kernel() {
export KATA_BUILD_CC=yes
info "build initramfs for cc kernel"
"${initramfs_builder}"
export KATA_BUILD_CC=yes
export kernel_version="$(yq r $versions_yaml assets.kernel.version)"
install_cached_component \
"kernel" \
"${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}" \
"${kernel_version}" \
"$(get_kernel_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
"${initramfs_builder}"
DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -f -v "${kernel_version}"
}
@ -155,6 +205,16 @@ install_cc_qemu() {
info "build static CC qemu"
export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)"
export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)"
install_cached_component \
"QEMU" \
"${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}" \
"${qemu_version}-$(calc_qemu_files_sha256sum)" \
"$(get_qemu_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
"${qemu_cc_builder}"
tar xvf "${builddir}/kata-static-qemu-cc.tar.gz" -C "${destdir}"
}
@ -177,6 +237,15 @@ install_cc_shimv2() {
# Install static CC virtiofsd asset
install_cc_virtiofsd() {
install_cached_component \
"virtiofsd" \
"${jenkins_url}/job/kata-containers-2.0-virtiofsd-cc-$(uname -m)/${cached_artifacts_path}" \
"$(get_from_kata_deps "assets.externals.virtiofsd.version")" \
"$(get_virtiofsd_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
info "build static CC virtiofsd"
"${virtiofsd_builder}"
info "Install static CC virtiofsd"
@ -198,6 +267,16 @@ install_cc_tee_kernel() {
info "build initramfs for tee kernel"
export kernel_version=${kernel_version}
install_cached_component \
"kernel" \
"${jenkins_url}/job/kata-containers-2.0-kernel-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \
"${kernel_version}" \
"$(get_kernel_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
"${initramfs_builder}"
kernel_url="$(yq r $versions_yaml assets.kernel.${tee}.url)"
DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -x "${tee}" -v "${kernel_version}" -u "${kernel_url}"
@ -222,6 +301,16 @@ install_cc_tee_qemu() {
export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.url)"
export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.tag)"
export tee="${tee}"
install_cached_component \
"QEMU ${tee}" \
"${jenkins_url}/job/kata-containers-2.0-qemu-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \
"${qemu_version}-$(calc_qemu_files_sha256sum)" \
"$(get_qemu_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
"${qemu_cc_builder}"
tar xvf "${builddir}/kata-static-${tee}-qemu-cc.tar.gz" -C "${destdir}"
}
@ -231,6 +320,15 @@ install_cc_tdx_qemu() {
}
install_cc_tdx_td_shim() {
install_cached_component \
"td-shim" \
"${jenkins_url}/job/kata-containers-2.0-td-shim-cc-$(uname -m)/${cached_artifacts_path}" \
"$(get_from_kata_deps "assets.externals.td-shim.version")" \
"$(get_td_shim_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${td_shim_builder}"
tar xvf "${builddir}/td-shim.tar.gz" -C "${destdir}"
}
@ -239,6 +337,18 @@ install_cc_tee_ovmf() {
tee="${1}"
tarball_name="${2}"
local component_name="ovmf"
local component_version="$(get_from_kata_deps "assets.external.ovmf.${tee}.version")"
[ "${tee}" == "tdx" ] && component_name="tdvf"
install_cached_component \
"${component_name}" \
"${jenkins_url}/job/kata-containers-2.0-${component_name}-cc-$(uname -m)/${cached_artifacts_path}" \
"$(component_version)" \
"$(get_ovmf_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
DESTDIR="${destdir}" PREFIX="${cc_prefix}" ovmf_build="${tee}" "${ovmf_builder}"
tar xvf "${builddir}/${tarball_name}" -C "${destdir}"
}
@ -349,6 +459,11 @@ handle_build() {
info "DESTDIR ${destdir}"
local build_target
build_target="$1"
export final_tarball_path="${workdir}/kata-static-${build_target}.tar.xz"
export final_tarball_name="$(basename ${final_tarball_path})"
rm -f ${final_tarball_name}
case "${build_target}" in
all)
install_clh
@ -428,12 +543,11 @@ handle_build() {
;;
esac
tarball_name="${workdir}/kata-static-${build_target}.tar.xz"
(
if [ ! -f "${final_tarball_path}" ]; then
cd "${destdir}"
sudo tar cvfJ "${tarball_name}" "."
)
tar tvf "${tarball_name}"
sudo tar cvfJ "${final_tarball_path}" "."
fi
tar tvf "${final_tarball_path}"
}
silent_mode_error_trap() {

70
tools/packaging/scripts/lib.sh
View File

@ -13,6 +13,8 @@ export PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}"
this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export repo_root_dir="$(cd "${this_script_dir}/../../../" && pwd)"
short_commit_length=10
hub_bin="hub-bin"
@ -121,12 +123,11 @@ get_config_version() {
fi
}
# $1 - Repo's root dir
# $2 - The file we're looking for the last modification
# $1 - The file we're looking for the last modification
get_last_modification() {
local repo_root_dir="${1}"
local file="${2}"
local file="${1}"
pushd ${repo_root_dir} &> /dev/null
# This is a workaround needed for when running this code on Jenkins
git config --global --add safe.directory ${repo_root_dir} &> /dev/null
@ -134,6 +135,7 @@ get_last_modification() {
[ $(git status --porcelain | grep "${file#${repo_root_dir}/}" | wc -l) -gt 0 ] && dirty="-dirty"
echo "$(git log -1 --pretty=format:"%H" ${file})${dirty}"
popd &> /dev/null
}
# $1 - The tag to be pushed to the registry
@ -173,9 +175,67 @@ sha256sum_from_files() {
files="$(echo $files | tr ' ' '\n' | LC_ALL=C sort -u)"
# Concate the files and calculate a hash.
shasum="$(cat $files | sha256sum -b)" || true
info "shasum of files $shasum"
if [ -n "$shasum" ];then
# Return only the SHA field.
echo $(awk '{ print $1 }' <<< $shasum)
fi
}
calc_qemu_files_sha256sum() {
local files="${this_script_dir}/../qemu \
${this_script_dir}/../static-build/qemu.blacklist \
${this_script_dir}/../static-build/scripts"
sha256sum_from_files "$files"
}
get_initramfs_image_name() {
initramfs_script_dir="${this_script_dir}/../static-build/initramfs"
echo "${CC_BUILDER_REGISTRY}:initramfs-cryptosetup$(get_from_kata_deps "externals.cryptsetup.version")-lvm2-$(get_from_kata_deps "externals.lvm2.version")-$(get_last_modification ${initramfs_script_dir})-$(uname -m)"
}
get_kernel_image_name() {
kernel_script_dir="${this_script_dir}/../static-build/kernel"
echo "${CC_BUILDER_REGISTRY}:kernel-$(get_last_modification ${kernel_script_dir})-$(uname -m)"
}
get_ovmf_image_name() {
ovmf_script_dir="${this_script_dir}/../static-build/ovmf"
echo "${CC_BUILDER_REGISTRY}:ovmf-$(get_last_modification ${ovmf_script_dir})-$(uname -m)"
}
get_qemu_image_name() {
qemu_script_dir="${this_script_dir}/../static-build/qemu"
echo "${CC_BUILDER_REGISTRY}:qemu-$(get_last_modification ${qemu_script_dir})-$(uname -m)"
}
get_shim_v2_image_name() {
shim_v2_script_dir="${this_script_dir}/../static-build/shim-v2"
echo "${CC_BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps "languages.golang.meta.newest-version")-rust-$(get_from_kata_deps "languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)"
}
get_td_shim_image_name() {
td_shim_script_dir="${this_script_dir}/../static-build/td-shim"
echo "${CC_BUILDER_REGISTRY}:td-shim-$(get_last_modification ${td_shim_script_dir})-$(uname -m)"
}
get_virtiofsd_image_name() {
ARCH=$(uname -m)
case ${ARCH} in
"aarch64")
libc="musl"
;;
"ppc64le")
libc="gnu"
;;
"s390x")
libc="gnu"
;;
"x86_64")
libc="musl"
;;
esac
virtiofsd_script_dir="${this_script_dir}/../static-build/virtiofsd"
echo "${CC_BUILDER_REGISTRY}:virtiofsd-$(get_from_kata_deps "externals.virtiofsd.toolchain")-${libc}-$(get_last_modification ${virtiofsd_script_dir})-$(uname -m)"
}

66
tools/packaging/static-build/cache_components.sh
View File

@ -13,61 +13,53 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${script_dir}/../scripts/lib.sh"
export KATA_BUILD_CC="${KATA_BUILD_CC:-}"
export qemu_cc_tarball_name="kata-static-qemu-cc.tar.gz"
export TEE="${TEE:-}"
cache_qemu_artifacts() {
source "${script_dir}/qemu/build-static-qemu-cc.sh"
local qemu_tarball_name="kata-static-cc-qemu.tar.xz"
local current_qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
create_cache_asset "${qemu_cc_tarball_name}" "${current_qemu_version}"
if [ -n "${TEE}" ]; then
qemu_tarball_name="kata-static-cc-${TEE}-qemu.tar.xz"
[ "${TEE}" == "tdx" ] && current_qemu_version=$(get_from_kata_deps "asserts.hypervisor.qemu.tdx.tag")
fi
local qemu_script_dir="${repo_root_dir}/tools/packaging/static-build/qemu"
local qemu_sha=$(calc_qemu_files_sha256sum)
echo "${current_qemu_version} ${qemu_sha}" > "latest"
local current_qemu_image="$(get_qemu_image_name)"
create_cache_asset "${qemu_tarball_name}" "${current_qemu_version}-${qemu_sha}" "${current_qemu_image}"
}
cache_clh_artifacts() {
local binary="cloud-hypervisor"
local binary_path="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
echo "binary path $binary_path"
local current_cloud_hypervisor_version=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")
local clh_binary_path="${binary_path}/tools/packaging/kata-deploy/local-build/build/cc-cloud-hypervisor/builddir/cloud-hypervisor"
if [ -f "${clh_binary_path}/cloud-hypervisor" ]; then
cp "${clh_binary_path}/${binary}" .
else
cloud_hypervisor_build_path="${binary_path}/cloud-hypervisor"
cp "${cloud_hypervisor_build_path}/${binary}" .
fi
create_cache_asset "${binary}" "${current_cloud_hypervisor_version}"
echo "${current_cloud_hypervisor_version}" > "latest"
local clh_tarball_name="kata-static-cc-clh.tar.xz"
[ -n "${TEE}" ] && clh_tarball_name="kata-static-cc-tdx-clh.tar.xz"
local current_clh_version=$(get_from_kata_deps "assets.cloud-hypervisor.version")
create_cache_asset "${clh_tarball_name}" "${current_clh_version}" ""
}
cache_kernel_artifacts() {
local current_kernel_version=$(get_from_kata_deps "assets.kernel.version" | cut -c2- )
local gral_path="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
local kernel_config_file="${gral_path}/tools/packaging/kernel/kata_config_version"
local kernel_config="$(cat ${kernel_config_file})"
echo "${current_kernel_version} ${kernel_config}" > "latest"
local kernel_path="${gral_path}/tools/packaging/kata-deploy/local-build/build/cc-kernel/destdir/opt/confidential-containers/share/kata-containers"
local vmlinux_binary_name="vmlinux-${current_kernel_version}-${kernel_config}"
ls ${kernel_path}
local vmlinux_file="${kernel_path}/${vmlinux_binary_name}"
if [ -f "${vmlinux_file}" ]; then
cp -a "${vmlinux_file}" .
create_cache_asset "${vmlinux_binary_name}" "${current_kernel_version}"
fi
local vmlinuz_binary_name="vmlinuz-${current_kernel_version}-${kernel_config}"
local vmlinuz_file="${kernel_path}/${vmlinuz_binary_name}"
if [ -f "${vmlinuz_file}" ]; then
cp -a "${vmlinuz_file}" .
create_cache_asset "${vmlinuz_binary_name}" "${current_kernel_version}"
local kernel_tarball_name="kata-static-cc-kernel.tar.xz"
local current_kernel_image="$(get_kernel_image_name)"
local current_kernel_version="$(get_from_kata_deps "assets.kernel.version")"
if [ -n "${TEE}" ]; then
kernel_tarball_name="kata-stastic-cc-${TEE}-kernel.tar.xz"
[ "${TEE}" == "tdx" ] && current_kernel_version="$(get_from_kata_deps "assets.kernel.${TEE}.tag")"
[ "${TEE}" == "sev" ] && current_kernel_version="$(get_from_kata_deps "assets.kernel.${TEE}.version")"
fi
create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}" "${current_kernel_image}"
}
create_cache_asset() {
local component_name="$1"
local component_version="$2"
local component_name="${1}"
local component_version="${2}"
local component_image="${3}"
sudo cp "${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/${component_name}" .
sudo chown -R "${USER}:${USER}" .
sha256sum "${component_name}" > "sha256sum-${component_name}"
cat "sha256sum-${component_name}"
echo "${component_version}" > "latest"
cat "latest"
echo "${component_image}" > "latest_image"
cat "latest_image"
}
help() {

39
tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh
View File

@ -23,7 +23,6 @@ cloud_hypervisor_repo="${cloud_hypervisor_repo:-}"
cloud_hypervisor_version="${cloud_hypervisor_version:-}"
cloud_hypervisor_pr="${cloud_hypervisor_pr:-}"
cloud_hypervisor_pull_ref_branch="${cloud_hypervisor_pull_ref_branch:-main}"
cloud_hypervisor_latest_build_url="${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}"
if [ -z "$cloud_hypervisor_repo" ]; then
info "Get cloud_hypervisor information from runtime versions.yaml"
@ -83,40 +82,6 @@ build_clh_from_source() {
popd
}
check_cached_cloud_hypervisor() {
local cached_cloud_hypervisor_version=$(curl -sfL "${cloud_hypervisor_latest_build_url}"/latest) || latest="none"
info "Current cloud hypervisor version: ${cloud_hypervisor_version}"
info "Cached cloud hypervisor version: ${cached_cloud_hypervisor_version}"
if [ "${cloud_hypervisor_version}" == "${cached_cloud_hypervisor_version}" ] && [ "${ARCH}" == "x86_64" ]; then
install_cached_cloud_hypervisor
else
build_clh_from_source
fi
}
install_cached_cloud_hypervisor() {
local cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
local clh_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-cloud-hypervisor/builddir/cloud-hypervisor"
mkdir cloud-hypervisor
pushd cloud-hypervisor
local checksum_file="sha256sum-cloud-hypervisor"
info "Downloading the cloud hypervisor binary"
curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/cloud-hypervisor" || return 1
info "Checking cloud hypervisor binary checksum"
curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/${checksum_file}" || return 1
info "Verify checksum"
sudo sha256sum -c "${checksum_file}" || return 1
chmod +x cloud-hypervisor
local clh_binary_path="${cached_path}/cloud-hypervisor"
if [ ! -d "${clh_binary_path}" ]; then
mkdir -p "${clh_binary_path}"
fi
if [ ! -f "${clh_binary_path}/cloud-hypervisor" ]; then
cp cloud-hypervisor "${clh_binary_path}"
fi
popd
}
if [ "${ARCH}" == "aarch64" ]; then
info "aarch64 binaries are not distributed as part of the Cloud Hypervisor releases, forcing to build from source"
force_build_from_source="true"
@ -129,8 +94,8 @@ fi
if [ "${force_build_from_source}" == "true" ]; then
info "Build cloud-hypervisor from source as it's been request via the force_build_from_source flag"
check_cached_cloud_hypervisor
build_clh_from_source
else
pull_clh_released_binary ||
(info "Failed to pull cloud-hypervisor released binary, trying to build from source" && check_cached_cloud_hypervisor)
(info "Failed to pull cloud-hypervisor released binary, trying to build from source" && build_clh_from_source)
fi

12
tools/packaging/static-build/initramfs/build.sh
View File

@ -9,30 +9,28 @@ set -o nounset
set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly initramfs_builder="${script_dir}/build-initramfs.sh"
readonly default_install_dir="$(cd "${script_dir}/../../kernel" && pwd)"
source "${script_dir}/../../scripts/lib.sh"
kata_version="${kata_version:-}"
cryptsetup_repo="${cryptsetup_repo:-}"
cryptsetup_version="${cryptsetup_version:-}"
lvm2_repo="${lvm2_repo:-}"
lvm2_version="${lvm2_version:-}"
package_output_dir="${package_output_dir:-}"
[ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps "externals.cryptsetup.url" "${kata_version}")
[ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps "externals.cryptsetup.version" "${kata_version}")
[ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps "externals.lvm2.url" "${kata_version}")
[ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps "externals.lvm2.version" "${kata_version}")
[ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps "externals.cryptsetup.url")
[ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps "externals.cryptsetup.version")
[ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps "externals.lvm2.url")
[ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps "externals.lvm2.version")
[ -n "${cryptsetup_repo}" ] || die "Failed to get cryptsetup repo"
[ -n "${cryptsetup_version}" ] || die "Failed to get cryptsetup version"
[ -n "${lvm2_repo}" ] || die "Failed to get lvm2 repo"
[ -n "${lvm2_version}" ] || die "Failed to get lvm2 version"
container_image="${INITRAMFS_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:initramfs-cryptsetup-${cryptsetup_version}-lvm2-${lvm2_version}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
container_image="${INITRAMFS_CONTAINER_BUILDER:-$(get_initramfs_image_name)}"
sudo docker pull ${container_image} || (sudo docker build \
--build-arg cryptsetup_repo="${cryptsetup_repo}" \

99
tools/packaging/static-build/kernel/build.sh
View File

@ -9,90 +9,33 @@ set -o nounset
set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
source "${script_dir}/../../scripts/lib.sh"
readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
DESTDIR=${DESTDIR:-${PWD}}
PREFIX=${PREFIX:-/opt/kata}
container_image="${KERNEL_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
kernel_latest_build_url="${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}"
current_kernel_version=${kernel_version:-$(get_from_kata_deps "assets.kernel.version")}
cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
current_kernel_config_file="${cached_path}/tools/packaging/kernel/kata_config_version"
current_kernel_config="$(cat $current_kernel_config_file)"
kernel_version="$(echo ${current_kernel_version} | cut -c2- )"
container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}"
build_from_source() {
sudo docker pull ${container_image} || \
(sudo docker build -t "${container_image}" "${script_dir}" && \
# No-op unless PUSH_TO_REGISTRY is exported as "yes"
push_to_registry "${container_image}")
sudo docker pull ${container_image} || \
(sudo docker build -t "${container_image}" "${script_dir}" && \
# No-op unless PUSH_TO_REGISTRY is exported as "yes"
push_to_registry "${container_image}")
sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
-w "${PWD}" \
--env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \
"${container_image}" \
bash -c "${kernel_builder} $* setup"
sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
-w "${PWD}" \
--env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \
"${container_image}" \
bash -c "${kernel_builder} $* setup"
sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
-w "${PWD}" \
"${container_image}" \
bash -c "${kernel_builder} $* build"
sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
-w "${PWD}" \
"${container_image}" \
bash -c "${kernel_builder} $* build"
sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
-w "${PWD}" \
--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
"${container_image}" \
bash -c "${kernel_builder} $* install"
}
check_cached_kernel() {
local latest=$(curl -sfL "${kernel_latest_build_url}"/latest) || latest="none"
local cached_kernel_version="$(echo ${latest} | awk '{print $1}')"
info "Current kernel version: ${kernel_version}"
info "Cached kernel version: ${cached_kernel_version}"
if [ "${kernel_version}" == "${cached_kernel_version}" ] && [ "$(uname -m)" == "x86_64" ]; then
local cached_kernel_config="$(echo ${latest} | awk '{print $2}')"
info "Cached kernel config: ${cached_kernel_config}"
info "Current kernel config: ${current_kernel_config}"
if [ -z "${cached_kernel_config}" ]; then
build_from_source $*
else
install_cached_kernel $*
fi
else
build_from_source $*
fi
}
install_cached_kernel() {
local kernel_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-kernel/destdir/opt/confidential-containers/share/kata-containers"
local vmlinux_kernel_name="vmlinux-${cached_kernel_version}-${cached_kernel_config}"
local vmlinuz_kernel_name="vmlinuz-${cached_kernel_version}-${cached_kernel_config}"
mkdir -p "${kernel_directory}"
pushd "${kernel_directory}"
ls
local vmlinux_url="${kernel_latest_build_url}/${vmlinux_kernel_name}"
if curl --output /dev/null --silent --head --fail "${vmlinux_url}"; then
info "Installing vmlinux cached kernel"
curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinux_kernel_name}" -o "${vmlinux_kernel_name}" || return 1
sudo -E ln -sf "${kernel_directory}/${vmlinux_kernel_name}" "${kernel_directory}/vmlinux.container"
fi
local vmlinuz_url="${kernel_latest_build_url}/${vmlinuz_kernel_name}"
if curl --output /dev/null --silent --head --fail "${vmlinuz_url}"; then
info "Installing vmlinuz cached kernel"
curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinuz_kernel_name}" -o "${vmlinuz_kernel_name}" || return 1
sudo -E ln -sf "${kernel_directory}/${vmlinuz_kernel_name}" "${kernel_directory}/vmlinuz.container"
fi
popd
}
main() {
check_cached_kernel $*
}
main $*
sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
-w "${PWD}" \
--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
"${container_image}" \
bash -c "${kernel_builder} $* install"

3
tools/packaging/static-build/ovmf/build.sh
View File

@ -9,14 +9,13 @@ set -o nounset
set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly ovmf_builder="${script_dir}/build-ovmf.sh"
source "${script_dir}/../../scripts/lib.sh"
DESTDIR=${DESTDIR:-${PWD}}
PREFIX=${PREFIX:-/opt/kata}
container_image="${OVMF_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:ovmf-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
container_image="${OVMF_CONTAINER_BUILDER:-$(get_ovmf_image_name)}"
ovmf_build="${ovmf_build:-x86_64}"
kata_version="${kata_version:-}"
ovmf_repo="${ovmf_repo:-}"

3
tools/packaging/static-build/qemu/build-base-qemu.sh
View File

@ -9,7 +9,6 @@ set -o nounset
set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly qemu_builder="${script_dir}/build-qemu.sh"
source "${script_dir}/../../scripts/lib.sh"
@ -39,7 +38,7 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d")
[ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu"
[ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static"
container_image="${QEMU_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:qemu-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
container_image="${QEMU_CONTAINER_BUILDER:-$(get_qemu_image_name)}"
sudo docker pull ${container_image} || \
(sudo "${container_engine}" build \

90
tools/packaging/static-build/qemu/build-static-qemu-cc.sh
View File

@ -12,83 +12,23 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${script_dir}/../../scripts/lib.sh"
export qemu_repo="${qemu_repo:-}"
export qemu_version="${qemu_version:-}"
export qemu_latest_build_url="${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}"
export katacontainers_repo="${katacontainers_repo:=github.com/kata-containers/kata-containers}"
export qemu_tarball_name="kata-static-qemu-cc.tar.gz"
export pkg_dir="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
export qemu_tarball_directory="${pkg_dir}/kata-deploy/local-build/build/cc-qemu/builddir"
export tee="${tee:-}"
qemu_repo="${qemu_repo:-}"
qemu_version="${qemu_version:-}"
tee="${tee:-}"
export prefix="/opt/confidential-containers/"
get_qemu_information() {
if [ -z "${qemu_repo}" ]; then
info "Get qemu information from runtime versions.yaml"
export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url")
[ -n "${qemu_url}" ] || die "failed to get qemu url"
export qemu_repo="${qemu_url}.git"
fi
if [ -z "${qemu_repo}" ]; then
info "Get qemu information from runtime versions.yaml"
export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url")
[ -n "${qemu_url}" ] || die "failed to get qemu url"
export qemu_repo="${qemu_url}.git"
fi
[ -n "${qemu_repo}" ] || die "failed to get qemu repo"
[ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
[ -n "${qemu_version}" ] || die "failed to get qemu version"
}
[ -n "${qemu_repo}" ] || die "failed to get qemu repo"
[ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
[ -n "${qemu_version}" ] || die "failed to get qemu version"
calc_qemu_files_sha256sum() {
info "pkg directory is at ${pkg_dir}"
local files="${pkg_dir}/qemu \
${pkg_dir}/static-build/qemu.blacklist \
${pkg_dir}/static-build/scripts"
sha256sum_from_files "$files"
}
cached_or_build_qemu_tar() {
# Check latest qemu cc tar version sha256sum
local latest=$(curl -sfL "${qemu_latest_build_url}/latest") || latest="none"
local cached_qemu_version="$(echo ${latest} | awk '{print $1}')"
info "Current qemu version: ${qemu_version}"
info "Cached qemu version: ${cached_qemu_version}"
if [ "${qemu_version}" == "${cached_qemu_version}" ]; then
info "Get latest cached information ${latest}"
local cached_sha256sum="$(echo ${latest} | awk '{print $2}')"
info "Cached sha256sum version: ${cached_sha256sum}"
local current_sha256sum="$(calc_qemu_files_sha256sum)"
info "Current sha256sum of the qemu directory ${current_sha256sum}"
if [ -z "${cached_sha256sum}" ]; then
build_qemu_tar
elif [ "${current_sha256sum}" == "${cached_sha256sum}" ]; then
install_cached_qemu_tar
else
build_qemu_tar
fi
else
build_qemu_tar
fi
}
build_qemu_tar() {
[ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz"
"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}"
}
install_cached_qemu_tar() {
info "Using cached tarball of qemu"
curl -fL --progress-bar "${qemu_latest_build_url}/${qemu_tarball_name}" -o "${qemu_tarball_name}" || return 1
curl -fsOL "${qemu_latest_build_url}/sha256sum-${qemu_tarball_name}" || return 1
sha256sum -c "sha256sum-${qemu_tarball_name}" || return 1
}
main() {
get_qemu_information
# Currently the cached for qemu cc only works in x86_64
if [ "$(uname -m)" == "x86_64" ]; then
cached_or_build_qemu_tar
else
build_qemu_tar
fi
}
main $@
qemu_tarball_name="kata-static-qemu-cc.tar.gz"
[ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz"
"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}"

6
tools/packaging/static-build/shim-v2/build.sh
View File

@ -9,17 +9,17 @@ set -o nounset
set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
source "${script_dir}/../../scripts/lib.sh"
readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
GO_VERSION=${GO_VERSION}
RUST_VERSION=${RUST_VERSION:-}
DESTDIR=${DESTDIR:-${PWD}}
PREFIX=${PREFIX:-/opt/kata}
container_image="${SHIM_V2_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
container_image="${SHIM_V2_CONTAINER_BUILDER:-$(get_shim_v2_image_name)}"
EXTRA_OPTS="${EXTRA_OPTS:-""}"
VMM_CONFIGS="qemu fc"

3
tools/packaging/static-build/td-shim/build.sh
View File

@ -9,7 +9,6 @@ set -o nounset
set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly tdshim_builder="${script_dir}/build-td-shim.sh"
source "${script_dir}/../../scripts/lib.sh"
@ -30,7 +29,7 @@ package_output_dir="${package_output_dir:-}"
[ -n "${tdshim_version}" ] || die "Failed to get TD-shim version or commit"
[ -n "${tdshim_toolchain}" ] || die "Failed to get TD-shim toolchain to be used to build the project"
container_image="${TDSHIM_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:td-shim-${tdshim_toolchain}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
container_image="${TDSHIM_CONTAINER_BUILDER:-$(get_td_shim_image_name)}"
sudo docker pull ${container_image} || \
(sudo docker build \

3
tools/packaging/static-build/virtiofsd/build.sh
View File

@ -9,7 +9,6 @@ set -o nounset
set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly virtiofsd_builder="${script_dir}/build-static-virtiofsd.sh"
source "${script_dir}/../../scripts/lib.sh"
@ -49,7 +48,7 @@ case ${ARCH} in
;;
esac
container_image="${VIRTIOFSD_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
container_image="${VIRTIOFSD_CONTAINER_BUILDER:-$(get_virtiofsd_image_name)}"
sudo docker pull ${container_image} || \
(sudo docker build \