Merge pull request #5784 from fidencio/topic/cached-components

Re-work the way we cache components for the CCv0 branch
This commit is contained in:
Fabiano Fidêncio 2022-12-01 23:19:59 +01:00 committed by GitHub
commit 72d2f19c98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 270 additions and 262 deletions

View File

@ -14,7 +14,9 @@ set -o pipefail
readonly script_name="$(basename "${BASH_SOURCE[0]}")" readonly script_name="$(basename "${BASH_SOURCE[0]}")"
readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly packaging_root_dir="$(cd "${script_dir}/../" && pwd)" readonly packaging_root_dir="$(cd "${script_dir}/../" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../" && pwd)"
source "${packaging_root_dir}/scripts/lib.sh"
readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)" readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)"
patches_path="" patches_path=""
@ -26,8 +28,6 @@ final_image_name="kata-containers"
final_initrd_name="kata-containers-initrd" final_initrd_name="kata-containers-initrd"
image_initrd_extension=".img" image_initrd_extension=".img"
source "${packaging_root_dir}/scripts/lib.sh"
arch_target="$(uname -m)" arch_target="$(uname -m)"
build_initrd() { build_initrd() {

View File

@ -15,8 +15,9 @@ readonly project="kata-containers"
readonly script_name="$(basename "${BASH_SOURCE[0]}")" readonly script_name="$(basename "${BASH_SOURCE[0]}")"
readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${script_dir}/../../scripts/lib.sh"
readonly prefix="/opt/kata" readonly prefix="/opt/kata"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly static_build_dir="${repo_root_dir}/tools/packaging/static-build" readonly static_build_dir="${repo_root_dir}/tools/packaging/static-build"
readonly version_file="${repo_root_dir}/VERSION" readonly version_file="${repo_root_dir}/VERSION"
readonly versions_yaml="${repo_root_dir}/versions.yaml" readonly versions_yaml="${repo_root_dir}/versions.yaml"
@ -37,6 +38,8 @@ readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_imag
readonly cc_prefix="/opt/confidential-containers" readonly cc_prefix="/opt/confidential-containers"
readonly qemu_cc_builder="${static_build_dir}/qemu/build-static-qemu-cc.sh" readonly qemu_cc_builder="${static_build_dir}/qemu/build-static-qemu-cc.sh"
source "${script_dir}/../../scripts/lib.sh"
ARCH=$(uname -m) ARCH=$(uname -m)
workdir="${WORKDIR:-$PWD}" workdir="${WORKDIR:-$PWD}"
@ -103,8 +106,45 @@ EOF
exit "${return_code}" exit "${return_code}"
} }
cleanup_and_fail() {
rm -f "${component_tarball_path}"
return 1
}
install_cached_component() {
local component="${1}"
local jenkins_build_url="${2}"
local current_version="${3}"
local current_image_version="${4}"
local component_tarball_name="${5}"
local component_tarball_path="${6}"
local cached_version=$(curl -sfL "${jenkins_build_url}/latest" | awk '{print $1}') || cached_version="none"
local cached_image_version=$(curl -sfL "${jenkins_build_url}/latest_image" | awk '{print $1}') || cached_image_version="none"
[ "${cached_image_version}" != "${current_image_version}" ] && return 1
[ "${cached_version}" != "${current_version}" ] && return 1
info "Using cached tarball of ${component}"
pushd ${workdir}
echo "Downloading tarball from: ${jenkins_build_url}/${component_tarball_name}"
curl -fL --progress-bar "${jenkins_build_url}/${component_tarball_name}" -o "${component_tarball_path}" || return cleanup_and_fail
curl -fsOL "${jenkins_build_url}/sha256sum-${component_tarball_name}" || return cleanup_and_fail
sha256sum -c "sha256sum-${component_tarball_name}" && return cleanup_and_fail
popd
}
# Install static CC cloud-hypervisor asset # Install static CC cloud-hypervisor asset
install_cc_clh() { install_cc_clh() {
install_cached_component \
"cloud-hypervisor" \
"${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}" \
"$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \
"" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
if [[ "${ARCH}" == "x86_64" ]]; then if [[ "${ARCH}" == "x86_64" ]]; then
export features="tdx" export features="tdx"
fi fi
@ -142,11 +182,21 @@ install_cc_tdx_image() {
#Install CC kernel asset #Install CC kernel asset
install_cc_kernel() { install_cc_kernel() {
export KATA_BUILD_CC=yes
info "build initramfs for cc kernel" info "build initramfs for cc kernel"
"${initramfs_builder}"
export KATA_BUILD_CC=yes
export kernel_version="$(yq r $versions_yaml assets.kernel.version)" export kernel_version="$(yq r $versions_yaml assets.kernel.version)"
install_cached_component \
"kernel" \
"${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}" \
"${kernel_version}" \
"$(get_kernel_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
"${initramfs_builder}"
DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -f -v "${kernel_version}" DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -f -v "${kernel_version}"
} }
@ -155,6 +205,16 @@ install_cc_qemu() {
info "build static CC qemu" info "build static CC qemu"
export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)" export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)"
export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)" export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)"
install_cached_component \
"QEMU" \
"${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}" \
"${qemu_version}-$(calc_qemu_files_sha256sum)" \
"$(get_qemu_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
"${qemu_cc_builder}" "${qemu_cc_builder}"
tar xvf "${builddir}/kata-static-qemu-cc.tar.gz" -C "${destdir}" tar xvf "${builddir}/kata-static-qemu-cc.tar.gz" -C "${destdir}"
} }
@ -177,6 +237,15 @@ install_cc_shimv2() {
# Install static CC virtiofsd asset # Install static CC virtiofsd asset
install_cc_virtiofsd() { install_cc_virtiofsd() {
install_cached_component \
"virtiofsd" \
"${jenkins_url}/job/kata-containers-2.0-virtiofsd-cc-$(uname -m)/${cached_artifacts_path}" \
"$(get_from_kata_deps "assets.externals.virtiofsd.version")" \
"$(get_virtiofsd_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
info "build static CC virtiofsd" info "build static CC virtiofsd"
"${virtiofsd_builder}" "${virtiofsd_builder}"
info "Install static CC virtiofsd" info "Install static CC virtiofsd"
@ -198,6 +267,16 @@ install_cc_tee_kernel() {
info "build initramfs for tee kernel" info "build initramfs for tee kernel"
export kernel_version=${kernel_version} export kernel_version=${kernel_version}
install_cached_component \
"kernel" \
"${jenkins_url}/job/kata-containers-2.0-kernel-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \
"${kernel_version}" \
"$(get_kernel_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
"${initramfs_builder}" "${initramfs_builder}"
kernel_url="$(yq r $versions_yaml assets.kernel.${tee}.url)" kernel_url="$(yq r $versions_yaml assets.kernel.${tee}.url)"
DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -x "${tee}" -v "${kernel_version}" -u "${kernel_url}" DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -x "${tee}" -v "${kernel_version}" -u "${kernel_url}"
@ -222,6 +301,16 @@ install_cc_tee_qemu() {
export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.url)" export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.url)"
export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.tag)" export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.tag)"
export tee="${tee}" export tee="${tee}"
install_cached_component \
"QEMU ${tee}" \
"${jenkins_url}/job/kata-containers-2.0-qemu-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \
"${qemu_version}-$(calc_qemu_files_sha256sum)" \
"$(get_qemu_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
"${qemu_cc_builder}" "${qemu_cc_builder}"
tar xvf "${builddir}/kata-static-${tee}-qemu-cc.tar.gz" -C "${destdir}" tar xvf "${builddir}/kata-static-${tee}-qemu-cc.tar.gz" -C "${destdir}"
} }
@ -231,6 +320,15 @@ install_cc_tdx_qemu() {
} }
install_cc_tdx_td_shim() { install_cc_tdx_td_shim() {
install_cached_component \
"td-shim" \
"${jenkins_url}/job/kata-containers-2.0-td-shim-cc-$(uname -m)/${cached_artifacts_path}" \
"$(get_from_kata_deps "assets.externals.td-shim.version")" \
"$(get_td_shim_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${td_shim_builder}" DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${td_shim_builder}"
tar xvf "${builddir}/td-shim.tar.gz" -C "${destdir}" tar xvf "${builddir}/td-shim.tar.gz" -C "${destdir}"
} }
@ -239,6 +337,18 @@ install_cc_tee_ovmf() {
tee="${1}" tee="${1}"
tarball_name="${2}" tarball_name="${2}"
local component_name="ovmf"
local component_version="$(get_from_kata_deps "assets.external.ovmf.${tee}.version")"
[ "${tee}" == "tdx" ] && component_name="tdvf"
install_cached_component \
"${component_name}" \
"${jenkins_url}/job/kata-containers-2.0-${component_name}-cc-$(uname -m)/${cached_artifacts_path}" \
"$(component_version)" \
"$(get_ovmf_image_name)" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
DESTDIR="${destdir}" PREFIX="${cc_prefix}" ovmf_build="${tee}" "${ovmf_builder}" DESTDIR="${destdir}" PREFIX="${cc_prefix}" ovmf_build="${tee}" "${ovmf_builder}"
tar xvf "${builddir}/${tarball_name}" -C "${destdir}" tar xvf "${builddir}/${tarball_name}" -C "${destdir}"
} }
@ -349,6 +459,11 @@ handle_build() {
info "DESTDIR ${destdir}" info "DESTDIR ${destdir}"
local build_target local build_target
build_target="$1" build_target="$1"
export final_tarball_path="${workdir}/kata-static-${build_target}.tar.xz"
export final_tarball_name="$(basename ${final_tarball_path})"
rm -f ${final_tarball_name}
case "${build_target}" in case "${build_target}" in
all) all)
install_clh install_clh
@ -428,12 +543,11 @@ handle_build() {
;; ;;
esac esac
tarball_name="${workdir}/kata-static-${build_target}.tar.xz" if [ ! -f "${final_tarball_path}" ]; then
(
cd "${destdir}" cd "${destdir}"
sudo tar cvfJ "${tarball_name}" "." sudo tar cvfJ "${final_tarball_path}" "."
) fi
tar tvf "${tarball_name}" tar tvf "${final_tarball_path}"
} }
silent_mode_error_trap() { silent_mode_error_trap() {

View File

@ -13,6 +13,8 @@ export PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}"
this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export repo_root_dir="$(cd "${this_script_dir}/../../../" && pwd)"
short_commit_length=10 short_commit_length=10
hub_bin="hub-bin" hub_bin="hub-bin"
@ -121,12 +123,11 @@ get_config_version() {
fi fi
} }
# $1 - Repo's root dir # $1 - The file we're looking for the last modification
# $2 - The file we're looking for the last modification
get_last_modification() { get_last_modification() {
local repo_root_dir="${1}" local file="${1}"
local file="${2}"
pushd ${repo_root_dir} &> /dev/null
# This is a workaround needed for when running this code on Jenkins # This is a workaround needed for when running this code on Jenkins
git config --global --add safe.directory ${repo_root_dir} &> /dev/null git config --global --add safe.directory ${repo_root_dir} &> /dev/null
@ -134,6 +135,7 @@ get_last_modification() {
[ $(git status --porcelain | grep "${file#${repo_root_dir}/}" | wc -l) -gt 0 ] && dirty="-dirty" [ $(git status --porcelain | grep "${file#${repo_root_dir}/}" | wc -l) -gt 0 ] && dirty="-dirty"
echo "$(git log -1 --pretty=format:"%H" ${file})${dirty}" echo "$(git log -1 --pretty=format:"%H" ${file})${dirty}"
popd &> /dev/null
} }
# $1 - The tag to be pushed to the registry # $1 - The tag to be pushed to the registry
@ -173,9 +175,67 @@ sha256sum_from_files() {
files="$(echo $files | tr ' ' '\n' | LC_ALL=C sort -u)" files="$(echo $files | tr ' ' '\n' | LC_ALL=C sort -u)"
# Concate the files and calculate a hash. # Concate the files and calculate a hash.
shasum="$(cat $files | sha256sum -b)" || true shasum="$(cat $files | sha256sum -b)" || true
info "shasum of files $shasum"
if [ -n "$shasum" ];then if [ -n "$shasum" ];then
# Return only the SHA field. # Return only the SHA field.
echo $(awk '{ print $1 }' <<< $shasum) echo $(awk '{ print $1 }' <<< $shasum)
fi fi
} }
calc_qemu_files_sha256sum() {
local files="${this_script_dir}/../qemu \
${this_script_dir}/../static-build/qemu.blacklist \
${this_script_dir}/../static-build/scripts"
sha256sum_from_files "$files"
}
get_initramfs_image_name() {
initramfs_script_dir="${this_script_dir}/../static-build/initramfs"
echo "${CC_BUILDER_REGISTRY}:initramfs-cryptosetup$(get_from_kata_deps "externals.cryptsetup.version")-lvm2-$(get_from_kata_deps "externals.lvm2.version")-$(get_last_modification ${initramfs_script_dir})-$(uname -m)"
}
get_kernel_image_name() {
kernel_script_dir="${this_script_dir}/../static-build/kernel"
echo "${CC_BUILDER_REGISTRY}:kernel-$(get_last_modification ${kernel_script_dir})-$(uname -m)"
}
get_ovmf_image_name() {
ovmf_script_dir="${this_script_dir}/../static-build/ovmf"
echo "${CC_BUILDER_REGISTRY}:ovmf-$(get_last_modification ${ovmf_script_dir})-$(uname -m)"
}
get_qemu_image_name() {
qemu_script_dir="${this_script_dir}/../static-build/qemu"
echo "${CC_BUILDER_REGISTRY}:qemu-$(get_last_modification ${qemu_script_dir})-$(uname -m)"
}
get_shim_v2_image_name() {
shim_v2_script_dir="${this_script_dir}/../static-build/shim-v2"
echo "${CC_BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps "languages.golang.meta.newest-version")-rust-$(get_from_kata_deps "languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)"
}
get_td_shim_image_name() {
td_shim_script_dir="${this_script_dir}/../static-build/td-shim"
echo "${CC_BUILDER_REGISTRY}:td-shim-$(get_last_modification ${td_shim_script_dir})-$(uname -m)"
}
get_virtiofsd_image_name() {
ARCH=$(uname -m)
case ${ARCH} in
"aarch64")
libc="musl"
;;
"ppc64le")
libc="gnu"
;;
"s390x")
libc="gnu"
;;
"x86_64")
libc="musl"
;;
esac
virtiofsd_script_dir="${this_script_dir}/../static-build/virtiofsd"
echo "${CC_BUILDER_REGISTRY}:virtiofsd-$(get_from_kata_deps "externals.virtiofsd.toolchain")-${libc}-$(get_last_modification ${virtiofsd_script_dir})-$(uname -m)"
}

View File

@ -13,61 +13,53 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${script_dir}/../scripts/lib.sh" source "${script_dir}/../scripts/lib.sh"
export KATA_BUILD_CC="${KATA_BUILD_CC:-}" export KATA_BUILD_CC="${KATA_BUILD_CC:-}"
export qemu_cc_tarball_name="kata-static-qemu-cc.tar.gz" export TEE="${TEE:-}"
cache_qemu_artifacts() { cache_qemu_artifacts() {
source "${script_dir}/qemu/build-static-qemu-cc.sh" local qemu_tarball_name="kata-static-cc-qemu.tar.xz"
local current_qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version") local current_qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
create_cache_asset "${qemu_cc_tarball_name}" "${current_qemu_version}" if [ -n "${TEE}" ]; then
qemu_tarball_name="kata-static-cc-${TEE}-qemu.tar.xz"
[ "${TEE}" == "tdx" ] && current_qemu_version=$(get_from_kata_deps "asserts.hypervisor.qemu.tdx.tag")
fi
local qemu_script_dir="${repo_root_dir}/tools/packaging/static-build/qemu"
local qemu_sha=$(calc_qemu_files_sha256sum) local qemu_sha=$(calc_qemu_files_sha256sum)
echo "${current_qemu_version} ${qemu_sha}" > "latest" local current_qemu_image="$(get_qemu_image_name)"
create_cache_asset "${qemu_tarball_name}" "${current_qemu_version}-${qemu_sha}" "${current_qemu_image}"
} }
cache_clh_artifacts() { cache_clh_artifacts() {
local binary="cloud-hypervisor" local clh_tarball_name="kata-static-cc-clh.tar.xz"
local binary_path="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" [ -n "${TEE}" ] && clh_tarball_name="kata-static-cc-tdx-clh.tar.xz"
echo "binary path $binary_path" local current_clh_version=$(get_from_kata_deps "assets.cloud-hypervisor.version")
local current_cloud_hypervisor_version=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version") create_cache_asset "${clh_tarball_name}" "${current_clh_version}" ""
local clh_binary_path="${binary_path}/tools/packaging/kata-deploy/local-build/build/cc-cloud-hypervisor/builddir/cloud-hypervisor"
if [ -f "${clh_binary_path}/cloud-hypervisor" ]; then
cp "${clh_binary_path}/${binary}" .
else
cloud_hypervisor_build_path="${binary_path}/cloud-hypervisor"
cp "${cloud_hypervisor_build_path}/${binary}" .
fi
create_cache_asset "${binary}" "${current_cloud_hypervisor_version}"
echo "${current_cloud_hypervisor_version}" > "latest"
} }
cache_kernel_artifacts() { cache_kernel_artifacts() {
local current_kernel_version=$(get_from_kata_deps "assets.kernel.version" | cut -c2- ) local kernel_tarball_name="kata-static-cc-kernel.tar.xz"
local gral_path="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" local current_kernel_image="$(get_kernel_image_name)"
local kernel_config_file="${gral_path}/tools/packaging/kernel/kata_config_version" local current_kernel_version="$(get_from_kata_deps "assets.kernel.version")"
local kernel_config="$(cat ${kernel_config_file})" if [ -n "${TEE}" ]; then
echo "${current_kernel_version} ${kernel_config}" > "latest" kernel_tarball_name="kata-stastic-cc-${TEE}-kernel.tar.xz"
local kernel_path="${gral_path}/tools/packaging/kata-deploy/local-build/build/cc-kernel/destdir/opt/confidential-containers/share/kata-containers" [ "${TEE}" == "tdx" ] && current_kernel_version="$(get_from_kata_deps "assets.kernel.${TEE}.tag")"
local vmlinux_binary_name="vmlinux-${current_kernel_version}-${kernel_config}" [ "${TEE}" == "sev" ] && current_kernel_version="$(get_from_kata_deps "assets.kernel.${TEE}.version")"
ls ${kernel_path}
local vmlinux_file="${kernel_path}/${vmlinux_binary_name}"
if [ -f "${vmlinux_file}" ]; then
cp -a "${vmlinux_file}" .
create_cache_asset "${vmlinux_binary_name}" "${current_kernel_version}"
fi
local vmlinuz_binary_name="vmlinuz-${current_kernel_version}-${kernel_config}"
local vmlinuz_file="${kernel_path}/${vmlinuz_binary_name}"
if [ -f "${vmlinuz_file}" ]; then
cp -a "${vmlinuz_file}" .
create_cache_asset "${vmlinuz_binary_name}" "${current_kernel_version}"
fi fi
create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}" "${current_kernel_image}"
} }
create_cache_asset() { create_cache_asset() {
local component_name="$1" local component_name="${1}"
local component_version="$2" local component_version="${2}"
local component_image="${3}"
sudo cp "${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/${component_name}" .
sudo chown -R "${USER}:${USER}" . sudo chown -R "${USER}:${USER}" .
sha256sum "${component_name}" > "sha256sum-${component_name}" sha256sum "${component_name}" > "sha256sum-${component_name}"
cat "sha256sum-${component_name}" cat "sha256sum-${component_name}"
echo "${component_version}" > "latest"
cat "latest"
echo "${component_image}" > "latest_image"
cat "latest_image"
} }
help() { help() {

View File

@ -23,7 +23,6 @@ cloud_hypervisor_repo="${cloud_hypervisor_repo:-}"
cloud_hypervisor_version="${cloud_hypervisor_version:-}" cloud_hypervisor_version="${cloud_hypervisor_version:-}"
cloud_hypervisor_pr="${cloud_hypervisor_pr:-}" cloud_hypervisor_pr="${cloud_hypervisor_pr:-}"
cloud_hypervisor_pull_ref_branch="${cloud_hypervisor_pull_ref_branch:-main}" cloud_hypervisor_pull_ref_branch="${cloud_hypervisor_pull_ref_branch:-main}"
cloud_hypervisor_latest_build_url="${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}"
if [ -z "$cloud_hypervisor_repo" ]; then if [ -z "$cloud_hypervisor_repo" ]; then
info "Get cloud_hypervisor information from runtime versions.yaml" info "Get cloud_hypervisor information from runtime versions.yaml"
@ -83,40 +82,6 @@ build_clh_from_source() {
popd popd
} }
check_cached_cloud_hypervisor() {
local cached_cloud_hypervisor_version=$(curl -sfL "${cloud_hypervisor_latest_build_url}"/latest) || latest="none"
info "Current cloud hypervisor version: ${cloud_hypervisor_version}"
info "Cached cloud hypervisor version: ${cached_cloud_hypervisor_version}"
if [ "${cloud_hypervisor_version}" == "${cached_cloud_hypervisor_version}" ] && [ "${ARCH}" == "x86_64" ]; then
install_cached_cloud_hypervisor
else
build_clh_from_source
fi
}
install_cached_cloud_hypervisor() {
local cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
local clh_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-cloud-hypervisor/builddir/cloud-hypervisor"
mkdir cloud-hypervisor
pushd cloud-hypervisor
local checksum_file="sha256sum-cloud-hypervisor"
info "Downloading the cloud hypervisor binary"
curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/cloud-hypervisor" || return 1
info "Checking cloud hypervisor binary checksum"
curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/${checksum_file}" || return 1
info "Verify checksum"
sudo sha256sum -c "${checksum_file}" || return 1
chmod +x cloud-hypervisor
local clh_binary_path="${cached_path}/cloud-hypervisor"
if [ ! -d "${clh_binary_path}" ]; then
mkdir -p "${clh_binary_path}"
fi
if [ ! -f "${clh_binary_path}/cloud-hypervisor" ]; then
cp cloud-hypervisor "${clh_binary_path}"
fi
popd
}
if [ "${ARCH}" == "aarch64" ]; then if [ "${ARCH}" == "aarch64" ]; then
info "aarch64 binaries are not distributed as part of the Cloud Hypervisor releases, forcing to build from source" info "aarch64 binaries are not distributed as part of the Cloud Hypervisor releases, forcing to build from source"
force_build_from_source="true" force_build_from_source="true"
@ -129,8 +94,8 @@ fi
if [ "${force_build_from_source}" == "true" ]; then if [ "${force_build_from_source}" == "true" ]; then
info "Build cloud-hypervisor from source as it's been request via the force_build_from_source flag" info "Build cloud-hypervisor from source as it's been request via the force_build_from_source flag"
check_cached_cloud_hypervisor build_clh_from_source
else else
pull_clh_released_binary || pull_clh_released_binary ||
(info "Failed to pull cloud-hypervisor released binary, trying to build from source" && check_cached_cloud_hypervisor) (info "Failed to pull cloud-hypervisor released binary, trying to build from source" && build_clh_from_source)
fi fi

View File

@ -9,30 +9,28 @@ set -o nounset
set -o pipefail set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly initramfs_builder="${script_dir}/build-initramfs.sh" readonly initramfs_builder="${script_dir}/build-initramfs.sh"
readonly default_install_dir="$(cd "${script_dir}/../../kernel" && pwd)" readonly default_install_dir="$(cd "${script_dir}/../../kernel" && pwd)"
source "${script_dir}/../../scripts/lib.sh" source "${script_dir}/../../scripts/lib.sh"
kata_version="${kata_version:-}"
cryptsetup_repo="${cryptsetup_repo:-}" cryptsetup_repo="${cryptsetup_repo:-}"
cryptsetup_version="${cryptsetup_version:-}" cryptsetup_version="${cryptsetup_version:-}"
lvm2_repo="${lvm2_repo:-}" lvm2_repo="${lvm2_repo:-}"
lvm2_version="${lvm2_version:-}" lvm2_version="${lvm2_version:-}"
package_output_dir="${package_output_dir:-}" package_output_dir="${package_output_dir:-}"
[ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps "externals.cryptsetup.url" "${kata_version}") [ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps "externals.cryptsetup.url")
[ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps "externals.cryptsetup.version" "${kata_version}") [ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps "externals.cryptsetup.version")
[ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps "externals.lvm2.url" "${kata_version}") [ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps "externals.lvm2.url")
[ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps "externals.lvm2.version" "${kata_version}") [ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps "externals.lvm2.version")
[ -n "${cryptsetup_repo}" ] || die "Failed to get cryptsetup repo" [ -n "${cryptsetup_repo}" ] || die "Failed to get cryptsetup repo"
[ -n "${cryptsetup_version}" ] || die "Failed to get cryptsetup version" [ -n "${cryptsetup_version}" ] || die "Failed to get cryptsetup version"
[ -n "${lvm2_repo}" ] || die "Failed to get lvm2 repo" [ -n "${lvm2_repo}" ] || die "Failed to get lvm2 repo"
[ -n "${lvm2_version}" ] || die "Failed to get lvm2 version" [ -n "${lvm2_version}" ] || die "Failed to get lvm2 version"
container_image="${INITRAMFS_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:initramfs-cryptsetup-${cryptsetup_version}-lvm2-${lvm2_version}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" container_image="${INITRAMFS_CONTAINER_BUILDER:-$(get_initramfs_image_name)}"
sudo docker pull ${container_image} || (sudo docker build \ sudo docker pull ${container_image} || (sudo docker build \
--build-arg cryptsetup_repo="${cryptsetup_repo}" \ --build-arg cryptsetup_repo="${cryptsetup_repo}" \

View File

@ -9,22 +9,15 @@ set -o nounset
set -o pipefail set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
source "${script_dir}/../../scripts/lib.sh" source "${script_dir}/../../scripts/lib.sh"
readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
DESTDIR=${DESTDIR:-${PWD}} DESTDIR=${DESTDIR:-${PWD}}
PREFIX=${PREFIX:-/opt/kata} PREFIX=${PREFIX:-/opt/kata}
container_image="${KERNEL_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}"
kernel_latest_build_url="${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}"
current_kernel_version=${kernel_version:-$(get_from_kata_deps "assets.kernel.version")}
cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
current_kernel_config_file="${cached_path}/tools/packaging/kernel/kata_config_version"
current_kernel_config="$(cat $current_kernel_config_file)"
kernel_version="$(echo ${current_kernel_version} | cut -c2- )"
build_from_source() {
sudo docker pull ${container_image} || \ sudo docker pull ${container_image} || \
(sudo docker build -t "${container_image}" "${script_dir}" && \ (sudo docker build -t "${container_image}" "${script_dir}" && \
# No-op unless PUSH_TO_REGISTRY is exported as "yes" # No-op unless PUSH_TO_REGISTRY is exported as "yes"
@ -46,53 +39,3 @@ build_from_source() {
--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
"${container_image}" \ "${container_image}" \
bash -c "${kernel_builder} $* install" bash -c "${kernel_builder} $* install"
}
check_cached_kernel() {
local latest=$(curl -sfL "${kernel_latest_build_url}"/latest) || latest="none"
local cached_kernel_version="$(echo ${latest} | awk '{print $1}')"
info "Current kernel version: ${kernel_version}"
info "Cached kernel version: ${cached_kernel_version}"
if [ "${kernel_version}" == "${cached_kernel_version}" ] && [ "$(uname -m)" == "x86_64" ]; then
local cached_kernel_config="$(echo ${latest} | awk '{print $2}')"
info "Cached kernel config: ${cached_kernel_config}"
info "Current kernel config: ${current_kernel_config}"
if [ -z "${cached_kernel_config}" ]; then
build_from_source $*
else
install_cached_kernel $*
fi
else
build_from_source $*
fi
}
install_cached_kernel() {
local kernel_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-kernel/destdir/opt/confidential-containers/share/kata-containers"
local vmlinux_kernel_name="vmlinux-${cached_kernel_version}-${cached_kernel_config}"
local vmlinuz_kernel_name="vmlinuz-${cached_kernel_version}-${cached_kernel_config}"
mkdir -p "${kernel_directory}"
pushd "${kernel_directory}"
ls
local vmlinux_url="${kernel_latest_build_url}/${vmlinux_kernel_name}"
if curl --output /dev/null --silent --head --fail "${vmlinux_url}"; then
info "Installing vmlinux cached kernel"
curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinux_kernel_name}" -o "${vmlinux_kernel_name}" || return 1
sudo -E ln -sf "${kernel_directory}/${vmlinux_kernel_name}" "${kernel_directory}/vmlinux.container"
fi
local vmlinuz_url="${kernel_latest_build_url}/${vmlinuz_kernel_name}"
if curl --output /dev/null --silent --head --fail "${vmlinuz_url}"; then
info "Installing vmlinuz cached kernel"
curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinuz_kernel_name}" -o "${vmlinuz_kernel_name}" || return 1
sudo -E ln -sf "${kernel_directory}/${vmlinuz_kernel_name}" "${kernel_directory}/vmlinuz.container"
fi
popd
}
main() {
check_cached_kernel $*
}
main $*

View File

@ -9,14 +9,13 @@ set -o nounset
set -o pipefail set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly ovmf_builder="${script_dir}/build-ovmf.sh" readonly ovmf_builder="${script_dir}/build-ovmf.sh"
source "${script_dir}/../../scripts/lib.sh" source "${script_dir}/../../scripts/lib.sh"
DESTDIR=${DESTDIR:-${PWD}} DESTDIR=${DESTDIR:-${PWD}}
PREFIX=${PREFIX:-/opt/kata} PREFIX=${PREFIX:-/opt/kata}
container_image="${OVMF_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:ovmf-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" container_image="${OVMF_CONTAINER_BUILDER:-$(get_ovmf_image_name)}"
ovmf_build="${ovmf_build:-x86_64}" ovmf_build="${ovmf_build:-x86_64}"
kata_version="${kata_version:-}" kata_version="${kata_version:-}"
ovmf_repo="${ovmf_repo:-}" ovmf_repo="${ovmf_repo:-}"

View File

@ -9,7 +9,6 @@ set -o nounset
set -o pipefail set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly qemu_builder="${script_dir}/build-qemu.sh" readonly qemu_builder="${script_dir}/build-qemu.sh"
source "${script_dir}/../../scripts/lib.sh" source "${script_dir}/../../scripts/lib.sh"
@ -39,7 +38,7 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d")
[ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu" [ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu"
[ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static" [ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static"
container_image="${QEMU_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:qemu-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" container_image="${QEMU_CONTAINER_BUILDER:-$(get_qemu_image_name)}"
sudo docker pull ${container_image} || \ sudo docker pull ${container_image} || \
(sudo "${container_engine}" build \ (sudo "${container_engine}" build \

View File

@ -12,18 +12,12 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${script_dir}/../../scripts/lib.sh" source "${script_dir}/../../scripts/lib.sh"
export qemu_repo="${qemu_repo:-}" qemu_repo="${qemu_repo:-}"
export qemu_version="${qemu_version:-}" qemu_version="${qemu_version:-}"
export qemu_latest_build_url="${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}" tee="${tee:-}"
export katacontainers_repo="${katacontainers_repo:=github.com/kata-containers/kata-containers}"
export qemu_tarball_name="kata-static-qemu-cc.tar.gz"
export pkg_dir="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')"
export qemu_tarball_directory="${pkg_dir}/kata-deploy/local-build/build/cc-qemu/builddir"
export tee="${tee:-}"
export prefix="/opt/confidential-containers/" export prefix="/opt/confidential-containers/"
get_qemu_information() {
if [ -z "${qemu_repo}" ]; then if [ -z "${qemu_repo}" ]; then
info "Get qemu information from runtime versions.yaml" info "Get qemu information from runtime versions.yaml"
export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url") export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url")
@ -34,61 +28,7 @@ get_qemu_information() {
[ -n "${qemu_repo}" ] || die "failed to get qemu repo" [ -n "${qemu_repo}" ] || die "failed to get qemu repo"
[ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version") [ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
[ -n "${qemu_version}" ] || die "failed to get qemu version" [ -n "${qemu_version}" ] || die "failed to get qemu version"
}
calc_qemu_files_sha256sum() { qemu_tarball_name="kata-static-qemu-cc.tar.gz"
info "pkg directory is at ${pkg_dir}"
local files="${pkg_dir}/qemu \
${pkg_dir}/static-build/qemu.blacklist \
${pkg_dir}/static-build/scripts"
sha256sum_from_files "$files"
}
cached_or_build_qemu_tar() {
# Check latest qemu cc tar version sha256sum
local latest=$(curl -sfL "${qemu_latest_build_url}/latest") || latest="none"
local cached_qemu_version="$(echo ${latest} | awk '{print $1}')"
info "Current qemu version: ${qemu_version}"
info "Cached qemu version: ${cached_qemu_version}"
if [ "${qemu_version}" == "${cached_qemu_version}" ]; then
info "Get latest cached information ${latest}"
local cached_sha256sum="$(echo ${latest} | awk '{print $2}')"
info "Cached sha256sum version: ${cached_sha256sum}"
local current_sha256sum="$(calc_qemu_files_sha256sum)"
info "Current sha256sum of the qemu directory ${current_sha256sum}"
if [ -z "${cached_sha256sum}" ]; then
build_qemu_tar
elif [ "${current_sha256sum}" == "${cached_sha256sum}" ]; then
install_cached_qemu_tar
else
build_qemu_tar
fi
else
build_qemu_tar
fi
}
build_qemu_tar() {
[ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz" [ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz"
"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}" "${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}"
}
install_cached_qemu_tar() {
info "Using cached tarball of qemu"
curl -fL --progress-bar "${qemu_latest_build_url}/${qemu_tarball_name}" -o "${qemu_tarball_name}" || return 1
curl -fsOL "${qemu_latest_build_url}/sha256sum-${qemu_tarball_name}" || return 1
sha256sum -c "sha256sum-${qemu_tarball_name}" || return 1
}
main() {
get_qemu_information
# Currently the cached for qemu cc only works in x86_64
if [ "$(uname -m)" == "x86_64" ]; then
cached_or_build_qemu_tar
else
build_qemu_tar
fi
}
main $@

View File

@ -9,17 +9,17 @@ set -o nounset
set -o pipefail set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
source "${script_dir}/../../scripts/lib.sh" source "${script_dir}/../../scripts/lib.sh"
readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
GO_VERSION=${GO_VERSION} GO_VERSION=${GO_VERSION}
RUST_VERSION=${RUST_VERSION:-} RUST_VERSION=${RUST_VERSION:-}
DESTDIR=${DESTDIR:-${PWD}} DESTDIR=${DESTDIR:-${PWD}}
PREFIX=${PREFIX:-/opt/kata} PREFIX=${PREFIX:-/opt/kata}
container_image="${SHIM_V2_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" container_image="${SHIM_V2_CONTAINER_BUILDER:-$(get_shim_v2_image_name)}"
EXTRA_OPTS="${EXTRA_OPTS:-""}" EXTRA_OPTS="${EXTRA_OPTS:-""}"
VMM_CONFIGS="qemu fc" VMM_CONFIGS="qemu fc"

View File

@ -9,7 +9,6 @@ set -o nounset
set -o pipefail set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly tdshim_builder="${script_dir}/build-td-shim.sh" readonly tdshim_builder="${script_dir}/build-td-shim.sh"
source "${script_dir}/../../scripts/lib.sh" source "${script_dir}/../../scripts/lib.sh"
@ -30,7 +29,7 @@ package_output_dir="${package_output_dir:-}"
[ -n "${tdshim_version}" ] || die "Failed to get TD-shim version or commit" [ -n "${tdshim_version}" ] || die "Failed to get TD-shim version or commit"
[ -n "${tdshim_toolchain}" ] || die "Failed to get TD-shim toolchain to be used to build the project" [ -n "${tdshim_toolchain}" ] || die "Failed to get TD-shim toolchain to be used to build the project"
container_image="${TDSHIM_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:td-shim-${tdshim_toolchain}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" container_image="${TDSHIM_CONTAINER_BUILDER:-$(get_td_shim_image_name)}"
sudo docker pull ${container_image} || \ sudo docker pull ${container_image} || \
(sudo docker build \ (sudo docker build \

View File

@ -9,7 +9,6 @@ set -o nounset
set -o pipefail set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
readonly virtiofsd_builder="${script_dir}/build-static-virtiofsd.sh" readonly virtiofsd_builder="${script_dir}/build-static-virtiofsd.sh"
source "${script_dir}/../../scripts/lib.sh" source "${script_dir}/../../scripts/lib.sh"
@ -49,7 +48,7 @@ case ${ARCH} in
;; ;;
esac esac
container_image="${VIRTIOFSD_CONTAINER_BUILDER:-${CC_BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}" container_image="${VIRTIOFSD_CONTAINER_BUILDER:-$(get_virtiofsd_image_name)}"
sudo docker pull ${container_image} || \ sudo docker pull ${container_image} || \
(sudo docker build \ (sudo docker build \