diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index d6577f9219..dc1ff041cc 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -38,6 +38,8 @@ readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_imag readonly cc_prefix="/opt/confidential-containers" readonly qemu_cc_builder="${static_build_dir}/qemu/build-static-qemu-cc.sh" +source "${script_dir}/../../scripts/lib.sh" + ARCH=$(uname -m) workdir="${WORKDIR:-$PWD}" @@ -104,8 +106,45 @@ EOF exit "${return_code}" } +cleanup_and_fail() { + rm -f "${component_tarball_path}" + return 1 +} + +install_cached_component() { + local component="${1}" + local jenkins_build_url="${2}" + local current_version="${3}" + local current_image_version="${4}" + local component_tarball_name="${5}" + local component_tarball_path="${6}" + + local cached_version=$(curl -sfL "${jenkins_build_url}/latest" | awk '{print $1}') || cached_version="none" + local cached_image_version=$(curl -sfL "${jenkins_build_url}/latest_image" | awk '{print $1}') || cached_image_version="none" + + [ "${cached_image_version}" != "${current_image_version}" ] && return 1 + [ "${cached_version}" != "${current_version}" ] && return 1 + + info "Using cached tarball of ${component}" + pushd ${workdir} + echo "Downloading tarball from: ${jenkins_build_url}/${component_tarball_name}" + curl -fL --progress-bar "${jenkins_build_url}/${component_tarball_name}" -o "${component_tarball_path}" || return cleanup_and_fail + curl -fsOL "${jenkins_build_url}/sha256sum-${component_tarball_name}" || return cleanup_and_fail + sha256sum -c "sha256sum-${component_tarball_name}" && return cleanup_and_fail + popd +} + # Install static CC cloud-hypervisor asset install_cc_clh() { + install_cached_component \ + "cloud-hypervisor" \ + "${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + if [[ "${ARCH}" == "x86_64" ]]; then export features="tdx" fi @@ -143,11 +182,21 @@ install_cc_tdx_image() { #Install CC kernel asset install_cc_kernel() { - export KATA_BUILD_CC=yes - info "build initramfs for cc kernel" - "${initramfs_builder}" + + export KATA_BUILD_CC=yes export kernel_version="$(yq r $versions_yaml assets.kernel.version)" + + install_cached_component \ + "kernel" \ + "${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + + "${initramfs_builder}" DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -f -v "${kernel_version}" } @@ -156,6 +205,16 @@ install_cc_qemu() { info "build static CC qemu" export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)" export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)" + + install_cached_component \ + "QEMU" \ + "${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}" \ + "${qemu_version}-$(calc_qemu_files_sha256sum)" \ + "$(get_qemu_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + "${qemu_cc_builder}" tar xvf "${builddir}/kata-static-qemu-cc.tar.gz" -C "${destdir}" } @@ -178,6 +237,15 @@ install_cc_shimv2() { # Install static CC virtiofsd asset install_cc_virtiofsd() { + install_cached_component \ + "virtiofsd" \ + "${jenkins_url}/job/kata-containers-2.0-virtiofsd-cc-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "assets.externals.virtiofsd.version")" \ + "$(get_virtiofsd_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "build static CC virtiofsd" "${virtiofsd_builder}" info "Install static CC virtiofsd" @@ -199,6 +267,16 @@ install_cc_tee_kernel() { info "build initramfs for tee kernel" export kernel_version=${kernel_version} + + install_cached_component \ + "kernel" \ + "${jenkins_url}/job/kata-containers-2.0-kernel-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + "${initramfs_builder}" kernel_url="$(yq r $versions_yaml assets.kernel.${tee}.url)" DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${kernel_builder}" -x "${tee}" -v "${kernel_version}" -u "${kernel_url}" @@ -223,6 +301,16 @@ install_cc_tee_qemu() { export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.url)" export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.tag)" export tee="${tee}" + + install_cached_component \ + "QEMU ${tee}" \ + "${jenkins_url}/job/kata-containers-2.0-qemu-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \ + "${qemu_version}-$(calc_qemu_files_sha256sum)" \ + "$(get_qemu_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + "${qemu_cc_builder}" tar xvf "${builddir}/kata-static-${tee}-qemu-cc.tar.gz" -C "${destdir}" } @@ -232,6 +320,15 @@ install_cc_tdx_qemu() { } install_cc_tdx_td_shim() { + install_cached_component \ + "td-shim" \ + "${jenkins_url}/job/kata-containers-2.0-td-shim-cc-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "assets.externals.td-shim.version")" \ + "$(get_td_shim_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${td_shim_builder}" tar xvf "${builddir}/td-shim.tar.gz" -C "${destdir}" } @@ -240,6 +337,18 @@ install_cc_tee_ovmf() { tee="${1}" tarball_name="${2}" + local component_name="ovmf" + local component_version="$(get_from_kata_deps "assets.external.ovmf.${tee}.version")" + [ "${tee}" == "tdx" ] && component_name="tdvf" + install_cached_component \ + "${component_name}" \ + "${jenkins_url}/job/kata-containers-2.0-${component_name}-cc-$(uname -m)/${cached_artifacts_path}" \ + "$(component_version)" \ + "$(get_ovmf_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + DESTDIR="${destdir}" PREFIX="${cc_prefix}" ovmf_build="${tee}" "${ovmf_builder}" tar xvf "${builddir}/${tarball_name}" -C "${destdir}" } @@ -350,6 +459,11 @@ handle_build() { info "DESTDIR ${destdir}" local build_target build_target="$1" + + export final_tarball_path="${workdir}/kata-static-${build_target}.tar.xz" + export final_tarball_name="$(basename ${final_tarball_path})" + rm -f ${final_tarball_name} + case "${build_target}" in all) install_clh @@ -429,12 +543,11 @@ handle_build() { ;; esac - tarball_name="${workdir}/kata-static-${build_target}.tar.xz" - ( + if [ ! -f "${final_tarball_path}" ]; then cd "${destdir}" - sudo tar cvfJ "${tarball_name}" "." - ) - tar tvf "${tarball_name}" + sudo tar cvfJ "${final_tarball_path}" "." + fi + tar tvf "${final_tarball_path}" } silent_mode_error_trap() { diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh index 1fda017402..0bee1ea041 100755 --- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh +++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh @@ -23,7 +23,6 @@ cloud_hypervisor_repo="${cloud_hypervisor_repo:-}" cloud_hypervisor_version="${cloud_hypervisor_version:-}" cloud_hypervisor_pr="${cloud_hypervisor_pr:-}" cloud_hypervisor_pull_ref_branch="${cloud_hypervisor_pull_ref_branch:-main}" -cloud_hypervisor_latest_build_url="${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}" if [ -z "$cloud_hypervisor_repo" ]; then info "Get cloud_hypervisor information from runtime versions.yaml" @@ -83,40 +82,6 @@ build_clh_from_source() { popd } -check_cached_cloud_hypervisor() { - local cached_cloud_hypervisor_version=$(curl -sfL "${cloud_hypervisor_latest_build_url}"/latest) || latest="none" - info "Current cloud hypervisor version: ${cloud_hypervisor_version}" - info "Cached cloud hypervisor version: ${cached_cloud_hypervisor_version}" - if [ "${cloud_hypervisor_version}" == "${cached_cloud_hypervisor_version}" ] && [ "${ARCH}" == "x86_64" ]; then - install_cached_cloud_hypervisor - else - build_clh_from_source - fi -} - -install_cached_cloud_hypervisor() { - local cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" - local clh_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-cloud-hypervisor/builddir/cloud-hypervisor" - mkdir cloud-hypervisor - pushd cloud-hypervisor - local checksum_file="sha256sum-cloud-hypervisor" - info "Downloading the cloud hypervisor binary" - curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/cloud-hypervisor" || return 1 - info "Checking cloud hypervisor binary checksum" - curl -fOL --progress-bar "${cloud_hypervisor_latest_build_url}/${checksum_file}" || return 1 - info "Verify checksum" - sudo sha256sum -c "${checksum_file}" || return 1 - chmod +x cloud-hypervisor - local clh_binary_path="${cached_path}/cloud-hypervisor" - if [ ! -d "${clh_binary_path}" ]; then - mkdir -p "${clh_binary_path}" - fi - if [ ! -f "${clh_binary_path}/cloud-hypervisor" ]; then - cp cloud-hypervisor "${clh_binary_path}" - fi - popd -} - if [ "${ARCH}" == "aarch64" ]; then info "aarch64 binaries are not distributed as part of the Cloud Hypervisor releases, forcing to build from source" force_build_from_source="true" @@ -129,8 +94,8 @@ fi if [ "${force_build_from_source}" == "true" ]; then info "Build cloud-hypervisor from source as it's been request via the force_build_from_source flag" - check_cached_cloud_hypervisor + build_clh_from_source else pull_clh_released_binary || - (info "Failed to pull cloud-hypervisor released binary, trying to build from source" && check_cached_cloud_hypervisor) + (info "Failed to pull cloud-hypervisor released binary, trying to build from source" && build_clh_from_source) fi diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index db0e65d77d..c9bb71be4c 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -17,82 +17,25 @@ readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}" -kernel_latest_build_url="${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}" -current_kernel_version=${kernel_version:-$(get_from_kata_deps "assets.kernel.version")} -cached_path="$(echo ${script_dir} | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" -current_kernel_config_file="${cached_path}/tools/packaging/kernel/kata_config_version" -current_kernel_config="$(cat $current_kernel_config_file)" -kernel_version="$(echo ${current_kernel_version} | cut -c2- )" -build_from_source() { - sudo docker pull ${container_image} || \ - (sudo docker build -t "${container_image}" "${script_dir}" && \ - # No-op unless PUSH_TO_REGISTRY is exported as "yes" - push_to_registry "${container_image}") +sudo docker pull ${container_image} || \ + (sudo docker build -t "${container_image}" "${script_dir}" && \ + # No-op unless PUSH_TO_REGISTRY is exported as "yes" + push_to_registry "${container_image}") - sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ - -w "${PWD}" \ - --env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \ - "${container_image}" \ - bash -c "${kernel_builder} $* setup" +sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + -w "${PWD}" \ + --env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \ + "${container_image}" \ + bash -c "${kernel_builder} $* setup" - sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ - -w "${PWD}" \ - "${container_image}" \ - bash -c "${kernel_builder} $* build" +sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + -w "${PWD}" \ + "${container_image}" \ + bash -c "${kernel_builder} $* build" - sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ - -w "${PWD}" \ - --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ - "${container_image}" \ - bash -c "${kernel_builder} $* install" -} - -check_cached_kernel() { - local latest=$(curl -sfL "${kernel_latest_build_url}"/latest) || latest="none" - local cached_kernel_version="$(echo ${latest} | awk '{print $1}')" - info "Current kernel version: ${kernel_version}" - info "Cached kernel version: ${cached_kernel_version}" - if [ "${kernel_version}" == "${cached_kernel_version}" ] && [ "$(uname -m)" == "x86_64" ]; then - local cached_kernel_config="$(echo ${latest} | awk '{print $2}')" - info "Cached kernel config: ${cached_kernel_config}" - info "Current kernel config: ${current_kernel_config}" - if [ -z "${cached_kernel_config}" ]; then - build_from_source $* - else - install_cached_kernel $* - fi - else - build_from_source $* - fi -} - -install_cached_kernel() { - local kernel_directory="${cached_path}/tools/packaging/kata-deploy/local-build/build/cc-kernel/destdir/opt/confidential-containers/share/kata-containers" - local vmlinux_kernel_name="vmlinux-${cached_kernel_version}-${cached_kernel_config}" - local vmlinuz_kernel_name="vmlinuz-${cached_kernel_version}-${cached_kernel_config}" - mkdir -p "${kernel_directory}" - pushd "${kernel_directory}" - ls - local vmlinux_url="${kernel_latest_build_url}/${vmlinux_kernel_name}" - if curl --output /dev/null --silent --head --fail "${vmlinux_url}"; then - info "Installing vmlinux cached kernel" - curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinux_kernel_name}" -o "${vmlinux_kernel_name}" || return 1 - sudo -E ln -sf "${kernel_directory}/${vmlinux_kernel_name}" "${kernel_directory}/vmlinux.container" - fi - - local vmlinuz_url="${kernel_latest_build_url}/${vmlinuz_kernel_name}" - if curl --output /dev/null --silent --head --fail "${vmlinuz_url}"; then - info "Installing vmlinuz cached kernel" - curl -fL --progress-bar "${kernel_latest_build_url}/${vmlinuz_kernel_name}" -o "${vmlinuz_kernel_name}" || return 1 - sudo -E ln -sf "${kernel_directory}/${vmlinuz_kernel_name}" "${kernel_directory}/vmlinuz.container" - fi - popd - -} - -main() { - check_cached_kernel $* -} - -main $* +sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + -w "${PWD}" \ + --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ + "${container_image}" \ + bash -c "${kernel_builder} $* install" diff --git a/tools/packaging/static-build/qemu/build-static-qemu-cc.sh b/tools/packaging/static-build/qemu/build-static-qemu-cc.sh index 724e606b08..4ec3dcfd16 100755 --- a/tools/packaging/static-build/qemu/build-static-qemu-cc.sh +++ b/tools/packaging/static-build/qemu/build-static-qemu-cc.sh @@ -12,74 +12,23 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${script_dir}/../../scripts/lib.sh" -export qemu_repo="${qemu_repo:-}" -export qemu_version="${qemu_version:-}" -export qemu_latest_build_url="${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}" -export katacontainers_repo="${katacontainers_repo:=github.com/kata-containers/kata-containers}" -export qemu_tarball_name="kata-static-qemu-cc.tar.gz" -export pkg_dir="$(echo $script_dir | sed 's,/*[^/]\+/*$,,' | sed 's,/*[^/]\+/*$,,')" -export qemu_tarball_directory="${pkg_dir}/kata-deploy/local-build/build/cc-qemu/builddir" -export tee="${tee:-}" +qemu_repo="${qemu_repo:-}" +qemu_version="${qemu_version:-}" +tee="${tee:-}" export prefix="/opt/confidential-containers/" -get_qemu_information() { - if [ -z "${qemu_repo}" ]; then - info "Get qemu information from runtime versions.yaml" - export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url") - [ -n "${qemu_url}" ] || die "failed to get qemu url" - export qemu_repo="${qemu_url}.git" - fi +if [ -z "${qemu_repo}" ]; then + info "Get qemu information from runtime versions.yaml" + export qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url") + [ -n "${qemu_url}" ] || die "failed to get qemu url" + export qemu_repo="${qemu_url}.git" +fi - [ -n "${qemu_repo}" ] || die "failed to get qemu repo" - [ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version") - [ -n "${qemu_version}" ] || die "failed to get qemu version" -} +[ -n "${qemu_repo}" ] || die "failed to get qemu repo" +[ -n "${qemu_version}" ] || export qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version") +[ -n "${qemu_version}" ] || die "failed to get qemu version" -cached_or_build_qemu_tar() { - # Check latest qemu cc tar version sha256sum - local latest=$(curl -sfL "${qemu_latest_build_url}/latest") || latest="none" - local cached_qemu_version="$(echo ${latest} | awk '{print $1}')" - info "Current qemu version: ${qemu_version}" - info "Cached qemu version: ${cached_qemu_version}" - if [ "${qemu_version}" == "${cached_qemu_version}" ]; then - info "Get latest cached information ${latest}" - local cached_sha256sum="$(echo ${latest} | awk '{print $2}')" - info "Cached sha256sum version: ${cached_sha256sum}" - local current_sha256sum="$(calc_qemu_files_sha256sum)" - info "Current sha256sum of the qemu directory ${current_sha256sum}" - if [ -z "${cached_sha256sum}" ]; then - build_qemu_tar - elif [ "${current_sha256sum}" == "${cached_sha256sum}" ]; then - install_cached_qemu_tar - else - build_qemu_tar - fi - else - build_qemu_tar - fi -} - -build_qemu_tar() { - [ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz" - "${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}" -} - -install_cached_qemu_tar() { - info "Using cached tarball of qemu" - curl -fL --progress-bar "${qemu_latest_build_url}/${qemu_tarball_name}" -o "${qemu_tarball_name}" || return 1 - curl -fsOL "${qemu_latest_build_url}/sha256sum-${qemu_tarball_name}" || return 1 - sha256sum -c "sha256sum-${qemu_tarball_name}" || return 1 -} - -main() { - get_qemu_information - # Currently the cached for qemu cc only works in x86_64 - if [ "$(uname -m)" == "x86_64" ]; then - cached_or_build_qemu_tar - else - build_qemu_tar - fi -} - -main $@ +qemu_tarball_name="kata-static-qemu-cc.tar.gz" +[ -n "${tee}" ] && qemu_tarball_name="kata-static-${tee}-qemu-cc.tar.gz" +"${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "${tee}" "${qemu_tarball_name}"