diff --git a/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile b/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile index 51e7ba4311..a29514968b 100644 --- a/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile +++ b/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile @@ -7,10 +7,11 @@ ENV DEBIAN_FRONTEND=noninteractive ENV INSTALL_IN_GOPATH=false COPY install_yq.sh /usr/bin/install_yq.sh +COPY install_oras.sh /usr/bin/install_oras.sh SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# Install yq and docker +# Install yq, oras, and docker RUN apt-get update && \ apt-get install -y --no-install-recommends \ ca-certificates \ @@ -18,6 +19,7 @@ RUN apt-get update && \ sudo && \ apt-get clean && rm -rf /var/lib/apt/lists/ && \ install_yq.sh && \ + install_oras.sh && \ curl -fsSL https://get.docker.com -o get-docker.sh && \ if uname -m | grep -Eq 's390x|ppc64le'; then export VERSION="v20.10" && \ sed -i 's/\//g' get-docker.sh; fi && \ diff --git a/tools/packaging/kata-deploy/local-build/dockerbuild/install_oras.sh b/tools/packaging/kata-deploy/local-build/dockerbuild/install_oras.sh new file mode 100755 index 0000000000..973a102050 --- /dev/null +++ b/tools/packaging/kata-deploy/local-build/dockerbuild/install_oras.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +install_dest="/usr/local/bin" + +function get_installed_oras_version() { + oras version | grep Version | sed -e s/Version:// | tr -d [:blank:] +} + +oras_required_version="v1.1.0" +if command -v oras; then + if [[ "${oras_required_version}" == "v$(get_installed_oras_version)" ]]; then + echo "ORAS is already installed in the system" + exit 0 + fi + + echo "Proceeding to cleanup the previous installed version of ORAS, and install the version specified in the versions.yaml file" + oras_system_path=$(which oras) + sudo rm -f ${oras_system_path} +fi + +arch=$(uname -m) +if [ "${arch}" = "ppc64le" ]; then + echo "An ORAS release for ppc64le is not available yet." + exit 0 +fi +if [ "${arch}" = "x86_64" ]; then + arch="amd64" +fi +if [ "${arch}" = "aarch64" ]; then + arch="arm64" +fi +oras_tarball="oras_${oras_required_version#v}_linux_${arch}.tar.gz" + +echo "Downloading ORAS ${oras_required_version}" +sudo curl -OL https://github.com/oras-project/oras/releases/download/${oras_required_version}/${oras_tarball} + +echo "Installing ORAS to ${install_dest}" +sudo mkdir -p "${install_dest}" +sudo tar -C "${install_dest}" -xzf "${oras_tarball}" +sudo rm -f "${oras_tarball}" diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index 5c336cc8b0..47cf2dd1dd 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -77,23 +77,44 @@ docker build -q -t build-kata-deploy \ --build-arg HOST_DOCKER_GID=${docker_gid} \ "${script_dir}/dockerbuild/" +CI="${CI:-}" +ARTEFACT_REGISTRY="${ARTEFACT_REGISTRY:-}" +ARTEFACT_REGISTRY_USERNAME="${ARTEFACT_REGISTRY_USERNAME:-}" +ARTEFACT_REGISTRY_PASSWORD="${ARTEFACT_REGISTRY_PASSWORD:-}" +TARGET_BRANCH="${TARGET_BRANCH:-}" +BUILDER_REGISTRY="${BUILDER_REGISTRY:-}" +PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}" +INITRAMFS_CONTAINER_BUILDER="${INITRAMFS_CONTAINER_BUILDER:-}" +KERNEL_CONTAINER_BUILDER="${KERNEL_CONTAINER_BUILDER:-}" +OVMF_CONTAINER_BUILDER="${OVMF_CONTAINER_BUILDER:-}" +QEMU_CONTAINER_BUILDER="${QEMU_CONTAINER_BUILDER:-}" +SHIM_V2_CONTAINER_BUILDER="${SHIM_V2_CONTAINER_BUILDER:-}" +TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER:-}" +VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" +MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" +USE_CACHE="${USE_CACHE:-}" + docker run \ -v $HOME/.docker:/root/.docker \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "${kata_dir}:${kata_dir}" \ - --env CI="${CI:-}" \ + --env CI="${CI}" \ --env USER=${USER} \ - --env BUILDER_REGISTRY="${BUILDER_REGISTRY:-}" \ - --env PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}" \ - --env INITRAMFS_CONTAINER_BUILDER="${INITRAMFS_CONTAINER_BUILDER:-}" \ - --env KERNEL_CONTAINER_BUILDER="${KERNEL_CONTAINER_BUILDER:-}" \ - --env OVMF_CONTAINER_BUILDER="${OVMF_CONTAINER_BUILDER:-}" \ - --env QEMU_CONTAINER_BUILDER="${QEMU_CONTAINER_BUILDER:-}" \ - --env SHIM_V2_CONTAINER_BUILDER="${SHIM_V2_CONTAINER_BUILDER:-}" \ - --env TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER:-}" \ - --env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" \ - --env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \ - --env USE_CACHE="${USE_CACHE:-}" \ + --env ARTEFACT_REGISTRY="${ARTEFACT_REGISTRY}" \ + --env ARTEFACT_REGISTRY_USERNAME="${ARTEFACT_REGISTRY_USERNAME}" \ + --env ARTEFACT_REGISTRY_PASSWORD="${ARTEFACT_REGISTRY_PASSWORD}" \ + --env TARGET_BRANCH="${TARGET_BRANCH}" \ + --env BUILDER_REGISTRY="${BUILDER_REGISTRY}" \ + --env PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY}" \ + --env INITRAMFS_CONTAINER_BUILDER="${INITRAMFS_CONTAINER_BUILDER}" \ + --env KERNEL_CONTAINER_BUILDER="${KERNEL_CONTAINER_BUILDER}" \ + --env OVMF_CONTAINER_BUILDER="${OVMF_CONTAINER_BUILDER}" \ + --env QEMU_CONTAINER_BUILDER="${QEMU_CONTAINER_BUILDER}" \ + --env SHIM_V2_CONTAINER_BUILDER="${SHIM_V2_CONTAINER_BUILDER}" \ + --env TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER}" \ + --env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER}" \ + --env MEASURED_ROOTFS="${MEASURED_ROOTFS}" \ + --env USE_CACHE="${USE_CACHE}" \ --env CROSS_BUILD="${CROSS_BUILD}" \ --env TARGET_ARCH="${TARGET_ARCH}" \ --env ARCH="${ARCH}" \ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 4a1b256f1a..3e80612332 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -41,6 +41,11 @@ readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts" ARCH=${ARCH:-$(uname -m)} MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} USE_CACHE="${USE_CACHE:-"yes"}" +ARTEFACT_REGISTRY="${ARTEFACT_REGISTRY:-}" +ARTEFACT_REGISTRY_USERNAME="${ARTEFACT_REGISTRY_USERNAME:-}" +ARTEFACT_REGISTRY_PASSWORD="${ARTEFACT_REGISTRY_PASSWORD:-}" +TARGET_BRANCH="${TARGET_BRANCH:=}" +PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-}" workdir="${WORKDIR:-$PWD}" @@ -161,11 +166,14 @@ install_image() { local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + latest_artefact="${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${image_type}" + latest_builder_image="" + install_cached_tarball_component \ "${component}" \ "${jenkins}" \ - "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${image_type}" \ - "" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 @@ -209,13 +217,16 @@ install_initrd() { local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + latest_artefact="${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${initrd_type}" + latest_builder_image="" + [[ "${ARCH}" == "aarch64" && "${CROSS_BUILD}" == "true" ]] && echo "warning: Don't cross build initrd for aarch64 as it's too slow" && exit 0 install_cached_tarball_component \ "${component}" \ "${jenkins}" \ - "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${initrd_type}" \ - "" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 @@ -250,11 +261,14 @@ install_cached_kernel_tarball_component() { local kernel_name=${1} local module_dir=${2:-""} + latest_artefact="${kernel_version}-${kernel_kata_config_version}-$(get_last_modification $(dirname $kernel_builder))" + latest_builder_image="$(get_kernel_image_name)" + install_cached_tarball_component \ "${kernel_name}" \ "${jenkins_url}/job/kata-containers-main-${kernel_name}-${ARCH}/${cached_artifacts_path}" \ - "${kernel_version}-${kernel_kata_config_version}-$(get_last_modification $(dirname $kernel_builder))" \ - "$(get_kernel_image_name)" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "${final_tarball_name}" \ "${final_tarball_path}" \ || return 1 @@ -267,8 +281,8 @@ install_cached_kernel_tarball_component() { install_cached_tarball_component \ "${kernel_name}" \ "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ - "${kernel_version}-${kernel_kata_config_version}-$(get_last_modification $(dirname $kernel_builder))" \ - "$(get_kernel_image_name)" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "kata-static-kernel-sev-modules.tar.xz" \ "${workdir}/kata-static-kernel-sev-modules.tar.xz" \ || return 1 @@ -387,11 +401,14 @@ install_qemu_helper() { export qemu_repo="$(get_from_kata_deps ${qemu_repo_yaml_path})" export qemu_version="$(get_from_kata_deps ${qemu_version_yaml_path})" + latest_artefact="${qemu_version}-$(calc_qemu_files_sha256sum)" + latest_builder_image="$(get_qemu_image_name)" + install_cached_tarball_component \ "${qemu_name}" \ "${jenkins_url}/job/kata-containers-main-${qemu_name}-${ARCH}/${cached_artifacts_path}" \ - "${qemu_version}-$(calc_qemu_files_sha256sum)" \ - "$(get_qemu_image_name)" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 @@ -436,11 +453,14 @@ install_qemu_snp_experimental() { install_firecracker() { local firecracker_version=$(get_from_kata_deps "assets.hypervisor.firecracker.version") + latest_artefact="${firecracker_version}" + latest_builder_image="" + install_cached_tarball_component \ "firecracker" \ "${jenkins_url}/job/kata-containers-main-firecracker-$(uname -m)/${cached_artifacts_path}" \ - "${firecracker_version}" \ - "" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 @@ -458,11 +478,14 @@ install_clh_helper() { features="${2}" suffix="${3:-""}" + latest_artefact="$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" + latest_builder_image="" + install_cached_tarball_component \ "cloud-hypervisor${suffix}" \ "${jenkins_url}/job/kata-containers-main-clh-$(uname -m)${suffix}/${cached_artifacts_path}" \ - "$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \ - "" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 @@ -498,11 +521,14 @@ install_clh_glibc() { # Install static virtiofsd asset install_virtiofsd() { + latest_artefact="$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" + latest_builder_image="$(get_virtiofsd_image_name)" + install_cached_tarball_component \ "virtiofsd" \ "${jenkins_url}/job/kata-containers-main-virtiofsd-${ARCH}/${cached_artifacts_path}" \ - "$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" \ - "$(get_virtiofsd_image_name)" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 @@ -518,11 +544,14 @@ install_virtiofsd() { install_nydus() { [ "${ARCH}" == "aarch64" ] && ARCH=arm64 + latest_artefact="$(get_from_kata_deps "externals.nydus.version")" + latest_builder_image="" + install_cached_tarball_component \ "nydus" \ "${jenkins_url}/job/kata-containers-main-nydus-$(uname -m)/${cached_artifacts_path}" \ - "$(get_from_kata_deps "externals.nydus.version")" \ - "" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 @@ -543,13 +572,15 @@ install_shimv2() { local protocols_last_commit="$(get_last_modification "${repo_root_dir}/src/libs/protocols")" local GO_VERSION="$(get_from_kata_deps "languages.golang.meta.newest-version")" local RUST_VERSION="$(get_from_kata_deps "languages.rust.meta.newest-version")" - local shim_v2_version="${shim_v2_last_commit}-${protocols_last_commit}-${runtime_rs_last_commit}-${GO_VERSION}-${RUST_VERSION}" + + latest_artefact="${shim_v2_last_commit}-${protocols_last_commit}-${runtime_rs_last_commit}-${GO_VERSION}-${RUST_VERSION}" + latest_builder_image="$(get_shim_v2_image_name)" install_cached_tarball_component \ "shim-v2" \ "${jenkins_url}/job/kata-containers-main-shim-v2-${ARCH}/${cached_artifacts_path}" \ - "${shim_v2_version}" \ - "$(get_shim_v2_image_name)" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 @@ -576,13 +607,16 @@ install_ovmf() { tarball_name="${2:-edk2-x86_64.tar.gz}" local component_name="ovmf" - local component_version="$(get_from_kata_deps "externals.ovmf.${ovmf_type}.version")" [ "${ovmf_type}" == "tdx" ] && component_name="tdvf" + + latest_artefact="$(get_from_kata_deps "externals.ovmf.${ovmf_type}.version")" + latest_builder_image="$(get_ovmf_image_name)" + install_cached_tarball_component \ "${component_name}" \ "${jenkins_url}/job/kata-containers-main-ovmf-${ovmf_type}-$(uname -m)/${cached_artifacts_path}" \ - "${component_version}" \ - "$(get_ovmf_image_name)" \ + "${latest_artefact}" \ + "${latest_builder_image}" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 @@ -609,6 +643,10 @@ get_kata_version() { handle_build() { info "DESTDIR ${destdir}" + + latest_artefact="" + latest_builder_image="" + local build_target build_target="$1" @@ -696,6 +734,25 @@ handle_build() { sudo tar cvfJ "${final_tarball_path}" "." fi tar tvf "${final_tarball_path}" + + echo "${latest_artefact}" > ${workdir}/${build_target}-version + echo "${latest_builder_image}" > ${workdir}/${build_target}-builder-image-version + + if [ "${PUSH_TO_REGISTRY}" = "yes" ]; then + if [ -z "${ARTEFACT_REGISTRY}" ] || + [ -z "${ARTEFACT_REGISTRY_USERNAME}" ] || + [ -z "${ARTEFACT_REGISTRY_PASSWORD}" ] || + [ -z "${TARGET_BRANCH}" ]; then + die "ARTEFACT_REGISTRY, ARTEFACT_REGISTRY_USERNAME, ARTEFACT_REGISTRY_PASSWORD and TARGET_BRANCH must be passed to the script when pushing the artefacts to the registry!" + fi + + pushd ${workdir} + echo "${ARTEFACT_REGISTRY_PASSWORD}" | oras login "${ARTEFACT_REGISTRY}" -u "${ARTEFACT_REGISTRY_USERNAME}" --password-stdin + + oras push ${ARTEFACT_REGISTRY}/kata-containers/cached-artefacts/${build_target}:latest-${TARGET_BRANCH}-$(uname -m) ${final_tarball_name} ${build_target}-version ${build_target}-builder-image-version + oras logout "${ARTEFACT_REGISTRY}" + popd + fi } silent_mode_error_trap() {