From 4d835f608911c8af5799827bd5148a3ffbbc0ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 5 Dec 2022 11:27:59 +0100 Subject: [PATCH 1/4] cache_components: Add the ability to cache the rootfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the most complex part to cache, as the cached component can be only used if: * There were no changes in the agent * There were no changes in the libs (used by the agent) * There were no changes in the rootfs build scripts * There is no change in the version of the following components: * attestation-agent (part of the rootfs) * gperf (used to build libseccomp) * libseccomp (used to build the agent) * pause image (part of the rootfs) * skopeo (part of the rootfs) * umoci (part of the rootfs) * rust (used to build the kata-containers and attestation agents) We're relying on the last commit merged on places related to the rootfs generation and using that as the rootfs version and that should be good enough for what we need. Apart from everything already mentioned, we've also added the ability to cache the `root_hash_vanilla.txt` and `root_hash_tdx.txt` files, as those are needed for when building the shim-v2, in order to have measured boot working there. It's important to note that we've added the ability to cache *both* files, and I've taken that path as the shim-v2 cache work (which will come soon) relies on both files. Signed-off-by: Fabiano Fidêncio --- .../static-build/cache_components.sh | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/tools/packaging/static-build/cache_components.sh b/tools/packaging/static-build/cache_components.sh index 4936af9011..d6c47782f0 100755 --- a/tools/packaging/static-build/cache_components.sh +++ b/tools/packaging/static-build/cache_components.sh @@ -74,10 +74,43 @@ cache_virtiofsd_artifacts() { create_cache_asset "${virtiofsd_tarball_name}" "${current_virtiofsd_version}" "${current_virtiofsd_image}" } +cache_rootfs_artifacts() { + # We need to remove `-dirty` from teh osbuilder_last_commit as the rootfs artefacts are generated on that folder + local osbuilder_last_commit="$(echo $(get_last_modification "${repo_root_dir}/tools/osbuilder") | sed s/-dirty//)" + local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" + local agent_last_commit="$(get_last_modification "${repo_root_dir}/src/agent")" + local libs_last_commit="$(get_last_modification "${repo_root_dir}/src/libs")" + local attestation_agent_version="$(get_from_kata_deps "externals.attestation-agent.version")" + local gperf_version="$(get_from_kata_deps "externals.gperf.version")" + local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" + local pause_version="$(get_from_kata_deps "externals.pause.version")" + local skopeo_version="$(get_from_kata_deps "externals.skopeo.branch")" + local umoci_version="$(get_from_kata_deps "externals.umoci.tag")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local rootfs_tarball_name="kata-static-cc-rootfs-image.tar.xz" + local aa_kbc="offline_fs_kbc" + local image_type="image" + local root_hash_vanilla="${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt" + local root_hash_tdx="" + if [ -n "${TEE}" ]; then + if [ "${TEE}" == "tdx" ]; then + rootfs_tarball_name="kata-static-cc-tdx-rootfs-image.tar.xz" + aa_kbc="eaa_kbc" + image_type="image" + root_hash_vanilla="" + root_hash_tdx="${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt" + fi + fi + local current_rootfs_version="${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${attestation_agent_version}-${gperf_version}-${libseccomp_version}-${pause_version}-${skopeo_version}-${umoci_version}-${rust_version}-${image_type}-${aa_kbc}" + create_cache_asset "${rootfs_tarball_name}" "${current_rootfs_version}" "" "${root_hash_vanilla}" "${root_hash_tdx}" +} + create_cache_asset() { local component_name="${1}" local component_version="${2}" local component_image="${3}" + local root_hash_vanilla="${4:-""}" + local root_hash_tdx="${5:-""}" sudo cp "${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/${component_name}" . sudo chown -R "${USER}:${USER}" . @@ -87,6 +120,18 @@ create_cache_asset() { cat "latest" echo "${component_image}" > "latest_image" cat "latest_image" + if [ -n "${root_hash_vanilla}" ]; then + local cached_root_hash_vanilla="$(basename ${root_hash_vanilla})" + sudo cp "${root_hash_vanilla}" "${cached_root_hash_vanilla}" + sudo chown -R "${USER}:${USER}" "${cached_root_hash_vanilla}" + echo "${cached_root_hash_vanilla}: $(cat "${cached_root_hash_vanilla}")" + fi + if [ -n "${root_hash_tdx}" ]; then + local cached_root_hash_tdx="$(basename ${root_hash_tdx})" + sudo cp "${root_hash_tdx}" "${cached_root_hash_tdx}" + sudo chown -R "${USER}:${USER}" "${cached_root_hash_tdx}" + echo "${cached_root_hash_tdx}: $(cat "${cached_root_hash_tdx}")" + fi } help() { @@ -109,6 +154,10 @@ Usage: $0 "[options]" * tdvf * td-shim -v Virtiofsd cache + -r Rootfs Cache + * can receive a TEE environment variable value, valid values are: + * tdx + If not TEE environment is passed, the Rootfs Image will be built without TEE support. -h Shows help EOF )" @@ -120,8 +169,9 @@ main() { local kernel_component="${kernel_component:-}" local firmware_component="${firmware_component:-}" local virtiofsd_component="${virtiofsd_component:-}" + local rootfs_component="${rootfs_component:-}" local OPTIND - while getopts ":ckqfvh:" opt + while getopts ":ckqfvrh:" opt do case "$opt" in c) @@ -139,6 +189,9 @@ main() { v) virtiofsd_component="1" ;; + r) + rootfs_component="1" + ;; h) help exit 0; @@ -157,6 +210,7 @@ main() { [[ -z "${qemu_component}" ]] && \ [[ -z "${firmware_component}" ]] && \ [[ -z "${virtiofsd_component}" ]] && \ + [[ -z "${rootfs_component}" ]] && \ help && die "Must choose at least one option" mkdir -p "${WORKSPACE}/artifacts" @@ -168,6 +222,7 @@ main() { [ "${qemu_component}" == "1" ] && cache_qemu_artifacts [ "${firmware_component}" == "1" ] && cache_firmware_artifacts [ "${virtiofsd_component}" == "1" ] && cache_virtiofsd_artifacts + [ "${rootfs_component}" == "1" ] && cache_rootfs_artifacts ls -la "${WORKSPACE}/artifacts/" popd From d9dd1ac9ec9e48eea1b7c11d95b816b84b806d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 5 Dec 2022 13:19:03 +0100 Subject: [PATCH 2/4] kata-deploy-binaries: Use cached rootfs when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As done for different components, let's also use a cached version of the rootfs whenever it's possible. Fixes: #5433 Signed-off-by: Fabiano Fidêncio --- .../local-build/kata-deploy-binaries.sh | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 7db6acba0f..00ce5052ab 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -118,6 +118,8 @@ install_cached_component() { local current_image_version="${4}" local component_tarball_name="${5}" local component_tarball_path="${6}" + local root_hash_vanilla="${7:-""}" + local root_hash_tdx="${8:-""}" local cached_version=$(curl -sfL "${jenkins_build_url}/latest" | awk '{print $1}') || cached_version="none" local cached_image_version=$(curl -sfL "${jenkins_build_url}/latest_image" | awk '{print $1}') || cached_image_version="none" @@ -130,6 +132,14 @@ install_cached_component() { wget "${jenkins_build_url}/${component_tarball_name}" || return cleanup_and_fail wget "${jenkins_build_url}/sha256sum-${component_tarball_name}" || return cleanup_and_fail sha256sum -c "sha256sum-${component_tarball_name}" || return cleanup_and_fail + if [ -n "${root_hash_vanilla}" ]; then + wget "${jenkins_build_url}/${root_hash_vanilla}" || return cleanup_and_fail + mv "${root_hash_vanilla}" "${repo_root_dir}/tools/osbuilder/" + fi + if [ -n "${root_hash_tdx}" ]; then + wget "${jenkins_build_url}/${root_hash_tdx}" || return cleanup_and_fail + mv "${root_hash_tdx}" "${repo_root_dir}/tools/osbuilder/" + fi mv "${component_tarball_name}" "${component_tarball_path}" } @@ -161,8 +171,45 @@ install_cc_image() { image_type="${2:-image}" image_initrd_suffix="${3:-""}" root_hash_suffix="${4:-""}" + tee="${5:-""}" export KATA_BUILD_CC=yes + local jenkins="${jenkins_url}/job/kata-containers-2.0-rootfs-image-cc-$(uname -m)/${cached_artifacts_path}" + local component="rootfs-image" + local root_hash_vanilla="root_hash_vanilla.txt" + local root_hash_tdx="" + if [ -n "${tee}" ]; then + if [ "${tee}" == "tdx" ]; then + jenkins="${jenkins_url}/job/kata-containers-2.0-rootfs-image-${tee}-cc-$(uname -m)/${cached_artifacts_path}" + component="${tee}-rootfs-image" + root_hash_vanilla="" + root_hash_tdx="root_hash_${tee}.txt" + fi + fi + + local osbuilder_last_commit="$(echo $(get_last_modification "${repo_root_dir}/tools/osbuilder") | sed s/-dirty//)" + local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" + local agent_last_commit="$(get_last_modification "${repo_root_dir}/src/agent")" + local libs_last_commit="$(get_last_modification "${repo_root_dir}/src/libs")" + local attestation_agent_version="$(get_from_kata_deps "externals.attestation-agent.version")" + local gperf_version="$(get_from_kata_deps "externals.gperf.version")" + local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" + local pause_version="$(get_from_kata_deps "externals.pause.version")" + local skopeo_version="$(get_from_kata_deps "externals.skopeo.branch")" + local umoci_version="$(get_from_kata_deps "externals.umoci.tag")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + + install_cached_component \ + "${component}" \ + "${jenkins}" \ + "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${attestation_agent_version}-${gperf_version}-${libseccomp_version}-${pause_version}-${skopeo_version}-${umoci_version}-${rust_version}-${image_type}-${AA_KBC}" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + "${root_hash_vanilla}" \ + "${root_hash_tdx}" \ + && return 0 + info "Create CC image configured with AA_KBC=${AA_KBC}" "${rootfs_builder}" \ --imagetype="${image_type}" \ @@ -175,7 +222,7 @@ install_cc_image() { install_cc_sev_image() { AA_KBC="offline_sev_kbc" image_type="initrd" - install_cc_image "${AA_KBC}" "${image_type}" + install_cc_image "${AA_KBC}" "${image_type}" "sev" } install_cc_tdx_image() { @@ -183,7 +230,7 @@ install_cc_tdx_image() { image_type="image" image_suffix="tdx" root_hash_suffix="tdx" - install_cc_image "${AA_KBC}" "${image_type}" "${image_suffix}" "${root_hash_suffix}" + install_cc_image "${AA_KBC}" "${image_type}" "${image_suffix}" "${root_hash_suffix}" "tdx" } #Install CC kernel asset From de69f9c83234f8b126e4453485c4c031ba829264 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 5 Dec 2022 16:48:13 +0000 Subject: [PATCH 3/4] cache_components: Add the ability to cache the shim-v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to cache the shim-v2 we're considering the the cached component can be used if: * There were no changes in the runtime directory * There were no changes in the golang version used * There were no changes in the rust version used * We don't build the rust agent, but better be prepared for the future * There were no changes in the following files that are provided by the rootfs builds: * root_hash_vanilla.txt * root_hash_tdx.txt Signed-off-by: Gabriela Cervantes Signed-off-by: Fabiano Fidêncio --- .../static-build/cache_components.sh | 19 ++++++++++++++++++- tools/packaging/static-build/shim-v2/build.sh | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tools/packaging/static-build/cache_components.sh b/tools/packaging/static-build/cache_components.sh index d6c47782f0..193cf07c55 100755 --- a/tools/packaging/static-build/cache_components.sh +++ b/tools/packaging/static-build/cache_components.sh @@ -105,6 +105,16 @@ cache_rootfs_artifacts() { create_cache_asset "${rootfs_tarball_name}" "${current_rootfs_version}" "" "${root_hash_vanilla}" "${root_hash_tdx}" } +cache_shim_v2_artifacts() { + local shim_v2_tarball_name="kata-static-cc-shim-v2.tar.xz" + local shim_v2_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime")" + local golang_version="$(get_from_kata_deps "languages.golang.meta.newest-version")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local current_shim_v2_version="${shim_v2_last_commit}-${golang_version}-${rust_version}" + local current_shim_v2_image="$(get_shim_v2_image_name)" + create_cache_asset "${shim_v2_tarball_name}" "${current_shim_v2_version}" "${current_shim_v2_image}" "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt" "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt" +} + create_cache_asset() { local component_name="${1}" local component_version="${2}" @@ -153,6 +163,7 @@ Usage: $0 "[options]" * Requires FIRMWARE environment variable set, valid values are: * tdvf * td-shim + -s Shim v2 cache -v Virtiofsd cache -r Rootfs Cache * can receive a TEE environment variable value, valid values are: @@ -168,10 +179,11 @@ main() { local qemu_component="${qemu_component:-}" local kernel_component="${kernel_component:-}" local firmware_component="${firmware_component:-}" + local shim_v2_component="${shim_v2_component:-}" local virtiofsd_component="${virtiofsd_component:-}" local rootfs_component="${rootfs_component:-}" local OPTIND - while getopts ":ckqfvrh:" opt + while getopts ":ckqfvrsh:" opt do case "$opt" in c) @@ -186,6 +198,9 @@ main() { f) firmware_component="1" ;; + s) + shim_v2_component="1" + ;; v) virtiofsd_component="1" ;; @@ -209,6 +224,7 @@ main() { [[ -z "${kernel_component}" ]] && \ [[ -z "${qemu_component}" ]] && \ [[ -z "${firmware_component}" ]] && \ + [[ -z "${shim_v2_component}" ]] && \ [[ -z "${virtiofsd_component}" ]] && \ [[ -z "${rootfs_component}" ]] && \ help && die "Must choose at least one option" @@ -221,6 +237,7 @@ main() { [ "${kernel_component}" == "1" ] && cache_kernel_artifacts [ "${qemu_component}" == "1" ] && cache_qemu_artifacts [ "${firmware_component}" == "1" ] && cache_firmware_artifacts + [ "${shim_v2_component}" == "1" ] && cache_shim_v2_artifacts [ "${virtiofsd_component}" == "1" ] && cache_virtiofsd_artifacts [ "${rootfs_component}" == "1" ] && cache_rootfs_artifacts diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index db78cc14cd..bedf55d92c 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -49,7 +49,7 @@ if [ -n "${RUST_VERSION}" ]; then "${container_image}" \ bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" install" fi - + sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${repo_root_dir}/src/runtime" \ "${container_image}" \ From aef3e5184bcd3a3ebe826147212f3f62368f0058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 12 Dec 2022 16:47:29 +0100 Subject: [PATCH 4/4] kata-deploy-binaries: Use cached shim-v2 when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As done for different components, let's also use a cached version of the shim-v2 whenever it's possible. Fixes: #5838 Signed-off-by: Fabiano Fidêncio --- .../local-build/kata-deploy-binaries.sh | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 00ce5052ab..21294aa109 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -143,6 +143,64 @@ install_cached_component() { mv "${component_tarball_name}" "${component_tarball_path}" } +# We've to add a different cached function here as for using the shim-v2 caching +# we have to rely and check some artefacts coming from the cc-rootfs-image and the +# cc-tdx-rootfs-image jobs. +install_cached_cc_shim_v2() { + local component="${1}" + local jenkins_build_url="${2}" + local current_version="${3}" + local current_image_version="${4}" + local component_tarball_name="${5}" + local component_tarball_path="${6}" + local root_hash_vanilla="${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt" + local root_hash_tdx="${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt" + + local rootfs_image_cached_root_hash="${jenkins_url}/job/kata-containers-2.0-rootfs-image-cc-$(uname -m)/${cached_artifacts_path}/root_hash_vanilla.txt" + local tdx_rootfs_image_cached_root_hash="${jenkins_url}/job/kata-containers-2.0-rootfs-image-tdx-cc-$(uname -m)/${cached_artifacts_path}/root_hash_tdx.txt" + + + wget "${rootfs_image_cached_root_hash}" -O "rootfs_root_hash_vanilla.txt" || return 1 + if [ -f "${root_hash_vanilla}" ]; then + # There's already a pre-existent root_hash_vanilla.txt, + # let's check whether this is the same one cached on the + # rootfs job. + + # In case it's not the same, let's proceed building the + # shim-v2 with what we have locally. + diff "${root_hash_vanilla}" "rootfs_root_hash_vanilla.txt" > /dev/null || return 1 + fi + mv "rootfs_root_hash_vanilla.txt" "${root_hash_vanilla}" + + wget "${rootfs_image_cached_root_hash}" -O "rootfs_root_hash_tdx.txt" || return 1 + if [ -f "${root_hash_tdx}" ]; then + # There's already a pre-existent root_hash_tdx.txt, + # let's check whether this is the same one cached on the + # rootfs job. + + # In case it's not the same, let's proceed building the + # shim-v2 with what we have locally. + diff "${root_hash_tdx}" "rootfs_root_hash_tdx.txt" > /dev/null || return 1 + fi + mv "rootfs_root_hash_tdx.txt" "${root_hash_tdx}" + + wget "${jenkins_build_url}/root_hash_vanilla.txt" -O "shim_v2_root_hash_vanilla.txt" || return 1 + diff "${root_hash_vanilla}" "shim_v2_root_hash_vanilla.txt" > /dev/null || return 1 + + wget "${jenkins_build_url}/root_hash_tdx.txt" -O "shim_v2_root_hash_tdx.txt" || return 1 + diff "${root_hash_tdx}" "shim_v2_root_hash_tdx.txt" > /dev/null || return 1 + + install_cached_component \ + "${component}" \ + "${jenkins_build_url}" \ + "${current_version}" \ + "${current_image_version}" \ + "${component_tarball_name}" \ + "${component_tarball_path}" \ + "$(basename ${root_hash_vanilla})" \ + "$(basename ${root_hash_tdx})" +} + # Install static CC cloud-hypervisor asset install_cc_clh() { install_cached_component \ @@ -273,6 +331,20 @@ install_cc_qemu() { #Install all components that are not assets install_cc_shimv2() { + local shim_v2_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime")" + local golang_version="$(get_from_kata_deps "languages.golang.meta.newest-version")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local shim_v2_version="${shim_v2_last_commit}-${golang_version}-${rust_version}" + + install_cached_cc_shim_v2 \ + "shim-v2" \ + "${jenkins_url}/job/kata-containers-2.0-shim-v2-cc-$(uname -m)/${cached_artifacts_path}" \ + "${shim_v2_version}" \ + "$(get_shim_v2_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + GO_VERSION="$(yq r ${versions_yaml} languages.golang.meta.newest-version)" export GO_VERSION export REMOVE_VMM_CONFIGS="acrn fc"