diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index 419db1b64a..f0f606850b 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -22,8 +22,6 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - stage: - - ${{ inputs.stage }} asset: - cloud-hypervisor - cloud-hypervisor-glibc @@ -49,9 +47,11 @@ jobs: - shim-v2 - tdvf - virtiofsd + stage: + - ${{ inputs.stage }} exclude: - - stage: release - asset: cloud-hypervisor-glibc + - asset: cloud-hypervisor-glibc + stage: release steps: - name: Login to Kata Containers quay.io if: ${{ inputs.push-to-registry == 'yes' }} diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh index 11499283fc..53d9da08a7 100755 --- a/tests/integration/gha-run.sh +++ b/tests/integration/gha-run.sh @@ -9,7 +9,8 @@ set -o nounset set -o pipefail integration_dir="$(dirname "$(readlink -f "$0")")" -tools_dir="${integration_dir}/../../tools" +repo_root_dir="$(cd "${integration_dir}/../../" && pwd)" +tools_dir="${repo_root_dir}/tools" function _print_cluster_name() { short_sha="$(git rev-parse --short=12 HEAD)" @@ -37,7 +38,7 @@ function create_cluster() { -s "Standard_D4s_v5" \ --node-count 1 \ --generate-ssh-keys \ - $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation") + $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation") } function install_bats() { @@ -55,8 +56,16 @@ function get_cluster_credentials() { -n "$(_print_cluster_name)" } +function ensure_yq() { + : "${GOPATH:=${GITHUB_WORKSPACE}}" + export GOPATH + export PATH="${GOPATH}/bin:${PATH}" + INSTALL_IN_GOPATH=true "${repo_root_dir}/ci/install_yq.sh" +} + function run_tests() { platform="${1}" + ensure_yq # Emsure we're in the default namespace kubectl config set-context --current --namespace=default @@ -65,6 +74,10 @@ function run_tests() { kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then + yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS" + yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}" + fi cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image" @@ -134,6 +147,8 @@ function delete_cluster() { } function main() { + export KATA_HOST_OS="${KATA_HOST_OS:-}" + action="${1:-}" case "${action}" in diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 0975ec0d5b..db1e16633c 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -54,10 +54,6 @@ else ) fi -if [ ${KATA_HOST_OS} == "cbl-mariner" ]; then - exit 0 -fi - # we may need to skip a few test cases when running on non-x86_64 arch arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml" if [ -f "${arch_config_file}" ]; then diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index 0c3baf2dc0..6984ad286c 100755 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -8,13 +8,30 @@ set -o nounset set -o pipefail kubernetes_dir=$(dirname "$(readlink -f "$0")") +repo_root_dir="$(cd "${kubernetes_dir}/../../../" && pwd)" set_runtime_class() { sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml } +set_kernel_path() { + if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then + mariner_kernel_path="/usr/share/cloud-hypervisor/vmlinux.bin" + find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.kernel]' "${mariner_kernel_path}" \; + fi +} + +set_initrd_path() { + if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then + initrd_path="/opt/kata/share/kata-containers/kata-containers-initrd-cbl-mariner.img" + find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.initrd]' "${initrd_path}" \; + fi +} + main() { set_runtime_class + set_kernel_path + set_initrd_path } main "$@" diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index 230538d1cc..fad6646517 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -22,45 +22,44 @@ readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)" export GOPATH=${GOPATH:-${HOME}/go} arch_target="$(uname -m)" -final_image_name="kata-containers" -final_initrd_name="kata-containers-initrd" +final_artifact_name="kata-containers" image_initrd_extension=".img" build_initrd() { info "Build initrd" - info "initrd os: $initrd_distro" - info "initrd os version: $initrd_os_version" + info "initrd os: $os_name" + info "initrd os version: $os_version" sudo -E PATH="$PATH" make initrd \ - DISTRO="$initrd_distro" \ + DISTRO="$os_name" \ DEBUG="${DEBUG:-}" \ - OS_VERSION="${initrd_os_version}" \ + OS_VERSION="${os_version}" \ ROOTFS_BUILD_DEST="${builddir}/initrd-image" \ USE_DOCKER=1 \ AGENT_INIT="yes" - mv "kata-containers-initrd.img" "${install_dir}/${initrd_name}" + mv "kata-containers-initrd.img" "${install_dir}/${artifact_name}" ( cd "${install_dir}" - ln -sf "${initrd_name}" "${final_initrd_name}${image_initrd_extension}" + ln -sf "${artifact_name}" "${final_artifact_name}${image_initrd_extension}" ) } build_image() { info "Build image" - info "image os: $img_distro" - info "image os version: $img_os_version" + info "image os: $os_name" + info "image os version: $os_version" sudo -E PATH="${PATH}" make image \ - DISTRO="${img_distro}" \ + DISTRO="${os_name}" \ DEBUG="${DEBUG:-}" \ USE_DOCKER="1" \ - IMG_OS_VERSION="${img_os_version}" \ + IMG_OS_VERSION="${os_version}" \ ROOTFS_BUILD_DEST="${builddir}/rootfs-image" - mv -f "kata-containers.img" "${install_dir}/${image_name}" + mv -f "kata-containers.img" "${install_dir}/${artifact_name}" if [ -e "root_hash.txt" ]; then cp root_hash.txt "${install_dir}/" fi ( cd "${install_dir}" - ln -sf "${image_name}" "${final_image_name}${image_initrd_extension}" + ln -sf "${artifact_name}" "${final_artifact_name}${image_initrd_extension}" ) } @@ -74,6 +73,8 @@ Usage: ${script_name} [options] Options: + --osname=${os_name} + --osversion=${os_version} --imagetype=${image_type} --prefix=${prefix} --destdir=${destdir} @@ -94,33 +95,20 @@ main() { case "$opt" in -) case "${OPTARG}" in + osname=*) + os_name=${OPTARG#*=} + ;; + osversion=*) + os_version=${OPTARG#*=} + ;; imagetype=image) image_type=image - #image information - img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name") - img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version") - image_name="kata-${img_distro}-${img_os_version}.${image_type}" ;; imagetype=initrd) image_type=initrd - #initrd information - initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.name") - initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.version") - initrd_name="kata-${initrd_distro}-${initrd_os_version}.${image_type}" ;; image_initrd_suffix=*) image_initrd_suffix=${OPTARG#*=} - if [ "${image_initrd_suffix}" == "sev" ]; then - initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.name") - initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.version") - initrd_name="kata-${initrd_distro}-${initrd_os_version}-${image_initrd_suffix}.${image_type}" - final_initrd_name="${final_initrd_name}-${image_initrd_suffix}" - elif [ "${image_initrd_suffix}" == "tdx" ]; then - img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name") - img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version") - image_name="kata-${img_distro}-${img_os_version}-${image_initrd_suffix}.${image_type}" - final_image_name="${final_image_name}-${image_initrd_suffix}" - fi ;; prefix=*) prefix=${OPTARG#*=} @@ -149,7 +137,16 @@ main() { echo "build ${image_type}" + if [ "${image_type}" = "initrd" ]; then + final_artifact_name+="-initrd" + fi + if [ -n "${image_initrd_suffix}" ]; then + artifact_name="kata-${os_name}-${os_version}-${image_initrd_suffix}.${image_type}" + final_artifact_name+="-${image_initrd_suffix}" + else + artifact_name="kata-${os_name}-${os_version}.${image_type}" + fi install_dir="${destdir}/${prefix}/share/kata-containers/" readonly install_dir diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index 11589c88a1..fc82082c4d 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -65,6 +65,7 @@ docker run \ --env TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER:-}" \ --env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" \ --env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \ + --env USE_CACHE="${USE_CACHE:-}" \ --rm \ -w ${script_dir} \ build-kata-deploy "${kata_deploy_create}" $@ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 3db8c76d0a..2342d4b02e 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -40,6 +40,7 @@ readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts" ARCH=$(uname -m) MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} +USE_CACHE="${USE_CACHE:-"yes"}" workdir="${WORKDIR:-$PWD}" @@ -79,6 +80,7 @@ options: --build= : all cloud-hypervisor + cloud-hypervisor-glibc firecracker kernel kernel-dragonball-experimental @@ -97,6 +99,7 @@ options: rootfs-image rootfs-image-tdx rootfs-initrd + rootfs-initrd-mariner rootfs-initrd-sev shim-v2 tdvf @@ -113,6 +116,10 @@ cleanup_and_fail() { } install_cached_tarball_component() { + if [ "${USE_CACHE}" != "yes" ]; then + return 1 + fi + local component="${1}" local jenkins_build_url="${2}" local current_version="${3}" @@ -136,8 +143,13 @@ install_cached_tarball_component() { #Install guest image install_image() { - local image_type="${1:-"image"}" - local initrd_suffix="${2:-""}" + local variant="${1:-}" + + image_type="image" + if [ -n "${variant}" ]; then + image_type+="-${variant}" + fi + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${image_type}-$(uname -m)/${cached_artifacts_path}" local component="rootfs-${image_type}" @@ -152,25 +164,39 @@ install_image() { install_cached_tarball_component \ "${component}" \ "${jenkins}" \ - "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-image" \ + "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${image_type}" \ "" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 info "Create image" - "${rootfs_builder}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}" + + if [ -n "${variant}" ]; then + os_name="$(get_from_kata_deps "assets.image.architecture.${ARCH}.${variant}.name")" + os_version="$(get_from_kata_deps "assets.image.architecture.${ARCH}.${variant}.version")" + else + os_name="$(get_from_kata_deps "assets.image.architecture.${ARCH}.name")" + os_version="$(get_from_kata_deps "assets.image.architecture.${ARCH}.version")" + fi + + "${rootfs_builder}" --osname="${os_name}" --osversion="${os_version}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${variant}" } #Install guest image for tdx install_image_tdx() { - install_image "image-tdx" "tdx" + install_image "tdx" } #Install guest initrd install_initrd() { - local initrd_type="${1:-"initrd"}" - local initrd_suffix="${2:-""}" + local variant="${1:-}" + + initrd_type="initrd" + if [ -n "${variant}" ]; then + initrd_type+="-${variant}" + fi + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-$(uname -m)/${cached_artifacts_path}" local component="rootfs-${initrd_type}" @@ -192,12 +218,26 @@ install_initrd() { && return 0 info "Create initrd" - "${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}" + + if [ -n "${variant}" ]; then + os_name="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.${variant}.name")" + os_version="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.${variant}.version")" + else + os_name="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.name")" + os_version="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.version")" + fi + + "${rootfs_builder}" --osname="${os_name}" --osversion="${os_version}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${variant}" +} + +#Install Mariner guest initrd +install_initrd_mariner() { + install_initrd "cbl-mariner" } #Install guest initrd for sev install_initrd_sev() { - install_initrd "initrd-sev" "sev" + install_initrd "sev" } #Install kernel component helper @@ -413,26 +453,47 @@ install_firecracker() { sudo install -D --owner root --group root --mode 0744 release-${firecracker_version}-${ARCH}/jailer-${firecracker_version}-${ARCH} "${destdir}/opt/kata/bin/jailer" } -# Install static cloud-hypervisor asset -install_clh() { +install_clh_helper() { + libc="${1}" + features="${2}" + suffix="${3:-""}" + install_cached_tarball_component \ - "cloud-hypervisor" \ - "${jenkins_url}/job/kata-containers-main-clh-$(uname -m)/${cached_artifacts_path}" \ + "cloud-hypervisor${suffix}" \ + "${jenkins_url}/job/kata-containers-main-clh-$(uname -m)${suffix}/${cached_artifacts_path}" \ "$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \ "" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 - if [[ "${ARCH}" == "x86_64" ]]; then - export features="tdx" - fi - info "build static cloud-hypervisor" - "${clh_builder}" + libc="${libc}" features="${features}" "${clh_builder}" info "Install static cloud-hypervisor" mkdir -p "${destdir}/opt/kata/bin/" - sudo install -D --owner root --group root --mode 0744 cloud-hypervisor/cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor" + sudo install -D --owner root --group root --mode 0744 cloud-hypervisor/cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor${suffix}" +} + +# Install static cloud-hypervisor asset +install_clh() { + if [[ "${ARCH}" == "x86_64" ]]; then + features="mshv,tdx" + else + features="" + fi + + install_clh_helper "musl" "${features}" +} + +# Install static cloud-hypervisor-glibc asset +install_clh_glibc() { + if [[ "${ARCH}" == "x86_64" ]]; then + features="mshv" + else + features="" + fi + + install_clh_helper "gnu" "${features}" "-glibc" } # Install static virtiofsd asset @@ -561,6 +622,7 @@ handle_build() { install_firecracker install_image install_initrd + install_initrd_mariner install_initrd_sev install_kernel install_kernel_dragonball_experimental @@ -578,7 +640,7 @@ handle_build() { cloud-hypervisor) install_clh ;; - cloud-hypervisor-glibc) ;; + cloud-hypervisor-glibc) install_clh_glibc ;; firecracker) install_firecracker ;; @@ -616,7 +678,7 @@ handle_build() { rootfs-initrd) install_initrd ;; - rootfs-initrd-mariner) ;; + rootfs-initrd-mariner) install_initrd_mariner ;; rootfs-initrd-sev) install_initrd_sev ;; @@ -662,6 +724,7 @@ main() { qemu rootfs-image rootfs-initrd + rootfs-initrd-mariner shim-v2 virtiofsd ) diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 6bb660198d..09d27cc654 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -64,6 +64,15 @@ function install_artifacts() { chmod +x /opt/kata/bin/* [ -d /opt/kata/runtime-rs/bin ] && \ chmod +x /opt/kata/runtime-rs/bin/* + + # Allow Mariner to use custom configuration. + if [ "${HOST_OS:-}" == "cbl-mariner" ]; then + config_path="/opt/kata/share/defaults/kata-containers/configuration-clh.toml" + clh_path="/opt/kata/bin/cloud-hypervisor-glibc" + sed -i -E 's|(enable_annotations) = .+|\1 = ["enable_iommu", "initrd", "kernel"]|' "${config_path}" + sed -i -E "s|(valid_hypervisor_paths) = .+|\1 = [\"${clh_path}\"]|" "${config_path}" + sed -i -E "s|(path) = \".+/cloud-hypervisor\"|\1 = \"${clh_path}\"|" "${config_path}" + fi } function wait_till_node_is_ready() { diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh index 975a517a13..f381897bcd 100755 --- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh +++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh @@ -76,12 +76,12 @@ build_clh_from_source() { if [ -n "${features}" ]; then info "Build cloud-hypervisor enabling the following features: ${features}" - ./scripts/dev_cli.sh build --release --libc musl --features "${features}" + ./scripts/dev_cli.sh build --release --libc "${libc}" --features "${features}" else - ./scripts/dev_cli.sh build --release --libc musl + ./scripts/dev_cli.sh build --release --libc "${libc}" fi rm -f cloud-hypervisor - cp build/cargo_target/$(uname -m)-unknown-linux-musl/release/cloud-hypervisor . + cp build/cargo_target/$(uname -m)-unknown-linux-${libc}/release/cloud-hypervisor . popd } diff --git a/versions.yaml b/versions.yaml index c26a68134b..6695834ca4 100644 --- a/versions.yaml +++ b/versions.yaml @@ -122,17 +122,20 @@ assets: url: "https://github.com/kata-containers/kata-containers/tools/osbuilder" architecture: aarch64: - name: "ubuntu" - version: "latest" - ppc64le: - name: "ubuntu" - version: "latest" - s390x: - name: "ubuntu" - version: "latest" - x86_64: name: &default-image-name "ubuntu" - version: "latest" + version: &default-image-version "latest" + ppc64le: + name: *default-image-name + version: *default-image-version + s390x: + name: *default-image-name + version: *default-image-version + x86_64: + name: *default-image-name + version: *default-image-version + tdx: + name: *default-image-name + version: *default-image-version meta: image-type: *default-image-name @@ -156,6 +159,9 @@ assets: x86_64: name: *default-initrd-name version: *default-initrd-version + cbl-mariner: + name: "cbl-mariner" + version: "2.0" sev: name: *glibc-initrd-name version: *glibc-initrd-version