github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-06-01 20:06:35 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #6839 from sprt/sprt/mariner-ci-tests

Browse Source 
tests: Enable running k8s tests on Mariner
This commit is contained in:
Fabiano Fidêncio 2023-07-07 13:36:28 +02:00 committed by GitHub
commit 18bd2d6e4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 181 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/build-kata-static-tarball-amd64.yaml vendored
View File

@ -22,8 +22,6 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
stage:
- ${{ inputs.stage }}
asset:
- cloud-hypervisor
- cloud-hypervisor-glibc
@ -49,9 +47,11 @@ jobs:
- shim-v2
- tdvf
- virtiofsd
stage:
- ${{ inputs.stage }}
exclude:
- stage: release
asset: cloud-hypervisor-glibc
- asset: cloud-hypervisor-glibc
stage: release
steps:
- name: Login to Kata Containers quay.io
if: ${{ inputs.push-to-registry == 'yes' }}

19
tests/integration/gha-run.sh
View File

@ -9,7 +9,8 @@ set -o nounset
set -o pipefail
integration_dir="$(dirname "$(readlink -f "$0")")"
tools_dir="${integration_dir}/../../tools"
repo_root_dir="$(cd "${integration_dir}/../../" && pwd)"
tools_dir="${repo_root_dir}/tools"
function _print_cluster_name() {
short_sha="$(git rev-parse --short=12 HEAD)"
@ -37,7 +38,7 @@ function create_cluster() {
-s "Standard_D4s_v5" \
--node-count 1 \
--generate-ssh-keys \
$([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation")
$([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation")
}
function install_bats() {
@ -55,8 +56,16 @@ function get_cluster_credentials() {
-n "$(_print_cluster_name)"
}
function ensure_yq() {
: "${GOPATH:=${GITHUB_WORKSPACE}}"
export GOPATH
export PATH="${GOPATH}/bin:${PATH}"
INSTALL_IN_GOPATH=true "${repo_root_dir}/ci/install_yq.sh"
}
function run_tests() {
platform="${1}"
ensure_yq
# Emsure we're in the default namespace
kubectl config set-context --current --namespace=default
@ -65,6 +74,10 @@ function run_tests() {
kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
fi
cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image"
@ -134,6 +147,8 @@ function delete_cluster() {
}
function main() {
export KATA_HOST_OS="${KATA_HOST_OS:-}"
action="${1:-}"
case "${action}" in

4
tests/integration/kubernetes/run_kubernetes_tests.sh
View File

@ -54,10 +54,6 @@ else
)
fi
if [ ${KATA_HOST_OS} == "cbl-mariner" ]; then
exit 0
fi
# we may need to skip a few test cases when running on non-x86_64 arch
arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml"
if [ -f "${arch_config_file}" ]; then

17
tests/integration/kubernetes/setup.sh
View File

@ -8,13 +8,30 @@ set -o nounset
set -o pipefail
kubernetes_dir=$(dirname "$(readlink -f "$0")")
repo_root_dir="$(cd "${kubernetes_dir}/../../../" && pwd)"
set_runtime_class() {
sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml
}
set_kernel_path() {
if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
mariner_kernel_path="/usr/share/cloud-hypervisor/vmlinux.bin"
find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.kernel]' "${mariner_kernel_path}" \;
fi
}
set_initrd_path() {
if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
initrd_path="/opt/kata/share/kata-containers/kata-containers-initrd-cbl-mariner.img"
find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.initrd]' "${initrd_path}" \;
fi
}
main() {
set_runtime_class
set_kernel_path
set_initrd_path
}
main "$@"

63
tools/packaging/guest-image/build_image.sh
View File

@ -22,45 +22,44 @@ readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)"
export GOPATH=${GOPATH:-${HOME}/go}
arch_target="$(uname -m)"
final_image_name="kata-containers"
final_initrd_name="kata-containers-initrd"
final_artifact_name="kata-containers"
image_initrd_extension=".img"
build_initrd() {
info "Build initrd"
info "initrd os: $initrd_distro"
info "initrd os version: $initrd_os_version"
info "initrd os: $os_name"
info "initrd os version: $os_version"
sudo -E PATH="$PATH" make initrd \
DISTRO="$initrd_distro" \
DISTRO="$os_name" \
DEBUG="${DEBUG:-}" \
OS_VERSION="${initrd_os_version}" \
OS_VERSION="${os_version}" \
ROOTFS_BUILD_DEST="${builddir}/initrd-image" \
USE_DOCKER=1 \
AGENT_INIT="yes"
mv "kata-containers-initrd.img" "${install_dir}/${initrd_name}"
mv "kata-containers-initrd.img" "${install_dir}/${artifact_name}"
(
cd "${install_dir}"
ln -sf "${initrd_name}" "${final_initrd_name}${image_initrd_extension}"
ln -sf "${artifact_name}" "${final_artifact_name}${image_initrd_extension}"
)
}
build_image() {
info "Build image"
info "image os: $img_distro"
info "image os version: $img_os_version"
info "image os: $os_name"
info "image os version: $os_version"
sudo -E PATH="${PATH}" make image \
DISTRO="${img_distro}" \
DISTRO="${os_name}" \
DEBUG="${DEBUG:-}" \
USE_DOCKER="1" \
IMG_OS_VERSION="${img_os_version}" \
IMG_OS_VERSION="${os_version}" \
ROOTFS_BUILD_DEST="${builddir}/rootfs-image"
mv -f "kata-containers.img" "${install_dir}/${image_name}"
mv -f "kata-containers.img" "${install_dir}/${artifact_name}"
if [ -e "root_hash.txt" ]; then
cp root_hash.txt "${install_dir}/"
fi
(
cd "${install_dir}"
ln -sf "${image_name}" "${final_image_name}${image_initrd_extension}"
ln -sf "${artifact_name}" "${final_artifact_name}${image_initrd_extension}"
)
}
@ -74,6 +73,8 @@ Usage:
${script_name} [options]
Options:
--osname=${os_name}
--osversion=${os_version}
--imagetype=${image_type}
--prefix=${prefix}
--destdir=${destdir}
@ -94,33 +95,20 @@ main() {
case "$opt" in
-)
case "${OPTARG}" in
osname=*)
os_name=${OPTARG#*=}
;;
osversion=*)
os_version=${OPTARG#*=}
;;
imagetype=image)
image_type=image
#image information
img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name")
img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version")
image_name="kata-${img_distro}-${img_os_version}.${image_type}"
;;
imagetype=initrd)
image_type=initrd
#initrd information
initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.name")
initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.version")
initrd_name="kata-${initrd_distro}-${initrd_os_version}.${image_type}"
;;
image_initrd_suffix=*)
image_initrd_suffix=${OPTARG#*=}
if [ "${image_initrd_suffix}" == "sev" ]; then
initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.name")
initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.version")
initrd_name="kata-${initrd_distro}-${initrd_os_version}-${image_initrd_suffix}.${image_type}"
final_initrd_name="${final_initrd_name}-${image_initrd_suffix}"
elif [ "${image_initrd_suffix}" == "tdx" ]; then
img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name")
img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version")
image_name="kata-${img_distro}-${img_os_version}-${image_initrd_suffix}.${image_type}"
final_image_name="${final_image_name}-${image_initrd_suffix}"
fi
;;
prefix=*)
prefix=${OPTARG#*=}
@ -149,7 +137,16 @@ main() {
echo "build ${image_type}"
if [ "${image_type}" = "initrd" ]; then
final_artifact_name+="-initrd"
fi
if [ -n "${image_initrd_suffix}" ]; then
artifact_name="kata-${os_name}-${os_version}-${image_initrd_suffix}.${image_type}"
final_artifact_name+="-${image_initrd_suffix}"
else
artifact_name="kata-${os_name}-${os_version}.${image_type}"
fi
install_dir="${destdir}/${prefix}/share/kata-containers/"
readonly install_dir

1
tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
View File

@ -65,6 +65,7 @@ docker run \
--env TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER:-}" \
--env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" \
--env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \
--env USE_CACHE="${USE_CACHE:-}" \
--rm \
-w ${script_dir} \
build-kata-deploy "${kata_deploy_create}" $@

105
tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
View File

@ -40,6 +40,7 @@ readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts"
ARCH=$(uname -m)
MEASURED_ROOTFS=${MEASURED_ROOTFS:-no}
USE_CACHE="${USE_CACHE:-"yes"}"
workdir="${WORKDIR:-$PWD}"
@ -79,6 +80,7 @@ options:
--build=<asset> :
all
cloud-hypervisor
cloud-hypervisor-glibc
firecracker
kernel
kernel-dragonball-experimental
@ -97,6 +99,7 @@ options:
rootfs-image
rootfs-image-tdx
rootfs-initrd
rootfs-initrd-mariner
rootfs-initrd-sev
shim-v2
tdvf
@ -113,6 +116,10 @@ cleanup_and_fail() {
}
install_cached_tarball_component() {
if [ "${USE_CACHE}" != "yes" ]; then
return 1
fi
local component="${1}"
local jenkins_build_url="${2}"
local current_version="${3}"
@ -136,8 +143,13 @@ install_cached_tarball_component() {
#Install guest image
install_image() {
local image_type="${1:-"image"}"
local initrd_suffix="${2:-""}"
local variant="${1:-}"
image_type="image"
if [ -n "${variant}" ]; then
image_type+="-${variant}"
fi
local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${image_type}-$(uname -m)/${cached_artifacts_path}"
local component="rootfs-${image_type}"
@ -152,25 +164,39 @@ install_image() {
install_cached_tarball_component \
"${component}" \
"${jenkins}" \
"${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-image" \
"${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${image_type}" \
"" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
info "Create image"
"${rootfs_builder}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}"
if [ -n "${variant}" ]; then
os_name="$(get_from_kata_deps "assets.image.architecture.${ARCH}.${variant}.name")"
os_version="$(get_from_kata_deps "assets.image.architecture.${ARCH}.${variant}.version")"
else
os_name="$(get_from_kata_deps "assets.image.architecture.${ARCH}.name")"
os_version="$(get_from_kata_deps "assets.image.architecture.${ARCH}.version")"
fi
"${rootfs_builder}" --osname="${os_name}" --osversion="${os_version}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${variant}"
}
#Install guest image for tdx
install_image_tdx() {
install_image "image-tdx" "tdx"
install_image "tdx"
}
#Install guest initrd
install_initrd() {
local initrd_type="${1:-"initrd"}"
local initrd_suffix="${2:-""}"
local variant="${1:-}"
initrd_type="initrd"
if [ -n "${variant}" ]; then
initrd_type+="-${variant}"
fi
local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-$(uname -m)/${cached_artifacts_path}"
local component="rootfs-${initrd_type}"
@ -192,12 +218,26 @@ install_initrd() {
&& return 0
info "Create initrd"
"${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}"
if [ -n "${variant}" ]; then
os_name="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.${variant}.name")"
os_version="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.${variant}.version")"
else
os_name="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.name")"
os_version="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.version")"
fi
"${rootfs_builder}" --osname="${os_name}" --osversion="${os_version}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${variant}"
}
#Install Mariner guest initrd
install_initrd_mariner() {
install_initrd "cbl-mariner"
}
#Install guest initrd for sev
install_initrd_sev() {
install_initrd "initrd-sev" "sev"
install_initrd "sev"
}
#Install kernel component helper
@ -413,26 +453,47 @@ install_firecracker() {
sudo install -D --owner root --group root --mode 0744 release-${firecracker_version}-${ARCH}/jailer-${firecracker_version}-${ARCH} "${destdir}/opt/kata/bin/jailer"
}
# Install static cloud-hypervisor asset
install_clh() {
install_clh_helper() {
libc="${1}"
features="${2}"
suffix="${3:-""}"
install_cached_tarball_component \
"cloud-hypervisor" \
"${jenkins_url}/job/kata-containers-main-clh-$(uname -m)/${cached_artifacts_path}" \
"cloud-hypervisor${suffix}" \
"${jenkins_url}/job/kata-containers-main-clh-$(uname -m)${suffix}/${cached_artifacts_path}" \
"$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \
"" \
"${final_tarball_name}" \
"${final_tarball_path}" \
&& return 0
if [[ "${ARCH}" == "x86_64" ]]; then
export features="tdx"
fi
info "build static cloud-hypervisor"
"${clh_builder}"
libc="${libc}" features="${features}" "${clh_builder}"
info "Install static cloud-hypervisor"
mkdir -p "${destdir}/opt/kata/bin/"
sudo install -D --owner root --group root --mode 0744 cloud-hypervisor/cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor"
sudo install -D --owner root --group root --mode 0744 cloud-hypervisor/cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor${suffix}"
}
# Install static cloud-hypervisor asset
install_clh() {
if [[ "${ARCH}" == "x86_64" ]]; then
features="mshv,tdx"
else
features=""
fi
install_clh_helper "musl" "${features}"
}
# Install static cloud-hypervisor-glibc asset
install_clh_glibc() {
if [[ "${ARCH}" == "x86_64" ]]; then
features="mshv"
else
features=""
fi
install_clh_helper "gnu" "${features}" "-glibc"
}
# Install static virtiofsd asset
@ -561,6 +622,7 @@ handle_build() {
install_firecracker
install_image
install_initrd
install_initrd_mariner
install_initrd_sev
install_kernel
install_kernel_dragonball_experimental
@ -578,7 +640,7 @@ handle_build() {
cloud-hypervisor) install_clh ;;
cloud-hypervisor-glibc) ;;
cloud-hypervisor-glibc) install_clh_glibc ;;
firecracker) install_firecracker ;;
@ -616,7 +678,7 @@ handle_build() {
rootfs-initrd) install_initrd ;;
rootfs-initrd-mariner) ;;
rootfs-initrd-mariner) install_initrd_mariner ;;
rootfs-initrd-sev) install_initrd_sev ;;
@ -662,6 +724,7 @@ main() {
qemu
rootfs-image
rootfs-initrd
rootfs-initrd-mariner
shim-v2
virtiofsd
)

9
tools/packaging/kata-deploy/scripts/kata-deploy.sh
View File

@ -64,6 +64,15 @@ function install_artifacts() {
chmod +x /opt/kata/bin/*
[ -d /opt/kata/runtime-rs/bin ] && \
chmod +x /opt/kata/runtime-rs/bin/*
# Allow Mariner to use custom configuration.
if [ "${HOST_OS:-}" == "cbl-mariner" ]; then
config_path="/opt/kata/share/defaults/kata-containers/configuration-clh.toml"
clh_path="/opt/kata/bin/cloud-hypervisor-glibc"
sed -i -E 's|(enable_annotations) = .+|\1 = ["enable_iommu", "initrd", "kernel"]|' "${config_path}"
sed -i -E "s|(valid_hypervisor_paths) = .+|\1 = [\"${clh_path}\"]|" "${config_path}"
sed -i -E "s|(path) = \".+/cloud-hypervisor\"|\1 = \"${clh_path}\"|" "${config_path}"
fi
}
function wait_till_node_is_ready() {

6
tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh
View File

@ -76,12 +76,12 @@ build_clh_from_source() {
if [ -n "${features}" ]; then
info "Build cloud-hypervisor enabling the following features: ${features}"
./scripts/dev_cli.sh build --release --libc musl --features "${features}"
./scripts/dev_cli.sh build --release --libc "${libc}" --features "${features}"
else
./scripts/dev_cli.sh build --release --libc musl
./scripts/dev_cli.sh build --release --libc "${libc}"
fi
rm -f cloud-hypervisor
cp build/cargo_target/$(uname -m)-unknown-linux-musl/release/cloud-hypervisor .
cp build/cargo_target/$(uname -m)-unknown-linux-${libc}/release/cloud-hypervisor .
popd
}

26
versions.yaml
View File

@ -122,17 +122,20 @@ assets:
url: "https://github.com/kata-containers/kata-containers/tools/osbuilder"
architecture:
aarch64:
name: "ubuntu"
version: "latest"
ppc64le:
name: "ubuntu"
version: "latest"
s390x:
name: "ubuntu"
version: "latest"
x86_64:
name: &default-image-name "ubuntu"
version: "latest"
version: &default-image-version "latest"
ppc64le:
name: *default-image-name
version: *default-image-version
s390x:
name: *default-image-name
version: *default-image-version
x86_64:
name: *default-image-name
version: *default-image-version
tdx:
name: *default-image-name
version: *default-image-version
meta:
image-type: *default-image-name
@ -156,6 +159,9 @@ assets:
x86_64:
name: *default-initrd-name
version: *default-initrd-version
cbl-mariner:
name: "cbl-mariner"
version: "2.0"
sev:
name: *glibc-initrd-name
version: *glibc-initrd-version