diff --git a/tests/integration/kubernetes/k8s-guest-pull-image.bats b/tests/integration/kubernetes/k8s-guest-pull-image.bats index 25434f06f6..abe4075c9f 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image.bats @@ -8,12 +8,17 @@ load "${BATS_TEST_DIRNAME}/lib.sh" load "${BATS_TEST_DIRNAME}/confidential_common.sh" +export SNAPSHOTTER="${SNAPSHOTTER:-}" +export EXPERIMENTAL_FORCE_GUEST_PULL="${EXPERIMENTAL_FORCE_GUEST_PULL:-}" + setup() { if ! is_confidential_runtime_class; then skip "Test not supported for ${KATA_HYPERVISOR}." fi - [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ]; then + skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test" + fi setup_common || die "setup_common failed" unencrypted_image="quay.io/prometheus/busybox:latest" @@ -87,9 +92,6 @@ setup() { } @test "Test we can pull an image inside the guest using trusted storage" { - [ "$(uname -m)" == "s390x" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-snp" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" # The image pulled in the guest will be downloaded and unpacked in the `/run/kata-containers/image` directory. # The tests will use `cryptsetup` to encrypt a block device and mount it at `/run/kata-containers/image`. @@ -107,14 +109,18 @@ setup() { pod_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${pod_config_template}").XXX") IMAGE="$image_pulled_time_less_than_default_time" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config" - - # Set CreateContainerRequest timeout for qemu-coco-dev - if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]]; then - create_container_timeout=300 - set_metadata_annotation "$pod_config" \ - "io.katacontainers.config.runtime.create_container_timeout" \ - "${create_container_timeout}" + # Set CreateContainerRequest timeout in the annotation to allow for enough time for guest-pull where + # the container remains in 'creating' state until the pull completes. Usually pulling this and the large image in + # below test takes 30-60 seconds, but we occasionally observe spikes on all our bare-metal runners. + create_container_timeout=300 + # On AKS, so far, these spikes have not been observed. Issue 10299, as referenced in other parts of this test, tells us + # that we cannot modify the runtimeRequestTimeout on AKS. We hence set the timeout to the 120s default value. + if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && [ "${KBS_INGRESS}" = "aks" ]; then + create_container_timeout=120 fi + set_metadata_annotation "$pod_config" \ + "io.katacontainers.config.runtime.create_container_timeout" \ + "${create_container_timeout}" # Set annotation to pull image in guest set_metadata_annotation "${pod_config}" \ @@ -126,16 +132,14 @@ setup() { cat $pod_config add_allow_all_policy_to_yaml "$pod_config" - local wait_time=120 - [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && wait_time=300 + local wait_time=300 + if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && [ "${KBS_INGRESS}" = "aks" ]; then + wait_time=120 + fi k8s_create_pod "$pod_config" "$wait_time" } @test "Test we cannot pull a large image that pull time exceeds createcontainer timeout inside the guest" { - [ "$(uname -m)" == "s390x" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-snp" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - storage_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${storage_config_template}").XXX") local_device=$(create_loop_device) LOCAL_DEVICE="$local_device" NODE_NAME="$node" envsubst < "$storage_config_template" > "$storage_config" @@ -181,10 +185,6 @@ setup() { } @test "Test we can pull a large image inside the guest with large createcontainer timeout" { - [ "$(uname -m)" == "s390x" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-snp" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && [ "${KBS_INGRESS}" = "aks" ]; then skip "skip this specific one due to issue https://github.com/kata-containers/kata-containers/issues/10299" fi @@ -203,8 +203,8 @@ setup() { IMAGE="$large_image" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config" # Set CreateContainerRequest timeout in the annotation to pull large image in guest - create_container_timeout=120 - [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && create_container_timeout=600 + # Bare-metal CI runners' kubelets are configured with an equivalent runtimeRequestTimeout of 600s + create_container_timeout=600 set_metadata_annotation "$pod_config" \ "io.katacontainers.config.runtime.create_container_timeout" \ "${create_container_timeout}" @@ -219,8 +219,7 @@ setup() { cat $pod_config add_allow_all_policy_to_yaml "$pod_config" - local wait_time=120 - [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && wait_time=600 + local wait_time=600 k8s_create_pod "$pod_config" "$wait_time" } @@ -229,7 +228,9 @@ teardown() { skip "Test not supported for ${KATA_HYPERVISOR}." fi - [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ]; then + skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test" + fi teardown_common "${node}" "${node_start_time:-}" kubectl delete --ignore-not-found pvc trusted-pvc diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index 3251cad3d4..68aad88b8a 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -100,6 +100,7 @@ TOOLS_CONTAINER_BUILDER="${TOOLS_CONTAINER_BUILDER:-}" VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" AGENT_INIT="${AGENT_INIT:-no}" MEASURED_ROOTFS="${MEASURED_ROOTFS:-no}" +CONFIDENTIAL_GUEST="${CONFIDENTIAL_GUEST:-no}" USE_CACHE="${USE_CACHE:-}" BUSYBOX_CONF_FILE=${BUSYBOX_CONF_FILE:-} NVIDIA_GPU_STACK="${NVIDIA_GPU_STACK:-}" @@ -141,6 +142,7 @@ docker run \ --env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER}" \ --env AGENT_INIT="${AGENT_INIT}" \ --env MEASURED_ROOTFS="${MEASURED_ROOTFS}" \ + --env CONFIDENTIAL_GUEST="${CONFIDENTIAL_GUEST}" \ --env USE_CACHE="${USE_CACHE}" \ --env BUSYBOX_CONF_FILE="${BUSYBOX_CONF_FILE}" \ --env NVIDIA_GPU_STACK="${NVIDIA_GPU_STACK}" \ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 667bbf21ba..3c535a242d 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -43,6 +43,7 @@ readonly se_image_builder="${repo_root_dir}/tools/packaging/guest-image/build_se ARCH=${ARCH:-$(uname -m)} BUSYBOX_CONF_FILE="${BUSYBOX_CONF_FILE:-}" MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} +CONFIDENTIAL_GUEST=${CONFIDENTIAL_GUEST:-no} USE_CACHE="${USE_CACHE:-"yes"}" ARTEFACT_REGISTRY="${ARTEFACT_REGISTRY:-ghcr.io}" ARTEFACT_REPOSITORY="${ARTEFACT_REPOSITORY:-kata-containers}" @@ -452,6 +453,7 @@ install_image() { #Install guest image for confidential guests install_image_confidential() { + export CONFIDENTIAL_GUEST="yes" if [ "${ARCH}" == "s390x" ]; then export MEASURED_ROOTFS="no" else @@ -563,6 +565,7 @@ install_initrd() { #Install guest initrd for confidential guests install_initrd_confidential() { + export CONFIDENTIAL_GUEST="yes" export MEASURED_ROOTFS="no" install_initrd "confidential" } @@ -609,6 +612,7 @@ install_initrd_nvidia_gpu() { # Instal NVIDIA GPU confidential image install_image_nvidia_gpu_confidential() { + export CONFIDENTIAL_GUEST="yes" export AGENT_POLICY export MEASURED_ROOTFS="yes" local version=$(get_from_kata_deps .externals.nvidia.driver.version) @@ -619,6 +623,7 @@ install_image_nvidia_gpu_confidential() { # Install NVIDIA GPU confidential initrd install_initrd_nvidia_gpu_confidential() { + export CONFIDENTIAL_GUEST="yes" export AGENT_POLICY export MEASURED_ROOTFS="no" local version=$(get_from_kata_deps .externals.nvidia.driver.version) @@ -726,10 +731,12 @@ install_kernel() { local extra_cmd="" case "${ARCH}" in s390x) + export CONFIDENTIAL_GUEST="yes" export MEASURED_ROOTFS="no" extra_cmd="-x" ;; x86_64) + export CONFIDENTIAL_GUEST="yes" export MEASURED_ROOTFS="yes" extra_cmd="-x" ;; @@ -741,6 +748,7 @@ install_kernel() { } install_kernel_cca_confidential() { + export CONFIDENTIAL_GUEST="yes" export MEASURED_ROOTFS="yes" install_kernel_helper \ @@ -765,6 +773,7 @@ install_kernel_nvidia_gpu_dragonball_experimental() { #Install GPU enabled kernel asset install_kernel_nvidia_gpu() { + export CONFIDENTIAL_GUEST="yes" export MEASURED_ROOTFS="yes" install_kernel_helper \ "assets.kernel.nvidia" \ diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index f8f1633b9e..c84fc493e8 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -26,11 +26,12 @@ DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}" MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} +CONFIDENTIAL_GUEST=${CONFIDENTIAL_GUEST:-no} KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN:-}" kernel_builder_args="-a ${ARCH:-} $*" KERNEL_DEBUG_ENABLED=${KERNEL_DEBUG_ENABLED:-"no"} -if [[ "${MEASURED_ROOTFS}" == "yes" ]]; then +if [[ "${MEASURED_ROOTFS}" == "yes" ]] || [[ "${CONFIDENTIAL_GUEST}" == "yes" ]]; then kernel_builder_args+=" -m" fi