From d3742ca877f1947aa49f4c73ce216bec1657c309 Mon Sep 17 00:00:00 2001 From: Manuel Huber Date: Thu, 5 Feb 2026 15:17:17 -0800 Subject: [PATCH 1/3] tests: enable guest pull bats for force guest pull Similar to k8s-guest-pull-image-authenticated and to k8s-guest-pull-image-signature, enabling k8s-guest-pull-image to run against the experimental force guest pull method. Only k8s-guest-pull-image-encrypted requires nydus. Signed-off-by: Manuel Huber --- .../integration/kubernetes/k8s-guest-pull-image.bats | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/integration/kubernetes/k8s-guest-pull-image.bats b/tests/integration/kubernetes/k8s-guest-pull-image.bats index 25434f06f6..e878c7d19e 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image.bats @@ -8,12 +8,17 @@ load "${BATS_TEST_DIRNAME}/lib.sh" load "${BATS_TEST_DIRNAME}/confidential_common.sh" +export SNAPSHOTTER="${SNAPSHOTTER:-}" +export EXPERIMENTAL_FORCE_GUEST_PULL="${EXPERIMENTAL_FORCE_GUEST_PULL:-}" + setup() { if ! is_confidential_runtime_class; then skip "Test not supported for ${KATA_HYPERVISOR}." fi - [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ]; then + skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test" + fi setup_common || die "setup_common failed" unencrypted_image="quay.io/prometheus/busybox:latest" @@ -229,7 +234,9 @@ teardown() { skip "Test not supported for ${KATA_HYPERVISOR}." fi - [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ]; then + skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test" + fi teardown_common "${node}" "${node_start_time:-}" kubectl delete --ignore-not-found pvc trusted-pvc From 4c760fd031258f67ed0ead1bce1cf18398001cb8 Mon Sep 17 00:00:00 2001 From: Manuel Huber Date: Fri, 6 Feb 2026 13:30:19 -0800 Subject: [PATCH 2/3] build: add CONFIDENTIAL_GUEST variable for kernel This change adds the CONFIDENTIAL_GUEST variable to the kernel build logic. Similar to commit 976df22119b1f26866b308658432c252f313f80c, we would like to enable the cryptsetup functionalities not only when building a measured root file system, but also when building for a confidential guest. The current state is that not all confidential guests use a measured root filesystem, and as a matter of fact, we should indeed decouple these aspects. With the current convention, a confidential guest is a user of CDH with its storage features. A better naming of the CONFIDENTIAL_GUEST variable could have been a naming related to CDH storage functionality. Further, the kernel build script's -m parameter could be improved too - as indicated by this change, not only measured rootfs builds will need the cryptsetup.conf file. Signed-off-by: Manuel Huber --- .../local-build/kata-deploy-binaries-in-docker.sh | 2 ++ .../kata-deploy/local-build/kata-deploy-binaries.sh | 9 +++++++++ tools/packaging/static-build/kernel/build.sh | 3 ++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index 3251cad3d4..68aad88b8a 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -100,6 +100,7 @@ TOOLS_CONTAINER_BUILDER="${TOOLS_CONTAINER_BUILDER:-}" VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" AGENT_INIT="${AGENT_INIT:-no}" MEASURED_ROOTFS="${MEASURED_ROOTFS:-no}" +CONFIDENTIAL_GUEST="${CONFIDENTIAL_GUEST:-no}" USE_CACHE="${USE_CACHE:-}" BUSYBOX_CONF_FILE=${BUSYBOX_CONF_FILE:-} NVIDIA_GPU_STACK="${NVIDIA_GPU_STACK:-}" @@ -141,6 +142,7 @@ docker run \ --env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER}" \ --env AGENT_INIT="${AGENT_INIT}" \ --env MEASURED_ROOTFS="${MEASURED_ROOTFS}" \ + --env CONFIDENTIAL_GUEST="${CONFIDENTIAL_GUEST}" \ --env USE_CACHE="${USE_CACHE}" \ --env BUSYBOX_CONF_FILE="${BUSYBOX_CONF_FILE}" \ --env NVIDIA_GPU_STACK="${NVIDIA_GPU_STACK}" \ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 7e26956cac..f584690bfb 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -43,6 +43,7 @@ readonly se_image_builder="${repo_root_dir}/tools/packaging/guest-image/build_se ARCH=${ARCH:-$(uname -m)} BUSYBOX_CONF_FILE="${BUSYBOX_CONF_FILE:-}" MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} +CONFIDENTIAL_GUEST=${CONFIDENTIAL_GUEST:-no} USE_CACHE="${USE_CACHE:-"yes"}" ARTEFACT_REGISTRY="${ARTEFACT_REGISTRY:-ghcr.io}" ARTEFACT_REPOSITORY="${ARTEFACT_REPOSITORY:-kata-containers}" @@ -452,6 +453,7 @@ install_image() { #Install guest image for confidential guests install_image_confidential() { + export CONFIDENTIAL_GUEST="yes" if [ "${ARCH}" == "s390x" ]; then export MEASURED_ROOTFS="no" else @@ -563,6 +565,7 @@ install_initrd() { #Install guest initrd for confidential guests install_initrd_confidential() { + export CONFIDENTIAL_GUEST="yes" export MEASURED_ROOTFS="no" install_initrd "confidential" } @@ -609,6 +612,7 @@ install_initrd_nvidia_gpu() { # Instal NVIDIA GPU confidential image install_image_nvidia_gpu_confidential() { + export CONFIDENTIAL_GUEST="yes" export AGENT_POLICY export MEASURED_ROOTFS="yes" local version=$(get_from_kata_deps .externals.nvidia.driver.version) @@ -619,6 +623,7 @@ install_image_nvidia_gpu_confidential() { # Install NVIDIA GPU confidential initrd install_initrd_nvidia_gpu_confidential() { + export CONFIDENTIAL_GUEST="yes" export AGENT_POLICY export MEASURED_ROOTFS="no" local version=$(get_from_kata_deps .externals.nvidia.driver.version) @@ -726,10 +731,12 @@ install_kernel() { local extra_cmd="" case "${ARCH}" in s390x) + export CONFIDENTIAL_GUEST="yes" export MEASURED_ROOTFS="no" extra_cmd="-x" ;; x86_64) + export CONFIDENTIAL_GUEST="yes" export MEASURED_ROOTFS="yes" extra_cmd="-x" ;; @@ -741,6 +748,7 @@ install_kernel() { } install_kernel_cca_confidential() { + export CONFIDENTIAL_GUEST="yes" export MEASURED_ROOTFS="yes" install_kernel_helper \ @@ -765,6 +773,7 @@ install_kernel_nvidia_gpu_dragonball_experimental() { #Install GPU enabled kernel asset install_kernel_nvidia_gpu() { + export CONFIDENTIAL_GUEST="yes" export MEASURED_ROOTFS="yes" install_kernel_helper \ "assets.kernel.nvidia" \ diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index f8f1633b9e..c84fc493e8 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -26,11 +26,12 @@ DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}" MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} +CONFIDENTIAL_GUEST=${CONFIDENTIAL_GUEST:-no} KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN:-}" kernel_builder_args="-a ${ARCH:-} $*" KERNEL_DEBUG_ENABLED=${KERNEL_DEBUG_ENABLED:-"no"} -if [[ "${MEASURED_ROOTFS}" == "yes" ]]; then +if [[ "${MEASURED_ROOTFS}" == "yes" ]] || [[ "${CONFIDENTIAL_GUEST}" == "yes" ]]; then kernel_builder_args+=" -m" fi From fd340ac91cfd29a74a2f5ff442755c244cb7f43f Mon Sep 17 00:00:00 2001 From: Manuel Huber Date: Fri, 6 Feb 2026 11:51:41 -0800 Subject: [PATCH 3/3] tests: remove skips for some guest-pull scenarios Issue 10838 is resolved by the prior commit, enabling the -m option of the kernel build for confidential guests which are not users of the measured rootfs, and by commit 976df22119b1f26866b308658432c252f313f80c, which ensures relevant user space packages are present. Not every confidential guest has the measured rootfs option enabled. Every confidential guest is assumed to support CDH's secure storage features, in contrast. We also adjust test timeouts to account for occasional spikes on our bare metal runners (e.g., SNP, TDX, s390x). Signed-off-by: Manuel Huber --- .../kubernetes/k8s-guest-pull-image.bats | 42 ++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/tests/integration/kubernetes/k8s-guest-pull-image.bats b/tests/integration/kubernetes/k8s-guest-pull-image.bats index e878c7d19e..abe4075c9f 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image.bats @@ -92,9 +92,6 @@ setup() { } @test "Test we can pull an image inside the guest using trusted storage" { - [ "$(uname -m)" == "s390x" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-snp" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" # The image pulled in the guest will be downloaded and unpacked in the `/run/kata-containers/image` directory. # The tests will use `cryptsetup` to encrypt a block device and mount it at `/run/kata-containers/image`. @@ -112,14 +109,18 @@ setup() { pod_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${pod_config_template}").XXX") IMAGE="$image_pulled_time_less_than_default_time" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config" - - # Set CreateContainerRequest timeout for qemu-coco-dev - if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]]; then - create_container_timeout=300 - set_metadata_annotation "$pod_config" \ - "io.katacontainers.config.runtime.create_container_timeout" \ - "${create_container_timeout}" + # Set CreateContainerRequest timeout in the annotation to allow for enough time for guest-pull where + # the container remains in 'creating' state until the pull completes. Usually pulling this and the large image in + # below test takes 30-60 seconds, but we occasionally observe spikes on all our bare-metal runners. + create_container_timeout=300 + # On AKS, so far, these spikes have not been observed. Issue 10299, as referenced in other parts of this test, tells us + # that we cannot modify the runtimeRequestTimeout on AKS. We hence set the timeout to the 120s default value. + if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && [ "${KBS_INGRESS}" = "aks" ]; then + create_container_timeout=120 fi + set_metadata_annotation "$pod_config" \ + "io.katacontainers.config.runtime.create_container_timeout" \ + "${create_container_timeout}" # Set annotation to pull image in guest set_metadata_annotation "${pod_config}" \ @@ -131,16 +132,14 @@ setup() { cat $pod_config add_allow_all_policy_to_yaml "$pod_config" - local wait_time=120 - [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && wait_time=300 + local wait_time=300 + if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && [ "${KBS_INGRESS}" = "aks" ]; then + wait_time=120 + fi k8s_create_pod "$pod_config" "$wait_time" } @test "Test we cannot pull a large image that pull time exceeds createcontainer timeout inside the guest" { - [ "$(uname -m)" == "s390x" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-snp" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - storage_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${storage_config_template}").XXX") local_device=$(create_loop_device) LOCAL_DEVICE="$local_device" NODE_NAME="$node" envsubst < "$storage_config_template" > "$storage_config" @@ -186,10 +185,6 @@ setup() { } @test "Test we can pull a large image inside the guest with large createcontainer timeout" { - [ "$(uname -m)" == "s390x" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-snp" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838" - if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && [ "${KBS_INGRESS}" = "aks" ]; then skip "skip this specific one due to issue https://github.com/kata-containers/kata-containers/issues/10299" fi @@ -208,8 +203,8 @@ setup() { IMAGE="$large_image" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config" # Set CreateContainerRequest timeout in the annotation to pull large image in guest - create_container_timeout=120 - [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && create_container_timeout=600 + # Bare-metal CI runners' kubelets are configured with an equivalent runtimeRequestTimeout of 600s + create_container_timeout=600 set_metadata_annotation "$pod_config" \ "io.katacontainers.config.runtime.create_container_timeout" \ "${create_container_timeout}" @@ -224,8 +219,7 @@ setup() { cat $pod_config add_allow_all_policy_to_yaml "$pod_config" - local wait_time=120 - [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && wait_time=600 + local wait_time=600 k8s_create_pod "$pod_config" "$wait_time" }