From f2de259387391badcf036c19b01532891429222a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 20 Mar 2024 09:39:45 +0100 Subject: [PATCH 01/10] runtime: tdx: Use shared_fs=none MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We shouldn't be using 9p, at all, with TEEs, as off right now we have no way to ensure the channels are encrypted. The way to work this around for now is using guest pull, either with containerd + nydus snapshotter or with CRI-O; or even tardev snapshotter for pulling on the host (which is the approach used by MSFT). This is only done for TDX for now, leaving the generic, AMD, and IBM related stuff for the folks working on those to switch and debug possible issues on their environment. Signed-off-by: Fabiano Fidêncio --- src/runtime/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 901bceb4e5..d7f058c3e1 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -246,7 +246,7 @@ DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs # Please keep DEFSHAREDFS_QEMU_COCO_DEV_VIRTIOFS in sync with TDX/SEV/SNP DEFSHAREDFS_QEMU_COCO_DEV_VIRTIOFS := virtio-9p DEFSHAREDFS_STRATOVIRT_VIRTIOFS := virtio-fs -DEFSHAREDFS_QEMU_TDX_VIRTIOFS := virtio-9p +DEFSHAREDFS_QEMU_TDX_VIRTIOFS := none DEFSHAREDFS_QEMU_SEV_VIRTIOFS := virtio-9p DEFSHAREDFS_QEMU_SNP_VIRTIOFS := virtio-9p DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd From e84619d54be6def81d361af94ce1ee02c4773be0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 19 May 2024 14:33:02 +0200 Subject: [PATCH 02/10] tests: k8s: tdx: Add `add_runtime_handler_annotations` function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This function will set the needed annotation for enforcing that the image pull will be handled by the snapshotter set for the runtime handler, instead of using the default one. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/setup.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index 41538537f5..c61614fc1e 100755 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -12,6 +12,7 @@ DEBUG="${DEBUG:-}" export AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-no}" export KATA_HOST_OS="${KATA_HOST_OS:-}" +export KATA_HYPERVISOR="${KATA_HYPERVISOR:-}" if [ -n "${K8S_TEST_POLICY_FILES:-}" ]; then K8S_TEST_POLICY_FILES=($K8S_TEST_POLICY_FILES) @@ -102,10 +103,25 @@ add_cbl_mariner_kernel_initrd_annotations() { fi } +add_runtime_handler_annotations() { + case "${KATA_HYPERVISOR}" in + qemu-tdx) + info "Add runtime handler annotations for ${KATA_HYPERVISOR}" + local handler_annotation="io.containerd.cri.runtime-handler" + local handler_value="kata-${KATA_HYPERVISOR}" + for K8S_TEST_YAML in runtimeclass_workloads_work/*.yaml + do + add_annotations_to_yaml "${K8S_TEST_YAML}" "${handler_annotation}" "${handler_value}" + done + ;; + esac +} + main() { ensure_yq reset_workloads_work_dir add_cbl_mariner_kernel_initrd_annotations + add_runtime_handler_annotations } main "$@" From 3f38309c39557668e1578559efa0682d2ce99408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 19 May 2024 14:34:51 +0200 Subject: [PATCH 03/10] tests: k8s: tdx: Stop running `k8s-guest-pull-image.bats` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're doing that as all tests are going to be running with `shared_fs=none`, meaning that we don't need any specific test for this case anymore. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/k8s-guest-pull-image.bats | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/kubernetes/k8s-guest-pull-image.bats b/tests/integration/kubernetes/k8s-guest-pull-image.bats index fb42974019..ba7d8da111 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image.bats @@ -9,6 +9,10 @@ load "${BATS_TEST_DIRNAME}/lib.sh" load "${BATS_TEST_DIRNAME}/confidential_common.sh" setup() { + if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then + skip "${KATA_HYPERVISOR} is already running all the tests with guest-pulling, skip this specific one" + fi + if is_confidential_hardware; then skip "Due to issues related to pull-image integration skip tests for ${KATA_HYPERVISOR}." fi @@ -226,6 +230,10 @@ setup() { } teardown() { + if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then + skip "${KATA_HYPERVISOR} is already running all the tests with guest-pulling, skip this specific one" + fi + if is_confidential_hardware; then skip "Due to issues related to pull-image integration skip tests for ${KATA_HYPERVISOR}." fi From 1a8a4d046db1215d0d0227792aa3e4e130e2f122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 19 May 2024 14:45:26 +0200 Subject: [PATCH 04/10] tests: k8s: setup: Improve / Fix logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's make sure the logs will print the correct annotation and its value, instead of always mentioning "kernel" and "initrd". Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/setup.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index c61614fc1e..ba33d68a96 100755 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -57,7 +57,7 @@ add_annotations_to_yaml() { case "${resource_kind}" in Pod) - echo "Adding kernel and initrd annotations to ${resource_kind} from ${yaml_file}" + info "Adding \"${annotation_name}=${annotation_value}\" to ${resource_kind} from ${yaml_file}" yq write -i \ "${K8S_TEST_YAML}" \ "metadata.annotations[${annotation_name}]" \ @@ -65,7 +65,7 @@ add_annotations_to_yaml() { ;; Deployment|Job|ReplicationController) - echo "Adding kernel and initrd annotations to ${resource_kind} from ${yaml_file}" + info "Adding \"${annotation_name}=${annotation_value}\" to ${resource_kind} from ${yaml_file}" yq write -i \ "${K8S_TEST_YAML}" \ "spec.template.metadata.annotations[${annotation_name}]" \ @@ -73,15 +73,15 @@ add_annotations_to_yaml() { ;; List) - echo "Issue #7765: adding kernel and initrd annotations to ${resource_kind} from ${yaml_file} is not implemented yet" + info "Issue #7765: adding annotations to ${resource_kind} from ${yaml_file} is not implemented yet" ;; ConfigMap|LimitRange|Namespace|PersistentVolume|PersistentVolumeClaim|RuntimeClass|Secret|Service) - echo "Kernel and initrd annotations are not required for ${resource_kind} from ${yaml_file}" + info "Annotations are not required for ${resource_kind} from ${yaml_file}" ;; *) - echo "k8s resource type ${resource_kind} from ${yaml_file} is not yet supported for kernel and initrd annotations testing" + info "k8s resource type ${resource_kind} from ${yaml_file} is not yet supported for annotations testing" return 1 ;; esac From 03ce41b7435deb9aceefb7d8b9d13c57daa0226e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 19 May 2024 15:29:11 +0200 Subject: [PATCH 05/10] tests: k8s: tdx: Skip "Check custom dns" test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test has been failing on TDX for a while, and an issue has been created to track it down, see: https://github.com/kata-containers/kata-containers/issues/9663 For now, let's have it skipped. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/k8s-custom-dns.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/kubernetes/k8s-custom-dns.bats b/tests/integration/kubernetes/k8s-custom-dns.bats index 02b6f1110f..1c52bf2094 100644 --- a/tests/integration/kubernetes/k8s-custom-dns.bats +++ b/tests/integration/kubernetes/k8s-custom-dns.bats @@ -9,6 +9,8 @@ load "${BATS_TEST_DIRNAME}/../../common.bash" load "${BATS_TEST_DIRNAME}/tests_common.sh" setup() { + [ "${KATA_HYPERVISOR}" = "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9663" + pod_name="custom-dns-test" file_name="/etc/resolv.conf" get_pod_config_dir @@ -37,6 +39,8 @@ setup() { } teardown() { + [ "${KATA_HYPERVISOR}" = "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9663" + # Debugging information kubectl describe "pod/$pod_name" From 937b2d580612567a7998ed9ebea368448c1bcca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 19 May 2024 16:28:59 +0200 Subject: [PATCH 06/10] tests: k8s: tdx: Skip "Kill all processes in container" test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test fails when using `shared_fs=none` with the nydus snapshotter, and we're tracking the issue here: https://github.com/kata-containers/kata-containers/issues/9664 For now, let's have it skipped. Signed-off-by: Fabiano Fidêncio --- .../kubernetes/k8s-kill-all-process-in-container.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/kubernetes/k8s-kill-all-process-in-container.bats b/tests/integration/kubernetes/k8s-kill-all-process-in-container.bats index 9174855c38..e50edd97a1 100644 --- a/tests/integration/kubernetes/k8s-kill-all-process-in-container.bats +++ b/tests/integration/kubernetes/k8s-kill-all-process-in-container.bats @@ -9,6 +9,8 @@ load "${BATS_TEST_DIRNAME}/../../common.bash" load "${BATS_TEST_DIRNAME}/tests_common.sh" setup() { + [ "${KATA_HYPERVISOR}" = "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9664" + pod_name="busybox" first_container_name="first-test-container" @@ -39,6 +41,8 @@ setup() { } teardown() { + [ "${KATA_HYPERVISOR}" = "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9664" + # Debugging information kubectl describe "pod/$pod_name" From 606a62a0a7bcce9425006a8dd4c5ee950728ea6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 19 May 2024 17:35:30 +0200 Subject: [PATCH 07/10] tests: k8s: tdx: Skip "Setting sysctl" test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test fails when using `shared_fs=none` with the nydus-snapshotter, and we're tracking the issue here: https://github.com/kata-containers/kata-containers/issues/9666 Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/k8s-sysctls.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/kubernetes/k8s-sysctls.bats b/tests/integration/kubernetes/k8s-sysctls.bats index cea2b9fb47..f7df754667 100644 --- a/tests/integration/kubernetes/k8s-sysctls.bats +++ b/tests/integration/kubernetes/k8s-sysctls.bats @@ -9,6 +9,8 @@ load "${BATS_TEST_DIRNAME}/../../common.bash" load "${BATS_TEST_DIRNAME}/tests_common.sh" setup() { + [ "${KATA_HYPERVISOR}" = "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9666" + pod_name="sysctl-test" get_pod_config_dir @@ -30,6 +32,8 @@ setup() { } teardown() { + [ "${KATA_HYPERVISOR}" = "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9666" + # Debugging information kubectl describe "pod/$pod_name" From 9d56145499a058bb989c1e0ad4138a915d6574f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 19 May 2024 18:25:09 +0200 Subject: [PATCH 08/10] tests: k8s: tdx: Skip volume related tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similarly to firecracker, which doesn't have support for virtio-fs / virtio-9p, TDX used with `shared_fs=none` will face the very same limitations. The tests affected are: * k8s-credentials-secrets.bats * k8s-file-volume.bats * k8s-inotify.bats * k8s-nested-configmap-secret.bats * k8s-projected-volume.bats * k8s-volume.bats Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/k8s-credentials-secrets.bats | 2 ++ tests/integration/kubernetes/k8s-file-volume.bats | 3 +++ tests/integration/kubernetes/k8s-inotify.bats | 2 ++ tests/integration/kubernetes/k8s-nested-configmap-secret.bats | 2 ++ tests/integration/kubernetes/k8s-projected-volume.bats | 2 ++ tests/integration/kubernetes/k8s-volume.bats | 2 ++ 6 files changed, 13 insertions(+) diff --git a/tests/integration/kubernetes/k8s-credentials-secrets.bats b/tests/integration/kubernetes/k8s-credentials-secrets.bats index a448608ff9..88db956337 100644 --- a/tests/integration/kubernetes/k8s-credentials-secrets.bats +++ b/tests/integration/kubernetes/k8s-credentials-secrets.bats @@ -11,6 +11,7 @@ load "${BATS_TEST_DIRNAME}/tests_common.sh" setup() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" get_pod_config_dir pod_yaml_file="${pod_config_dir}/pod-secret.yaml" @@ -62,6 +63,7 @@ setup() { teardown() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" # Debugging information kubectl describe "pod/$pod_name" diff --git a/tests/integration/kubernetes/k8s-file-volume.bats b/tests/integration/kubernetes/k8s-file-volume.bats index 17f9725673..f794c6e3db 100644 --- a/tests/integration/kubernetes/k8s-file-volume.bats +++ b/tests/integration/kubernetes/k8s-file-volume.bats @@ -12,6 +12,8 @@ TEST_INITRD="${TEST_INITRD:-no}" setup() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" + pod_name="test-file-volume" container_name="busybox-file-volume-container" node="$(get_one_kata_node)" @@ -57,6 +59,7 @@ setup() { teardown() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" kubectl describe pod "$pod_name" diff --git a/tests/integration/kubernetes/k8s-inotify.bats b/tests/integration/kubernetes/k8s-inotify.bats index 193cf99dc3..7b73f30313 100644 --- a/tests/integration/kubernetes/k8s-inotify.bats +++ b/tests/integration/kubernetes/k8s-inotify.bats @@ -13,6 +13,7 @@ setup() { [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" issue_url="https://github.com/kata-containers/kata-containers/issues/8906" [ "${KATA_HYPERVISOR}" == "qemu-se" ] && skip "test not working for IBM Z LPAR (see ${issue_url})" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" get_pod_config_dir pod_yaml="${pod_config_dir}"/inotify-configmap-pod.yaml @@ -47,6 +48,7 @@ teardown() { [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" issue_url="https://github.com/kata-containers/kata-containers/issues/8906" [ "${KATA_HYPERVISOR}" == "qemu-se" ] && skip "test not working for IBM Z LPAR (see ${issue_url})" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" # Debugging information kubectl describe "pod/$pod_name" kubectl delete pod "$pod_name" diff --git a/tests/integration/kubernetes/k8s-nested-configmap-secret.bats b/tests/integration/kubernetes/k8s-nested-configmap-secret.bats index ebffb6e187..5c49f11c97 100644 --- a/tests/integration/kubernetes/k8s-nested-configmap-secret.bats +++ b/tests/integration/kubernetes/k8s-nested-configmap-secret.bats @@ -11,6 +11,7 @@ load "${BATS_TEST_DIRNAME}/tests_common.sh" setup() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" get_pod_config_dir @@ -32,6 +33,7 @@ setup() { teardown() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" # Debugging information kubectl describe "pod/$pod_name" diff --git a/tests/integration/kubernetes/k8s-projected-volume.bats b/tests/integration/kubernetes/k8s-projected-volume.bats index 5d9f336a79..d448d62d17 100644 --- a/tests/integration/kubernetes/k8s-projected-volume.bats +++ b/tests/integration/kubernetes/k8s-projected-volume.bats @@ -11,6 +11,7 @@ load "${BATS_TEST_DIRNAME}/tests_common.sh" setup() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" get_pod_config_dir @@ -56,6 +57,7 @@ setup() { teardown() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" # Debugging information kubectl describe "pod/$pod_name" diff --git a/tests/integration/kubernetes/k8s-volume.bats b/tests/integration/kubernetes/k8s-volume.bats index 4178f8b1e1..1f75398efb 100644 --- a/tests/integration/kubernetes/k8s-volume.bats +++ b/tests/integration/kubernetes/k8s-volume.bats @@ -12,6 +12,7 @@ TEST_INITRD="${TEST_INITRD:-no}" setup() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" get_pod_config_dir @@ -62,6 +63,7 @@ setup() { teardown() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}" + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9667" # Debugging information kubectl describe "pod/$pod_name" From a2c70222a8b34d0b828e714ca2bca90df8030fa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 19 May 2024 20:53:58 +0200 Subject: [PATCH 09/10] tests: k8s: tdx: Skip initContainerd shared vol test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is another one that is related to initContainers not being properly handled with the guest image pulling. Let's skip it for now as we have https://github.com/kata-containers/kata-containers/issues/9668 to track it down. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/k8s-shared-volume.bats | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/kubernetes/k8s-shared-volume.bats b/tests/integration/kubernetes/k8s-shared-volume.bats index 360ea998c3..9af567eddd 100644 --- a/tests/integration/kubernetes/k8s-shared-volume.bats +++ b/tests/integration/kubernetes/k8s-shared-volume.bats @@ -40,6 +40,8 @@ setup() { } @test "initContainer with shared volume" { + [ "${KATA_HYPERVISOR}" = "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9668" + pod_name="initcontainer-shared-volume" last_container="last" cmd='test $(cat /volume/initContainer) -lt $(cat /volume/container)' From e8ebe188689b01250443ec745f4611594bbb7a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 19 May 2024 20:56:29 +0200 Subject: [PATCH 10/10] tests: k8s: tdx: Skip liveness probe test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test doesn't fail with the guest image pulling, but it for sure should. :-) We can see in the bats logs, something like: ``` Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled 31s default-scheduler Successfully assigned kata-containers-k8s-tests/liveness-exec to 984fee00bd70.jf.intel.com Normal Pulled 23s kubelet Successfully pulled image "quay.io/prometheus/busybox:latest" in 345ms (345ms including waiting) Normal Started 21s kubelet Started container liveness Warning Unhealthy 7s (x3 over 13s) kubelet Liveness probe failed: cat: can't open '/tmp/healthy': No such file or directory Normal Killing 7s kubelet Container liveness failed liveness probe, will be restarted Normal Pulled 7s kubelet Successfully pulled image "quay.io/prometheus/busybox:latest" in 389ms (389ms including waiting) Warning Failed 5s kubelet Error: failed to create containerd task: failed to create shim task: the file /bin/sh was not found: unknown Normal Pulling 5s (x3 over 23s) kubelet Pulling image "quay.io/prometheus/busybox:latest" Normal Pulled 4s kubelet Successfully pulled image "quay.io/prometheus/busybox:latest" in 342ms (342ms including waiting) Normal Created 4s (x3 over 23s) kubelet Created container liveness Warning Failed 3s kubelet Error: failed to create containerd task: failed to create shim task: failed to mount /run/kata-containers/f0ec86fb156a578964007f7773a3ccbdaf60023106634fe030f039e2e154cd11/rootfs to /run/kata-containers/liveness/rootfs, with error: ENOENT: No such file or directory: unknown Warning BackOff 1s (x3 over 3s) kubelet Back-off restarting failed container liveness in pod liveness-exec_kata-containers-k8s-tests(b1a980bf-a5b3-479d-97c2-ebdb45773eff) ``` Let's skip it for now as we have an issue opened to track it down: https://github.com/kata-containers/kata-containers/issues/9665 Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/k8s-liveness-probes.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/kubernetes/k8s-liveness-probes.bats b/tests/integration/kubernetes/k8s-liveness-probes.bats index ea8d51c41e..62c67f7af5 100644 --- a/tests/integration/kubernetes/k8s-liveness-probes.bats +++ b/tests/integration/kubernetes/k8s-liveness-probes.bats @@ -9,6 +9,8 @@ load "${BATS_TEST_DIRNAME}/../../common.bash" load "${BATS_TEST_DIRNAME}/tests_common.sh" setup() { + [ "${KATA_HYPERVISOR}" = "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9665" + sleep_liveness=20 agnhost_name="${container_images_agnhost_name}" agnhost_version="${container_images_agnhost_version}" @@ -89,6 +91,8 @@ setup() { } teardown() { + [ "${KATA_HYPERVISOR}" = "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/9665" + # Debugging information kubectl describe "pod/$pod_name"