diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 9a2244ebff..204c66587a 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -62,7 +62,14 @@ function enable_cluster_http_application_routing() { } function install_azure_cli() { - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + # This is a workaround for https://github.com/Azure/azure-cli/issues/28984 + # which ended up breaking our CI. + curl -sL https://aka.ms/InstallAzureCLIDeb -o installAzureCli.sh + sed -i '/curl -sLS https:\/\/packages.microsoft.com\/keys\/microsoft.asc |/d' installAzureCli.sh + sed -i '/gpg --dearmor -o \/etc\/apt\/keyrings\/microsoft.gpg/d' installAzureCli.sh + sed -i '/chmod go+r \/etc\/apt\/keyrings\/microsoft.gpg/d' installAzureCli.sh + sudo bash installAzureCli.sh + # The aks-preview extension is required while the Mariner Kata host is in preview. az extension add --name aks-preview } diff --git a/tests/integration/kubernetes/confidential_common.sh b/tests/integration/kubernetes/confidential_common.sh index b1bc9208aa..dd59e75eec 100644 --- a/tests/integration/kubernetes/confidential_common.sh +++ b/tests/integration/kubernetes/confidential_common.sh @@ -44,7 +44,7 @@ function check_hypervisor_for_confidential_tests() { local kata_hypervisor="${1}" # This check must be done with "${KATA_HYPERVISOR}" to avoid # having substrings, like qemu, being matched with qemu-$something. - if [[ " ${SUPPORTED_TEE_HYPERVISORS[*]} " =~ " ${kata_hypervisor} " ]] ||\ + if check_hypervisor_for_confidential_tests_tee_only "${kata_hypervisor}" ||\ [[ " ${SUPPORTED_NON_TEE_HYPERVISORS[*]} " =~ " ${kata_hypervisor} " ]]; then return 0 else @@ -52,10 +52,33 @@ function check_hypervisor_for_confidential_tests() { fi } -# Common setup for confidential tests. -function confidential_setup() { - ensure_yq - if ! check_hypervisor_for_confidential_tests "${KATA_HYPERVISOR}"; then - return 1 - fi +# This function verifies whether the input hypervisor supports confidential tests and +# relies on `KATA_HYPERVISOR` being an environment variable +function check_hypervisor_for_confidential_tests_tee_only() { + local kata_hypervisor="${1}" + # This check must be done with "${KATA_HYPERVISOR}" to avoid + # having substrings, like qemu, being matched with qemu-$something. + if [[ " ${SUPPORTED_TEE_HYPERVISORS[*]} " =~ " ${kata_hypervisor} " ]]; then + return 0 + fi + + return 1 +} + +# Common check for confidential tests. +function is_confidential_runtime_class() { + if check_hypervisor_for_confidential_tests "${KATA_HYPERVISOR}"; then + return 0 + fi + + return 1 +} + +# Common check for confidential hardware tests. +function is_confidential_hardware() { + if check_hypervisor_for_confidential_tests_tee_only "${KATA_HYPERVISOR}"; then + return 0 + fi + + return 1 } diff --git a/tests/integration/kubernetes/k8s-confidential-attestation.bats b/tests/integration/kubernetes/k8s-confidential-attestation.bats index c2035e1bb7..2be9a37ae0 100644 --- a/tests/integration/kubernetes/k8s-confidential-attestation.bats +++ b/tests/integration/kubernetes/k8s-confidential-attestation.bats @@ -15,7 +15,7 @@ export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" export AA_KBC="${AA_KBC:-cc_kbc}" setup() { - confidential_setup || skip "Test not supported for ${KATA_HYPERVISOR}." + is_confidential_runtime_class || skip "Test not supported for ${KATA_HYPERVISOR}." if [ "${KBS}" = "false" ]; then skip "Test skipped as KBS not setup" @@ -82,7 +82,7 @@ setup() { } teardown() { - check_hypervisor_for_confidential_tests ${KATA_HYPERVISOR} || skip "Test not supported for ${KATA_HYPERVISOR}." + is_confidential_runtime_class || skip "Test not supported for ${KATA_HYPERVISOR}." if [ "${KBS}" = "false" ]; then skip "Test skipped as KBS not setup" diff --git a/tests/integration/kubernetes/k8s-confidential.bats b/tests/integration/kubernetes/k8s-confidential.bats index 01abec96a0..afb464b047 100644 --- a/tests/integration/kubernetes/k8s-confidential.bats +++ b/tests/integration/kubernetes/k8s-confidential.bats @@ -10,7 +10,9 @@ load "${BATS_TEST_DIRNAME}/confidential_common.sh" load "${BATS_TEST_DIRNAME}/tests_common.sh" setup() { - confidential_setup || skip "Test not supported for ${KATA_HYPERVISOR}." + if ! is_confidential_hardware; then + skip "Test is supported only on confidential hardware (which ${KATA_HYPERVISOR} is not)" + fi setup_unencrypted_confidential_pod } @@ -41,7 +43,9 @@ setup() { } teardown() { - check_hypervisor_for_confidential_tests ${KATA_HYPERVISOR} || skip "Test not supported for ${KATA_HYPERVISOR}." + if ! is_confidential_hardware; then + skip "Test is supported only on confidential hardware (which ${KATA_HYPERVISOR} is not)" + fi kubectl describe "pod/${pod_name}" || true kubectl delete -f "${pod_config_dir}/pod-confidential-unencrypted.yaml" || true diff --git a/tests/integration/kubernetes/k8s-guest-pull-image.bats b/tests/integration/kubernetes/k8s-guest-pull-image.bats index 93d9120a68..fb42974019 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image.bats @@ -9,18 +9,30 @@ load "${BATS_TEST_DIRNAME}/lib.sh" load "${BATS_TEST_DIRNAME}/confidential_common.sh" setup() { - confidential_setup && skip "Due to issues related to pull-image integration skip tests for ${KATA_HYPERVISOR}." + if is_confidential_hardware; then + skip "Due to issues related to pull-image integration skip tests for ${KATA_HYPERVISOR}." + fi + + if ! is_confidential_runtime_class; then + skip "Test not supported for ${KATA_HYPERVISOR}." + fi [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" - setup_common + setup_common unencrypted_image_1="quay.io/sjenning/nginx:1.15-alpine" unencrypted_image_2="quay.io/prometheus/busybox:latest" large_image="quay.io/confidential-containers/test-images:largeimage" } @test "Test we can pull an unencrypted image outside the guest with runc and then inside the guest successfully" { - confidential_setup && skip "Due to issues related to pull-image integration skip tests for ${KATA_HYPERVISOR}." + if is_confidential_hardware; then + skip "Due to issues related to pull-image integration skip tests for ${KATA_HYPERVISOR}." + fi + + if ! is_confidential_runtime_class; then + skip "Test not supported for ${KATA_HYPERVISOR}." + fi # 1. Create one runc pod with the $unencrypted_image_1 image # We want to have one runc pod, so we pass a fake runtimeclass "runc" and then delete the runtimeClassName, @@ -82,7 +94,7 @@ setup() { echo "Pod $kata_pod_with_nydus_config file:" cat $kata_pod_with_nydus_config - # The pod should be failed because the default timeout of CreateContainerRequest is 60s + # The pod should be failed because the default timeout of CreateContainerRequest is 60s assert_pod_fail "$kata_pod_with_nydus_config" assert_logs_contain "$node" kata "$node_start_time" \ 'context deadline exceeded' @@ -104,7 +116,6 @@ setup() { } @test "Test we can pull an unencrypted image inside the guest twice in a row and then outside the guest successfully" { - skip "Skip this test until we use containerd 2.0 with 'image pull per runtime class' feature: https://github.com/containerd/containerd/issues/9377" # 1. Create one kata pod with the $unencrypted_image_1 image and nydus annotation twice kata_pod_with_nydus_config="$(new_pod_config "$unencrypted_image_1" "kata-${KATA_HYPERVISOR}")" set_node "$kata_pod_with_nydus_config" "$node" @@ -121,7 +132,7 @@ setup() { add_allow_all_policy_to_yaml "$kata_pod_with_nydus_config" k8s_create_pod "$kata_pod_with_nydus_config" - + echo "Kata pod test-e2e with nydus annotation is running" echo "Checking the image was pulled in the guest" @@ -160,7 +171,6 @@ setup() { } @test "Test we can pull an other unencrypted image outside the guest and then inside the guest successfully" { - skip "Skip this test until we use containerd 2.0 with 'image pull per runtime class' feature: https://github.com/containerd/containerd/issues/9377" # 1. Create one kata pod with the $unencrypted_image_2 image and without nydus annotation kata_pod_without_nydus_config="$(new_pod_config "$unencrypted_image_2" "kata-${KATA_HYPERVISOR}")" set_node "$kata_pod_without_nydus_config" "$node" @@ -172,7 +182,7 @@ setup() { add_allow_all_policy_to_yaml "$kata_pod_without_nydus_config" k8s_create_pod "$kata_pod_without_nydus_config" - + echo "Kata pod test-e2e without nydus annotation is running" echo "Checking the image was pulled in the host" @@ -199,7 +209,7 @@ setup() { add_allow_all_policy_to_yaml "$kata_pod_with_nydus_config" k8s_create_pod "$kata_pod_with_nydus_config" - + echo "Kata pod test-e2e with nydus annotation is running" echo "Checking the image was pulled in the guest" sandbox_id=$(get_node_kata_sandbox_id $node) @@ -216,7 +226,13 @@ setup() { } teardown() { - confidential_setup && skip "Due to issues related to pull-image integration skip tests for ${KATA_HYPERVISOR}." + if is_confidential_hardware; then + skip "Due to issues related to pull-image integration skip tests for ${KATA_HYPERVISOR}." + fi + + if ! is_confidential_runtime_class; then + skip "Test not supported for ${KATA_HYPERVISOR}." + fi [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" diff --git a/tests/integration/kubernetes/lib.sh b/tests/integration/kubernetes/lib.sh index 95eb161620..b4c2eaa0a6 100644 --- a/tests/integration/kubernetes/lib.sh +++ b/tests/integration/kubernetes/lib.sh @@ -111,7 +111,7 @@ assert_rootfs_count() { local expect_count="$3" local allrootfs="" - # verify that the sandbox_id is not empty; + # verify that the sandbox_id is not empty; # otherwise, the command $(exec_host $node "find /run/kata-containers/shared/sandboxes/${sandbox_id} -name rootfs -type d") # may yield an unexpected count of rootfs. if [ -z "$sandbox_id" ]; then @@ -130,8 +130,8 @@ assert_rootfs_count() { done echo "allrootfs is: $allrootfs" count=$(echo $allrootfs | grep -o "rootfs" | wc -l) - echo "count of container rootfs in host is: $count, expect count is: $expect_count" - [ $expect_count -eq $count ] + echo "count of container rootfs in host is: $count, expect count is less than, or equal to: $expect_count" + [ $expect_count -ge $count ] } # Create a pod configuration out of a template file.