diff --git a/.github/workflows/run-kata-coco-tests.yaml b/.github/workflows/run-kata-coco-tests.yaml index 6618437216..8561859012 100644 --- a/.github/workflows/run-kata-coco-tests.yaml +++ b/.github/workflows/run-kata-coco-tests.yaml @@ -186,3 +186,78 @@ jobs: - name: Delete Snapshotter if: always() run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter + + # Generate jobs for testing CoCo on non-TEE environments + run-k8s-tests-coco-nontee: + strategy: + fail-fast: false + matrix: + vmm: + - qemu + snapshotter: + - nydus + pull-type: + - guest-pull + runs-on: ubuntu-latest + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + GH_PR_NUMBER: ${{ inputs.pr-number }} + KATA_HOST_OS: ${{ matrix.host_os }} + KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: "vanilla" + PULL_TYPE: ${{ matrix.pull-type }} + SNAPSHOTTER: ${{ matrix.snapshotter }} + USING_NFD: "false" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.commit-hash }} + fetch-depth: 0 + + - name: Rebase atop of the latest target branch + run: | + ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" + env: + TARGET_BRANCH: ${{ inputs.target-branch }} + + - name: Download Azure CLI + run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli + + - name: Log into the Azure account + run: bash tests/integration/kubernetes/gha-run.sh login-azure + env: + AZ_APPID: ${{ secrets.AZ_APPID }} + AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} + AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} + AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} + + - name: Create AKS cluster + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh create-cluster + + - name: Install `bats` + run: bash tests/integration/kubernetes/gha-run.sh install-bats + + - name: Install `kubectl` + run: bash tests/integration/kubernetes/gha-run.sh install-kubectl + + - name: Download credentials for the Kubernetes CLI to use them + run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials + + - name: Deploy Snapshotter + timeout-minutes: 5 + run: bash tests/integration/kubernetes/gha-run.sh deploy-snapshotter + + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks + + - name: Run tests + timeout-minutes: 60 + run: bash tests/integration/kubernetes/gha-run.sh run-tests + + - name: Delete AKS cluster + if: always() + run: bash tests/integration/kubernetes/gha-run.sh delete-cluster diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 9c930bd7ab..6ce754bc13 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -146,6 +146,7 @@ function get_cluster_credentials() { test_type="${1:-k8s}" az aks get-credentials \ + --overwrite-existing \ -g "$(_print_rg_name ${test_type})" \ -n "$(_print_cluster_name ${test_type})" } diff --git a/tests/integration/kubernetes/k8s-guest-pull-image.bats b/tests/integration/kubernetes/k8s-guest-pull-image.bats index 329c868e1a..35d412c567 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image.bats @@ -10,6 +10,8 @@ load "${BATS_TEST_DIRNAME}/confidential_common.sh" setup() { confidential_setup || skip "Test not supported for ${KATA_HYPERVISOR}." + [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + setup_common unencrypted_image_1="quay.io/sjenning/nginx:1.15-alpine" unencrypted_image_2="quay.io/prometheus/busybox:latest" @@ -17,7 +19,6 @@ setup() { } @test "Test we can pull an unencrypted image outside the guest with runc and then inside the guest successfully" { - [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && skip "Test not supported for ${KATA_HYPERVISOR}." # 1. Create one runc pod with the $unencrypted_image_1 image # We want to have one runc pod, so we pass a fake runtimeclass "runc" and then delete the runtimeClassName, # because the runtimeclass is not optional in new_pod_config function. @@ -46,6 +47,11 @@ setup() { "io.containerd.cri.runtime-handler" \ "kata-${KATA_HYPERVISOR}" + [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && \ + set_metadata_annotation "$kata_pod_with_nydus_config" \ + "io.katacontainers.config.hypervisor.image" \ + "/opt/kata/share/kata-containers/kata-containers-confidential.img" + # For debug sake echo "Pod $kata_pod_with_nydus_config file:" cat $kata_pod_with_nydus_config @@ -100,7 +106,6 @@ setup() { } @test "Test we can pull an unencrypted image inside the guest twice in a row and then outside the guest successfully" { - [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && skip "Test not supported for ${KATA_HYPERVISOR}." skip "Skip this test until we use containerd 2.0 with 'image pull per runtime class' feature: https://github.com/containerd/containerd/issues/9377" # 1. Create one kata pod with the $unencrypted_image_1 image and nydus annotation twice kata_pod_with_nydus_config="$(new_pod_config "$unencrypted_image_1" "kata-${KATA_HYPERVISOR}")" @@ -112,6 +117,11 @@ setup() { "io.containerd.cri.runtime-handler" \ "kata-${KATA_HYPERVISOR}" + [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && \ + set_metadata_annotation "$kata_pod_with_nydus_config" \ + "io.katacontainers.config.hypervisor.image" \ + "/opt/kata/share/kata-containers/kata-containers-confidential.img" + # For debug sake echo "Pod $kata_pod_with_nydus_config file:" cat $kata_pod_with_nydus_config @@ -157,7 +167,6 @@ setup() { } @test "Test we can pull an other unencrypted image outside the guest and then inside the guest successfully" { - [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && skip "Test not supported for ${KATA_HYPERVISOR}." skip "Skip this test until we use containerd 2.0 with 'image pull per runtime class' feature: https://github.com/containerd/containerd/issues/9377" # 1. Create one kata pod with the $unencrypted_image_2 image and without nydus annotation kata_pod_without_nydus_config="$(new_pod_config "$unencrypted_image_2" "kata-${KATA_HYPERVISOR}")" @@ -191,6 +200,11 @@ setup() { "io.containerd.cri.runtime-handler" \ "kata-${KATA_HYPERVISOR}" + [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && \ + set_metadata_annotation "$kata_pod_with_nydus_config" \ + "io.katacontainers.config.hypervisor.image" \ + "/opt/kata/share/kata-containers/kata-containers-confidential.img" + # For debug sake echo "Pod $kata_pod_with_nydus_config file:" cat $kata_pod_with_nydus_config @@ -215,6 +229,8 @@ setup() { teardown() { check_hypervisor_for_confidential_tests ${KATA_HYPERVISOR} || skip "Test not supported for ${KATA_HYPERVISOR}." + [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + kubectl describe pod "$pod_name" k8s_delete_all_pods_if_any_exists || true }