From 0d5399ba923a17f81525db9878df775f47ae39c6 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 5 Apr 2024 14:17:43 -0300 Subject: [PATCH 1/4] gha: Create CoCo tests jobs on non-TEE Created the new run-k8s-tests-coco-nontee jobs for running CoCo tests on non-TEE. It currently generates the run-k8s-tests-coco-nontee(qemu, nydus, guest-pull) job only to run the guest-pull tests. Fixes: #9410 Signed-off-by: Wainer dos Santos Moschetta --- .github/workflows/run-kata-coco-tests.yaml | 75 ++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/.github/workflows/run-kata-coco-tests.yaml b/.github/workflows/run-kata-coco-tests.yaml index 6618437216..8561859012 100644 --- a/.github/workflows/run-kata-coco-tests.yaml +++ b/.github/workflows/run-kata-coco-tests.yaml @@ -186,3 +186,78 @@ jobs: - name: Delete Snapshotter if: always() run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter + + # Generate jobs for testing CoCo on non-TEE environments + run-k8s-tests-coco-nontee: + strategy: + fail-fast: false + matrix: + vmm: + - qemu + snapshotter: + - nydus + pull-type: + - guest-pull + runs-on: ubuntu-latest + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + GH_PR_NUMBER: ${{ inputs.pr-number }} + KATA_HOST_OS: ${{ matrix.host_os }} + KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: "vanilla" + PULL_TYPE: ${{ matrix.pull-type }} + SNAPSHOTTER: ${{ matrix.snapshotter }} + USING_NFD: "false" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.commit-hash }} + fetch-depth: 0 + + - name: Rebase atop of the latest target branch + run: | + ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" + env: + TARGET_BRANCH: ${{ inputs.target-branch }} + + - name: Download Azure CLI + run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli + + - name: Log into the Azure account + run: bash tests/integration/kubernetes/gha-run.sh login-azure + env: + AZ_APPID: ${{ secrets.AZ_APPID }} + AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} + AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} + AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} + + - name: Create AKS cluster + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh create-cluster + + - name: Install `bats` + run: bash tests/integration/kubernetes/gha-run.sh install-bats + + - name: Install `kubectl` + run: bash tests/integration/kubernetes/gha-run.sh install-kubectl + + - name: Download credentials for the Kubernetes CLI to use them + run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials + + - name: Deploy Snapshotter + timeout-minutes: 5 + run: bash tests/integration/kubernetes/gha-run.sh deploy-snapshotter + + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks + + - name: Run tests + timeout-minutes: 60 + run: bash tests/integration/kubernetes/gha-run.sh run-tests + + - name: Delete AKS cluster + if: always() + run: bash tests/integration/kubernetes/gha-run.sh delete-cluster From c24f13431dd182038e9909459d41eddb7b999b18 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 5 Apr 2024 16:56:31 -0300 Subject: [PATCH 2/4] tests/k8s: enable guest-pull tests on non-TEE Enabled guest-pull tests on non-TEE environment. It know requires the SNAPSHOTTER environment variable to avoid it running on jobs where nydus-snapshotter is not installed Fixes: #9410 Signed-off-by: Wainer dos Santos Moschetta --- tests/integration/kubernetes/k8s-guest-pull-image.bats | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/kubernetes/k8s-guest-pull-image.bats b/tests/integration/kubernetes/k8s-guest-pull-image.bats index 329c868e1a..4a9a133ad6 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image.bats @@ -10,6 +10,8 @@ load "${BATS_TEST_DIRNAME}/confidential_common.sh" setup() { confidential_setup || skip "Test not supported for ${KATA_HYPERVISOR}." + [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + setup_common unencrypted_image_1="quay.io/sjenning/nginx:1.15-alpine" unencrypted_image_2="quay.io/prometheus/busybox:latest" @@ -17,7 +19,6 @@ setup() { } @test "Test we can pull an unencrypted image outside the guest with runc and then inside the guest successfully" { - [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && skip "Test not supported for ${KATA_HYPERVISOR}." # 1. Create one runc pod with the $unencrypted_image_1 image # We want to have one runc pod, so we pass a fake runtimeclass "runc" and then delete the runtimeClassName, # because the runtimeclass is not optional in new_pod_config function. @@ -100,7 +101,6 @@ setup() { } @test "Test we can pull an unencrypted image inside the guest twice in a row and then outside the guest successfully" { - [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && skip "Test not supported for ${KATA_HYPERVISOR}." skip "Skip this test until we use containerd 2.0 with 'image pull per runtime class' feature: https://github.com/containerd/containerd/issues/9377" # 1. Create one kata pod with the $unencrypted_image_1 image and nydus annotation twice kata_pod_with_nydus_config="$(new_pod_config "$unencrypted_image_1" "kata-${KATA_HYPERVISOR}")" @@ -157,7 +157,6 @@ setup() { } @test "Test we can pull an other unencrypted image outside the guest and then inside the guest successfully" { - [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && skip "Test not supported for ${KATA_HYPERVISOR}." skip "Skip this test until we use containerd 2.0 with 'image pull per runtime class' feature: https://github.com/containerd/containerd/issues/9377" # 1. Create one kata pod with the $unencrypted_image_2 image and without nydus annotation kata_pod_without_nydus_config="$(new_pod_config "$unencrypted_image_2" "kata-${KATA_HYPERVISOR}")" @@ -215,6 +214,8 @@ setup() { teardown() { check_hypervisor_for_confidential_tests ${KATA_HYPERVISOR} || skip "Test not supported for ${KATA_HYPERVISOR}." + [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + kubectl describe pod "$pod_name" k8s_delete_all_pods_if_any_exists || true } From 3508f3a43add52ccb6c3fd7e483744beaaf79600 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 5 Apr 2024 20:40:28 -0300 Subject: [PATCH 3/4] tests/k8s: use CoCo image on guest-pull when non-TEE When running on non-TEE environments (e.g. KATA_HYPERVISOR=qemu) the tests should be stressing the CoCo image (/opt/kata/share/kata-containers/kata-containers-confidential.img) although currently the default image/initrd is built to be able to do guest-pull as well. Signed-off-by: Wainer dos Santos Moschetta --- .../kubernetes/k8s-guest-pull-image.bats | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/integration/kubernetes/k8s-guest-pull-image.bats b/tests/integration/kubernetes/k8s-guest-pull-image.bats index 4a9a133ad6..35d412c567 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image.bats @@ -47,6 +47,11 @@ setup() { "io.containerd.cri.runtime-handler" \ "kata-${KATA_HYPERVISOR}" + [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && \ + set_metadata_annotation "$kata_pod_with_nydus_config" \ + "io.katacontainers.config.hypervisor.image" \ + "/opt/kata/share/kata-containers/kata-containers-confidential.img" + # For debug sake echo "Pod $kata_pod_with_nydus_config file:" cat $kata_pod_with_nydus_config @@ -112,6 +117,11 @@ setup() { "io.containerd.cri.runtime-handler" \ "kata-${KATA_HYPERVISOR}" + [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && \ + set_metadata_annotation "$kata_pod_with_nydus_config" \ + "io.katacontainers.config.hypervisor.image" \ + "/opt/kata/share/kata-containers/kata-containers-confidential.img" + # For debug sake echo "Pod $kata_pod_with_nydus_config file:" cat $kata_pod_with_nydus_config @@ -190,6 +200,11 @@ setup() { "io.containerd.cri.runtime-handler" \ "kata-${KATA_HYPERVISOR}" + [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && \ + set_metadata_annotation "$kata_pod_with_nydus_config" \ + "io.katacontainers.config.hypervisor.image" \ + "/opt/kata/share/kata-containers/kata-containers-confidential.img" + # For debug sake echo "Pod $kata_pod_with_nydus_config file:" cat $kata_pod_with_nydus_config From 4f74617897a026c1e22b58eba21a88f46e7996db Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 5 Apr 2024 20:44:47 -0300 Subject: [PATCH 4/4] tests: pass --overwrite-existing to aks get-credentials By passing --overwrite-existing to `aks get-credentials` it will stop asking if I want to overwrite the existing credentials. This is handy for running the scripts locally. Signed-off-by: Wainer dos Santos Moschetta --- tests/gha-run-k8s-common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 9c930bd7ab..6ce754bc13 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -146,6 +146,7 @@ function get_cluster_credentials() { test_type="${1:-k8s}" az aks get-credentials \ + --overwrite-existing \ -g "$(_print_rg_name ${test_type})" \ -n "$(_print_cluster_name ${test_type})" }