ci: Add autogenerated policy tests on k0s, k3s, rke2 and microk8s

These tests run only on nightly and when triggering the dev CI manually.
They cover both nydus snapshotter with guest-pull and experimental-force-guest-pull,
using qemu-coco-dev and qemu-coco-dev-runtime-rs, and are included in the
run-kata-coco-tests workflow behind the extensive-matrix-autogenerated-policy flag.

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>

.github/workflows/ci-devel.yaml

@@ -17,6 +17,7 @@ jobs:
       pr-number: "dev"
       tag: ${{ github.sha }}-dev
       target-branch: ${{ github.ref_name }}
+      extensive-matrix-autogenerated-policy: "yes"
 
     secrets:
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}

.github/workflows/ci-nightly.yaml

@@ -22,6 +22,7 @@ jobs:
       pr-number: "nightly"
       tag: ${{ github.sha }}-nightly
       target-branch: ${{ github.ref_name }}
+      extensive-matrix-autogenerated-policy: "yes"
     secrets:
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
       AZ_APPID: ${{ secrets.AZ_APPID }}

.github/workflows/ci.yaml

@@ -19,6 +19,10 @@ on:
         required: false
         type: string
         default: no
+      extensive-matrix-autogenerated-policy:
+        required: false
+        type: string
+        default: no
     secrets:
       AUTHENTICATED_IMAGE_PASSWORD:
         required: true
@@ -358,6 +362,7 @@ jobs:
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
+      extensive-matrix-autogenerated-policy: ${{ inputs.extensive-matrix-autogenerated-policy }}
     secrets:
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
       AZ_APPID: ${{ secrets.AZ_APPID }}

.github/workflows/run-kata-coco-tests.yaml

@@ -24,6 +24,10 @@ on:
         required: false
         type: string
         default: ""
+      extensive-matrix-autogenerated-policy:
+        required: false
+        type: string
+        default: no
     secrets:
       AUTHENTICATED_IMAGE_PASSWORD:
         required: true
@@ -258,6 +262,136 @@ jobs:
         timeout-minutes: 5
         run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
 
+  # Extensive matrix: autogenerated policy tests (nydus + experimental-force-guest-pull) on k0s, k3s, rke2, microk8s with qemu-coco-dev / qemu-coco-dev-runtime-rs
+  run-k8s-tests-coco-nontee-extensive-matrix:
+    if: ${{ inputs.extensive-matrix-autogenerated-policy == 'yes' }}
+    name: run-k8s-tests-coco-nontee-extensive-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: [
+          { k8s: k0s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
+          { k8s: k0s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
+          { k8s: k0s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
+          { k8s: k3s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
+          { k8s: k3s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
+          { k8s: k3s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
+          { k8s: rke2, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
+          { k8s: rke2, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
+          { k8s: rke2, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
+          { k8s: microk8s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
+          { k8s: microk8s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
+          { k8s: microk8s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
+        ]
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    environment: ci
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
+      KBS: "true"
+      KBS_INGRESS: "nodeport"
+      KUBERNETES: ${{ matrix.environment.k8s }}
+      SNAPSHOTTER: ${{ matrix.environment.snapshotter }}
+      PULL_TYPE: ${{ matrix.environment.pull_type }}
+      EXPERIMENTAL_FORCE_GUEST_PULL: ${{ matrix.environment.pull_type == 'experimental-force-guest-pull' && matrix.environment.vmm || '' }}
+      AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      AUTO_GENERATE_POLICY: "yes"
+      K8S_TEST_HOST_TYPE: "all"
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tools-tarball
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-tools-artifacts
+
+      - name: Install kata-tools
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
+
+      - name: Remove unnecessary directories to free up space
+        run: |
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf /usr/lib/jvm
+          sudo rm -rf /usr/share/swift
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /usr/local/julia*
+          sudo rm -rf /opt/az
+          sudo rm -rf /usr/local/share/chromium
+          sudo rm -rf /opt/microsoft
+          sudo rm -rf /opt/google
+          sudo rm -rf /usr/lib/firefox
+
+      - name: Deploy ${{ matrix.environment.k8s }}
+        timeout-minutes: 15
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+
+      - name: Deploy Kata
+        timeout-minutes: 20
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
+        env:
+          USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: ${{ matrix.environment.snapshotter == 'nydus' }}
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
+      - name: Run tests
+        timeout-minutes: 80
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Report tests
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh report-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        timeout-minutes: 15
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup
+
+      - name: Delete CoCo KBS
+        if: always()
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+
+      - name: Delete CSI driver
+        if: always()
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
   # Generate jobs for testing CoCo on non-TEE environments with erofs-snapshotter
   run-k8s-tests-coco-nontee-with-erofs-snapshotter:
     name: run-k8s-tests-coco-nontee-with-erofs-snapshotter