diff --git a/.github/workflows/ci-devel.yaml b/.github/workflows/ci-devel.yaml index 3fa673e641..7042a2ccee 100644 --- a/.github/workflows/ci-devel.yaml +++ b/.github/workflows/ci-devel.yaml @@ -17,6 +17,7 @@ jobs: pr-number: "dev" tag: ${{ github.sha }}-dev target-branch: ${{ github.ref_name }} + extensive-matrix-autogenerated-policy: "yes" secrets: AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }} diff --git a/.github/workflows/ci-nightly.yaml b/.github/workflows/ci-nightly.yaml index 5d335e3cff..57ccd74f85 100644 --- a/.github/workflows/ci-nightly.yaml +++ b/.github/workflows/ci-nightly.yaml @@ -22,6 +22,7 @@ jobs: pr-number: "nightly" tag: ${{ github.sha }}-nightly target-branch: ${{ github.ref_name }} + extensive-matrix-autogenerated-policy: "yes" secrets: AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }} AZ_APPID: ${{ secrets.AZ_APPID }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2f32fd5b4a..d270298ca5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,6 +19,10 @@ on: required: false type: string default: no + extensive-matrix-autogenerated-policy: + required: false + type: string + default: no secrets: AUTHENTICATED_IMAGE_PASSWORD: required: true @@ -358,6 +362,7 @@ jobs: commit-hash: ${{ inputs.commit-hash }} pr-number: ${{ inputs.pr-number }} target-branch: ${{ inputs.target-branch }} + extensive-matrix-autogenerated-policy: ${{ inputs.extensive-matrix-autogenerated-policy }} secrets: AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }} AZ_APPID: ${{ secrets.AZ_APPID }} diff --git a/.github/workflows/run-kata-coco-tests.yaml b/.github/workflows/run-kata-coco-tests.yaml index ef0970a5c8..65f21ac7e7 100644 --- a/.github/workflows/run-kata-coco-tests.yaml +++ b/.github/workflows/run-kata-coco-tests.yaml @@ -24,6 +24,10 @@ on: required: false type: string default: "" + extensive-matrix-autogenerated-policy: + required: false + type: string + default: no secrets: AUTHENTICATED_IMAGE_PASSWORD: required: true @@ -258,6 +262,136 @@ jobs: timeout-minutes: 5 run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver + # Extensive matrix: autogenerated policy tests (nydus + experimental-force-guest-pull) on k0s, k3s, rke2, microk8s with qemu-coco-dev / qemu-coco-dev-runtime-rs + run-k8s-tests-coco-nontee-extensive-matrix: + if: ${{ inputs.extensive-matrix-autogenerated-policy == 'yes' }} + name: run-k8s-tests-coco-nontee-extensive-matrix + strategy: + fail-fast: false + matrix: + environment: [ + { k8s: k0s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull }, + { k8s: k0s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull }, + { k8s: k0s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull }, + { k8s: k3s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull }, + { k8s: k3s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull }, + { k8s: k3s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull }, + { k8s: rke2, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull }, + { k8s: rke2, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull }, + { k8s: rke2, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull }, + { k8s: microk8s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull }, + { k8s: microk8s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull }, + { k8s: microk8s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull }, + ] + runs-on: ubuntu-24.04 + permissions: + contents: read + environment: ci + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + GH_PR_NUMBER: ${{ inputs.pr-number }} + KATA_HYPERVISOR: ${{ matrix.environment.vmm }} + KBS: "true" + KBS_INGRESS: "nodeport" + KUBERNETES: ${{ matrix.environment.k8s }} + SNAPSHOTTER: ${{ matrix.environment.snapshotter }} + PULL_TYPE: ${{ matrix.environment.pull_type }} + EXPERIMENTAL_FORCE_GUEST_PULL: ${{ matrix.environment.pull_type == 'experimental-force-guest-pull' && matrix.environment.vmm || '' }} + AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }} + AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }} + AUTO_GENERATE_POLICY: "yes" + K8S_TEST_HOST_TYPE: "all" + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.commit-hash }} + fetch-depth: 0 + persist-credentials: false + + - name: Rebase atop of the latest target branch + run: | + ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" + env: + TARGET_BRANCH: ${{ inputs.target-branch }} + + - name: get-kata-tools-tarball + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }} + path: kata-tools-artifacts + + - name: Install kata-tools + run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts + + - name: Remove unnecessary directories to free up space + run: | + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + sudo rm -rf /usr/lib/jvm + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/julia* + sudo rm -rf /opt/az + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /opt/microsoft + sudo rm -rf /opt/google + sudo rm -rf /usr/lib/firefox + + - name: Deploy ${{ matrix.environment.k8s }} + timeout-minutes: 15 + run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s + + - name: Install `bats` + run: bash tests/integration/kubernetes/gha-run.sh install-bats + + - name: Deploy Kata + timeout-minutes: 20 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata + env: + USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: ${{ matrix.environment.snapshotter == 'nydus' }} + + - name: Deploy CoCo KBS + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs + + - name: Install `kbs-client` + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client + + - name: Deploy CSI driver + timeout-minutes: 5 + run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver + + - name: Run tests + timeout-minutes: 80 + run: bash tests/integration/kubernetes/gha-run.sh run-tests + + - name: Report tests + if: always() + run: bash tests/integration/kubernetes/gha-run.sh report-tests + + - name: Delete kata-deploy + if: always() + timeout-minutes: 15 + run: bash tests/integration/kubernetes/gha-run.sh cleanup + + - name: Delete CoCo KBS + if: always() + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs + + - name: Delete CSI driver + if: always() + timeout-minutes: 5 + run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver + # Generate jobs for testing CoCo on non-TEE environments with erofs-snapshotter run-k8s-tests-coco-nontee-with-erofs-snapshotter: name: run-k8s-tests-coco-nontee-with-erofs-snapshotter