kata-containers/.github/workflows/run-kata-coco-tests.yaml

name: CI | Run kata coco tests
on:
  workflow_call:
    inputs:
      tarball-suffix:
        required: false
        type: string
      registry:
        required: true
        type: string
      repo:
        required: true
        type: string
      tag:
        required: true
        type: string
      pr-number:
        required: true
        type: string
      commit-hash:
        required: false
        type: string
      target-branch:
        required: false
        type: string
        default: ""
      extensive-matrix-autogenerated-policy:
        required: false
        type: string
        default: no
      tee-test-scope:
        required: false
        type: string
        default: small
    secrets:
      AUTHENTICATED_IMAGE_PASSWORD:
        required: true
      AZ_APPID:
        required: true
      AZ_TENANT_ID:
        required: true
      AZ_SUBSCRIPTION_ID:
        required: true

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-coco
  cancel-in-progress: true

permissions: {}

jobs:
  run-k8s-tests-on-tee:
    name: run-k8s-tests-on-tee
    strategy:
      fail-fast: false
      matrix:
        include:
          - runner: tdx
            vmm: qemu-tdx
          - runner: tdx
            vmm: qemu-tdx-runtime-rs
          - runner: sev-snp
            vmm: qemu-snp
          - runner: sev-snp
            vmm: qemu-snp-runtime-rs
    concurrency:
      group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-k8s-tee-${{ toJSON(matrix) }}
      cancel-in-progress: true
    runs-on: ${{ matrix.runner }}
    env:
      DOCKER_REGISTRY: ${{ inputs.registry }}
      DOCKER_REPO: ${{ inputs.repo }}
      DOCKER_TAG: ${{ inputs.tag }}
      GH_PR_NUMBER: ${{ inputs.pr-number }}
      KATA_HYPERVISOR: ${{ matrix.vmm }}
      KUBERNETES: "vanilla"
      KBS: "true"
      K8S_TEST_HOST_TYPE: ${{ inputs.tee-test-scope == 'full' && 'baremetal' || 'baremetal-small-tee' }}
      KBS_INGRESS: "nodeport"
      SNAPSHOTTER: "nydus"
      PULL_TYPE: "guest-pull"
      AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
      AUTO_GENERATE_POLICY: "yes"
    steps:
      - name: Skip non-nightly TDX runtime-rs
        if: ${{ matrix.vmm == 'qemu-tdx-runtime-rs' && inputs.pr-number != 'nightly' }}
        run: echo "SKIP_JOB=true" >> "${GITHUB_ENV}"

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        if: ${{ env.SKIP_JOB != 'true' }}
        with:
          ref: ${{ inputs.commit-hash }}
          fetch-depth: 0
          persist-credentials: false

      - name: Rebase atop of the latest target branch
        if: ${{ env.SKIP_JOB != 'true' }}
        run: |
          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
        env:
          TARGET_BRANCH: ${{ inputs.target-branch }}

      - name: get-kata-tools-tarball
        if: ${{ env.SKIP_JOB != 'true' }}
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
          path: kata-tools-artifacts

      - name: Install kata-tools
        if: ${{ env.SKIP_JOB != 'true' }}
        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts

      - name: Deploy Kata
        if: ${{ env.SKIP_JOB != 'true' }}
        timeout-minutes: 20
        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata

      - name: Uninstall previous `kbs-client`
        if: ${{ env.SKIP_JOB != 'true' }}
        timeout-minutes: 10
        run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client

      - name: Deploy CoCo KBS
        if: ${{ env.SKIP_JOB != 'true' }}
        timeout-minutes: 10
        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs

      - name: Install `kbs-client`
        if: ${{ env.SKIP_JOB != 'true' }}
        timeout-minutes: 10
        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client

      - name: Run tests
        if: ${{ env.SKIP_JOB != 'true' }}
        timeout-minutes: 100
        run: bash tests/integration/kubernetes/gha-run.sh run-tests

      - name: Report tests
        if: ${{ always() && env.SKIP_JOB != 'true' }}
        run: bash tests/integration/kubernetes/gha-run.sh report-tests

      - name: Delete kata-deploy
        if: ${{ always() && env.SKIP_JOB != 'true' }}
        timeout-minutes: 15
        run: bash tests/integration/kubernetes/gha-run.sh cleanup

      - name: Delete CoCo KBS
        if: ${{ always() && env.SKIP_JOB != 'true' }}
        timeout-minutes: 10
        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs

  # Generate jobs for testing CoCo on non-TEE environments
  run-k8s-tests-coco-nontee:
    name: run-k8s-tests-coco-nontee
    strategy:
      fail-fast: false
      matrix:
        environment: [
          { vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
          { vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
          { vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
        ]
    concurrency:
      group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-${{ toJSON(matrix) }}
      cancel-in-progress: true
    runs-on: ubuntu-24.04
    permissions:
      contents: read
    environment:
      name: ci
      deployment: false
    env:
      DOCKER_REGISTRY: ${{ inputs.registry }}
      DOCKER_REPO: ${{ inputs.repo }}
      DOCKER_TAG: ${{ inputs.tag }}
      GH_PR_NUMBER: ${{ inputs.pr-number }}
      KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
      # Some tests rely on that variable to run (or not)
      KBS: "true"
      # Set the KBS ingress handler (empty string disables handling)
      KBS_INGRESS: "nodeport"
      KUBERNETES: "vanilla"
      PULL_TYPE: ${{ matrix.environment.pull_type }}
      AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
      SNAPSHOTTER: ${{ matrix.environment.snapshotter }}
      EXPERIMENTAL_FORCE_GUEST_PULL: ${{ matrix.environment.pull_type == 'experimental-force-guest-pull' && matrix.environment.vmm || '' }}
      AUTO_GENERATE_POLICY: "yes"
      K8S_TEST_HOST_TYPE: "all"
      CONTAINER_ENGINE: "containerd"
      CONTAINER_ENGINE_VERSION: "active"
      GH_TOKEN: ${{ github.token }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.commit-hash }}
          fetch-depth: 0
          persist-credentials: false

      - name: Rebase atop of the latest target branch
        run: |
          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
        env:
          TARGET_BRANCH: ${{ inputs.target-branch }}

      - name: get-kata-tools-tarball
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
          path: kata-tools-artifacts

      - name: Install kata-tools
        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts

      - name: Remove unnecessary directories to free up space
        run: |
          sudo rm -rf /usr/local/.ghcup
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          sudo rm -rf /usr/lib/jvm
          sudo rm -rf /usr/share/swift
          sudo rm -rf /usr/local/share/powershell
          sudo rm -rf /usr/local/julia*
          sudo rm -rf /opt/az
          sudo rm -rf /usr/local/share/chromium
          sudo rm -rf /opt/microsoft
          sudo rm -rf /opt/google
          sudo rm -rf /usr/lib/firefox

      - name: Deploy kubernetes
        timeout-minutes: 15
        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s

      - name: Install `bats`
        run: bash tests/integration/kubernetes/gha-run.sh install-bats

      - name: Deploy Kata
        timeout-minutes: 20
        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
        env:
          USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: ${{ matrix.environment.snapshotter == 'nydus' }}

      - name: Deploy CoCo KBS
        timeout-minutes: 10
        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs

      - name: Install `kbs-client`
        timeout-minutes: 10
        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client

      - name: Run tests
        timeout-minutes: 80
        run: bash tests/integration/kubernetes/gha-run.sh run-tests

      - name: Report tests
        if: always()
        run: bash tests/integration/kubernetes/gha-run.sh report-tests

      - name: Delete kata-deploy
        if: always()
        timeout-minutes: 15
        run: bash tests/integration/kubernetes/gha-run.sh cleanup

      - name: Delete CoCo KBS
        if: always()
        timeout-minutes: 10
        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs

  # Extensive matrix: autogenerated policy tests (nydus + experimental-force-guest-pull) on k0s, k3s, rke2, microk8s with qemu-coco-dev / qemu-coco-dev-runtime-rs
  run-k8s-tests-coco-nontee-extensive-matrix:
    if: ${{ inputs.extensive-matrix-autogenerated-policy == 'yes' }}
    name: run-k8s-tests-coco-nontee-extensive-matrix
    strategy:
      fail-fast: false
      matrix:
        environment: [
          { k8s: k0s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
          { k8s: k0s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
          { k8s: k0s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
          { k8s: k3s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
          { k8s: k3s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
          { k8s: k3s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
          { k8s: rke2, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
          { k8s: rke2, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
          { k8s: rke2, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
          { k8s: microk8s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
          { k8s: microk8s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
          { k8s: microk8s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
        ]
    concurrency:
      group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-${{ toJSON(matrix) }}
      cancel-in-progress: true
    runs-on: ubuntu-24.04
    permissions:
      contents: read
    environment:
      name: ci
      deployment: false
    env:
      DOCKER_REGISTRY: ${{ inputs.registry }}
      DOCKER_REPO: ${{ inputs.repo }}
      DOCKER_TAG: ${{ inputs.tag }}
      GH_PR_NUMBER: ${{ inputs.pr-number }}
      KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
      KBS: "true"
      KBS_INGRESS: "nodeport"
      KUBERNETES: ${{ matrix.environment.k8s }}
      SNAPSHOTTER: ${{ matrix.environment.snapshotter }}
      PULL_TYPE: ${{ matrix.environment.pull_type }}
      EXPERIMENTAL_FORCE_GUEST_PULL: ${{ matrix.environment.pull_type == 'experimental-force-guest-pull' && matrix.environment.vmm || '' }}
      AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
      AUTO_GENERATE_POLICY: "yes"
      K8S_TEST_HOST_TYPE: "all"
      GH_TOKEN: ${{ github.token }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.commit-hash }}
          fetch-depth: 0
          persist-credentials: false

      - name: Rebase atop of the latest target branch
        run: |
          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
        env:
          TARGET_BRANCH: ${{ inputs.target-branch }}

      - name: get-kata-tools-tarball
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
          path: kata-tools-artifacts

      - name: Install kata-tools
        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts

      - name: Remove unnecessary directories to free up space
        run: |
          sudo rm -rf /usr/local/.ghcup
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          sudo rm -rf /usr/lib/jvm
          sudo rm -rf /usr/share/swift
          sudo rm -rf /usr/local/share/powershell
          sudo rm -rf /usr/local/julia*
          sudo rm -rf /opt/az
          sudo rm -rf /usr/local/share/chromium
          sudo rm -rf /opt/microsoft
          sudo rm -rf /opt/google
          sudo rm -rf /usr/lib/firefox

      - name: Deploy ${{ matrix.environment.k8s }}
        timeout-minutes: 15
        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s

      - name: Install `bats`
        run: bash tests/integration/kubernetes/gha-run.sh install-bats

      - name: Deploy Kata
        timeout-minutes: 20
        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
        env:
          USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: ${{ matrix.environment.snapshotter == 'nydus' }}

      - name: Deploy CoCo KBS
        timeout-minutes: 10
        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs

      - name: Install `kbs-client`
        timeout-minutes: 10
        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client

      - name: Run tests
        timeout-minutes: 80
        run: bash tests/integration/kubernetes/gha-run.sh run-tests

      - name: Report tests
        if: always()
        run: bash tests/integration/kubernetes/gha-run.sh report-tests

      - name: Delete kata-deploy
        if: always()
        timeout-minutes: 15
        run: bash tests/integration/kubernetes/gha-run.sh cleanup

      - name: Delete CoCo KBS
        if: always()
        timeout-minutes: 10
        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs

  # Generate jobs for testing CoCo on non-TEE environments with erofs-snapshotter
  run-k8s-tests-coco-nontee-with-erofs-snapshotter:
    name: run-k8s-tests-coco-nontee-with-erofs-snapshotter
    strategy:
      fail-fast: false
      matrix:
        vmm:
          - qemu-coco-dev-runtime-rs
        snapshotter:
          - erofs
        pull-type:
          - default
    concurrency:
      group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}
      cancel-in-progress: true
    runs-on: ubuntu-24.04
    environment:
      name: ci
      deployment: false
    env:
      DOCKER_REGISTRY: ${{ inputs.registry }}
      DOCKER_REPO: ${{ inputs.repo }}
      DOCKER_TAG: ${{ inputs.tag }}
      GH_PR_NUMBER: ${{ inputs.pr-number }}
      KATA_HYPERVISOR: ${{ matrix.vmm }}
      # Some tests rely on that variable to run (or not)
      KBS: "false"
      # Set the KBS ingress handler (empty string disables handling)
      KBS_INGRESS: ""
      KUBERNETES: "vanilla"
      CONTAINER_ENGINE: "containerd"
      CONTAINER_ENGINE_VERSION: "v2.3"
      PULL_TYPE: ${{ matrix.pull-type }}
      SNAPSHOTTER: ${{ matrix.snapshotter }}
      USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: "true"
      K8S_TEST_HOST_TYPE: "all"
      # We are skipping the auto generated policy tests for now,
      # but those should be enabled as soon as we work on that.
      AUTO_GENERATE_POLICY: "no"
      GH_TOKEN: ${{ github.token }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.commit-hash }}
          fetch-depth: 0
          persist-credentials: false

      - name: Rebase atop of the latest target branch
        run: |
          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
        env:
          TARGET_BRANCH: ${{ inputs.target-branch }}

      - name: get-kata-tools-tarball
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
          path: kata-tools-artifacts

      - name: Install kata-tools
        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts

      - name: Remove unnecessary directories to free up space
        run: |
          sudo rm -rf /usr/local/.ghcup
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          sudo rm -rf /usr/lib/jvm
          sudo rm -rf /usr/share/swift
          sudo rm -rf /usr/local/share/powershell
          sudo rm -rf /usr/local/julia*
          sudo rm -rf /opt/az
          sudo rm -rf /usr/local/share/chromium
          sudo rm -rf /opt/microsoft
          sudo rm -rf /opt/google
          sudo rm -rf /usr/lib/firefox

      - name: Deploy kubernetes
        timeout-minutes: 15
        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s

      - name: Install `bats`
        run: bash tests/integration/kubernetes/gha-run.sh install-bats

      - name: Deploy Kata
        timeout-minutes: 20
        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata

      - name: Run tests
        timeout-minutes: 80
        run: bash tests/integration/kubernetes/gha-run.sh run-tests

      - name: Report tests
        if: always()
        run: bash tests/integration/kubernetes/gha-run.sh report-tests

      - name: Delete kata-deploy
        if: always()
        timeout-minutes: 15
        run: bash tests/integration/kubernetes/gha-run.sh cleanup