mirror of
https://github.com/kata-containers/kata-containers.git
synced 2026-07-13 20:27:41 +00:00
4e024bfb43
This reverts commit ab25592533, as now
we're deploying k3s/rke2 in a way that we properly test them.
Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
502 lines
18 KiB
YAML
502 lines
18 KiB
YAML
name: CI | Run kata coco tests
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
tarball-suffix:
|
|
required: false
|
|
type: string
|
|
registry:
|
|
required: true
|
|
type: string
|
|
repo:
|
|
required: true
|
|
type: string
|
|
tag:
|
|
required: true
|
|
type: string
|
|
pr-number:
|
|
required: true
|
|
type: string
|
|
commit-hash:
|
|
required: false
|
|
type: string
|
|
target-branch:
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
extensive-matrix-autogenerated-policy:
|
|
required: false
|
|
type: string
|
|
default: no
|
|
secrets:
|
|
AUTHENTICATED_IMAGE_PASSWORD:
|
|
required: true
|
|
AZ_APPID:
|
|
required: true
|
|
AZ_TENANT_ID:
|
|
required: true
|
|
AZ_SUBSCRIPTION_ID:
|
|
required: true
|
|
ITA_KEY:
|
|
required: true
|
|
|
|
permissions: {}
|
|
|
|
jobs:
|
|
run-k8s-tests-on-tee:
|
|
name: run-k8s-tests-on-tee
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- runner: tdx
|
|
vmm: qemu-tdx
|
|
- runner: sev-snp
|
|
vmm: qemu-snp
|
|
runs-on: ${{ matrix.runner }}
|
|
env:
|
|
DOCKER_REGISTRY: ${{ inputs.registry }}
|
|
DOCKER_REPO: ${{ inputs.repo }}
|
|
DOCKER_TAG: ${{ inputs.tag }}
|
|
GH_PR_NUMBER: ${{ inputs.pr-number }}
|
|
KATA_HYPERVISOR: ${{ matrix.vmm }}
|
|
KUBERNETES: "vanilla"
|
|
KBS: "true"
|
|
K8S_TEST_HOST_TYPE: "baremetal"
|
|
KBS_INGRESS: "nodeport"
|
|
SNAPSHOTTER: "nydus"
|
|
PULL_TYPE: "guest-pull"
|
|
AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
|
|
AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
|
|
GH_ITA_KEY: ${{ secrets.ITA_KEY }}
|
|
AUTO_GENERATE_POLICY: "yes"
|
|
steps:
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
ref: ${{ inputs.commit-hash }}
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Rebase atop of the latest target branch
|
|
run: |
|
|
./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
|
|
env:
|
|
TARGET_BRANCH: ${{ inputs.target-branch }}
|
|
|
|
- name: get-kata-tools-tarball
|
|
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
with:
|
|
name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
|
|
path: kata-tools-artifacts
|
|
|
|
- name: Install kata-tools
|
|
run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
|
|
|
|
- name: Deploy Kata
|
|
timeout-minutes: 20
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
|
|
|
|
- name: Uninstall previous `kbs-client`
|
|
timeout-minutes: 10
|
|
run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
|
|
|
|
- name: Deploy CoCo KBS
|
|
timeout-minutes: 10
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
|
|
env:
|
|
ITA_KEY: ${{ env.KATA_HYPERVISOR == 'qemu-tdx' && env.GH_ITA_KEY || '' }}
|
|
|
|
- name: Install `kbs-client`
|
|
timeout-minutes: 10
|
|
run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
|
|
|
|
- name: Deploy CSI driver
|
|
timeout-minutes: 5
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
|
|
|
|
- name: Run tests
|
|
timeout-minutes: 100
|
|
run: bash tests/integration/kubernetes/gha-run.sh run-tests
|
|
|
|
- name: Report tests
|
|
if: always()
|
|
run: bash tests/integration/kubernetes/gha-run.sh report-tests
|
|
|
|
- name: Delete kata-deploy
|
|
if: always()
|
|
timeout-minutes: 15
|
|
run: bash tests/integration/kubernetes/gha-run.sh cleanup
|
|
|
|
- name: Delete CoCo KBS
|
|
if: always()
|
|
timeout-minutes: 10
|
|
run: |
|
|
[[ "${KATA_HYPERVISOR}" == "qemu-tdx" ]] && echo "ITA_KEY=${GH_ITA_KEY}" >> "${GITHUB_ENV}"
|
|
bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
|
|
|
|
- name: Delete CSI driver
|
|
timeout-minutes: 5
|
|
run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
|
|
|
|
# Generate jobs for testing CoCo on non-TEE environments
|
|
run-k8s-tests-coco-nontee:
|
|
name: run-k8s-tests-coco-nontee
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
environment: [
|
|
{ vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
|
|
{ vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
|
|
{ vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
|
|
]
|
|
runs-on: ubuntu-24.04
|
|
permissions:
|
|
contents: read
|
|
environment: ci
|
|
env:
|
|
DOCKER_REGISTRY: ${{ inputs.registry }}
|
|
DOCKER_REPO: ${{ inputs.repo }}
|
|
DOCKER_TAG: ${{ inputs.tag }}
|
|
GH_PR_NUMBER: ${{ inputs.pr-number }}
|
|
KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
|
|
# Some tests rely on that variable to run (or not)
|
|
KBS: "true"
|
|
# Set the KBS ingress handler (empty string disables handling)
|
|
KBS_INGRESS: "nodeport"
|
|
KUBERNETES: "vanilla"
|
|
PULL_TYPE: ${{ matrix.environment.pull_type }}
|
|
AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
|
|
AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
|
|
SNAPSHOTTER: ${{ matrix.environment.snapshotter }}
|
|
EXPERIMENTAL_FORCE_GUEST_PULL: ${{ matrix.environment.pull_type == 'experimental-force-guest-pull' && matrix.environment.vmm || '' }}
|
|
AUTO_GENERATE_POLICY: "yes"
|
|
K8S_TEST_HOST_TYPE: "all"
|
|
CONTAINER_ENGINE: "containerd"
|
|
CONTAINER_ENGINE_VERSION: "active"
|
|
GH_TOKEN: ${{ github.token }}
|
|
steps:
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
ref: ${{ inputs.commit-hash }}
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Rebase atop of the latest target branch
|
|
run: |
|
|
./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
|
|
env:
|
|
TARGET_BRANCH: ${{ inputs.target-branch }}
|
|
|
|
- name: get-kata-tools-tarball
|
|
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
with:
|
|
name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
|
|
path: kata-tools-artifacts
|
|
|
|
- name: Install kata-tools
|
|
run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
|
|
|
|
- name: Remove unnecessary directories to free up space
|
|
run: |
|
|
sudo rm -rf /usr/local/.ghcup
|
|
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf /usr/local/share/boost
|
|
sudo rm -rf /usr/lib/jvm
|
|
sudo rm -rf /usr/share/swift
|
|
sudo rm -rf /usr/local/share/powershell
|
|
sudo rm -rf /usr/local/julia*
|
|
sudo rm -rf /opt/az
|
|
sudo rm -rf /usr/local/share/chromium
|
|
sudo rm -rf /opt/microsoft
|
|
sudo rm -rf /opt/google
|
|
sudo rm -rf /usr/lib/firefox
|
|
|
|
- name: Deploy kubernetes
|
|
timeout-minutes: 15
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
|
|
|
|
- name: Install `bats`
|
|
run: bash tests/integration/kubernetes/gha-run.sh install-bats
|
|
|
|
- name: Deploy Kata
|
|
timeout-minutes: 20
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
|
|
env:
|
|
USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: ${{ matrix.environment.snapshotter == 'nydus' }}
|
|
|
|
- name: Deploy CoCo KBS
|
|
timeout-minutes: 10
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
|
|
|
|
- name: Install `kbs-client`
|
|
timeout-minutes: 10
|
|
run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
|
|
|
|
- name: Deploy CSI driver
|
|
timeout-minutes: 5
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
|
|
|
|
- name: Run tests
|
|
timeout-minutes: 80
|
|
run: bash tests/integration/kubernetes/gha-run.sh run-tests
|
|
|
|
- name: Report tests
|
|
if: always()
|
|
run: bash tests/integration/kubernetes/gha-run.sh report-tests
|
|
|
|
- name: Delete kata-deploy
|
|
if: always()
|
|
timeout-minutes: 15
|
|
run: bash tests/integration/kubernetes/gha-run.sh cleanup
|
|
|
|
- name: Delete CoCo KBS
|
|
if: always()
|
|
timeout-minutes: 10
|
|
run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
|
|
|
|
- name: Delete CSI driver
|
|
if: always()
|
|
timeout-minutes: 5
|
|
run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
|
|
|
|
# Extensive matrix: autogenerated policy tests (nydus + experimental-force-guest-pull) on k0s, k3s, rke2, microk8s with qemu-coco-dev / qemu-coco-dev-runtime-rs
|
|
run-k8s-tests-coco-nontee-extensive-matrix:
|
|
if: ${{ inputs.extensive-matrix-autogenerated-policy == 'yes' }}
|
|
name: run-k8s-tests-coco-nontee-extensive-matrix
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
environment: [
|
|
{ k8s: k0s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
|
|
{ k8s: k0s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
|
|
{ k8s: k0s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
|
|
{ k8s: k3s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
|
|
{ k8s: k3s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
|
|
{ k8s: k3s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
|
|
{ k8s: rke2, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
|
|
{ k8s: rke2, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
|
|
{ k8s: rke2, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
|
|
{ k8s: microk8s, vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
|
|
{ k8s: microk8s, vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
|
|
{ k8s: microk8s, vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
|
|
]
|
|
runs-on: ubuntu-24.04
|
|
permissions:
|
|
contents: read
|
|
environment: ci
|
|
env:
|
|
DOCKER_REGISTRY: ${{ inputs.registry }}
|
|
DOCKER_REPO: ${{ inputs.repo }}
|
|
DOCKER_TAG: ${{ inputs.tag }}
|
|
GH_PR_NUMBER: ${{ inputs.pr-number }}
|
|
KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
|
|
KBS: "true"
|
|
KBS_INGRESS: "nodeport"
|
|
KUBERNETES: ${{ matrix.environment.k8s }}
|
|
SNAPSHOTTER: ${{ matrix.environment.snapshotter }}
|
|
PULL_TYPE: ${{ matrix.environment.pull_type }}
|
|
EXPERIMENTAL_FORCE_GUEST_PULL: ${{ matrix.environment.pull_type == 'experimental-force-guest-pull' && matrix.environment.vmm || '' }}
|
|
AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
|
|
AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
|
|
AUTO_GENERATE_POLICY: "yes"
|
|
K8S_TEST_HOST_TYPE: "all"
|
|
GH_TOKEN: ${{ github.token }}
|
|
steps:
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
ref: ${{ inputs.commit-hash }}
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Rebase atop of the latest target branch
|
|
run: |
|
|
./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
|
|
env:
|
|
TARGET_BRANCH: ${{ inputs.target-branch }}
|
|
|
|
- name: get-kata-tools-tarball
|
|
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
with:
|
|
name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
|
|
path: kata-tools-artifacts
|
|
|
|
- name: Install kata-tools
|
|
run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
|
|
|
|
- name: Remove unnecessary directories to free up space
|
|
run: |
|
|
sudo rm -rf /usr/local/.ghcup
|
|
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf /usr/local/share/boost
|
|
sudo rm -rf /usr/lib/jvm
|
|
sudo rm -rf /usr/share/swift
|
|
sudo rm -rf /usr/local/share/powershell
|
|
sudo rm -rf /usr/local/julia*
|
|
sudo rm -rf /opt/az
|
|
sudo rm -rf /usr/local/share/chromium
|
|
sudo rm -rf /opt/microsoft
|
|
sudo rm -rf /opt/google
|
|
sudo rm -rf /usr/lib/firefox
|
|
|
|
- name: Deploy ${{ matrix.environment.k8s }}
|
|
timeout-minutes: 15
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
|
|
|
|
- name: Install `bats`
|
|
run: bash tests/integration/kubernetes/gha-run.sh install-bats
|
|
|
|
- name: Deploy Kata
|
|
timeout-minutes: 20
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
|
|
env:
|
|
USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: ${{ matrix.environment.snapshotter == 'nydus' }}
|
|
|
|
- name: Deploy CoCo KBS
|
|
timeout-minutes: 10
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
|
|
|
|
- name: Install `kbs-client`
|
|
timeout-minutes: 10
|
|
run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
|
|
|
|
- name: Deploy CSI driver
|
|
timeout-minutes: 5
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
|
|
|
|
- name: Run tests
|
|
timeout-minutes: 80
|
|
run: bash tests/integration/kubernetes/gha-run.sh run-tests
|
|
|
|
- name: Report tests
|
|
if: always()
|
|
run: bash tests/integration/kubernetes/gha-run.sh report-tests
|
|
|
|
- name: Delete kata-deploy
|
|
if: always()
|
|
timeout-minutes: 15
|
|
run: bash tests/integration/kubernetes/gha-run.sh cleanup
|
|
|
|
- name: Delete CoCo KBS
|
|
if: always()
|
|
timeout-minutes: 10
|
|
run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
|
|
|
|
- name: Delete CSI driver
|
|
if: always()
|
|
timeout-minutes: 5
|
|
run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
|
|
|
|
# Generate jobs for testing CoCo on non-TEE environments with erofs-snapshotter
|
|
run-k8s-tests-coco-nontee-with-erofs-snapshotter:
|
|
name: run-k8s-tests-coco-nontee-with-erofs-snapshotter
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
vmm:
|
|
- qemu-coco-dev
|
|
snapshotter:
|
|
- erofs
|
|
pull-type:
|
|
- default
|
|
runs-on: ubuntu-24.04
|
|
environment: ci
|
|
env:
|
|
DOCKER_REGISTRY: ${{ inputs.registry }}
|
|
DOCKER_REPO: ${{ inputs.repo }}
|
|
DOCKER_TAG: ${{ inputs.tag }}
|
|
GH_PR_NUMBER: ${{ inputs.pr-number }}
|
|
KATA_HYPERVISOR: ${{ matrix.vmm }}
|
|
# Some tests rely on that variable to run (or not)
|
|
KBS: "false"
|
|
# Set the KBS ingress handler (empty string disables handling)
|
|
KBS_INGRESS: ""
|
|
KUBERNETES: "vanilla"
|
|
CONTAINER_ENGINE: "containerd"
|
|
CONTAINER_ENGINE_VERSION: "active"
|
|
PULL_TYPE: ${{ matrix.pull-type }}
|
|
SNAPSHOTTER: ${{ matrix.snapshotter }}
|
|
USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: "true"
|
|
K8S_TEST_HOST_TYPE: "all"
|
|
# We are skipping the auto generated policy tests for now,
|
|
# but those should be enabled as soon as we work on that.
|
|
AUTO_GENERATE_POLICY: "no"
|
|
GH_TOKEN: ${{ github.token }}
|
|
steps:
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
ref: ${{ inputs.commit-hash }}
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Rebase atop of the latest target branch
|
|
run: |
|
|
./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
|
|
env:
|
|
TARGET_BRANCH: ${{ inputs.target-branch }}
|
|
|
|
- name: get-kata-tools-tarball
|
|
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
with:
|
|
name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
|
|
path: kata-tools-artifacts
|
|
|
|
- name: Install kata-tools
|
|
run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
|
|
|
|
- name: Remove unnecessary directories to free up space
|
|
run: |
|
|
sudo rm -rf /usr/local/.ghcup
|
|
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf /usr/local/share/boost
|
|
sudo rm -rf /usr/lib/jvm
|
|
sudo rm -rf /usr/share/swift
|
|
sudo rm -rf /usr/local/share/powershell
|
|
sudo rm -rf /usr/local/julia*
|
|
sudo rm -rf /opt/az
|
|
sudo rm -rf /usr/local/share/chromium
|
|
sudo rm -rf /opt/microsoft
|
|
sudo rm -rf /opt/google
|
|
sudo rm -rf /usr/lib/firefox
|
|
|
|
- name: Deploy kubernetes
|
|
timeout-minutes: 15
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
|
|
|
|
- name: Install `bats`
|
|
run: bash tests/integration/kubernetes/gha-run.sh install-bats
|
|
|
|
- name: Deploy Kata
|
|
timeout-minutes: 20
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
|
|
|
|
- name: Deploy CSI driver
|
|
timeout-minutes: 5
|
|
run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
|
|
|
|
- name: Run tests
|
|
timeout-minutes: 80
|
|
run: bash tests/integration/kubernetes/gha-run.sh run-tests
|
|
|
|
- name: Report tests
|
|
if: always()
|
|
run: bash tests/integration/kubernetes/gha-run.sh report-tests
|
|
|
|
- name: Delete kata-deploy
|
|
if: always()
|
|
timeout-minutes: 15
|
|
run: bash tests/integration/kubernetes/gha-run.sh cleanup
|
|
|
|
- name: Delete CSI driver
|
|
if: always()
|
|
timeout-minutes: 5
|
|
run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
|