From 1fa3475e36119c8eddbd269ef9e8190386ef4711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 10 Feb 2026 15:27:56 +0100 Subject: [PATCH] tests: k8s: rely more on free runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We were running most of the k8s integration tests on AKS. The ones that don't actually depend on AKS's environment now run on normal ubuntu-24.04 GitHub runners instead: we bring up a kubeadm cluster there, test with both containerd lts and active, and skip attestation tests since those runtimes don't need them. AKS is left only for the jobs that do depend on it. Signed-off-by: Fabiano Fidêncio Co-authored-by: Cursor --- .github/workflows/ci.yaml | 15 +++ .github/workflows/run-k8s-tests-on-aks.yaml | 12 +- .../run-k8s-tests-on-free-runner.yaml | 127 ++++++++++++++++++ tests/gha-run-k8s-common.sh | 8 ++ tests/integration/kubernetes/gha-run.sh | 1 + tests/integration/kubernetes/setup.sh | 1 + tests/integration/kubernetes/tests_common.sh | 13 +- 7 files changed, 159 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/run-k8s-tests-on-free-runner.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1c8d849d62..2f32fd5b4a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -297,6 +297,21 @@ jobs: AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} + run-k8s-tests-on-free-runner: + if: ${{ inputs.skip-test != 'yes' }} + needs: publish-kata-deploy-payload-amd64 + permissions: + contents: read + uses: ./.github/workflows/run-k8s-tests-on-free-runner.yaml + with: + tarball-suffix: -${{ inputs.tag }} + registry: ghcr.io + repo: ${{ github.repository_owner }}/kata-deploy-ci + tag: ${{ inputs.tag }}-amd64 + commit-hash: ${{ inputs.commit-hash }} + pr-number: ${{ inputs.pr-number }} + target-branch: ${{ inputs.target-branch }} + run-k8s-tests-on-arm64: if: ${{ inputs.skip-test != 'yes' }} needs: publish-kata-deploy-payload-arm64 diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 1a347c487c..7649aed6eb 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -42,17 +42,6 @@ jobs: strategy: fail-fast: false matrix: - host_os: - - ubuntu - vmm: - - clh - - dragonball - - qemu - - qemu-runtime-rs - - cloud-hypervisor - instance-type: - - small - - normal include: - host_os: cbl-mariner vmm: clh @@ -80,6 +69,7 @@ jobs: KUBERNETES: "vanilla" K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} GENPOLICY_PULL_METHOD: ${{ matrix.genpolicy-pull-method }} + RUNS_ON_AKS: "true" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: diff --git a/.github/workflows/run-k8s-tests-on-free-runner.yaml b/.github/workflows/run-k8s-tests-on-free-runner.yaml new file mode 100644 index 0000000000..65ac992b29 --- /dev/null +++ b/.github/workflows/run-k8s-tests-on-free-runner.yaml @@ -0,0 +1,127 @@ +# Run Kubernetes integration tests on free GitHub runners with a locally +# deployed cluster (kubeadm). +name: CI | Run kubernetes tests on free runner +on: + workflow_call: + inputs: + tarball-suffix: + required: false + type: string + registry: + required: true + type: string + repo: + required: true + type: string + tag: + required: true + type: string + pr-number: + required: true + type: string + commit-hash: + required: false + type: string + target-branch: + required: false + type: string + default: "" + +permissions: {} + +jobs: + run-k8s-tests: + name: run-k8s-tests + strategy: + fail-fast: false + matrix: + environment: [ + { vmm: clh, containerd_version: lts }, + { vmm: clh, containerd_version: active }, + { vmm: dragonball, containerd_version: lts }, + { vmm: dragonball, containerd_version: active }, + { vmm: qemu, containerd_version: lts }, + { vmm: qemu, containerd_version: active }, + { vmm: qemu-runtime-rs, containerd_version: lts }, + { vmm: qemu-runtime-rs, containerd_version: active }, + { vmm: cloud-hypervisor, containerd_version: lts }, + { vmm: cloud-hypervisor, containerd_version: active }, + ] + runs-on: ubuntu-24.04 + permissions: + contents: read + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + GH_PR_NUMBER: ${{ inputs.pr-number }} + KATA_HOST_OS: ubuntu + KATA_HYPERVISOR: ${{ matrix.environment.vmm }} + KUBERNETES: vanilla + K8S_TEST_HOST_TYPE: baremetal-no-attestation + CONTAINER_ENGINE: containerd + CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }} + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.commit-hash }} + fetch-depth: 0 + persist-credentials: false + + - name: Rebase atop of the latest target branch + run: | + ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" + env: + TARGET_BRANCH: ${{ inputs.target-branch }} + + - name: get-kata-tools-tarball + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }} + path: kata-tools-artifacts + + - name: Install kata-tools + run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts + + - name: Remove unnecessary directories to free up space + run: | + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + sudo rm -rf /usr/lib/jvm + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/julia* + sudo rm -rf /opt/az + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /opt/microsoft + sudo rm -rf /opt/google + sudo rm -rf /usr/lib/firefox + + - name: Deploy k8s (kubeadm) + run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s + + - name: Install `bats` + run: bash tests/integration/kubernetes/gha-run.sh install-bats + + - name: Deploy Kata + timeout-minutes: 20 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata + + - name: Run tests + timeout-minutes: 60 + run: bash tests/integration/kubernetes/gha-run.sh run-tests + + - name: Report tests + if: always() + run: bash tests/integration/kubernetes/gha-run.sh report-tests + + - name: Delete kata-deploy + if: always() + timeout-minutes: 15 + run: bash tests/integration/kubernetes/gha-run.sh cleanup + diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 9580f931d5..a867910c8e 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -417,6 +417,14 @@ function deploy_vanilla_k8s() { [[ -z "${container_engine}" ]] && die "container_engine is required" [[ -z "${container_engine_version}" ]] && die "container_engine_version is required" + # Resolve lts/active to the actual version from versions.yaml (e.g. v1.7, v2.1) + case "${container_engine_version}" in + lts|active) + container_engine_version=$(get_from_kata_deps ".externals.containerd.${container_engine_version}") + ;; + *) ;; + esac + install_system_dependencies "runc" load_k8s_needed_modules set_k8s_network_parameters diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 0b2be19f30..ae28d1c595 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -36,6 +36,7 @@ export PULL_TYPE="${PULL_TYPE:-default}" export TEST_CLUSTER_NAMESPACE="${TEST_CLUSTER_NAMESPACE:-kata-containers-k8s-tests}" export GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}" export TARGET_ARCH="${TARGET_ARCH:-x86_64}" +export RUNS_ON_AKS="${RUNS_ON_AKS:-false}" function configure_devmapper() { sudo mkdir -p /var/lib/containerd/devmapper diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index be8203d70b..3c090243e8 100644 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -14,6 +14,7 @@ export AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-no}" export KATA_HOST_OS="${KATA_HOST_OS:-}" export KATA_HYPERVISOR="${KATA_HYPERVISOR:-}" export PULL_TYPE="${PULL_TYPE:-default}" +export RUNS_ON_AKS="${RUNS_ON_AKS:-false}" declare -r kubernetes_dir=$(dirname "$(readlink -f "$0")") declare -r runtimeclass_workloads_work_dir="${kubernetes_dir}/runtimeclass_workloads_work" diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 3e69a1dbb8..5a2984deeb 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -39,6 +39,7 @@ AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-}" GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-}" KATA_HOST_OS="${KATA_HOST_OS:-}" +RUNS_ON_AKS="${RUNS_ON_AKS:-false}" # Common setup for tests. # @@ -98,13 +99,11 @@ is_nvidia_gpu_platform() { } is_aks_cluster() { - case "${KATA_HYPERVISOR}" in - "qemu-tdx"|"qemu-snp"|qemu-nvidia-gpu*) - return 1 - ;; - *) - return 0 - esac + if [[ "${RUNS_ON_AKS}" = "true" ]]; then + return 0 + fi + + return 1 } adapt_common_policy_settings_for_non_coco() {