tests: k8s: rely more on free runners

We were running most of the k8s integration tests on AKS. The ones that don't actually depend on AKS's environment now run on normal ubuntu-24.04 GitHub runners instead: we bring up a kubeadm cluster there, test with both containerd lts and active, and skip attestation tests since those runtimes don't need them. AKS is left only for the jobs that do depend on it. Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com> Co-authored-by: Cursor <cursoragent@cursor.com>
2026-02-21 22:34:29 +00:00 · 2026-02-10 15:27:56 +01:00
parent 2f056484f3
commit 1fa3475e36
7 changed files with 159 additions and 18 deletions
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -297,6 +297,21 @@ jobs:
      AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
      AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}

+  run-k8s-tests-on-free-runner:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: publish-kata-deploy-payload-amd64
+    permissions:
+      contents: read
+    uses: ./.github/workflows/run-k8s-tests-on-free-runner.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+
  run-k8s-tests-on-arm64:
    if: ${{ inputs.skip-test != 'yes' }}
    needs: publish-kata-deploy-payload-arm64
--- a/.github/workflows/run-k8s-tests-on-aks.yaml
+++ b/.github/workflows/run-k8s-tests-on-aks.yaml
@@ -42,17 +42,6 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        host_os:
-          - ubuntu
-        vmm:
-          - clh
-          - dragonball
-          - qemu
-          - qemu-runtime-rs
-          - cloud-hypervisor
-        instance-type:
-          - small
-          - normal
        include:
          - host_os: cbl-mariner
            vmm: clh
@@ -80,6 +69,7 @@ jobs:
      KUBERNETES: "vanilla"
      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
      GENPOLICY_PULL_METHOD: ${{ matrix.genpolicy-pull-method }}
+      RUNS_ON_AKS: "true"
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
--- a/.github/workflows/run-k8s-tests-on-free-runner.yaml
+++ b/.github/workflows/run-k8s-tests-on-free-runner.yaml
@@ -0,0 +1,127 @@
+# Run Kubernetes integration tests on free GitHub runners  with a locally
+# deployed cluster (kubeadm).
+name: CI | Run kubernetes tests on free runner
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+permissions: {}
+
+jobs:
+  run-k8s-tests:
+    name: run-k8s-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: [
+          { vmm: clh, containerd_version: lts },
+          { vmm: clh, containerd_version: active },
+          { vmm: dragonball, containerd_version: lts },
+          { vmm: dragonball, containerd_version: active },
+          { vmm: qemu, containerd_version: lts },
+          { vmm: qemu, containerd_version: active },
+          { vmm: qemu-runtime-rs, containerd_version: lts },
+          { vmm: qemu-runtime-rs, containerd_version: active },
+          { vmm: cloud-hypervisor, containerd_version: lts },
+          { vmm: cloud-hypervisor, containerd_version: active },
+        ]
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ubuntu
+      KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
+      KUBERNETES: vanilla
+      K8S_TEST_HOST_TYPE: baremetal-no-attestation
+      CONTAINER_ENGINE: containerd
+      CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }}
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tools-tarball
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-tools-artifacts
+
+      - name: Install kata-tools
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
+
+      - name: Remove unnecessary directories to free up space
+        run: |
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf /usr/lib/jvm
+          sudo rm -rf /usr/share/swift
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /usr/local/julia*
+          sudo rm -rf /opt/az
+          sudo rm -rf /usr/local/share/chromium
+          sudo rm -rf /opt/microsoft
+          sudo rm -rf /opt/google
+          sudo rm -rf /usr/lib/firefox
+
+      - name: Deploy k8s (kubeadm)
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+
+      - name: Deploy Kata
+        timeout-minutes: 20
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
+
+      - name: Run tests
+        timeout-minutes: 60
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Report tests
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh report-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        timeout-minutes: 15
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup
+
--- a/tests/gha-run-k8s-common.sh
+++ b/tests/gha-run-k8s-common.sh
@@ -417,6 +417,14 @@ function deploy_vanilla_k8s() {
 	[[ -z "${container_engine}" ]] && die "container_engine is required"
 	[[ -z "${container_engine_version}" ]] && die "container_engine_version is required"

+	# Resolve lts/active to the actual version from versions.yaml (e.g. v1.7, v2.1)
+	case "${container_engine_version}" in
+		lts|active)
+			container_engine_version=$(get_from_kata_deps ".externals.containerd.${container_engine_version}")
+			;;
+		*) ;;
+	esac
+
 	install_system_dependencies "runc"
 	load_k8s_needed_modules
 	set_k8s_network_parameters
--- a/tests/integration/kubernetes/gha-run.sh
+++ b/tests/integration/kubernetes/gha-run.sh
@@ -36,6 +36,7 @@ export PULL_TYPE="${PULL_TYPE:-default}"
 export TEST_CLUSTER_NAMESPACE="${TEST_CLUSTER_NAMESPACE:-kata-containers-k8s-tests}"
 export GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}"
 export TARGET_ARCH="${TARGET_ARCH:-x86_64}"
+export RUNS_ON_AKS="${RUNS_ON_AKS:-false}"

 function configure_devmapper() {
 	sudo mkdir -p /var/lib/containerd/devmapper
--- a/tests/integration/kubernetes/setup.sh
+++ b/tests/integration/kubernetes/setup.sh
@@ -14,6 +14,7 @@ export AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-no}"
 export KATA_HOST_OS="${KATA_HOST_OS:-}"
 export KATA_HYPERVISOR="${KATA_HYPERVISOR:-}"
 export PULL_TYPE="${PULL_TYPE:-default}"
+export RUNS_ON_AKS="${RUNS_ON_AKS:-false}"

 declare -r kubernetes_dir=$(dirname "$(readlink -f "$0")")
 declare -r runtimeclass_workloads_work_dir="${kubernetes_dir}/runtimeclass_workloads_work"
--- a/tests/integration/kubernetes/tests_common.sh
+++ b/tests/integration/kubernetes/tests_common.sh
@@ -39,6 +39,7 @@ AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-}"
 GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-}"
 KATA_HYPERVISOR="${KATA_HYPERVISOR:-}"
 KATA_HOST_OS="${KATA_HOST_OS:-}"
+RUNS_ON_AKS="${RUNS_ON_AKS:-false}"

 # Common setup for tests.
 #
@@ -98,13 +99,11 @@ is_nvidia_gpu_platform() {
 }

 is_aks_cluster() {
-	case "${KATA_HYPERVISOR}" in
-		"qemu-tdx"|"qemu-snp"|qemu-nvidia-gpu*)
-			return 1
-			;;
-		*)
-			return 0
-	esac
+	if [[ "${RUNS_ON_AKS}" = "true" ]]; then
+		return 0
+	fi
+
+	return 1
 }

 adapt_common_policy_settings_for_non_coco() {