diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1c8d849d62..2f32fd5b4a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -297,6 +297,21 @@ jobs: AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} + run-k8s-tests-on-free-runner: + if: ${{ inputs.skip-test != 'yes' }} + needs: publish-kata-deploy-payload-amd64 + permissions: + contents: read + uses: ./.github/workflows/run-k8s-tests-on-free-runner.yaml + with: + tarball-suffix: -${{ inputs.tag }} + registry: ghcr.io + repo: ${{ github.repository_owner }}/kata-deploy-ci + tag: ${{ inputs.tag }}-amd64 + commit-hash: ${{ inputs.commit-hash }} + pr-number: ${{ inputs.pr-number }} + target-branch: ${{ inputs.target-branch }} + run-k8s-tests-on-arm64: if: ${{ inputs.skip-test != 'yes' }} needs: publish-kata-deploy-payload-arm64 diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 1a347c487c..7649aed6eb 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -42,17 +42,6 @@ jobs: strategy: fail-fast: false matrix: - host_os: - - ubuntu - vmm: - - clh - - dragonball - - qemu - - qemu-runtime-rs - - cloud-hypervisor - instance-type: - - small - - normal include: - host_os: cbl-mariner vmm: clh @@ -80,6 +69,7 @@ jobs: KUBERNETES: "vanilla" K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} GENPOLICY_PULL_METHOD: ${{ matrix.genpolicy-pull-method }} + RUNS_ON_AKS: "true" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: diff --git a/.github/workflows/run-k8s-tests-on-free-runner.yaml b/.github/workflows/run-k8s-tests-on-free-runner.yaml new file mode 100644 index 0000000000..65ac992b29 --- /dev/null +++ b/.github/workflows/run-k8s-tests-on-free-runner.yaml @@ -0,0 +1,127 @@ +# Run Kubernetes integration tests on free GitHub runners with a locally +# deployed cluster (kubeadm). +name: CI | Run kubernetes tests on free runner +on: + workflow_call: + inputs: + tarball-suffix: + required: false + type: string + registry: + required: true + type: string + repo: + required: true + type: string + tag: + required: true + type: string + pr-number: + required: true + type: string + commit-hash: + required: false + type: string + target-branch: + required: false + type: string + default: "" + +permissions: {} + +jobs: + run-k8s-tests: + name: run-k8s-tests + strategy: + fail-fast: false + matrix: + environment: [ + { vmm: clh, containerd_version: lts }, + { vmm: clh, containerd_version: active }, + { vmm: dragonball, containerd_version: lts }, + { vmm: dragonball, containerd_version: active }, + { vmm: qemu, containerd_version: lts }, + { vmm: qemu, containerd_version: active }, + { vmm: qemu-runtime-rs, containerd_version: lts }, + { vmm: qemu-runtime-rs, containerd_version: active }, + { vmm: cloud-hypervisor, containerd_version: lts }, + { vmm: cloud-hypervisor, containerd_version: active }, + ] + runs-on: ubuntu-24.04 + permissions: + contents: read + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + GH_PR_NUMBER: ${{ inputs.pr-number }} + KATA_HOST_OS: ubuntu + KATA_HYPERVISOR: ${{ matrix.environment.vmm }} + KUBERNETES: vanilla + K8S_TEST_HOST_TYPE: baremetal-no-attestation + CONTAINER_ENGINE: containerd + CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }} + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.commit-hash }} + fetch-depth: 0 + persist-credentials: false + + - name: Rebase atop of the latest target branch + run: | + ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" + env: + TARGET_BRANCH: ${{ inputs.target-branch }} + + - name: get-kata-tools-tarball + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }} + path: kata-tools-artifacts + + - name: Install kata-tools + run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts + + - name: Remove unnecessary directories to free up space + run: | + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + sudo rm -rf /usr/lib/jvm + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/julia* + sudo rm -rf /opt/az + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /opt/microsoft + sudo rm -rf /opt/google + sudo rm -rf /usr/lib/firefox + + - name: Deploy k8s (kubeadm) + run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s + + - name: Install `bats` + run: bash tests/integration/kubernetes/gha-run.sh install-bats + + - name: Deploy Kata + timeout-minutes: 20 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata + + - name: Run tests + timeout-minutes: 60 + run: bash tests/integration/kubernetes/gha-run.sh run-tests + + - name: Report tests + if: always() + run: bash tests/integration/kubernetes/gha-run.sh report-tests + + - name: Delete kata-deploy + if: always() + timeout-minutes: 15 + run: bash tests/integration/kubernetes/gha-run.sh cleanup + diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 9580f931d5..a867910c8e 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -417,6 +417,14 @@ function deploy_vanilla_k8s() { [[ -z "${container_engine}" ]] && die "container_engine is required" [[ -z "${container_engine_version}" ]] && die "container_engine_version is required" + # Resolve lts/active to the actual version from versions.yaml (e.g. v1.7, v2.1) + case "${container_engine_version}" in + lts|active) + container_engine_version=$(get_from_kata_deps ".externals.containerd.${container_engine_version}") + ;; + *) ;; + esac + install_system_dependencies "runc" load_k8s_needed_modules set_k8s_network_parameters diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 0b2be19f30..ae28d1c595 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -36,6 +36,7 @@ export PULL_TYPE="${PULL_TYPE:-default}" export TEST_CLUSTER_NAMESPACE="${TEST_CLUSTER_NAMESPACE:-kata-containers-k8s-tests}" export GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}" export TARGET_ARCH="${TARGET_ARCH:-x86_64}" +export RUNS_ON_AKS="${RUNS_ON_AKS:-false}" function configure_devmapper() { sudo mkdir -p /var/lib/containerd/devmapper diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index be8203d70b..3c090243e8 100644 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -14,6 +14,7 @@ export AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-no}" export KATA_HOST_OS="${KATA_HOST_OS:-}" export KATA_HYPERVISOR="${KATA_HYPERVISOR:-}" export PULL_TYPE="${PULL_TYPE:-default}" +export RUNS_ON_AKS="${RUNS_ON_AKS:-false}" declare -r kubernetes_dir=$(dirname "$(readlink -f "$0")") declare -r runtimeclass_workloads_work_dir="${kubernetes_dir}/runtimeclass_workloads_work" diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 3e69a1dbb8..5a2984deeb 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -39,6 +39,7 @@ AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-}" GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-}" KATA_HOST_OS="${KATA_HOST_OS:-}" +RUNS_ON_AKS="${RUNS_ON_AKS:-false}" # Common setup for tests. # @@ -98,13 +99,11 @@ is_nvidia_gpu_platform() { } is_aks_cluster() { - case "${KATA_HYPERVISOR}" in - "qemu-tdx"|"qemu-snp"|qemu-nvidia-gpu*) - return 1 - ;; - *) - return 0 - esac + if [[ "${RUNS_ON_AKS}" = "true" ]]; then + return 0 + fi + + return 1 } adapt_common_policy_settings_for_non_coco() {