diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dffb6ff1f7..9611fdc4b1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -217,7 +217,7 @@ jobs: tags: ghcr.io/kata-containers/test-images:unencrypted-${{ inputs.pr-number }} push: true context: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/ - platforms: linux/amd64, linux/s390x + platforms: linux/amd64, linux/arm64, linux/s390x file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile run-kata-monitor-tests: @@ -277,6 +277,23 @@ jobs: pr-number: ${{ inputs.pr-number }} target-branch: ${{ inputs.target-branch }} + run-kata-coco-tests-on-arm64: + if: ${{ inputs.skip-test != 'yes' }} + needs: + - publish-kata-deploy-payload-arm64 + - build-and-publish-tee-confidential-unencrypted-image + uses: ./.github/workflows/run-kata-coco-tests-arm64-k8s.yaml + with: + tarball-suffix: -${{ inputs.tag }} + registry: ghcr.io + repo: ${{ github.repository_owner }}/kata-deploy-ci + tag: ${{ inputs.tag }}-arm64 + commit-hash: ${{ inputs.commit-hash }} + pr-number: ${{ inputs.pr-number }} + target-branch: ${{ inputs.target-branch }} + secrets: + AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }} + run-k8s-tests-on-nvidia-gpu: if: ${{ inputs.skip-test != 'yes' }} needs: publish-kata-deploy-payload-amd64 diff --git a/.github/workflows/run-kata-coco-tests-arm64-k8s.yaml b/.github/workflows/run-kata-coco-tests-arm64-k8s.yaml new file mode 100644 index 0000000000..68c0c6ad35 --- /dev/null +++ b/.github/workflows/run-kata-coco-tests-arm64-k8s.yaml @@ -0,0 +1,128 @@ +name: CI | Run kata coco tests on arm64 k8s +on: + workflow_call: + inputs: + tarball-suffix: + required: false + type: string + registry: + required: true + type: string + repo: + required: true + type: string + tag: + required: true + type: string + pr-number: + required: true + type: string + commit-hash: + required: false + type: string + target-branch: + required: false + type: string + default: "" + secrets: + AUTHENTICATED_IMAGE_PASSWORD: + required: false + +permissions: {} + +jobs: + run-k8s-tests-coco-arm64-k8s: + name: run-k8s-tests-coco-arm64-k8s (${{ matrix.vmm }}, ${{ matrix.k8s }}) + strategy: + fail-fast: false + matrix: + vmm: + - qemu-coco-dev-runtime-rs + k8s: + - kubeadm + runs-on: arm64-k8s + environment: + name: ci + deployment: false + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + GH_PR_NUMBER: ${{ inputs.pr-number }} + KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: ${{ matrix.k8s }} + K8S_TEST_HOST_TYPE: all + TARGET_ARCH: "aarch64" + KBS: "true" + KBS_INGRESS: "nodeport" + AUTO_GENERATE_POLICY: "yes" + PULL_TYPE: "guest-pull" + SNAPSHOTTER: "nydus" + AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }} + AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }} + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.commit-hash }} + fetch-depth: 0 + persist-credentials: false + + - name: Rebase atop of the latest target branch + run: | + ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" + env: + TARGET_BRANCH: ${{ inputs.target-branch }} + + - name: get-kata-tools-tarball + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: kata-tools-static-tarball-arm64${{ inputs.tarball-suffix }} + path: kata-tools-artifacts + + - name: Install kata-tools + run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts + + - name: Deploy Kata + timeout-minutes: 20 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata + + - name: Uninstall previous `kbs-client` + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client + + - name: Deploy CoCo KBS + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs + + - name: Install `kbs-client` + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client + + - name: Install `bats` + run: bash tests/integration/kubernetes/gha-run.sh install-bats + + - name: Run tests + timeout-minutes: 90 + run: bash tests/integration/kubernetes/gha-run.sh run-tests + + - name: Report tests + if: always() + run: bash tests/integration/kubernetes/gha-run.sh report-tests + + - name: Collect artifacts ${{ matrix.vmm }} + if: always() + run: bash tests/integration/kubernetes/gha-run.sh collect-artifacts + continue-on-error: true + + - name: Archive artifacts ${{ matrix.vmm }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: k8s-coco-arm64-${{ matrix.vmm }}-${{ matrix.k8s }}-${{ inputs.tag }} + path: /tmp/artifacts + retention-days: 1 + + - name: Delete kata-deploy + if: always() + timeout-minutes: 15 + run: bash tests/integration/kubernetes/gha-run.sh cleanup diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh index 4923fb6fdc..c71cdce808 100644 --- a/tests/integration/kubernetes/confidential_kbs.sh +++ b/tests/integration/kubernetes/confidential_kbs.sh @@ -227,7 +227,7 @@ kbs_install_cli() { source /etc/os-release || source /usr/lib/os-release case "${ID}" in - ubuntu) + debian|ubuntu) local pkgs="build-essential pkg-config libssl-dev" sudo apt-get update -y diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index c5ae2b115c..a2609da84a 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -453,7 +453,7 @@ function main() { if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then AUTO_GENERATE_POLICY="yes" elif [[ "${KATA_HYPERVISOR}" = qemu-coco-dev* && \ - "${TARGET_ARCH}" = "x86_64" && \ + ( "${TARGET_ARCH}" = "x86_64" || "${TARGET_ARCH}" = "aarch64" ) && \ "${PULL_TYPE}" != "experimental-force-guest-pull" ]]; then AUTO_GENERATE_POLICY="yes" elif [[ "${KATA_HYPERVISOR}" = qemu-nvidia-gpu-* ]]; then diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index be647ecaa3..34fce8cf17 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -112,6 +112,12 @@ is_k3s_or_rke2() { esac } +# The arm64 runner owners keep containerd updates synced across all runners. +is_arm64_host() { + [[ "$(uname -m)" == "aarch64" ]] && return 0 + return 1 +} + # Return the kubelet data directory, which varies by Kubernetes distribution. get_kubelet_data_dir() { case "${KUBERNETES:-}" in @@ -145,7 +151,7 @@ install_genpolicy_drop_ins() { # 20-* OCI version overlay if [[ "${KATA_HOST_OS:-}" == "cbl-mariner" ]]; then cp "${examples_dir}/20-oci-1.2.0-drop-in.json" "${settings_d}/" - elif is_k3s_or_rke2 || is_nvidia_gpu_platform || is_snp_hypervisor "${KATA_HYPERVISOR}" || is_tdx_hypervisor "${KATA_HYPERVISOR}" || [[ -n "${CONTAINER_ENGINE_VERSION:-}" ]]; then + elif is_k3s_or_rke2 || is_nvidia_gpu_platform || is_snp_hypervisor "${KATA_HYPERVISOR}" || is_tdx_hypervisor "${KATA_HYPERVISOR}" || [[ -n "${CONTAINER_ENGINE_VERSION:-}" ]] || is_arm64_host; then cp "${examples_dir}/20-oci-1.3.0-drop-in.json" "${settings_d}/" fi