ci: run qemu-coco-dev-runtime-rs tests on arm64

Add qemu-coco-dev-runtime-rs to the arm64 k8s test matrix so that the
CoCo non-TEE configuration is exercised on aarch64 runners.

Also enable auto-generated policy for qemu-coco-dev on aarch64 (matching
the existing x86_64 behavior) and register the new job as a required
gatekeeper check.

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
Made-with: Cursor

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -43,6 +43,7 @@ jobs:
           - agent
           - busybox
           - cloud-hypervisor
+          - coco-guest-components
           - firecracker
           - kernel
           - kernel-debug
@@ -51,6 +52,7 @@ jobs:
           - kernel-cca-confidential
           - nydus
           - ovmf
+          - pause-image
           - qemu
           - virtiofsd
     env:
@@ -151,6 +153,7 @@ jobs:
       matrix:
         asset:
           - rootfs-image
+          - rootfs-image-confidential
           - rootfs-image-nvidia-gpu
           - rootfs-initrd
     steps:
@@ -216,7 +219,9 @@ jobs:
       matrix:
         asset:
           - busybox
+          - coco-guest-components
           - kernel-nvidia-gpu-modules
+          - pause-image
     steps:
       - uses: geekyeggo/delete-artifact@f275313e70c08f6120db482d7a6b98377786765b # v5.1.0
         with:
@@ -288,6 +293,7 @@ jobs:
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
 
       - name: store-artifact shim-v2
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
@@ -297,6 +303,116 @@ jobs:
           retention-days: 15
           if-no-files-found: error
 
+  build-tools-asset:
+    name: build-tools-asset
+    runs-on: ubuntu-24.04-arm
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        asset:
+          - agent-ctl
+          - genpolicy
+          - kata-ctl
+          - kata-manager
+          - trace-forwarder
+        stage:
+          - ${{ inputs.stage }}
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: quay.io
+          username: ${{ vars.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+          persist-credentials: false
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-tools-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-tools-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: kata-tools-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-tools-build/kata-static-${{ matrix.asset }}.tar.zst
+          retention-days: 15
+          if-no-files-found: error
+
+  create-kata-tools-tarball:
+    name: create-kata-tools-tarball
+    runs-on: ubuntu-24.04-arm
+    needs: [build-tools-asset]
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+          fetch-tags: true
+          persist-credentials: false
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+      - name: get-artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          pattern: kata-tools-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-tools-artifacts
+          merge-multiple: true
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-tools-artifacts versions.yaml kata-tools-static.tar.zst
+        env:
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+      - name: Check kata-tools tarball size (GitHub release asset limit)
+        run: |
+          # https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases#storage-and-bandwidth-quotas
+          GITHUB_ASSET_MAX_BYTES=2147483648
+          tarball_size=$(stat -c "%s" kata-tools-static.tar.zst)
+          if [[ "${tarball_size}" -ge "${GITHUB_ASSET_MAX_BYTES}" ]]; then
+            echo "::error::tarball size (${tarball_size} bytes) >= GitHub release asset limit (${GITHUB_ASSET_MAX_BYTES} bytes)"
+            exit 1
+          fi
+          echo "tarball size: ${tarball_size} bytes"
+      - name: store-artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: kata-tools-static-tarball-arm64${{ inputs.tarball-suffix }}
+          path: kata-tools-static.tar.zst
+          retention-days: 15
+          if-no-files-found: error
+
   create-kata-tarball:
     name: create-kata-tarball
     runs-on: ubuntu-24.04-arm

.github/workflows/ci.yaml

@@ -266,6 +266,7 @@ jobs:
     needs: publish-kata-deploy-payload-arm64
     uses: ./.github/workflows/run-k8s-tests-on-arm64.yaml
     with:
+      tarball-suffix: -${{ inputs.tag }}
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-arm64

.github/workflows/run-k8s-tests-on-arm64.yaml

@@ -21,6 +21,9 @@ on:
         required: false
         type: string
         default: ""
+      tarball-suffix:
+        required: false
+        type: string
 
 permissions: {}
 
@@ -32,6 +35,7 @@ jobs:
       matrix:
         vmm:
           - qemu
+          - qemu-coco-dev-runtime-rs
           - qemu-runtime-rs
         k8s:
           - kubeadm
@@ -58,6 +62,17 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-kata-tools-tarball
+        if: ${{ startsWith(matrix.vmm, 'qemu-coco-dev') }}
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: kata-tools-static-tarball-arm64${{ inputs.tarball-suffix }}
+          path: kata-tools-artifacts
+
+      - name: Install kata-tools
+        if: ${{ startsWith(matrix.vmm, 'qemu-coco-dev') }}
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
+
       - name: Deploy Kata
         timeout-minutes: 20
         run: bash tests/integration/kubernetes/gha-run.sh deploy-kata

docs/helm-configuration.md

@@ -108,8 +108,8 @@ Includes:
 - `qemu-se` - IBM Secure Execution for Linux (SEL) (s390x)
 - `qemu-se-runtime-rs` - IBM Secure Execution for Linux (SEL) Rust runtime (s390x)
 - `qemu-cca` - Arm Confidential Compute Architecture (arm64)
-- `qemu-coco-dev` - Confidential Containers development (amd64, s390x)
-- `qemu-coco-dev-runtime-rs` - Confidential Containers development Rust runtime (amd64, s390x)
+- `qemu-coco-dev` - Confidential Containers development (amd64, arm64, s390x)
+- `qemu-coco-dev-runtime-rs` - Confidential Containers development Rust runtime (amd64, arm64, s390x)
 
 ### [`try-kata-nvidia-gpu.values.yaml`](https://github.com/kata-containers/kata-containers/blob/main/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml)
 

tests/gha-run-k8s-common.sh

@@ -715,7 +715,7 @@ function helm_helper() {
 						;;
 					qemu-coco-dev|qemu-coco-dev-runtime-rs)
 						yq -i ".shims.${shim}.enabled = true" "${values_yaml}"
-						yq -i ".shims.${shim}.supportedArches = [\"amd64\", \"s390x\"]" "${values_yaml}"
+						yq -i ".shims.${shim}.supportedArches = [\"amd64\", \"arm64\", \"s390x\"]" "${values_yaml}"
 						;;
 					qemu-nvidia-gpu)
 						yq -i ".shims.${shim}.enabled = true" "${values_yaml}"

tests/integration/kubernetes/gha-run.sh

@@ -493,7 +493,7 @@ function main() {
 		if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
 			AUTO_GENERATE_POLICY="yes"
 		elif [[ "${KATA_HYPERVISOR}" = qemu-coco-dev* && \
-		        "${TARGET_ARCH}" = "x86_64" && \
+		        ( "${TARGET_ARCH}" = "x86_64" || "${TARGET_ARCH}" = "aarch64" ) && \
 		        "${PULL_TYPE}" != "experimental-force-guest-pull" ]]; then
 			AUTO_GENERATE_POLICY="yes"
 		elif [[ "${KATA_HYPERVISOR}" = qemu-nvidia-gpu-* ]]; then

tools/packaging/kata-deploy/binary/src/config.rs

@@ -724,7 +724,7 @@ fn parse_custom_runtimes() -> Result<Vec<CustomRuntime>> {
 fn get_default_shims_for_arch(arch: &str) -> &'static str {
     match arch {
         "x86_64" => "clh cloud-hypervisor dragonball fc qemu qemu-coco-dev qemu-coco-dev-runtime-rs qemu-runtime-rs qemu-nvidia-gpu qemu-nvidia-gpu-snp qemu-nvidia-gpu-tdx qemu-snp qemu-snp-runtime-rs qemu-tdx qemu-tdx-runtime-rs",
-        "aarch64" => "clh cloud-hypervisor dragonball fc qemu qemu-runtime-rs qemu-nvidia-gpu qemu-cca",
+        "aarch64" => "clh cloud-hypervisor dragonball fc qemu qemu-coco-dev qemu-coco-dev-runtime-rs qemu-runtime-rs qemu-nvidia-gpu qemu-cca",
         "s390x" => "qemu qemu-runtime-rs qemu-se qemu-se-runtime-rs qemu-coco-dev qemu-coco-dev-runtime-rs",
         "ppc64le" => "qemu",
         _ => "qemu", // Fallback to qemu for unknown architectures

tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-tee.values.yaml

@@ -120,6 +120,7 @@ shims:
     enabled: true
     supportedArches:
       - amd64
+      - arm64
       - s390x
     allowedHypervisorAnnotations: []
     containerd:
@@ -135,6 +136,7 @@ shims:
     enabled: true
     supportedArches:
       - amd64
+      - arm64
       - s390x
     allowedHypervisorAnnotations: []
     containerd:

tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml

@@ -295,6 +295,7 @@ shims:
     enabled: ~
     supportedArches:
       - amd64
+      - arm64
       - s390x
     allowedHypervisorAnnotations: []
     containerd:
@@ -310,6 +311,7 @@ shims:
     enabled: ~
     supportedArches:
       - amd64
+      - arm64
       - s390x
     allowedHypervisorAnnotations: []
     containerd:

tools/packaging/kata-deploy/local-build/Makefile

@@ -60,6 +60,7 @@ BASE_TARBALLS = serial-targets \
 	shim-v2-tarball \
 	virtiofsd-tarball
 BASE_SERIAL_TARBALLS = rootfs-image-tarball \
+	rootfs-image-confidential-tarball \
 	rootfs-cca-confidential-image-tarball \
 	rootfs-cca-confidential-initrd-tarball \
 	rootfs-initrd-tarball

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -705,6 +705,11 @@ install_kernel() {
 			export MEASURED_ROOTFS="no"
 			extra_cmd="-x"
 			;;
+		aarch64)
+			export CONFIDENTIAL_GUEST="yes"
+			export MEASURED_ROOTFS="yes"
+			extra_cmd="-x"
+			;;
 		x86_64)
 			export CONFIDENTIAL_GUEST="yes"
 			export MEASURED_ROOTFS="yes"

tools/packaging/kernel/build-kernel.sh

@@ -578,8 +578,11 @@ install_kata() {
 	if [[ ${gpu_vendor} != "" ]]; then
 		suffix="-${gpu_vendor}-gpu${suffix}"
 	elif [[ ${conf_guest} != "" ]]; then
-		# CCA on aarch64 uses -confidential suffix; x86_64/s390x unified kernel does not
-		if [[ "${arch_target}" == "aarch64" ]]; then
+		# CCA kernel on aarch64 needs a -confidential suffix to coexist
+		# with the unified kernel; the regular kernel with -x does not
+		# get the suffix (matching x86_64/s390x unified kernel behavior).
+		# CCA builds are identified by -H (linux_headers) being set.
+		if [[ "${arch_target}" == "aarch64" ]] && [[ -n "${linux_headers}" ]]; then
 			suffix="-${conf_guest}${suffix}"
 		fi
 	fi

tools/packaging/static-build/tools/Dockerfile

@@ -61,10 +61,21 @@ RUN ARCH=$(uname -m) && \
     rm /tmp/oras.tar.gz && \
     oras version
 
-# Tools only build for x86_64
-RUN rustup target add x86_64-unknown-linux-musl
+RUN ARCH=$(uname -m) && \
+    case "${ARCH}" in \
+        x86_64) MUSL_TARGET="x86_64-unknown-linux-musl" ;; \
+        aarch64) MUSL_TARGET="aarch64-unknown-linux-musl" ;; \
+        *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; \
+    esac && \
+    rustup target add "${MUSL_TARGET}"
 
-RUN kernelname=$(uname -s | tr '[:upper:]' '[:lower:]'); \
-    curl -fsSOL "https://go.dev/dl/go${GO_TOOLCHAIN}.${kernelname}-amd64.tar.gz" && \
-    tar -C "${GO_HOME}" -xzf "go${GO_TOOLCHAIN}.${kernelname}-amd64.tar.gz" && \
-    rm "go${GO_TOOLCHAIN}.${kernelname}-amd64.tar.gz"
+RUN ARCH=$(uname -m) && \
+    case "${ARCH}" in \
+        x86_64) GO_ARCH="amd64" ;; \
+        aarch64) GO_ARCH="arm64" ;; \
+        *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; \
+    esac && \
+    kernelname=$(uname -s | tr '[:upper:]' '[:lower:]') && \
+    curl -fsSOL "https://go.dev/dl/go${GO_TOOLCHAIN}.${kernelname}-${GO_ARCH}.tar.gz" && \
+    tar -C "${GO_HOME}" -xzf "go${GO_TOOLCHAIN}.${kernelname}-${GO_ARCH}.tar.gz" && \
+    rm "go${GO_TOOLCHAIN}.${kernelname}-${GO_ARCH}.tar.gz"

tools/testing/gatekeeper/required-tests.yaml

@@ -78,6 +78,7 @@ mapping:
       - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (lts, qemu) / run-cri-containerd-amd64 (lts, qemu)
       - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-s390x (active, qemu) / run-cri-containerd-s390x (active, qemu)
       - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-arm64 / run-k8s-tests-on-arm64 (qemu, kubeadm)
+      - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-arm64 / run-k8s-tests-on-arm64 (qemu-coco-dev-runtime-rs, kubeadm)
       - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh, normal)
       - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh, small, containerd)
       - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh, small, oci-distribution)