genpolicy: Adapt to CRI-O

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>

.github/dependabot.yml

@@ -15,8 +15,6 @@ updates:
       - "/src/tools/trace-forwarder"
     schedule:
       interval: "daily"
-    cooldown:
-      default-days: 7
     ignore:
     # rust-vmm repos might cause incompatibilities on patch versions, so
     # lets handle them manually for now.
@@ -87,12 +85,8 @@ updates:
       - "src/tools/csi-kata-directvolume"
     schedule:
       interval: "daily"
-    cooldown:
-      default-days: 7
 
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "monthly"
-    cooldown:
-      default-days: 7

.github/workflows/ci.yaml

@@ -297,6 +297,21 @@ jobs:
       AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
       AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
 
+  run-k8s-tests-on-free-runner:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: publish-kata-deploy-payload-amd64
+    permissions:
+      contents: read
+    uses: ./.github/workflows/run-k8s-tests-on-free-runner.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+
   run-k8s-tests-on-arm64:
     if: ${{ inputs.skip-test != 'yes' }}
     needs: publish-kata-deploy-payload-arm64

.github/workflows/codeql.yml

@@ -72,7 +72,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@4bdb89f48054571735e3792627da6195c57459e2 # v3.31.10
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
@@ -95,6 +95,6 @@ jobs:
         make -C src/runtime
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@4bdb89f48054571735e3792627da6195c57459e2 # v3.31.10
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/docs.yaml

@@ -16,17 +16,17 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0
-      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+      - uses: actions/configure-pages@v5
+      - uses: actions/checkout@v5
         with:
           persist-credentials: false
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.x
       - run: pip install zensical
       - run: zensical build --clean
-      - uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0
+      - uses: actions/upload-pages-artifact@v4
         with:
           path: site
-      - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5
+      - uses: actions/deploy-pages@v4
         id: deployment

.github/workflows/push-oras-tarball-cache.yaml

@@ -0,0 +1,43 @@
+# Push gperf and busybox tarballs to the ORAS cache (ghcr.io) so that
+# download-with-oras-cache.sh can pull them instead of hitting upstream.
+# Runs when versions.yaml changes on main (e.g. after a PR merge) or manually.
+name: CI | Push ORAS tarball cache
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'versions.yaml'
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  push-oras-cache:
+    name: push-oras-cache
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Install yq
+        run: ./ci/install_yq.sh
+
+      - name: Install ORAS
+        uses: oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6 # v1.2.4
+        with:
+          version: "1.2.0"
+
+      - name: Populate ORAS tarball cache
+        run: ./tools/packaging/scripts/populate-oras-tarball-cache.sh all
+        env:
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REPOSITORY: kata-containers
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/run-cri-containerd-tests.yaml

@@ -35,6 +35,8 @@ on:
 jobs:
   run-cri-containerd:
     name: run-cri-containerd-${{ inputs.arch }} (${{ inputs.containerd_version }}, ${{ inputs.vmm }})
+    strategy:
+      fail-fast: false
     runs-on: ${{ inputs.runner }}
     env:
       CONTAINERD_VERSION: ${{ inputs.containerd_version }}

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -42,17 +42,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        host_os:
-          - ubuntu
-        vmm:
-          - clh
-          - dragonball
-          - qemu
-          - qemu-runtime-rs
-          - cloud-hypervisor
-        instance-type:
-          - small
-          - normal
         include:
           - host_os: cbl-mariner
             vmm: clh
@@ -80,6 +69,7 @@ jobs:
       KUBERNETES: "vanilla"
       K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
       GENPOLICY_PULL_METHOD: ${{ matrix.genpolicy-pull-method }}
+      RUNS_ON_AKS: "true"
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:

.github/workflows/run-k8s-tests-on-free-runner.yaml

@@ -0,0 +1,127 @@
+# Run Kubernetes integration tests on free GitHub runners  with a locally
+# deployed cluster (kubeadm).
+name: CI | Run kubernetes tests on free runner
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+permissions: {}
+
+jobs:
+  run-k8s-tests:
+    name: run-k8s-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: [
+          { vmm: clh, containerd_version: lts },
+          { vmm: clh, containerd_version: active },
+          { vmm: dragonball, containerd_version: lts },
+          { vmm: dragonball, containerd_version: active },
+          { vmm: qemu, containerd_version: lts },
+          { vmm: qemu, containerd_version: active },
+          { vmm: qemu-runtime-rs, containerd_version: lts },
+          { vmm: qemu-runtime-rs, containerd_version: active },
+          { vmm: cloud-hypervisor, containerd_version: lts },
+          { vmm: cloud-hypervisor, containerd_version: active },
+        ]
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ubuntu
+      KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
+      KUBERNETES: vanilla
+      K8S_TEST_HOST_TYPE: baremetal-no-attestation
+      CONTAINER_ENGINE: containerd
+      CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }}
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tools-tarball
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-tools-artifacts
+
+      - name: Install kata-tools
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
+
+      - name: Remove unnecessary directories to free up space
+        run: |
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf /usr/lib/jvm
+          sudo rm -rf /usr/share/swift
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /usr/local/julia*
+          sudo rm -rf /opt/az
+          sudo rm -rf /usr/local/share/chromium
+          sudo rm -rf /opt/microsoft
+          sudo rm -rf /opt/google
+          sudo rm -rf /usr/lib/firefox
+
+      - name: Deploy k8s (kubeadm)
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+
+      - name: Deploy Kata
+        timeout-minutes: 20
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
+
+      - name: Run tests
+        timeout-minutes: 60
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Report tests
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh report-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        timeout-minutes: 15
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup
+

.github/workflows/run-kata-coco-tests.yaml

@@ -140,165 +140,36 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        vmm:
-          - qemu-coco-dev
-          - qemu-coco-dev-runtime-rs
-        snapshotter:
-          - nydus
-        pull-type:
-          - guest-pull
-        include:
-          - pull-type: experimental-force-guest-pull
-            vmm: qemu-coco-dev
-            snapshotter: ""
-    runs-on: ubuntu-22.04
+        environment: [
+          { vmm: qemu-coco-dev, snapshotter: nydus, pull_type: guest-pull },
+          { vmm: qemu-coco-dev-runtime-rs, snapshotter: nydus, pull_type: guest-pull },
+          { vmm: qemu-coco-dev, snapshotter: "", pull_type: experimental-force-guest-pull },
+        ]
+    runs-on: ubuntu-24.04
     permissions:
-      id-token: write # Used for OIDC access to log into Azure
+      contents: read
     environment: ci
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
       DOCKER_TAG: ${{ inputs.tag }}
       GH_PR_NUMBER: ${{ inputs.pr-number }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
       # Some tests rely on that variable to run (or not)
       KBS: "true"
       # Set the KBS ingress handler (empty string disables handling)
-      KBS_INGRESS: "aks"
+      KBS_INGRESS: "nodeport"
       KUBERNETES: "vanilla"
-      PULL_TYPE: ${{ matrix.pull-type }}
+      PULL_TYPE: ${{ matrix.environment.pull_type }}
       AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
-      SNAPSHOTTER: ${{ matrix.snapshotter }}
-      EXPERIMENTAL_FORCE_GUEST_PULL: ${{ matrix.pull-type == 'experimental-force-guest-pull' && matrix.vmm || '' }}
-      # Caution: current ingress controller used to expose the KBS service
-      # requires much vCPUs, lefting only a few for the tests. Depending on the
-      # host type chose it will result on the creation of a cluster with
-      # insufficient resources.
+      SNAPSHOTTER: ${{ matrix.environment.snapshotter }}
+      EXPERIMENTAL_FORCE_GUEST_PULL: ${{ matrix.environment.pull_type == 'experimental-force-guest-pull' && matrix.environment.vmm || '' }}
+      AUTO_GENERATE_POLICY: "yes"
       K8S_TEST_HOST_TYPE: "all"
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          ref: ${{ inputs.commit-hash }}
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Rebase atop of the latest target branch
-        run: |
-          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
-        env:
-          TARGET_BRANCH: ${{ inputs.target-branch }}
-
-      - name: get-kata-tools-tarball
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-        with:
-          name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-tools-artifacts
-
-      - name: Install kata-tools
-        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
-
-      - name: Log into the Azure account
-        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
-        with:
-          client-id: ${{ secrets.AZ_APPID }}
-          tenant-id: ${{ secrets.AZ_TENANT_ID }}
-          subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-
-      - name: Create AKS cluster
-        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
-        with:
-          timeout_minutes: 15
-          max_attempts: 20
-          retry_on: error
-          retry_wait_seconds: 10
-          command: bash tests/integration/kubernetes/gha-run.sh create-cluster
-
-      - name: Install `bats`
-        run: bash tests/integration/kubernetes/gha-run.sh install-bats
-
-      - name: Install `kubectl`
-        uses: azure/setup-kubectl@776406bce94f63e41d621b960d78ee25c8b76ede # v4.0.1
-        with:
-          version: 'latest'
-
-      - name: Download credentials for the Kubernetes CLI to use them
-        run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials
-
-      - name: Deploy Kata
-        timeout-minutes: 20
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks
-        env:
-          USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: ${{ env.SNAPSHOTTER == 'nydus' }}
-          AUTO_GENERATE_POLICY: ${{ env.PULL_TYPE == 'experimental-force-guest-pull' && 'no' || 'yes' }}
-
-      - name: Deploy CoCo KBS
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
-
-      - name: Install `kbs-client`
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
-
-      - name: Deploy CSI driver
-        timeout-minutes: 5
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
-
-      - name: Run tests
-        timeout-minutes: 80
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests
-
-      - name: Report tests
-        if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh report-tests
-
-      - name: Refresh OIDC token in case access token expired
-        if: always()
-        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
-        with:
-          client-id: ${{ secrets.AZ_APPID }}
-          tenant-id: ${{ secrets.AZ_TENANT_ID }}
-          subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-
-      - name: Delete AKS cluster
-        if: always()
-        timeout-minutes: 15
-        run: bash tests/integration/kubernetes/gha-run.sh delete-cluster
-
-  # Generate jobs for testing CoCo on non-TEE environments with erofs-snapshotter
-  run-k8s-tests-coco-nontee-with-erofs-snapshotter:
-    name: run-k8s-tests-coco-nontee-with-erofs-snapshotter
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-coco-dev
-        snapshotter:
-          - erofs
-        pull-type:
-          - default
-    runs-on: ubuntu-24.04
-    environment: ci
-    env:
-      DOCKER_REGISTRY: ${{ inputs.registry }}
-      DOCKER_REPO: ${{ inputs.repo }}
-      DOCKER_TAG: ${{ inputs.tag }}
-      GH_PR_NUMBER: ${{ inputs.pr-number }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-      # Some tests rely on that variable to run (or not)
-      KBS: "false"
-      # Set the KBS ingress handler (empty string disables handling)
-      KBS_INGRESS: ""
-      KUBERNETES: "vanilla"
       CONTAINER_ENGINE: "containerd"
-      CONTAINER_ENGINE_VERSION: "v2.2"
-      PULL_TYPE: ${{ matrix.pull-type }}
-      SNAPSHOTTER: ${{ matrix.snapshotter }}
-      USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: "true"
-      K8S_TEST_HOST_TYPE: "all"
-      # We are skipping the auto generated policy tests for now,
-      # but those should be enabled as soon as we work on that.
-      AUTO_GENERATE_POLICY: "no"
+      CONTAINER_ENGINE_VERSION: "active"
+      GH_TOKEN: ${{ github.token }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -342,8 +213,221 @@ jobs:
       - name: Deploy kubernetes
         timeout-minutes: 15
         run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+
+      - name: Deploy Kata
+        timeout-minutes: 20
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
         env:
-          GH_TOKEN: ${{ github.token }}
+          USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: ${{ matrix.environment.snapshotter == 'nydus' }}
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
+      - name: Run tests
+        timeout-minutes: 80
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Report tests
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh report-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        timeout-minutes: 15
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup
+
+      - name: Delete CoCo KBS
+        if: always()
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+
+      - name: Delete CSI driver
+        if: always()
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
+  run-k8s-tests-coco-nontee-crio:
+    name: run-k8s-tests-coco-nontee-crio
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-coco-dev
+    runs-on: fidencio-crio
+    permissions:
+      contents: read
+    environment: ci
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KBS: "true"
+      KBS_INGRESS: "nodeport"
+      KUBERNETES: "vanilla"
+      PULL_TYPE: "guest-pull"
+      AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      SNAPSHOTTER: ""
+      EXPERIMENTAL_FORCE_GUEST_PULL: ""
+      AUTO_GENERATE_POLICY: "yes"
+      K8S_TEST_HOST_TYPE: "all"
+      CONTAINER_ENGINE: "crio"
+      CONTAINER_RUNTIME: "crio"
+      CONTAINER_ENGINE_VERSION: "active"
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tools-tarball
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-tools-artifacts
+
+      - name: Install kata-tools
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
+
+      - name: Deploy Kata
+        timeout-minutes: 20
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
+      - name: Run tests
+        timeout-minutes: 80
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Report tests
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh report-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        timeout-minutes: 15
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup
+
+      - name: Delete CoCo KBS
+        if: always()
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+
+      - name: Delete CSI driver
+        if: always()
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
+  # Generate jobs for testing CoCo on non-TEE environments with erofs-snapshotter
+  run-k8s-tests-coco-nontee-with-erofs-snapshotter:
+    name: run-k8s-tests-coco-nontee-with-erofs-snapshotter
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-coco-dev
+        snapshotter:
+          - erofs
+        pull-type:
+          - default
+    runs-on: ubuntu-24.04
+    environment: ci
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      # Some tests rely on that variable to run (or not)
+      KBS: "false"
+      # Set the KBS ingress handler (empty string disables handling)
+      KBS_INGRESS: ""
+      KUBERNETES: "vanilla"
+      CONTAINER_ENGINE: "containerd"
+      CONTAINER_ENGINE_VERSION: "active"
+      PULL_TYPE: ${{ matrix.pull-type }}
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      USE_EXPERIMENTAL_SETUP_SNAPSHOTTER: "true"
+      K8S_TEST_HOST_TYPE: "all"
+      # We are skipping the auto generated policy tests for now,
+      # but those should be enabled as soon as we work on that.
+      AUTO_GENERATE_POLICY: "no"
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tools-tarball
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-tools-artifacts
+
+      - name: Install kata-tools
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
+
+      - name: Remove unnecessary directories to free up space
+        run: |
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf /usr/lib/jvm
+          sudo rm -rf /usr/share/swift
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /usr/local/julia*
+          sudo rm -rf /opt/az
+          sudo rm -rf /usr/local/share/chromium
+          sudo rm -rf /opt/microsoft
+          sudo rm -rf /opt/google
+          sudo rm -rf /usr/lib/firefox
+
+      - name: Deploy kubernetes
+        timeout-minutes: 15
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
 
       - name: Install `bats`
         run: bash tests/integration/kubernetes/gha-run.sh install-bats
@@ -363,3 +447,13 @@ jobs:
       - name: Report tests
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh report-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        timeout-minutes: 15
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup
+
+      - name: Delete CSI driver
+        if: always()
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver

.github/workflows/scorecard.yaml

@@ -55,6 +55,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@4bdb89f48054571735e3792627da6195c57459e2 # v3.31.10
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: results.sarif

Cargo.lock

@@ -3945,9 +3945,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv"
-version = "0.7.45"
+version = "0.7.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b"
+checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1"
 dependencies = [
  "bitvec",
  "bytecheck",
@@ -3963,9 +3963,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv_derive"
-version = "0.7.45"
+version = "0.7.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0"
+checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5"
 dependencies = [
  "proc-macro2",
  "quote",

VERSION

@@ -1 +1 @@
-3.26.0
+3.27.0

docs/how-to/how-to-use-k8s-with-containerd-and-kata.md

@@ -49,6 +49,8 @@ In order to allow Kubelet to use containerd (using the CRI interface), configure
   EOF
   ```
 
+  For Kata Containers (and especially CoCo / Confidential Containers tests), use at least `--runtime-request-timeout=600s` (10m) so CRI CreateContainerRequest does not time out.
+
 - Inform systemd about the new configuration
 
   ```bash

src/agent/Cargo.lock

@@ -3488,9 +3488,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv"
-version = "0.7.45"
+version = "0.7.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b"
+checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1"
 dependencies = [
  "bitvec",
  "bytecheck",
@@ -3506,9 +3506,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv_derive"
-version = "0.7.45"
+version = "0.7.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0"
+checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5"
 dependencies = [
  "proc-macro2",
  "quote",

src/agent/rustjail/src/mount.rs

@@ -857,7 +857,7 @@ fn mount_from(
         dest.as_str(),
         Some(mount_typ.as_str()),
         flags,
-        Some(d.as_str()),
+        Some(d.as_str()).filter(|s| !s.is_empty()),
     )
     .inspect_err(|e| log_child!(cfd_log, "mount error: {:?}", e))?;
 

src/runtime-rs/Makefile

@@ -298,7 +298,7 @@ ifneq (,$(CLHCMD))
     KERNELTYPE_CLH = uncompressed
     KERNEL_NAME_CLH = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_CLH))
     KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH)
-    VMROOTFSDRIVER_CLH := virtio-pmem
+    VMROOTFSDRIVER_CLH := virtio-blk-pci
 
     DEFSANDBOXCGROUPONLY_CLH := true
     DEFSTATICRESOURCEMGMT_CLH := false

src/runtime-rs/config/configuration-cloud-hypervisor.toml.in

@@ -22,6 +22,8 @@ rootfs_type = @DEFROOTFSTYPE@
 
 # Block storage driver to be used for the VM rootfs is backed
 # by a block device.
+#
+# virtio-pmem is not supported with Cloud Hypervisor.
 vm_rootfs_driver = "@VMROOTFSDRIVER_CLH@"
 
 # Path to the firmware.

src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs

@@ -118,13 +118,11 @@ impl TryFrom<NamedHypervisorConfig> for VmConfig {
 
         // Note how CH handles the different image types:
         //
-        // - A standard image is specified in PmemConfig.
         // - An initrd/initramfs is specified in PayloadConfig.
-        // - A confidential guest image is specified by a DiskConfig.
+        // - An image is specified in DiskConfig.
+        //   Note: pmem is not used as it's not properly supported by Cloud Hypervisor.
         //   - If TDX is enabled, the firmware (`td-shim` [1]) must be
         //     specified in PayloadConfig.
-        // - A confidential guest initrd is specified by a PayloadConfig with
-        //   firmware.
         //
         // [1] - https://github.com/confidential-containers/td-shim
         let boot_info = cfg.boot_info;
@@ -140,14 +138,6 @@ impl TryFrom<NamedHypervisorConfig> for VmConfig {
             return Err(VmConfigError::NoBootFile);
         }
 
-        let pmem = if use_initrd || guest_protection_is_tdx(guest_protection_to_use.clone()) {
-            None
-        } else {
-            let pmem = PmemConfig::try_from(&boot_info).map_err(VmConfigError::PmemError)?;
-
-            Some(vec![pmem])
-        };
-
         let payload = Some(
             PayloadConfig::try_from((
                 boot_info.clone(),
@@ -159,7 +149,7 @@ impl TryFrom<NamedHypervisorConfig> for VmConfig {
 
         let mut disks: Vec<DiskConfig> = vec![];
 
-        if use_image && guest_protection_is_tdx(guest_protection_to_use.clone()) {
+        if use_image {
             let disk = DiskConfig::try_from(boot_info).map_err(VmConfigError::DiskError)?;
 
             disks.push(disk);
@@ -199,7 +189,6 @@ impl TryFrom<NamedHypervisorConfig> for VmConfig {
             fs,
             net,
             devices: host_devices,
-            pmem,
             disks,
             vsock: Some(vsock),
             rng,
@@ -1656,7 +1645,6 @@ mod tests {
         let (memory_info_confidential_guest, mem_config_confidential_guest) =
             make_memory_objects(79, usable_max_mem_bytes, true);
 
-        let (_, pmem_config_with_image) = make_bootinfo_pmemconfig_objects(image);
         let (machine_info, rng_config) = make_machineinfo_rngconfig_objects(entropy_source);
 
         let payload_firmware = None;
@@ -1664,6 +1652,7 @@ mod tests {
         let (boot_info_with_initrd, payload_config_with_initrd) =
             make_bootinfo_payloadconfig_objects(kernel, initramfs, payload_firmware, None);
 
+        let (_, disk_config_with_image) = make_bootinfo_diskconfig_objects(image);
         let (_, disk_config_confidential_guest_image) = make_bootinfo_diskconfig_objects(image);
 
         let boot_info_tdx_image = BootInfo {
@@ -1762,7 +1751,7 @@ mod tests {
             vsock: Some(valid_vsock.clone()),
 
             // rootfs image specific
-            pmem: Some(vec![pmem_config_with_image]),
+            disks: Some(vec![disk_config_with_image]),
 
             payload: Some(PayloadConfig {
                 kernel: Some(PathBuf::from(kernel)),

src/runtime-rs/crates/hypervisor/src/ch/inner.rs

@@ -123,7 +123,12 @@ impl CloudHypervisorInner {
         }
     }
 
-    pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) {
+    pub fn set_hypervisor_config(&mut self, mut config: HypervisorConfig) {
+        // virtio-pmem is not supported for Cloud Hypervisor.
+        if config.boot_info.vm_rootfs_driver == crate::VM_ROOTFS_DRIVER_PMEM {
+            config.boot_info.vm_rootfs_driver = crate::VM_ROOTFS_DRIVER_BLK.to_string();
+        }
+
         self.config = config;
     }
 

src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs

@@ -15,7 +15,6 @@ use crate::utils::vm_cleanup;
 use crate::utils::{bytes_to_megs, get_jailer_root, get_sandbox_path, megs_to_bytes};
 use crate::MemoryConfig;
 use crate::VM_ROOTFS_DRIVER_BLK;
-use crate::VM_ROOTFS_DRIVER_PMEM;
 use crate::{VcpuThreadIds, VmmState};
 use anyhow::{anyhow, Context, Result};
 use ch_config::ch_api::cloud_hypervisor_vm_netdev_add_with_fds;
@@ -130,12 +129,8 @@ impl CloudHypervisorInner {
         let confidential_guest = cfg.security_info.confidential_guest;
 
         // Note that the configuration option hypervisor.block_device_driver is not used.
-        let rootfs_driver = if confidential_guest {
-            // PMEM is not available with TDX.
-            VM_ROOTFS_DRIVER_BLK
-        } else {
-            VM_ROOTFS_DRIVER_PMEM
-        };
+        // NVDIMM is not supported for Cloud Hypervisor.
+        let rootfs_driver = VM_ROOTFS_DRIVER_BLK;
 
         let rootfs_type = match cfg.boot_info.rootfs_type.is_empty() {
             true => DEFAULT_CH_ROOTFS_TYPE,
@@ -155,6 +150,7 @@ impl CloudHypervisorInner {
             &cfg.boot_info.kernel_verity_params,
             rootfs_driver,
             rootfs_type,
+            true,
         )?;
 
         let mut console_params = if enable_debug {
@@ -1104,7 +1100,7 @@ fn get_guest_protection() -> Result<GuestProtection> {
     Ok(guest_protection)
 }
 
-// Return a TID/VCPU map from a specified /proc/{pid} path.
+// Return a VCPU/TID map from a specified /proc/{pid} path.
 fn get_ch_vcpu_tids(proc_path: &str) -> Result<HashMap<u32, u32>> {
     const VCPU_STR: &str = "vcpu";
 
@@ -1147,7 +1143,7 @@ fn get_ch_vcpu_tids(proc_path: &str) -> Result<HashMap<u32, u32>> {
             .parse::<u32>()
             .map_err(|e| anyhow!(e).context("Invalid vcpu id."))?;
 
-        vcpus.insert(tid, vcpu_id);
+        vcpus.insert(vcpu_id, tid);
     }
 
     if vcpus.is_empty() {
@@ -1613,4 +1609,65 @@ mod tests {
             assert!(actual_error == expected_error, "{}", msg);
         }
     }
+
+    #[actix_rt::test]
+    async fn test_get_ch_vcpu_tids_mapping() {
+        let tmp_dir = Builder::new().prefix("fake-proc-pid").tempdir().unwrap();
+        let task_dir = tmp_dir.path().join("task");
+        fs::create_dir_all(&task_dir).unwrap();
+
+        #[derive(Debug)]
+        struct ThreadInfo<'a> {
+            tid: &'a str,
+            comm: &'a str,
+        }
+
+        let threads = &[
+            // Non-vcpu thread, should be skipped.
+            ThreadInfo {
+                tid: "1000",
+                comm: "main_thread\n",
+            },
+            ThreadInfo {
+                tid: "2001",
+                comm: "vcpu0\n",
+            },
+            ThreadInfo {
+                tid: "2002",
+                comm: "vcpu1\n",
+            },
+            ThreadInfo {
+                tid: "2003",
+                comm: "vcpu2\n",
+            },
+        ];
+
+        for t in threads {
+            let tid_dir = task_dir.join(t.tid);
+            fs::create_dir_all(&tid_dir).unwrap();
+            fs::write(tid_dir.join("comm"), t.comm).unwrap();
+        }
+
+        let proc_path = tmp_dir.path().to_str().unwrap();
+        let result = get_ch_vcpu_tids(proc_path);
+
+        let msg = format!("result: {result:?}");
+
+        if std::env::var("DEBUG").is_ok() {
+            println!("DEBUG: {msg}");
+        }
+
+        let vcpus = result.unwrap();
+
+        // The mapping must be vcpu_id -> tid.
+        assert_eq!(vcpus.len(), 3, "non-vcpu threads should be excluded");
+        assert_eq!(vcpus[&0], 2001, "vcpu 0 should map to tid 2001");
+        assert_eq!(vcpus[&1], 2002, "vcpu 1 should map to tid 2002");
+        assert_eq!(vcpus[&2], 2003, "vcpu 2 should map to tid 2003");
+
+        assert!(
+            !vcpus.contains_key(&1000),
+            "non-vcpu thread should not be in the map"
+        );
+    }
 }

src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs

@@ -13,17 +13,17 @@ use crate::device::DeviceType;
 use crate::Hypervisor as hypervisor;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
+pub use kata_types::device::{
+    DRIVER_BLK_CCW_TYPE as KATA_CCW_DEV_TYPE, DRIVER_BLK_MMIO_TYPE as KATA_MMIO_BLK_DEV_TYPE,
+    DRIVER_BLK_PCI_TYPE as KATA_BLK_DEV_TYPE, DRIVER_NVDIMM_TYPE as KATA_NVDIMM_DEV_TYPE,
+    DRIVER_SCSI_TYPE as KATA_SCSI_DEV_TYPE,
+};
 
 /// VIRTIO_BLOCK_PCI indicates block driver is virtio-pci based
 pub const VIRTIO_BLOCK_PCI: &str = "virtio-blk-pci";
 pub const VIRTIO_BLOCK_MMIO: &str = "virtio-blk-mmio";
 pub const VIRTIO_BLOCK_CCW: &str = "virtio-blk-ccw";
 pub const VIRTIO_PMEM: &str = "virtio-pmem";
-pub const KATA_MMIO_BLK_DEV_TYPE: &str = "mmioblk";
-pub const KATA_BLK_DEV_TYPE: &str = "blk";
-pub const KATA_CCW_DEV_TYPE: &str = "ccw";
-pub const KATA_NVDIMM_DEV_TYPE: &str = "nvdimm";
-pub const KATA_SCSI_DEV_TYPE: &str = "scsi";
 
 #[derive(Clone, Copy, Debug, Default)]
 pub enum BlockDeviceAio {
@@ -95,6 +95,9 @@ pub struct BlockConfig {
     /// scsi_addr is of the format SCSI-Id:LUN
     pub scsi_addr: Option<String>,
 
+    /// CCW device address for virtio-blk-ccw on s390x (e.g., "0.0.0005")
+    pub ccw_addr: Option<String>,
+
     /// device attach count
     pub attach_count: u64,
 

src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs

@@ -150,6 +150,7 @@ impl DragonballInner {
                 &self.config.boot_info.kernel_verity_params,
                 &rootfs_driver,
                 &self.config.boot_info.rootfs_type,
+                true,
             )?;
             kernel_params.append(&mut rootfs_params);
         }

src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs

@@ -90,6 +90,7 @@ impl FcInner {
             &self.config.boot_info.kernel_verity_params,
             &self.config.blockdev_info.block_device_driver,
             &self.config.boot_info.rootfs_type,
+            true,
         )?;
         kernel_params.append(&mut rootfs_params);
         kernel_params.append(&mut KernelParams::from_string(

src/runtime-rs/crates/hypervisor/src/kernel_param.rs

@@ -10,8 +10,8 @@ use crate::{
     VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_BLK_CCW, VM_ROOTFS_DRIVER_MMIO, VM_ROOTFS_DRIVER_PMEM,
     VM_ROOTFS_ROOT_BLK, VM_ROOTFS_ROOT_PMEM,
 };
-use kata_types::config::LOG_VPORT_OPTION;
 use kata_types::config::hypervisor::{parse_kernel_verity_params, VERITY_BLOCK_SIZE_BYTES};
+use kata_types::config::LOG_VPORT_OPTION;
 use kata_types::fs::{
     VM_ROOTFS_FILESYSTEM_EROFS, VM_ROOTFS_FILESYSTEM_EXT4, VM_ROOTFS_FILESYSTEM_XFS,
 };
@@ -66,8 +66,7 @@ struct KernelVerityConfig {
 }
 
 fn new_kernel_verity_params(params_string: &str) -> Result<Option<KernelVerityConfig>> {
-    let cfg = parse_kernel_verity_params(params_string)
-        .map_err(|err| anyhow!(err.to_string()))?;
+    let cfg = parse_kernel_verity_params(params_string).map_err(|err| anyhow!(err.to_string()))?;
 
     Ok(cfg.map(|params| KernelVerityConfig {
         root_hash: params.root_hash,
@@ -145,6 +144,7 @@ impl KernelParams {
         kernel_verity_params: &str,
         rootfs_driver: &str,
         rootfs_type: &str,
+        use_dax: bool,
     ) -> Result<Self> {
         let mut params = vec![];
 
@@ -153,16 +153,29 @@ impl KernelParams {
                 params.push(Param::new("root", VM_ROOTFS_ROOT_PMEM));
                 match rootfs_type {
                     VM_ROOTFS_FILESYSTEM_EXT4 => {
-                        params.push(Param::new(
-                            "rootflags",
-                            "dax,data=ordered,errors=remount-ro ro",
-                        ));
+                        if use_dax {
+                            params.push(Param::new(
+                                "rootflags",
+                                "dax,data=ordered,errors=remount-ro ro",
+                            ));
+                        } else {
+                            params
+                                .push(Param::new("rootflags", "data=ordered,errors=remount-ro ro"));
+                        }
                     }
                     VM_ROOTFS_FILESYSTEM_XFS => {
-                        params.push(Param::new("rootflags", "dax ro"));
+                        if use_dax {
+                            params.push(Param::new("rootflags", "dax ro"));
+                        } else {
+                            params.push(Param::new("rootflags", "ro"));
+                        }
                     }
                     VM_ROOTFS_FILESYSTEM_EROFS => {
-                        params.push(Param::new("rootflags", "dax ro"));
+                        if use_dax {
+                            params.push(Param::new("rootflags", "dax ro"));
+                        } else {
+                            params.push(Param::new("rootflags", "ro"));
+                        }
                     }
                     _ => {
                         return Err(anyhow!("Unsupported rootfs type {}", rootfs_type));
@@ -346,6 +359,7 @@ mod tests {
     struct TestData<'a> {
         rootfs_driver: &'a str,
         rootfs_type: &'a str,
+        use_dax: bool,
         expect_params: KernelParams,
         result: Result<()>,
     }
@@ -353,10 +367,11 @@ mod tests {
     #[test]
     fn test_rootfs_kernel_params() {
         let tests = &[
-            // EXT4
+            // EXT4 with DAX
             TestData {
                 rootfs_driver: VM_ROOTFS_DRIVER_PMEM,
                 rootfs_type: VM_ROOTFS_FILESYSTEM_EXT4,
+                use_dax: true,
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_PMEM),
@@ -370,6 +385,7 @@ mod tests {
             TestData {
                 rootfs_driver: VM_ROOTFS_DRIVER_BLK,
                 rootfs_type: VM_ROOTFS_FILESYSTEM_EXT4,
+                use_dax: true,
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_BLK),
@@ -380,14 +396,15 @@ mod tests {
                 },
                 result: Ok(()),
             },
-            // XFS
+            // XFS without DAX
             TestData {
                 rootfs_driver: VM_ROOTFS_DRIVER_PMEM,
                 rootfs_type: VM_ROOTFS_FILESYSTEM_XFS,
+                use_dax: false,
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_PMEM),
-                        Param::new("rootflags", "dax ro"),
+                        Param::new("rootflags", "ro"),
                         Param::new("rootfstype", VM_ROOTFS_FILESYSTEM_XFS),
                     ]
                     .to_vec(),
@@ -397,6 +414,7 @@ mod tests {
             TestData {
                 rootfs_driver: VM_ROOTFS_DRIVER_BLK,
                 rootfs_type: VM_ROOTFS_FILESYSTEM_XFS,
+                use_dax: true,
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_BLK),
@@ -407,10 +425,11 @@ mod tests {
                 },
                 result: Ok(()),
             },
-            // EROFS
+            // EROFS with DAX
             TestData {
                 rootfs_driver: VM_ROOTFS_DRIVER_PMEM,
                 rootfs_type: VM_ROOTFS_FILESYSTEM_EROFS,
+                use_dax: true,
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_PMEM),
@@ -424,6 +443,7 @@ mod tests {
             TestData {
                 rootfs_driver: VM_ROOTFS_DRIVER_BLK,
                 rootfs_type: VM_ROOTFS_FILESYSTEM_EROFS,
+                use_dax: true,
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_BLK),
@@ -438,6 +458,7 @@ mod tests {
             TestData {
                 rootfs_driver: "foo",
                 rootfs_type: VM_ROOTFS_FILESYSTEM_EXT4,
+                use_dax: true,
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_BLK),
@@ -452,6 +473,7 @@ mod tests {
             TestData {
                 rootfs_driver: VM_ROOTFS_DRIVER_BLK,
                 rootfs_type: "foo",
+                use_dax: true,
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_BLK),
@@ -466,8 +488,12 @@ mod tests {
 
         for (i, t) in tests.iter().enumerate() {
             let msg = format!("test[{i}]: {t:?}");
-            let result =
-                KernelParams::new_rootfs_kernel_params("", t.rootfs_driver, t.rootfs_type);
+            let result = KernelParams::new_rootfs_kernel_params(
+                "",
+                t.rootfs_driver,
+                t.rootfs_type,
+                t.use_dax,
+            );
             let msg = format!("{msg}, result: {result:?}");
             if t.result.is_ok() {
                 assert!(result.is_ok(), "{}", msg);
@@ -486,6 +512,7 @@ mod tests {
             "root_hash=abc,salt=def,data_blocks=1,data_block_size=4096,hash_block_size=4096",
             VM_ROOTFS_DRIVER_BLK,
             VM_ROOTFS_FILESYSTEM_EXT4,
+            false,
         )?;
         let params_string = params.to_string()?;
         assert!(params_string.contains("dm-mod.create="));
@@ -496,6 +523,7 @@ mod tests {
             "root_hash=abc,data_blocks=1,data_block_size=4096,hash_block_size=4096",
             VM_ROOTFS_DRIVER_BLK,
             VM_ROOTFS_FILESYSTEM_EXT4,
+            false,
         )
         .err()
         .expect("expected missing salt error");
@@ -505,6 +533,7 @@ mod tests {
             "root_hash=abc,salt=def,data_block_size=4096,hash_block_size=4096",
             VM_ROOTFS_DRIVER_BLK,
             VM_ROOTFS_FILESYSTEM_EXT4,
+            false,
         )
         .err()
         .expect("expected missing data_blocks error");
@@ -514,6 +543,7 @@ mod tests {
             "root_hash=abc,salt=def,data_blocks=foo,data_block_size=4096,hash_block_size=4096",
             VM_ROOTFS_DRIVER_BLK,
             VM_ROOTFS_FILESYSTEM_EXT4,
+            false,
         )
         .err()
         .expect("expected invalid data_blocks error");
@@ -523,6 +553,7 @@ mod tests {
             "root_hash=abc,salt=def,data_blocks=1,data_block_size=4096,hash_block_size=4096,badfield",
             VM_ROOTFS_DRIVER_BLK,
             VM_ROOTFS_FILESYSTEM_EXT4,
+            false,
         )
         .err()
         .expect("expected invalid entry error");

src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs

@@ -179,10 +179,17 @@ impl Kernel {
         let mut kernel_params = KernelParams::new(config.debug_info.enable_debug);
 
         if config.boot_info.initrd.is_empty() {
+            // DAX is disabled on ARM due to a kernel panic in caches_clean_inval_pou.
+            #[cfg(target_arch = "aarch64")]
+            let use_dax = false;
+            #[cfg(not(target_arch = "aarch64"))]
+            let use_dax = true;
+
             let mut rootfs_params = KernelParams::new_rootfs_kernel_params(
                 &config.boot_info.kernel_verity_params,
                 &config.boot_info.vm_rootfs_driver,
                 &config.boot_info.rootfs_type,
+                use_dax,
             )
             .context("adding rootfs/verity params failed")?;
             kernel_params.append(&mut rootfs_params);
@@ -385,7 +392,7 @@ impl ToQemuParams for Cpu {
 /// Error type for CCW Subchannel operations
 #[derive(Debug)]
 #[allow(dead_code)]
-enum CcwError {
+pub enum CcwError {
     DeviceAlreadyExists(String), // Error when trying to add an existing device
     #[allow(dead_code)]
     DeviceNotFound(String), // Error when trying to remove a nonexistent device
@@ -416,7 +423,7 @@ impl CcwSubChannel {
     /// # Returns
     /// - `Result<u32, CcwError>`: slot index of the added device
     ///   or an error if the device already exists
-    fn add_device(&mut self, dev_id: &str) -> Result<u32, CcwError> {
+    pub fn add_device(&mut self, dev_id: &str) -> Result<u32, CcwError> {
         if self.devices.contains_key(dev_id) {
             Err(CcwError::DeviceAlreadyExists(dev_id.to_owned()))
         } else {
@@ -435,8 +442,7 @@ impl CcwSubChannel {
     /// # Returns
     /// - `Result<(), CcwError>`: Ok(()) if the device was removed
     ///   or an error if the device was not found
-    #[allow(dead_code)]
-    fn remove_device(&mut self, dev_id: &str) -> Result<(), CcwError> {
+    pub fn remove_device(&mut self, dev_id: &str) -> Result<(), CcwError> {
         if self.devices.remove(dev_id).is_some() {
             Ok(())
         } else {
@@ -444,17 +450,30 @@ impl CcwSubChannel {
         }
     }
 
-    /// Formats the CCW address for a given slot
+    /// Formats the CCW address for a given slot.
+    /// Uses the 0xfe channel subsystem ID used by QEMU.
     ///
     /// # Arguments
     /// - `slot`: slot index
     ///
     /// # Returns
     /// - `String`: formatted CCW address (e.g. `fe.0.0000`)
-    fn address_format_ccw(&self, slot: u32) -> String {
+    pub fn address_format_ccw(&self, slot: u32) -> String {
         format!("fe.{:x}.{:04x}", self.addr, slot)
     }
 
+    /// Formats the guest-visible CCW address for a given slot.
+    /// Uses channel subsystem ID 0 (guest perspective).
+    ///
+    /// # Arguments
+    /// - `slot`: slot index
+    ///
+    /// # Returns
+    /// - `String`: formatted guest-visible CCW address (e.g. `0.0.0000`)
+    pub fn address_format_ccw_for_virt_server(&self, slot: u32) -> String {
+        format!("0.{:x}.{:04x}", self.addr, slot)
+    }
+
     /// Sets the address of the subchannel.
     /// # Arguments
     /// - `addr`: subchannel address to set
@@ -2267,6 +2286,12 @@ impl<'a> QemuCmdLine<'a> {
         Ok(qemu_cmd_line)
     }
 
+    /// Takes ownership of the CCW subchannel, leaving `None` in its place.
+    /// Used to transfer boot-time CCW state to Qmp for hotplug allocation.
+    pub fn take_ccw_subchannel(&mut self) -> Option<CcwSubChannel> {
+        self.ccw_subchannel.take()
+    }
+
     fn add_monitor(&mut self, proto: &str) -> Result<()> {
         let monitor = QmpSocket::new(self.id.as_str(), MonitorProtocol::new(proto))?;
         self.devices.push(Box::new(monitor));

src/runtime-rs/crates/hypervisor/src/qemu/inner.rs

@@ -10,6 +10,7 @@ use crate::qemu::qmp::get_qmp_socket_path;
 use crate::{
     device::driver::ProtectionDeviceConfig, hypervisor_persist::HypervisorState, selinux,
     HypervisorConfig, MemoryConfig, VcpuThreadIds, VsockDevice, HYPERVISOR_QEMU,
+    KATA_BLK_DEV_TYPE, KATA_CCW_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, KATA_SCSI_DEV_TYPE,
 };
 
 use crate::utils::{
@@ -21,7 +22,7 @@ use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use kata_sys_util::netns::NetnsGuard;
 use kata_types::build_path;
-use kata_types::config::hypervisor::RootlessUser;
+use kata_types::config::hypervisor::{RootlessUser, VIRTIO_BLK_CCW};
 use kata_types::rootless::is_rootless;
 use kata_types::{
     capabilities::{Capabilities, CapabilityBits},
@@ -133,18 +134,18 @@ impl QemuInner {
                         continue;
                     }
                     match block_dev.config.driver_option.as_str() {
-                        "nvdimm" => cmdline.add_nvdimm(
+                        KATA_NVDIMM_DEV_TYPE => cmdline.add_nvdimm(
                             &block_dev.config.path_on_host,
                             block_dev.config.is_readonly,
                         )?,
-                        "ccw" | "blk" | "scsi" => cmdline.add_block_device(
+                        KATA_CCW_DEV_TYPE | KATA_BLK_DEV_TYPE | KATA_SCSI_DEV_TYPE => cmdline.add_block_device(
                             block_dev.device_id.as_str(),
                             &block_dev.config.path_on_host,
                             block_dev
                                 .config
                                 .is_direct
                                 .unwrap_or(self.config.blockdev_info.block_device_cache_direct),
-                            block_dev.config.driver_option.as_str() == "scsi",
+                            block_dev.config.driver_option.as_str() == KATA_SCSI_DEV_TYPE,
                         )?,
                         unsupported => {
                             info!(sl!(), "unsupported block device driver: {}", unsupported)
@@ -285,7 +286,12 @@ impl QemuInner {
         let qmp_socket_path = get_qmp_socket_path(self.id.as_str());
 
         match Qmp::new(&qmp_socket_path) {
-            Ok(qmp) => self.qmp = Some(qmp),
+            Ok(mut qmp) => {
+                if let Some(subchannel) = cmdline.take_ccw_subchannel() {
+                    qmp.set_ccw_subchannel(subchannel);
+                }
+                self.qmp = Some(qmp);
+            }
             Err(e) => {
                 error!(sl!(), "couldn't initialise QMP: {:?}", e);
                 return Err(e);
@@ -842,9 +848,10 @@ impl QemuInner {
                 qmp.hotplug_network_device(&netdev, &virtio_net_device)?
             }
             DeviceType::Block(mut block_device) => {
-                let (pci_path, scsi_addr) = qmp
+                let block_driver = &self.config.blockdev_info.block_device_driver;
+                let (pci_path, addr_str) = qmp
                     .hotplug_block_device(
-                        &self.config.blockdev_info.block_device_driver,
+                        block_driver,
                         block_device.config.index,
                         &block_device.config.path_on_host,
                         &block_device.config.blkdev_aio.to_string(),
@@ -857,8 +864,12 @@ impl QemuInner {
                 if pci_path.is_some() {
                     block_device.config.pci_path = pci_path;
                 }
-                if scsi_addr.is_some() {
-                    block_device.config.scsi_addr = scsi_addr;
+                if let Some(addr) = addr_str {
+                    if block_driver == VIRTIO_BLK_CCW {
+                        block_device.config.ccw_addr = Some(addr);
+                    } else {
+                        block_device.config.scsi_addr = Some(addr);
+                    }
                 }
 
                 return Ok(DeviceType::Block(block_device));

src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs

@@ -4,12 +4,12 @@
 //
 
 use crate::device::pci_path::PciPath;
-use crate::qemu::cmdline_generator::{DeviceVirtioNet, Netdev, QMP_SOCKET_FILE};
+use crate::qemu::cmdline_generator::{CcwSubChannel, DeviceVirtioNet, Netdev, QMP_SOCKET_FILE};
 use crate::utils::get_jailer_root;
 use crate::VcpuThreadIds;
 
 use anyhow::{anyhow, Context, Result};
-use kata_types::config::hypervisor::VIRTIO_SCSI;
+use kata_types::config::hypervisor::{VIRTIO_BLK_CCW, VIRTIO_SCSI};
 use kata_types::rootless::is_rootless;
 use nix::sys::socket::{sendmsg, ControlMessage, MsgFlags};
 use qapi_qmp::{
@@ -50,6 +50,11 @@ pub struct Qmp {
     // blocks seem ever to be onlined in the guest by kata-agent.
     // Store as u64 to keep up the convention of bytes being represented as u64.
     guest_memory_block_size: u64,
+
+    // CCW subchannel for s390x device address management.
+    // Transferred from QemuCmdLine after boot so that hotplug allocations
+    // continue from where boot-time allocations left off.
+    ccw_subchannel: Option<CcwSubChannel>,
 }
 
 // We have to implement Debug since the Hypervisor trait requires it and Qmp
@@ -76,6 +81,7 @@ impl Qmp {
                     stream,
                 )),
                 guest_memory_block_size: 0,
+                ccw_subchannel: None,
             };
 
             let info = qmp.qmp.handshake().context("qmp handshake failed")?;
@@ -102,6 +108,10 @@ impl Qmp {
             .with_context(|| format!("timed out waiting for QMP ready: {}", qmp_sock_path))
     }
 
+    pub fn set_ccw_subchannel(&mut self, subchannel: CcwSubChannel) {
+        self.ccw_subchannel = Some(subchannel);
+    }
+
     pub fn set_ignore_shared_memory_capability(&mut self) -> Result<()> {
         self.qmp
             .execute(&migrate_set_capabilities {
@@ -605,6 +615,13 @@ impl Qmp {
     /// {"execute":"device_add","arguments":{"driver":"scsi-hd","drive":"virtio-scsi0","id":"scsi_device_0","bus":"virtio-scsi1.0"}}
     /// {"return": {}}
     ///
+    /// Hotplug virtio-blk-ccw block device on s390x
+    /// # virtio-blk-ccw0
+    /// {"execute":"blockdev_add", "arguments": {"file":"/path/to/block.image","format":"qcow2","id":"virtio-blk-ccw0"}}
+    /// {"return": {}}
+    /// {"execute":"device_add","arguments":{"driver":"virtio-blk-ccw","id":"virtio-blk-ccw0","drive":"virtio-blk-ccw0","devno":"fe.0.0005","share-rw":true}}
+    /// {"return": {}}
+    ///
     #[allow(clippy::too_many_arguments)]
     pub fn hotplug_block_device(
         &mut self,
@@ -711,6 +728,14 @@ impl Qmp {
             blkdev_add_args.insert("lun".to_string(), lun.into());
             blkdev_add_args.insert("share-rw".to_string(), true.into());
 
+            info!(
+                sl!(),
+                "hotplug_block_device(): device_add arguments: bus: {}, id: {}, driver: {}, blkdev_add_args: {:#?}",
+                "scsi0.0",
+                node_name,
+                "scsi-hd",
+                blkdev_add_args
+            );
             self.qmp
                 .execute(&qmp::device_add {
                     bus: Some("scsi0.0".to_string()),
@@ -727,11 +752,60 @@ impl Qmp {
             );
 
             Ok((None, Some(scsi_addr)))
+        } else if block_driver == VIRTIO_BLK_CCW {
+            let subchannel = self
+                .ccw_subchannel
+                .as_mut()
+                .ok_or_else(|| anyhow!("CCW subchannel not available for virtio-blk-ccw hotplug"))?;
+
+            let slot = subchannel
+                .add_device(&node_name)
+                .map_err(|e| anyhow!("CCW subchannel add_device failed: {:?}", e))?;
+            let devno = subchannel.address_format_ccw(slot);
+            let ccw_addr = subchannel.address_format_ccw_for_virt_server(slot);
+
+            blkdev_add_args.insert("devno".to_owned(), devno.clone().into());
+            blkdev_add_args.insert("share-rw".to_string(), true.into());
+
+            info!(
+                sl!(),
+                "hotplug_block_device(): CCW device_add: id: {}, driver: {}, blkdev_add_args: {:#?}, ccw_addr: {}",
+                node_name,
+                block_driver,
+                blkdev_add_args,
+                ccw_addr
+            );
+            let device_add_result = self.qmp.execute(&qmp::device_add {
+                bus: None,
+                id: Some(node_name.clone()),
+                driver: block_driver.to_string(),
+                arguments: blkdev_add_args,
+            });
+            if let Err(e) = device_add_result {
+                // Roll back CCW subchannel state if QMP device_add fails
+                let _ = subchannel.remove_device(&node_name);
+                return Err(anyhow!("device_add {:?}", e));
+            }
+
+            info!(
+                sl!(),
+                "hotplug CCW block device return ccw address: {:?}", &ccw_addr
+            );
+
+            Ok((None, Some(ccw_addr)))
         } else {
             let (bus, slot) = self.find_free_slot()?;
             blkdev_add_args.insert("addr".to_owned(), format!("{slot:02}").into());
             blkdev_add_args.insert("share-rw".to_string(), true.into());
 
+            info!(
+                sl!(),
+                "hotplug_block_device(): device_add arguments: bus: {}, id: {}, driver: {}, blkdev_add_args: {:#?}",
+                bus,
+                node_name,
+                block_driver,
+                blkdev_add_args
+            );
             self.qmp
                 .execute(&qmp::device_add {
                     bus: Some(bus),

src/runtime-rs/crates/resource/src/manager_inner.rs

@@ -429,14 +429,16 @@ impl ResourceManagerInner {
                         .await
                         .context("do handle device")?;
 
-                    // create block device for kata agent,
-                    // if driver is virtio-blk-pci, the id will be pci address.
+                    // create block device for kata agent.
+                    // The device ID is derived from the available address: PCI, SCSI,
+                    // CCW, or virtual path, depending on the driver and configuration.
                     if let DeviceType::Block(device) = device_info {
-                        // The following would work for drivers virtio-blk-pci and virtio-mmio and virtio-scsi.
                         let id = if let Some(pci_path) = device.config.pci_path {
                             pci_path.to_string()
                         } else if let Some(scsi_address) = device.config.scsi_addr {
                             scsi_address
+                        } else if let Some(ccw_addr) = device.config.ccw_addr {
+                            ccw_addr
                         } else {
                             device.config.virt_path.clone()
                         };

src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs

@@ -100,7 +100,13 @@ impl BlockRootfs {
                 VIRTIO_BLK_MMIO => {
                     storage.source = device.config.virt_path;
                 }
-                VIRTIO_SCSI | VIRTIO_BLK_CCW | VIRTIO_PMEM => {
+                VIRTIO_BLK_CCW => {
+                    storage.source = device
+                        .config
+                        .ccw_addr
+                        .ok_or_else(|| anyhow!("CCW address missing for ccw block device"))?;
+                }
+                VIRTIO_SCSI | VIRTIO_PMEM => {
                     return Err(anyhow!(
                         "Complete support for block driver {} has not been implemented yet",
                         block_driver

src/runtime-rs/crates/resource/src/volume/utils.rs

@@ -15,6 +15,10 @@ use crate::{
 };
 use anyhow::{anyhow, Context, Result};
 use kata_sys_util::mount::{get_mount_options, get_mount_path};
+use kata_types::device::{
+    DRIVER_BLK_CCW_TYPE as KATA_CCW_DEV_TYPE, DRIVER_BLK_PCI_TYPE as KATA_BLK_DEV_TYPE,
+    DRIVER_SCSI_TYPE as KATA_SCSI_DEV_TYPE,
+};
 use oci_spec::runtime as oci;
 
 use hypervisor::device::DeviceType;
@@ -22,9 +26,6 @@ use hypervisor::device::DeviceType;
 pub const DEFAULT_VOLUME_FS_TYPE: &str = "ext4";
 pub const KATA_MOUNT_BIND_TYPE: &str = "bind";
 
-pub const KATA_BLK_DEV_TYPE: &str = "blk";
-pub const KATA_SCSI_DEV_TYPE: &str = "scsi";
-
 pub fn get_file_name<P: AsRef<Path>>(src: P) -> Result<String> {
     let file_name = src
         .as_ref()
@@ -104,6 +105,13 @@ pub async fn handle_block_volume(
                     return Err(anyhow!("block driver is scsi but no scsi address exists"));
                 }
             }
+            KATA_CCW_DEV_TYPE => {
+                if let Some(ccw_addr) = device.config.ccw_addr {
+                    ccw_addr.to_string()
+                } else {
+                    return Err(anyhow!("block driver is ccw but no ccw address exists"));
+                }
+            }
             _ => device.config.virt_path,
         };
         device_id = device.device_id;

src/runtime/Makefile

@@ -288,6 +288,7 @@ DEFSTATICRESOURCEMGMT_NV = true
 
 DEFDISABLEIMAGENVDIMM ?= false
 DEFDISABLEIMAGENVDIMM_NV = true
+DEFDISABLEIMAGENVDIMM_CLH ?= true
 
 DEFBINDMOUNTS := []
 
@@ -788,6 +789,7 @@ USER_VARS += DEFVFIOMODE_SE
 USER_VARS += BUILDFLAGS
 USER_VARS += DEFDISABLEIMAGENVDIMM
 USER_VARS += DEFDISABLEIMAGENVDIMM_NV
+USER_VARS += DEFDISABLEIMAGENVDIMM_CLH
 USER_VARS += DEFCCAMEASUREMENTALGO
 USER_VARS += DEFSHAREDFS_QEMU_CCA_VIRTIOFS
 USER_VARS += DEFPODRESOURCEAPISOCK

src/runtime/cmd/kata-runtime/kata-check_amd64_test.go

@@ -348,6 +348,15 @@ func TestCheckHostIsVMContainerCapable(t *testing.T) {
 
 	defer func() {
 		os.Remove(denylistModuleConf)
+		// reload removed modules
+		for mod := range archRequiredKernelModules {
+			cmd := exec.Command(modProbeCmd, mod)
+			if output, err := cmd.CombinedOutput(); err == nil {
+				kataLog.WithField("output", string(output)).Info("module loaded")
+			} else {
+				kataLog.WithField("output", string(output)).Warn("failed to load module")
+			}
+		}
 	}()
 
 	// remove the modules to force a failure

src/runtime/config/configuration-clh.toml.in

@@ -222,8 +222,8 @@ hypervisor_loglevel = 1
 # If false and nvdimm is supported, use nvdimm device to plug guest image.
 # Otherwise virtio-block device is used.
 #
-# nvdimm is not supported when `confidential_guest = true`.
-disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM@
+# nvdimm is not supported with Cloud Hypervisor or when `confidential_guest = true`.
+disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM_CLH@
 
 # Enable hot-plugging of VFIO devices to a root-port.
 # The default setting is  "no-port"

src/runtime/pkg/govmm/.github/workflows/main.yml

@@ -14,11 +14,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Install Go
-        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
       - name: golangci-lint

src/runtime/pkg/resourcecontrol/cgroups.go

@@ -8,6 +8,7 @@
 package resourcecontrol
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -50,7 +51,7 @@ type LinuxCgroup struct {
 	sync.Mutex
 }
 
-func sandboxDevices() []specs.LinuxDeviceCgroup {
+func sandboxDevices() ([]specs.LinuxDeviceCgroup, error) {
 	devices := []specs.LinuxDeviceCgroup{}
 
 	defaultDevices := []string{
@@ -68,14 +69,33 @@ func sandboxDevices() []specs.LinuxDeviceCgroup {
 	// In order to run Virtual Machines and create virtqueues, hypervisors
 	// need access to certain character devices in the host, like kvm and vhost-net.
 	hypervisorDevices := []string{
-		"/dev/kvm",         // To run virtual machines with KVM
-		"/dev/mshv",        // To run virtual machines with Hyper-V
+		"/dev/kvm",  // To run virtual machines with KVM
+		"/dev/mshv", // To run virtual machines with Hyper-V
+	}
+	virtualDevices := []string{
 		"/dev/vhost-net",   // To create virtqueues
 		"/dev/vfio/vfio",   // To access VFIO devices
 		"/dev/vhost-vsock", // To interact with vsock if
 	}
 
-	defaultDevices = append(defaultDevices, hypervisorDevices...)
+	hypervisorDeviceAdded := false
+	for _, hypervisor := range hypervisorDevices {
+		hypervisorDevice, err := DeviceToLinuxDevice(hypervisor)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				controllerLogger.WithField("source", "cgroups").Warnf("Failed to add %s to the devices cgroup: %v", hypervisor, err)
+			}
+			continue
+		}
+		devices = append(devices, hypervisorDevice)
+		hypervisorDeviceAdded = true
+		controllerLogger.WithField("source", "cgroups").Infof("Adding %s to the devices cgroup", hypervisor)
+		break
+	}
+	if !hypervisorDeviceAdded {
+		return []specs.LinuxDeviceCgroup{}, errors.New("failed to add any hypervisor device to devices cgroup")
+	}
+	defaultDevices = append(defaultDevices, virtualDevices...)
 
 	for _, device := range defaultDevices {
 		ldevice, err := DeviceToLinuxDevice(device)
@@ -128,7 +148,7 @@ func sandboxDevices() []specs.LinuxDeviceCgroup {
 
 	devices = append(devices, wildcardDevices...)
 
-	return devices
+	return devices, nil
 }
 
 func NewResourceController(path string, resources *specs.LinuxResources) (ResourceController, error) {
@@ -168,7 +188,11 @@ func NewResourceController(path string, resources *specs.LinuxResources) (Resour
 
 func NewSandboxResourceController(path string, resources *specs.LinuxResources, sandboxCgroupOnly bool) (ResourceController, error) {
 	sandboxResources := *resources
-	sandboxResources.Devices = append(sandboxResources.Devices, sandboxDevices()...)
+	sandboxDevices, err := sandboxDevices()
+	if err != nil {
+		return nil, err
+	}
+	sandboxResources.Devices = append(sandboxResources.Devices, sandboxDevices...)
 
 	// Currently we know to handle systemd cgroup path only when it's the only cgroup (no overhead group), hence,
 	// if sandboxCgroupOnly is not true we treat it as cgroupfs path as it used to be, although it may be incorrect.

src/runtime/virtcontainers/api_test.go

@@ -131,6 +131,11 @@ func newTestSandboxConfigKataAgent() SandboxConfig {
 }
 
 func TestCreateSandboxNoopAgentSuccessful(t *testing.T) {
+	// GITHUB_RUNNER_CI_NON_VIRT is set to true in .github/workflows/build-checks.yaml file for ARM64 runners because the self hosted runners do not support Virtualization
+	if os.Getenv("GITHUB_RUNNER_CI_NON_VIRT") == "true" {
+		t.Skip("Skipping the test as the GitHub self hosted runners for ARM64 do not support Virtualization")
+	}
+
 	assert := assert.New(t)
 	if tc.NotValid(ktu.NeedRoot()) {
 		t.Skip(testDisabledAsNonRoot)
@@ -159,6 +164,11 @@ func TestCreateSandboxNoopAgentSuccessful(t *testing.T) {
 }
 
 func TestCreateSandboxKataAgentSuccessful(t *testing.T) {
+	// GITHUB_RUNNER_CI_NON_VIRT is set to true in .github/workflows/build-checks.yaml file for ARM64 runners because the self hosted runners do not support Virtualization
+	if os.Getenv("GITHUB_RUNNER_CI_NON_VIRT") == "true" {
+		t.Skip("Skipping the test as the GitHub self hosted runners for ARM64 do not support Virtualization")
+	}
+
 	assert := assert.New(t)
 	if tc.NotValid(ktu.NeedRoot()) {
 		t.Skip(testDisabledAsNonRoot)
@@ -252,6 +262,11 @@ func createAndStartSandbox(ctx context.Context, config SandboxConfig) (sandbox V
 }
 
 func TestReleaseSandbox(t *testing.T) {
+	// GITHUB_RUNNER_CI_NON_VIRT is set to true in .github/workflows/build-checks.yaml file for ARM64 runners because the self hosted runners do not support Virtualization
+	if os.Getenv("GITHUB_RUNNER_CI_NON_VIRT") == "true" {
+		t.Skip("Skipping the test as the GitHub self hosted runners for ARM64 do not support Virtualization")
+	}
+
 	if tc.NotValid(ktu.NeedRoot()) {
 		t.Skip(testDisabledAsNonRoot)
 	}
@@ -269,6 +284,11 @@ func TestReleaseSandbox(t *testing.T) {
 }
 
 func TestCleanupContainer(t *testing.T) {
+	// GITHUB_RUNNER_CI_NON_VIRT is set to true in .github/workflows/build-checks.yaml file for ARM64 runners because the self hosted runners do not support Virtualization
+	if os.Getenv("GITHUB_RUNNER_CI_NON_VIRT") == "true" {
+		t.Skip("Skipping the test as the GitHub self hosted runners for ARM64 do not support Virtualization")
+	}
+
 	if tc.NotValid(ktu.NeedRoot()) {
 		t.Skip(testDisabledAsNonRoot)
 	}

src/runtime/virtcontainers/clh.go

@@ -332,6 +332,9 @@ func (clh *cloudHypervisor) getClhStopSandboxTimeout() time.Duration {
 func (clh *cloudHypervisor) setConfig(config *HypervisorConfig) error {
 	clh.config = *config
 
+	// We don't support NVDIMM with Cloud Hypervisor.
+	clh.config.DisableImageNvdimm = true
+
 	return nil
 }
 
@@ -584,8 +587,8 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 	// Set initial amount of cpu's for the virtual machine
 	clh.vmconfig.Cpus = chclient.NewCpusConfig(int32(clh.config.NumVCPUs()), int32(clh.config.DefaultMaxVCPUs))
 
-	disableNvdimm := (clh.config.DisableImageNvdimm || clh.config.ConfidentialGuest)
-	enableDax := !disableNvdimm
+	disableNvdimm := true
+	enableDax := false
 
 	params, err := getNonUserDefinedKernelParams(hypervisorConfig.RootfsType, disableNvdimm, enableDax, clh.config.Debug, clh.config.ConfidentialGuest, clh.config.IOMMU, hypervisorConfig.KernelVerityParams)
 	if err != nil {
@@ -607,31 +610,19 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 	}
 
 	if assetType == types.ImageAsset {
-		if clh.config.DisableImageNvdimm || clh.config.ConfidentialGuest {
-			disk := chclient.NewDiskConfig()
-			disk.Path = &assetPath
-			disk.SetReadonly(true)
+		disk := chclient.NewDiskConfig()
+		disk.Path = &assetPath
+		disk.SetReadonly(true)
 
-			diskRateLimiterConfig := clh.getDiskRateLimiterConfig()
-			if diskRateLimiterConfig != nil {
-				disk.SetRateLimiterConfig(*diskRateLimiterConfig)
-			}
+		diskRateLimiterConfig := clh.getDiskRateLimiterConfig()
+		if diskRateLimiterConfig != nil {
+			disk.SetRateLimiterConfig(*diskRateLimiterConfig)
+		}
 
-			if clh.vmconfig.Disks != nil {
-				*clh.vmconfig.Disks = append(*clh.vmconfig.Disks, *disk)
-			} else {
-				clh.vmconfig.Disks = &[]chclient.DiskConfig{*disk}
-			}
+		if clh.vmconfig.Disks != nil {
+			*clh.vmconfig.Disks = append(*clh.vmconfig.Disks, *disk)
 		} else {
-			pmem := chclient.NewPmemConfig(assetPath)
-			*pmem.DiscardWrites = true
-			pmem.SetIommu(clh.config.IOMMU)
-
-			if clh.vmconfig.Pmem != nil {
-				*clh.vmconfig.Pmem = append(*clh.vmconfig.Pmem, *pmem)
-			} else {
-				clh.vmconfig.Pmem = &[]chclient.PmemConfig{*pmem}
-			}
+			clh.vmconfig.Disks = &[]chclient.DiskConfig{*disk}
 		}
 	} else {
 		// assetType == types.InitrdAsset

src/runtime/virtcontainers/clh_test.go

@@ -69,6 +69,7 @@ func newClhConfig() (HypervisorConfig, error) {
 		NetRateLimiterOpsMaxRate:      int64(0),
 		NetRateLimiterOpsOneTimeBurst: int64(0),
 		HotPlugVFIO:                   config.NoPort,
+		DisableImageNvdimm:            true,
 	}, nil
 }
 

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/FILES

@@ -45,6 +45,7 @@ docs/VmCoredumpData.md
 docs/VmInfo.md
 docs/VmRemoveDevice.md
 docs/VmResize.md
+docs/VmResizeDisk.md
 docs/VmResizeZone.md
 docs/VmSnapshotConfig.md
 docs/VmmPingResponse.md
@@ -90,6 +91,7 @@ model_vm_coredump_data.go
 model_vm_info.go
 model_vm_remove_device.go
 model_vm_resize.go
+model_vm_resize_disk.go
 model_vm_resize_zone.go
 model_vm_snapshot_config.go
 model_vmm_ping_response.go

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/README.md

@@ -16,7 +16,6 @@ Install the following dependencies:
 ```shell
 go get github.com/stretchr/testify/assert
 go get golang.org/x/oauth2
-go get golang.org/x/net/context
 ```
 
 Put the package under your project folder and add the following in import:
@@ -100,6 +99,7 @@ Class | Method | HTTP request | Description
 *DefaultApi* | [**VmInfoGet**](docs/DefaultApi.md#vminfoget) | **Get** /vm.info | Returns general information about the cloud-hypervisor Virtual Machine (VM) instance.
 *DefaultApi* | [**VmReceiveMigrationPut**](docs/DefaultApi.md#vmreceivemigrationput) | **Put** /vm.receive-migration | Receive a VM migration from URL
 *DefaultApi* | [**VmRemoveDevicePut**](docs/DefaultApi.md#vmremovedeviceput) | **Put** /vm.remove-device | Remove a device from the VM
+*DefaultApi* | [**VmResizeDiskPut**](docs/DefaultApi.md#vmresizediskput) | **Put** /vm.resize-disk | Resize a disk
 *DefaultApi* | [**VmResizePut**](docs/DefaultApi.md#vmresizeput) | **Put** /vm.resize | Resize the VM
 *DefaultApi* | [**VmResizeZonePut**](docs/DefaultApi.md#vmresizezoneput) | **Put** /vm.resize-zone | Resize a memory zone
 *DefaultApi* | [**VmRestorePut**](docs/DefaultApi.md#vmrestoreput) | **Put** /vm.restore | Restore a VM from a snapshot.
@@ -149,6 +149,7 @@ Class | Method | HTTP request | Description
  - [VmInfo](docs/VmInfo.md)
  - [VmRemoveDevice](docs/VmRemoveDevice.md)
  - [VmResize](docs/VmResize.md)
+ - [VmResizeDisk](docs/VmResizeDisk.md)
  - [VmResizeZone](docs/VmResizeZone.md)
  - [VmSnapshotConfig](docs/VmSnapshotConfig.md)
  - [VmmPingResponse](docs/VmmPingResponse.md)
@@ -177,6 +178,3 @@ Each of these functions takes a value of the given basic type and returns a poin
 * `PtrTime`
 
 ## Author
-
-
-

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml

@@ -153,6 +153,21 @@ paths:
           description: The VM instance could not be resized because a cpu removal
             is still pending.
       summary: Resize the VM
+  /vm.resize-disk:
+    put:
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/VmResizeDisk'
+        description: Resizes a disk attached to the VM
+        required: true
+      responses:
+        "204":
+          description: The disk was successfully resized.
+        "500":
+          description: The disk could not be resized.
+      summary: Resize a disk
   /vm.resize-zone:
     put:
       requestBody:
@@ -649,7 +664,9 @@ components:
           - tap: tap
             host_mac: host_mac
             num_queues: 6
+            offload_ufo: true
             queue_size: 1
+            offload_csum: true
             ip: 192.168.249.1
             rate_limiter_config:
               ops:
@@ -663,6 +680,7 @@ components:
             mac: mac
             mtu: 3
             pci_segment: 2
+            offload_tso: true
             vhost_mode: Client
             iommu: false
             vhost_socket: vhost_socket
@@ -672,7 +690,9 @@ components:
           - tap: tap
             host_mac: host_mac
             num_queues: 6
+            offload_ufo: true
             queue_size: 1
+            offload_csum: true
             ip: 192.168.249.1
             rate_limiter_config:
               ops:
@@ -686,6 +706,7 @@ components:
             mac: mac
             mtu: 3
             pci_segment: 2
+            offload_tso: true
             vhost_mode: Client
             iommu: false
             vhost_socket: vhost_socket
@@ -1079,7 +1100,9 @@ components:
         - tap: tap
           host_mac: host_mac
           num_queues: 6
+          offload_ufo: true
           queue_size: 1
+          offload_csum: true
           ip: 192.168.249.1
           rate_limiter_config:
             ops:
@@ -1093,6 +1116,7 @@ components:
           mac: mac
           mtu: 3
           pci_segment: 2
+          offload_tso: true
           vhost_mode: Client
           iommu: false
           vhost_socket: vhost_socket
@@ -1102,7 +1126,9 @@ components:
         - tap: tap
           host_mac: host_mac
           num_queues: 6
+          offload_ufo: true
           queue_size: 1
+          offload_csum: true
           ip: 192.168.249.1
           rate_limiter_config:
             ops:
@@ -1116,6 +1142,7 @@ components:
           mac: mac
           mtu: 3
           pci_segment: 2
+          offload_tso: true
           vhost_mode: Client
           iommu: false
           vhost_socket: vhost_socket
@@ -1741,7 +1768,9 @@ components:
         tap: tap
         host_mac: host_mac
         num_queues: 6
+        offload_ufo: true
         queue_size: 1
+        offload_csum: true
         ip: 192.168.249.1
         rate_limiter_config:
           ops:
@@ -1755,6 +1784,7 @@ components:
         mac: mac
         mtu: 3
         pci_segment: 2
+        offload_tso: true
         vhost_mode: Client
         iommu: false
         vhost_socket: vhost_socket
@@ -1803,6 +1833,15 @@ components:
           type: integer
         rate_limiter_config:
           $ref: '#/components/schemas/RateLimiterConfig'
+        offload_tso:
+          default: true
+          type: boolean
+        offload_ufo:
+          default: true
+          type: boolean
+        offload_csum:
+          default: true
+          type: boolean
       type: object
     RngConfig:
       example:
@@ -2103,6 +2142,19 @@ components:
           format: int64
           type: integer
       type: object
+    VmResizeDisk:
+      example:
+        desired_size: 0
+        id: id
+      properties:
+        id:
+          description: disk identifier
+          type: string
+        desired_size:
+          description: desired disk size in bytes
+          format: int64
+          type: integer
+      type: object
     VmResizeZone:
       example:
         id: id

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go

@@ -2226,6 +2226,106 @@ func (a *DefaultApiService) VmRemoveDevicePutExecute(r ApiVmRemoveDevicePutReque
 	return localVarHTTPResponse, nil
 }
 
+type ApiVmResizeDiskPutRequest struct {
+	ctx          _context.Context
+	ApiService   *DefaultApiService
+	vmResizeDisk *VmResizeDisk
+}
+
+// Resizes a disk attached to the VM
+func (r ApiVmResizeDiskPutRequest) VmResizeDisk(vmResizeDisk VmResizeDisk) ApiVmResizeDiskPutRequest {
+	r.vmResizeDisk = &vmResizeDisk
+	return r
+}
+
+func (r ApiVmResizeDiskPutRequest) Execute() (*_nethttp.Response, error) {
+	return r.ApiService.VmResizeDiskPutExecute(r)
+}
+
+/*
+VmResizeDiskPut Resize a disk
+
+	@param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+	@return ApiVmResizeDiskPutRequest
+*/
+func (a *DefaultApiService) VmResizeDiskPut(ctx _context.Context) ApiVmResizeDiskPutRequest {
+	return ApiVmResizeDiskPutRequest{
+		ApiService: a,
+		ctx:        ctx,
+	}
+}
+
+// Execute executes the request
+func (a *DefaultApiService) VmResizeDiskPutExecute(r ApiVmResizeDiskPutRequest) (*_nethttp.Response, error) {
+	var (
+		localVarHTTPMethod   = _nethttp.MethodPut
+		localVarPostBody     interface{}
+		localVarFormFileName string
+		localVarFileName     string
+		localVarFileBytes    []byte
+	)
+
+	localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "DefaultApiService.VmResizeDiskPut")
+	if err != nil {
+		return nil, GenericOpenAPIError{error: err.Error()}
+	}
+
+	localVarPath := localBasePath + "/vm.resize-disk"
+
+	localVarHeaderParams := make(map[string]string)
+	localVarQueryParams := _neturl.Values{}
+	localVarFormParams := _neturl.Values{}
+	if r.vmResizeDisk == nil {
+		return nil, reportError("vmResizeDisk is required and must be specified")
+	}
+
+	// to determine the Content-Type header
+	localVarHTTPContentTypes := []string{"application/json"}
+
+	// set Content-Type header
+	localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+	if localVarHTTPContentType != "" {
+		localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+	}
+
+	// to determine the Accept header
+	localVarHTTPHeaderAccepts := []string{}
+
+	// set Accept header
+	localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+	if localVarHTTPHeaderAccept != "" {
+		localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+	}
+	// body params
+	localVarPostBody = r.vmResizeDisk
+	req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	localVarHTTPResponse, err := a.client.callAPI(req)
+	if err != nil || localVarHTTPResponse == nil {
+		return localVarHTTPResponse, err
+	}
+
+	localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body)
+	localVarHTTPResponse.Body.Close()
+	localVarHTTPResponse.Body = _ioutil.NopCloser(bytes.NewBuffer(localVarBody))
+	if err != nil {
+		return localVarHTTPResponse, err
+	}
+
+	if localVarHTTPResponse.StatusCode >= 300 {
+		newErr := GenericOpenAPIError{
+			body:  localVarBody,
+			error: localVarHTTPResponse.Status,
+		}
+		return localVarHTTPResponse, newErr
+	}
+
+	return localVarHTTPResponse, nil
+}
+
 type ApiVmResizePutRequest struct {
 	ctx        _context.Context
 	ApiService *DefaultApiService

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md

@@ -26,6 +26,7 @@ Method | HTTP request | Description
 [**VmInfoGet**](DefaultApi.md#VmInfoGet) | **Get** /vm.info | Returns general information about the cloud-hypervisor Virtual Machine (VM) instance.
 [**VmReceiveMigrationPut**](DefaultApi.md#VmReceiveMigrationPut) | **Put** /vm.receive-migration | Receive a VM migration from URL
 [**VmRemoveDevicePut**](DefaultApi.md#VmRemoveDevicePut) | **Put** /vm.remove-device | Remove a device from the VM
+[**VmResizeDiskPut**](DefaultApi.md#VmResizeDiskPut) | **Put** /vm.resize-disk | Resize a disk
 [**VmResizePut**](DefaultApi.md#VmResizePut) | **Put** /vm.resize | Resize the VM
 [**VmResizeZonePut**](DefaultApi.md#VmResizeZonePut) | **Put** /vm.resize-zone | Resize a memory zone
 [**VmRestorePut**](DefaultApi.md#VmRestorePut) | **Put** /vm.restore | Restore a VM from a snapshot.
@@ -1370,6 +1371,68 @@ No authorization required
 [[Back to README]](../README.md)
 
 
+## VmResizeDiskPut
+
+> VmResizeDiskPut(ctx).VmResizeDisk(vmResizeDisk).Execute()
+
+Resize a disk
+
+### Example
+
+```go
+package main
+
+import (
+    "context"
+    "fmt"
+    "os"
+    openapiclient "./openapi"
+)
+
+func main() {
+    vmResizeDisk := *openapiclient.NewVmResizeDisk() // VmResizeDisk | Resizes a disk attached to the VM
+
+    configuration := openapiclient.NewConfiguration()
+    api_client := openapiclient.NewAPIClient(configuration)
+    resp, r, err := api_client.DefaultApi.VmResizeDiskPut(context.Background()).VmResizeDisk(vmResizeDisk).Execute()
+    if err != nil {
+        fmt.Fprintf(os.Stderr, "Error when calling `DefaultApi.VmResizeDiskPut``: %v\n", err)
+        fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
+    }
+}
+```
+
+### Path Parameters
+
+
+
+### Other Parameters
+
+Other parameters are passed through a pointer to a apiVmResizeDiskPutRequest struct via the builder pattern
+
+
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------
+ **vmResizeDisk** | [**VmResizeDisk**](VmResizeDisk.md) | Resizes a disk attached to the VM | 
+
+### Return type
+
+ (empty response body)
+
+### Authorization
+
+No authorization required
+
+### HTTP request headers
+
+- **Content-Type**: application/json
+- **Accept**: Not defined
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints)
+[[Back to Model list]](../README.md#documentation-for-models)
+[[Back to README]](../README.md)
+
+
 ## VmResizePut
 
 > VmResizePut(ctx).VmResize(vmResize).Execute()

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/NetConfig.md

@@ -19,6 +19,9 @@ Name | Type | Description | Notes
 **Id** | Pointer to **string** |  | [optional] 
 **PciSegment** | Pointer to **int32** |  | [optional] 
 **RateLimiterConfig** | Pointer to [**RateLimiterConfig**](RateLimiterConfig.md) |  | [optional] 
+**OffloadTso** | Pointer to **bool** |  | [optional] [default to true]
+**OffloadUfo** | Pointer to **bool** |  | [optional] [default to true]
+**OffloadCsum** | Pointer to **bool** |  | [optional] [default to true]
 
 ## Methods
 
@@ -414,6 +417,81 @@ SetRateLimiterConfig sets RateLimiterConfig field to given value.
 
 HasRateLimiterConfig returns a boolean if a field has been set.
 
+### GetOffloadTso
+
+`func (o *NetConfig) GetOffloadTso() bool`
+
+GetOffloadTso returns the OffloadTso field if non-nil, zero value otherwise.
+
+### GetOffloadTsoOk
+
+`func (o *NetConfig) GetOffloadTsoOk() (*bool, bool)`
+
+GetOffloadTsoOk returns a tuple with the OffloadTso field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetOffloadTso
+
+`func (o *NetConfig) SetOffloadTso(v bool)`
+
+SetOffloadTso sets OffloadTso field to given value.
+
+### HasOffloadTso
+
+`func (o *NetConfig) HasOffloadTso() bool`
+
+HasOffloadTso returns a boolean if a field has been set.
+
+### GetOffloadUfo
+
+`func (o *NetConfig) GetOffloadUfo() bool`
+
+GetOffloadUfo returns the OffloadUfo field if non-nil, zero value otherwise.
+
+### GetOffloadUfoOk
+
+`func (o *NetConfig) GetOffloadUfoOk() (*bool, bool)`
+
+GetOffloadUfoOk returns a tuple with the OffloadUfo field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetOffloadUfo
+
+`func (o *NetConfig) SetOffloadUfo(v bool)`
+
+SetOffloadUfo sets OffloadUfo field to given value.
+
+### HasOffloadUfo
+
+`func (o *NetConfig) HasOffloadUfo() bool`
+
+HasOffloadUfo returns a boolean if a field has been set.
+
+### GetOffloadCsum
+
+`func (o *NetConfig) GetOffloadCsum() bool`
+
+GetOffloadCsum returns the OffloadCsum field if non-nil, zero value otherwise.
+
+### GetOffloadCsumOk
+
+`func (o *NetConfig) GetOffloadCsumOk() (*bool, bool)`
+
+GetOffloadCsumOk returns a tuple with the OffloadCsum field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetOffloadCsum
+
+`func (o *NetConfig) SetOffloadCsum(v bool)`
+
+SetOffloadCsum sets OffloadCsum field to given value.
+
+### HasOffloadCsum
+
+`func (o *NetConfig) HasOffloadCsum() bool`
+
+HasOffloadCsum returns a boolean if a field has been set.
+
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmResizeDisk.md

@@ -0,0 +1,82 @@
+# VmResizeDisk
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**Id** | Pointer to **string** | disk identifier | [optional] 
+**DesiredSize** | Pointer to **int64** | desired disk size in bytes | [optional] 
+
+## Methods
+
+### NewVmResizeDisk
+
+`func NewVmResizeDisk() *VmResizeDisk`
+
+NewVmResizeDisk instantiates a new VmResizeDisk object
+This constructor will assign default values to properties that have it defined,
+and makes sure properties required by API are set, but the set of arguments
+will change when the set of required properties is changed
+
+### NewVmResizeDiskWithDefaults
+
+`func NewVmResizeDiskWithDefaults() *VmResizeDisk`
+
+NewVmResizeDiskWithDefaults instantiates a new VmResizeDisk object
+This constructor will only assign default values to properties that have it defined,
+but it doesn't guarantee that properties required by API are set
+
+### GetId
+
+`func (o *VmResizeDisk) GetId() string`
+
+GetId returns the Id field if non-nil, zero value otherwise.
+
+### GetIdOk
+
+`func (o *VmResizeDisk) GetIdOk() (*string, bool)`
+
+GetIdOk returns a tuple with the Id field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetId
+
+`func (o *VmResizeDisk) SetId(v string)`
+
+SetId sets Id field to given value.
+
+### HasId
+
+`func (o *VmResizeDisk) HasId() bool`
+
+HasId returns a boolean if a field has been set.
+
+### GetDesiredSize
+
+`func (o *VmResizeDisk) GetDesiredSize() int64`
+
+GetDesiredSize returns the DesiredSize field if non-nil, zero value otherwise.
+
+### GetDesiredSizeOk
+
+`func (o *VmResizeDisk) GetDesiredSizeOk() (*int64, bool)`
+
+GetDesiredSizeOk returns a tuple with the DesiredSize field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetDesiredSize
+
+`func (o *VmResizeDisk) SetDesiredSize(v int64)`
+
+SetDesiredSize sets DesiredSize field to given value.
+
+### HasDesiredSize
+
+`func (o *VmResizeDisk) HasDesiredSize() bool`
+
+HasDesiredSize returns a boolean if a field has been set.
+
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_net_config.go

@@ -33,6 +33,9 @@ type NetConfig struct {
 	Id                *string            `json:"id,omitempty"`
 	PciSegment        *int32             `json:"pci_segment,omitempty"`
 	RateLimiterConfig *RateLimiterConfig `json:"rate_limiter_config,omitempty"`
+	OffloadTso        *bool              `json:"offload_tso,omitempty"`
+	OffloadUfo        *bool              `json:"offload_ufo,omitempty"`
+	OffloadCsum       *bool              `json:"offload_csum,omitempty"`
 }
 
 // NewNetConfig instantiates a new NetConfig object
@@ -55,6 +58,12 @@ func NewNetConfig() *NetConfig {
 	this.VhostUser = &vhostUser
 	var vhostMode string = "Client"
 	this.VhostMode = &vhostMode
+	var offloadTso bool = true
+	this.OffloadTso = &offloadTso
+	var offloadUfo bool = true
+	this.OffloadUfo = &offloadUfo
+	var offloadCsum bool = true
+	this.OffloadCsum = &offloadCsum
 	return &this
 }
 
@@ -77,6 +86,12 @@ func NewNetConfigWithDefaults() *NetConfig {
 	this.VhostUser = &vhostUser
 	var vhostMode string = "Client"
 	this.VhostMode = &vhostMode
+	var offloadTso bool = true
+	this.OffloadTso = &offloadTso
+	var offloadUfo bool = true
+	this.OffloadUfo = &offloadUfo
+	var offloadCsum bool = true
+	this.OffloadCsum = &offloadCsum
 	return &this
 }
 
@@ -560,6 +575,102 @@ func (o *NetConfig) SetRateLimiterConfig(v RateLimiterConfig) {
 	o.RateLimiterConfig = &v
 }
 
+// GetOffloadTso returns the OffloadTso field value if set, zero value otherwise.
+func (o *NetConfig) GetOffloadTso() bool {
+	if o == nil || o.OffloadTso == nil {
+		var ret bool
+		return ret
+	}
+	return *o.OffloadTso
+}
+
+// GetOffloadTsoOk returns a tuple with the OffloadTso field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *NetConfig) GetOffloadTsoOk() (*bool, bool) {
+	if o == nil || o.OffloadTso == nil {
+		return nil, false
+	}
+	return o.OffloadTso, true
+}
+
+// HasOffloadTso returns a boolean if a field has been set.
+func (o *NetConfig) HasOffloadTso() bool {
+	if o != nil && o.OffloadTso != nil {
+		return true
+	}
+
+	return false
+}
+
+// SetOffloadTso gets a reference to the given bool and assigns it to the OffloadTso field.
+func (o *NetConfig) SetOffloadTso(v bool) {
+	o.OffloadTso = &v
+}
+
+// GetOffloadUfo returns the OffloadUfo field value if set, zero value otherwise.
+func (o *NetConfig) GetOffloadUfo() bool {
+	if o == nil || o.OffloadUfo == nil {
+		var ret bool
+		return ret
+	}
+	return *o.OffloadUfo
+}
+
+// GetOffloadUfoOk returns a tuple with the OffloadUfo field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *NetConfig) GetOffloadUfoOk() (*bool, bool) {
+	if o == nil || o.OffloadUfo == nil {
+		return nil, false
+	}
+	return o.OffloadUfo, true
+}
+
+// HasOffloadUfo returns a boolean if a field has been set.
+func (o *NetConfig) HasOffloadUfo() bool {
+	if o != nil && o.OffloadUfo != nil {
+		return true
+	}
+
+	return false
+}
+
+// SetOffloadUfo gets a reference to the given bool and assigns it to the OffloadUfo field.
+func (o *NetConfig) SetOffloadUfo(v bool) {
+	o.OffloadUfo = &v
+}
+
+// GetOffloadCsum returns the OffloadCsum field value if set, zero value otherwise.
+func (o *NetConfig) GetOffloadCsum() bool {
+	if o == nil || o.OffloadCsum == nil {
+		var ret bool
+		return ret
+	}
+	return *o.OffloadCsum
+}
+
+// GetOffloadCsumOk returns a tuple with the OffloadCsum field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *NetConfig) GetOffloadCsumOk() (*bool, bool) {
+	if o == nil || o.OffloadCsum == nil {
+		return nil, false
+	}
+	return o.OffloadCsum, true
+}
+
+// HasOffloadCsum returns a boolean if a field has been set.
+func (o *NetConfig) HasOffloadCsum() bool {
+	if o != nil && o.OffloadCsum != nil {
+		return true
+	}
+
+	return false
+}
+
+// SetOffloadCsum gets a reference to the given bool and assigns it to the OffloadCsum field.
+func (o *NetConfig) SetOffloadCsum(v bool) {
+	o.OffloadCsum = &v
+}
+
 func (o NetConfig) MarshalJSON() ([]byte, error) {
 	toSerialize := map[string]interface{}{}
 	if o.Tap != nil {
@@ -607,6 +718,15 @@ func (o NetConfig) MarshalJSON() ([]byte, error) {
 	if o.RateLimiterConfig != nil {
 		toSerialize["rate_limiter_config"] = o.RateLimiterConfig
 	}
+	if o.OffloadTso != nil {
+		toSerialize["offload_tso"] = o.OffloadTso
+	}
+	if o.OffloadUfo != nil {
+		toSerialize["offload_ufo"] = o.OffloadUfo
+	}
+	if o.OffloadCsum != nil {
+		toSerialize["offload_csum"] = o.OffloadCsum
+	}
 	return json.Marshal(toSerialize)
 }
 

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_resize_disk.go

@@ -0,0 +1,151 @@
+/*
+Cloud Hypervisor API
+
+Local HTTP based API for managing and inspecting a cloud-hypervisor virtual machine.
+
+API version: 0.3.0
+*/
+
+// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
+
+package openapi
+
+import (
+	"encoding/json"
+)
+
+// VmResizeDisk struct for VmResizeDisk
+type VmResizeDisk struct {
+	// disk identifier
+	Id *string `json:"id,omitempty"`
+	// desired disk size in bytes
+	DesiredSize *int64 `json:"desired_size,omitempty"`
+}
+
+// NewVmResizeDisk instantiates a new VmResizeDisk object
+// This constructor will assign default values to properties that have it defined,
+// and makes sure properties required by API are set, but the set of arguments
+// will change when the set of required properties is changed
+func NewVmResizeDisk() *VmResizeDisk {
+	this := VmResizeDisk{}
+	return &this
+}
+
+// NewVmResizeDiskWithDefaults instantiates a new VmResizeDisk object
+// This constructor will only assign default values to properties that have it defined,
+// but it doesn't guarantee that properties required by API are set
+func NewVmResizeDiskWithDefaults() *VmResizeDisk {
+	this := VmResizeDisk{}
+	return &this
+}
+
+// GetId returns the Id field value if set, zero value otherwise.
+func (o *VmResizeDisk) GetId() string {
+	if o == nil || o.Id == nil {
+		var ret string
+		return ret
+	}
+	return *o.Id
+}
+
+// GetIdOk returns a tuple with the Id field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *VmResizeDisk) GetIdOk() (*string, bool) {
+	if o == nil || o.Id == nil {
+		return nil, false
+	}
+	return o.Id, true
+}
+
+// HasId returns a boolean if a field has been set.
+func (o *VmResizeDisk) HasId() bool {
+	if o != nil && o.Id != nil {
+		return true
+	}
+
+	return false
+}
+
+// SetId gets a reference to the given string and assigns it to the Id field.
+func (o *VmResizeDisk) SetId(v string) {
+	o.Id = &v
+}
+
+// GetDesiredSize returns the DesiredSize field value if set, zero value otherwise.
+func (o *VmResizeDisk) GetDesiredSize() int64 {
+	if o == nil || o.DesiredSize == nil {
+		var ret int64
+		return ret
+	}
+	return *o.DesiredSize
+}
+
+// GetDesiredSizeOk returns a tuple with the DesiredSize field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *VmResizeDisk) GetDesiredSizeOk() (*int64, bool) {
+	if o == nil || o.DesiredSize == nil {
+		return nil, false
+	}
+	return o.DesiredSize, true
+}
+
+// HasDesiredSize returns a boolean if a field has been set.
+func (o *VmResizeDisk) HasDesiredSize() bool {
+	if o != nil && o.DesiredSize != nil {
+		return true
+	}
+
+	return false
+}
+
+// SetDesiredSize gets a reference to the given int64 and assigns it to the DesiredSize field.
+func (o *VmResizeDisk) SetDesiredSize(v int64) {
+	o.DesiredSize = &v
+}
+
+func (o VmResizeDisk) MarshalJSON() ([]byte, error) {
+	toSerialize := map[string]interface{}{}
+	if o.Id != nil {
+		toSerialize["id"] = o.Id
+	}
+	if o.DesiredSize != nil {
+		toSerialize["desired_size"] = o.DesiredSize
+	}
+	return json.Marshal(toSerialize)
+}
+
+type NullableVmResizeDisk struct {
+	value *VmResizeDisk
+	isSet bool
+}
+
+func (v NullableVmResizeDisk) Get() *VmResizeDisk {
+	return v.value
+}
+
+func (v *NullableVmResizeDisk) Set(val *VmResizeDisk) {
+	v.value = val
+	v.isSet = true
+}
+
+func (v NullableVmResizeDisk) IsSet() bool {
+	return v.isSet
+}
+
+func (v *NullableVmResizeDisk) Unset() {
+	v.value = nil
+	v.isSet = false
+}
+
+func NewNullableVmResizeDisk(val *VmResizeDisk) *NullableVmResizeDisk {
+	return &NullableVmResizeDisk{value: val, isSet: true}
+}
+
+func (v NullableVmResizeDisk) MarshalJSON() ([]byte, error) {
+	return json.Marshal(v.value)
+}
+
+func (v *NullableVmResizeDisk) UnmarshalJSON(src []byte) error {
+	v.isSet = true
+	return json.Unmarshal(src, &v.value)
+}

src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml

@@ -163,6 +163,22 @@ paths:
         429:
           description: The VM instance could not be resized because a cpu removal is still pending.
 
+  /vm.resize-disk:
+    put:
+      summary: Resize a disk
+      requestBody:
+        description: Resizes a disk attached to the VM
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/VmResizeDisk"
+        required: true
+      responses:
+        204:
+          description: The disk was successfully resized.
+        500:
+          description: The disk could not be resized.
+
   /vm.resize-zone:
     put:
       summary: Resize a memory zone
@@ -966,6 +982,15 @@ components:
           format: int16
         rate_limiter_config:
           $ref: "#/components/schemas/RateLimiterConfig"
+        offload_tso:
+          type: boolean
+          default: true
+        offload_ufo:
+          type: boolean
+          default: true
+        offload_csum:
+          type: boolean
+          default: true
 
     RngConfig:
       required:
@@ -1194,6 +1219,17 @@ components:
           type: integer
           format: int64
 
+    VmResizeDisk:
+      type: object
+      properties:
+        id:
+          description: disk identifier
+          type: string
+        desired_size:
+          description: desired disk size in bytes
+          type: integer
+          format: int64
+
     VmResizeZone:
       type: object
       properties:

src/runtime/virtcontainers/qemu.go

@@ -1098,8 +1098,10 @@ func (q *qemu) LogAndWait(qemuCmd *exec.Cmd, reader io.ReadCloser) {
 			q.Logger().WithField("qemuPid", pid).Error(text)
 		}
 	}
-	q.Logger().Infof("Stop logging QEMU (qemuPid=%d)", pid)
-	qemuCmd.Wait()
+	q.Logger().WithField("qemuPid", pid).Infof("Stop logging QEMU")
+	if err := qemuCmd.Wait(); err != nil {
+		q.Logger().WithField("qemuPid", pid).WithField("error", err).Warn("QEMU exited with an error")
+	}
 }
 
 // StartVM will start the Sandbox's VM.

src/runtime/virtcontainers/qemu_arch_base_test.go

@@ -332,6 +332,38 @@ func TestQemuArchBaseAppendImage(t *testing.T) {
 	assert.Equal(expectedOut, devices)
 }
 
+func TestQemuArchBaseAppendNvdimmImage(t *testing.T) {
+	var devices []govmmQemu.Device
+	assert := assert.New(t)
+	qemuArchBase := newQemuArchBase()
+
+	image, err := os.CreateTemp("", "img")
+	assert.NoError(err)
+	defer image.Close()
+	defer os.Remove(image.Name())
+
+	imageStat, err := image.Stat()
+	assert.NoError(err)
+
+	devices, err = qemuArchBase.appendNvdimmImage(devices, image.Name())
+	assert.NoError(err)
+	assert.Len(devices, 1)
+
+	expectedOut := []govmmQemu.Device{
+		govmmQemu.Object{
+			Driver:   govmmQemu.NVDIMM,
+			Type:     govmmQemu.MemoryBackendFile,
+			DeviceID: "nv0",
+			ID:       "mem0",
+			MemPath:  image.Name(),
+			Size:     (uint64)(imageStat.Size()),
+			ReadOnly: true,
+		},
+	}
+
+	assert.Equal(expectedOut, devices)
+}
+
 func TestQemuArchBaseAppendBridges(t *testing.T) {
 	var devices []govmmQemu.Device
 	assert := assert.New(t)

src/runtime/virtcontainers/qemu_arm64.go

@@ -10,7 +10,6 @@ package virtcontainers
 import (
 	"context"
 	"fmt"
-	"os"
 	"time"
 
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
@@ -69,9 +68,10 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) {
 			kernelParamsDebug:    kernelParamsDebug,
 			kernelParams:         kernelParams,
 			disableNvdimm:        config.DisableImageNvdimm,
-			dax:                  true,
-			protection:           noneProtection,
-			legacySerial:         config.LegacySerial,
+			// DAX is disabled on ARM due to a kernel panic in caches_clean_inval_pou.
+			dax:          false,
+			protection:   noneProtection,
+			legacySerial: config.LegacySerial,
 		},
 		measurementAlgo: config.MeasurementAlgo,
 	}
@@ -109,35 +109,6 @@ func (q *qemuArm64) appendImage(ctx context.Context, devices []govmmQemu.Device,
 	return q.appendBlockImage(ctx, devices, path)
 }
 
-// There is no nvdimm/readonly feature in qemu 5.1 which is used by arm64 for now,
-// so we temporarily add this specific implementation for arm64 here until
-// the qemu used by arm64 is capable for that feature
-func (q *qemuArm64) appendNvdimmImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {
-	imageFile, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer imageFile.Close()
-
-	imageStat, err := imageFile.Stat()
-	if err != nil {
-		return nil, err
-	}
-
-	object := govmmQemu.Object{
-		Driver:   govmmQemu.NVDIMM,
-		Type:     govmmQemu.MemoryBackendFile,
-		DeviceID: "nv0",
-		ID:       "mem0",
-		MemPath:  path,
-		Size:     (uint64)(imageStat.Size()),
-	}
-
-	devices = append(devices, object)
-
-	return devices, nil
-}
-
 func (q *qemuArm64) setIgnoreSharedMemoryMigrationCaps(_ context.Context, _ *govmmQemu.QMP) error {
 	// x-ignore-shared not support in arm64 for now
 	return nil

src/runtime/virtcontainers/qemu_arm64_test.go

@@ -130,39 +130,6 @@ func TestQemuArm64AppendImage(t *testing.T) {
 	assert.Equal(expectedOut, devices)
 }
 
-func TestQemuArm64AppendNvdimmImage(t *testing.T) {
-	var devices []govmmQemu.Device
-	assert := assert.New(t)
-
-	f, err := os.CreateTemp("", "img")
-	assert.NoError(err)
-	defer func() { _ = f.Close() }()
-	defer func() { _ = os.Remove(f.Name()) }()
-
-	imageStat, err := f.Stat()
-	assert.NoError(err)
-
-	cfg := qemuConfig(QemuVirt)
-	cfg.ImagePath = f.Name()
-	arm64, err := newQemuArch(cfg)
-	assert.NoError(err)
-
-	expectedOut := []govmmQemu.Device{
-		govmmQemu.Object{
-			Driver:   govmmQemu.NVDIMM,
-			Type:     govmmQemu.MemoryBackendFile,
-			DeviceID: "nv0",
-			ID:       "mem0",
-			MemPath:  f.Name(),
-			Size:     (uint64)(imageStat.Size()),
-		},
-	}
-
-	devices, err = arm64.appendNvdimmImage(devices, f.Name())
-	assert.NoError(err)
-	assert.Equal(expectedOut, devices)
-}
-
 func TestQemuArm64WithInitrd(t *testing.T) {
 	assert := assert.New(t)
 

src/runtime/virtcontainers/sandbox_test.go

@@ -50,6 +50,11 @@ func testCreateSandbox(t *testing.T, id string,
 	nconfig NetworkConfig, containers []ContainerConfig,
 	volumes []types.Volume) (*Sandbox, error) {
 
+	// GITHUB_RUNNER_CI_NON_VIRT is set to true in .github/workflows/build-checks.yaml file for ARM64 runners because the self hosted runners do not support Virtualization
+	if os.Getenv("GITHUB_RUNNER_CI_NON_VIRT") == "true" {
+		t.Skip("Skipping the test as the GitHub self hosted runners for ARM64 do not support Virtualization")
+	}
+
 	if tc.NotValid(ktu.NeedRoot()) {
 		t.Skip(testDisabledAsNonRoot)
 	}
@@ -1307,6 +1312,10 @@ func checkSandboxRemains() error {
 }
 
 func TestSandboxCreationFromConfigRollbackFromCreateSandbox(t *testing.T) {
+	// GITHUB_RUNNER_CI_NON_VIRT is set to true in .github/workflows/build-checks.yaml file for ARM64 runners because the self hosted runners do not support Virtualization
+	if os.Getenv("GITHUB_RUNNER_CI_NON_VIRT") == "true" {
+		t.Skip("Skipping the test as the GitHub self hosted runners for ARM64 do not support Virtualization")
+	}
 	defer cleanUp()
 	assert := assert.New(t)
 	ctx := context.Background()
@@ -1398,6 +1407,10 @@ func TestSandboxExperimentalFeature(t *testing.T) {
 }
 
 func TestSandbox_Cgroups(t *testing.T) {
+	// GITHUB_RUNNER_CI_NON_VIRT is set to true in .github/workflows/build-checks.yaml file for ARM64 runners because the self hosted runners do not support Virtualization
+	if os.Getenv("GITHUB_RUNNER_CI_NON_VIRT") == "true" {
+		t.Skip("Skipping the test as the GitHub self hosted runners for ARM64 do not support Virtualization")
+	}
 	sandboxContainer := ContainerConfig{}
 	sandboxContainer.Annotations = make(map[string]string)
 	sandboxContainer.Annotations[annotations.ContainerTypeKey] = string(PodSandbox)

src/tools/agent-ctl/Cargo.lock

@@ -4544,9 +4544,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv"
-version = "0.7.43"
+version = "0.7.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "527a97cdfef66f65998b5f3b637c26f5a5ec09cc52a3f9932313ac645f4190f5"
+checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1"
 dependencies = [
  "bitvec",
  "bytecheck",
@@ -4562,9 +4562,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv_derive"
-version = "0.7.43"
+version = "0.7.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5c462a1328c8e67e4d6dbad1eb0355dd43e8ab432c6e227a43657f16ade5033"
+checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5"
 dependencies = [
  "proc-macro2",
  "quote",

src/tools/csi-kata-directvolume/go.mod

@@ -14,8 +14,8 @@ require (
 	github.com/kubernetes-csi/csi-lib-utils v0.16.0
 	github.com/pborman/uuid v1.2.1
 	github.com/stretchr/testify v1.8.4
-	golang.org/x/net v0.38.0
-	golang.org/x/sys v0.31.0
+	golang.org/x/net v0.50.0
+	golang.org/x/sys v0.41.0
 	google.golang.org/grpc v1.63.2
 	k8s.io/apimachinery v0.28.2
 	k8s.io/klog/v2 v2.110.1
@@ -36,7 +36,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/ulikunitz/xz v0.5.15 // indirect
-	golang.org/x/text v0.23.0 // indirect
+	golang.org/x/text v0.34.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect

src/tools/csi-kata-directvolume/go.sum

@@ -68,6 +68,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
 golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -79,10 +81,14 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
 golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
 golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=

src/tools/csi-kata-directvolume/pkg/directvolume/controllerserver.go

@@ -8,13 +8,13 @@
 package directvolume
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"strings"
 
 	"github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/pborman/uuid"
-	"golang.org/x/net/context"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 

src/tools/csi-kata-directvolume/pkg/directvolume/controllerserver_test.go

@@ -6,13 +6,13 @@
 package directvolume
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"testing"
 
 	csi "github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/stretchr/testify/require"
-	"golang.org/x/net/context"
 
 	"kata-containers/csi-kata-directvolume/pkg/spdkrpc"
 	"kata-containers/csi-kata-directvolume/pkg/utils"

src/tools/csi-kata-directvolume/pkg/directvolume/identityserver.go

@@ -8,8 +8,9 @@
 package directvolume
 
 import (
+	"context"
+
 	"github.com/container-storage-interface/spec/lib/go/csi"
-	"golang.org/x/net/context"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"k8s.io/klog/v2"

src/tools/csi-kata-directvolume/pkg/directvolume/nodeserver.go

@@ -8,6 +8,7 @@
 package directvolume
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -17,7 +18,6 @@ import (
 	"kata-containers/csi-kata-directvolume/pkg/utils"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
-	"golang.org/x/net/context"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"k8s.io/klog/v2"

src/tools/csi-kata-directvolume/pkg/directvolume/nodeserver_test.go

@@ -6,6 +6,7 @@
 package directvolume
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"strings"
@@ -13,7 +14,6 @@ import (
 
 	csi "github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/stretchr/testify/require"
-	"golang.org/x/net/context"
 
 	"kata-containers/csi-kata-directvolume/pkg/spdkrpc"
 	"kata-containers/csi-kata-directvolume/pkg/utils"

src/tools/csi-kata-directvolume/pkg/directvolume/server.go

@@ -8,9 +8,9 @@
 package directvolume
 
 import (
+	"context"
 	"sync"
 
-	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"k8s.io/klog/v2"
 

src/tools/genpolicy/Cargo.lock

@@ -2449,9 +2449,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv"
-version = "0.7.45"
+version = "0.7.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b"
+checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1"
 dependencies = [
  "bitvec",
  "bytecheck",
@@ -2467,9 +2467,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv_derive"
-version = "0.7.45"
+version = "0.7.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0"
+checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5"
 dependencies = [
  "proc-macro2",
  "quote",

src/tools/genpolicy/rules.rego

@@ -51,12 +51,33 @@ default WriteStreamRequest := false
 # them and inspect OPA logs for the root cause of a failure.
 default AllowRequestsFailingPolicy := false
 
-# Constants
+# Constants (containerd keys; CRI-O uses different keys, see *_CRIO below)
 S_NAME_KEY = "io.kubernetes.cri.sandbox-name"
 S_NAMESPACE_KEY = "io.kubernetes.cri.sandbox-namespace"
+S_NAME_KEY_CRIO = "io.kubernetes.cri-o.SandboxName"
+S_NAMESPACE_KEY_CRIO = "io.kubernetes.cri-o.Namespace"
+SANDBOX_ID_KEY = "io.kubernetes.cri.sandbox-id"
+SANDBOX_ID_KEY_CRIO = "io.kubernetes.cri-o.SandboxID"
+C_TYPE_KEY = "io.kubernetes.cri.container-type"
+C_TYPE_KEY_CRIO = "io.kubernetes.cri-o.ContainerType"
+CONTAINER_NAME_KEY = "io.kubernetes.cri.container-name"
+CONTAINER_NAME_KEY_CRIO = "io.kubernetes.cri-o.ContainerName"
+IMAGE_NAME_KEY = "io.kubernetes.cri.image-name"
+IMAGE_NAME_KEY_CRIO = "io.kubernetes.cri-o.ImageName"
+SANDBOX_LOG_DIR_KEY = "io.kubernetes.cri.sandbox-log-directory"
+SANDBOX_LOG_DIR_KEY_CRIO = "io.kubernetes.cri-o.LogPath"
 CDI_VFIO_ANNOTATION_PREFIX = "cdi.k8s.io/vfio"
 VFIO_PCI_ADDRESS_REGEX = "^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[01][0-9a-fA-F]\\.[0-7]=[0-9a-fA-F]{2}/[0-9a-fA-F]{2}$"
 
+# Get annotation value from input OCI: accept either CRI (containerd) or CRI-O key.
+get_input_anno(i_oci, cri_key, crio_key) := v if {
+    v := i_oci.Annotations[cri_key]
+}
+get_input_anno(i_oci, cri_key, crio_key) := v if {
+    not i_oci.Annotations[cri_key]
+    v := i_oci.Annotations[crio_key]
+}
+
 CreateContainerRequest := {"ops": ops, "allowed": true} if {
     # Check if the input request should be rejected even before checking the
     # policy_data.containers information.
@@ -69,8 +90,8 @@ CreateContainerRequest := {"ops": ops, "allowed": true} if {
     # array of possible state operations
     ops_builder := []
 
-    # check sandbox name
-    sandbox_name = i_oci.Annotations[S_NAME_KEY]
+    # check sandbox name (containerd or CRI-O)
+    sandbox_name := get_input_anno(i_oci, S_NAME_KEY, S_NAME_KEY_CRIO)
     add_sandbox_name_to_state := state_allows("sandbox_name", sandbox_name)
     ops_builder1 := concat_op_if_not_null(ops_builder, add_sandbox_name_to_state)
 
@@ -85,9 +106,9 @@ CreateContainerRequest := {"ops": ops, "allowed": true} if {
 
     p_oci := p_container.OCI
 
-    # check namespace
+    # check namespace (containerd or CRI-O)
     p_namespace := p_oci.Annotations[S_NAMESPACE_KEY]
-    i_namespace := i_oci.Annotations[S_NAMESPACE_KEY]
+    i_namespace := get_input_anno(i_oci, S_NAMESPACE_KEY, S_NAMESPACE_KEY_CRIO)
     print("CreateContainerRequest: p_namespace =", p_namespace, "i_namespace =", i_namespace)
     add_namespace_to_state := allow_namespace(p_namespace, i_namespace)
     ops_builder2 := concat_op_if_not_null(ops_builder1, add_namespace_to_state)
@@ -249,9 +270,13 @@ allow_anno_key_value(i_key, i_value, p_container) if {
     print("allow_anno_key_value 1: i key =", i_key)
 
     startswith(i_key, "io.kubernetes.cri.")
-
     print("allow_anno_key_value 1: true")
 }
+allow_anno_key_value(i_key, i_value, p_container) if {
+    print("allow_anno_key_value 1b: i key =", i_key)
+    startswith(i_key, "io.kubernetes.cri-o.")
+    print("allow_anno_key_value 1b: true")
+}
 allow_anno_key_value(i_key, i_value, p_container) if {
     print("allow_anno_key_value 2: i key =", i_key)
 
@@ -272,17 +297,17 @@ allow_anno_key_value(i_key, i_value, p_container) if {
     print("allow_anno_key_value 3: true")
 }
 
-# Get the value of the S_NAME_KEY annotation and
-# correlate it with other annotations and process fields.
+# Get the value of the sandbox name/namespace annotations (containerd or CRI-O) and
+# correlate with other annotations and process fields.
 allow_by_anno(p_oci, i_oci, p_storages, i_storages) if {
     print("allow_by_anno 1: start")
 
     not p_oci.Annotations[S_NAME_KEY]
 
-    i_s_name := i_oci.Annotations[S_NAME_KEY]
+    i_s_name := get_input_anno(i_oci, S_NAME_KEY, S_NAME_KEY_CRIO)
     print("allow_by_anno 1: i_s_name =", i_s_name)
 
-    i_s_namespace := i_oci.Annotations[S_NAMESPACE_KEY]
+    i_s_namespace := get_input_anno(i_oci, S_NAMESPACE_KEY, S_NAMESPACE_KEY_CRIO)
     print("allow_by_anno 1: i_s_namespace =", i_s_namespace)
 
     allow_by_sandbox_name(p_oci, i_oci, p_storages, i_storages, i_s_name, i_s_namespace)
@@ -293,12 +318,12 @@ allow_by_anno(p_oci, i_oci, p_storages, i_storages) if {
     print("allow_by_anno 2: start")
 
     p_s_name := p_oci.Annotations[S_NAME_KEY]
-    i_s_name := i_oci.Annotations[S_NAME_KEY]
+    i_s_name := get_input_anno(i_oci, S_NAME_KEY, S_NAME_KEY_CRIO)
     print("allow_by_anno 2: i_s_name =", i_s_name, "p_s_name =", p_s_name)
 
     allow_sandbox_name(p_s_name, i_s_name)
 
-    i_s_namespace := i_oci.Annotations[S_NAMESPACE_KEY]
+    i_s_namespace := get_input_anno(i_oci, S_NAMESPACE_KEY, S_NAMESPACE_KEY_CRIO)
     print("allow_by_anno 2: i_s_namespace =", i_s_namespace)
 
     allow_by_sandbox_name(p_oci, i_oci, p_storages, i_storages, i_s_name, i_s_namespace)
@@ -309,7 +334,7 @@ allow_by_anno(p_oci, i_oci, p_storages, i_storages) if {
 allow_by_sandbox_name(p_oci, i_oci, p_storages, i_storages, s_name, s_namespace) if {
     print("allow_by_sandbox_name: start")
 
-    i_namespace := i_oci.Annotations[S_NAMESPACE_KEY]
+    i_namespace := get_input_anno(i_oci, S_NAMESPACE_KEY, S_NAMESPACE_KEY_CRIO)
 
     allow_by_container_types(p_oci, i_oci, s_name, i_namespace)
     allow_by_bundle_or_sandbox_id(p_oci, i_oci, p_storages, i_storages)
@@ -325,18 +350,14 @@ allow_sandbox_name(p_s_name, i_s_name) if {
     print("allow_sandbox_name: true")
 }
 
-# Check that the "io.kubernetes.cri.container-type" and
-# "io.katacontainers.pkg.oci.container_type" annotations designate the
-# expected type - either a "sandbox" or a "container". Then, validate
-# other annotations based on the actual "sandbox" or "container" value
-# from the input container.
+# Check that the container-type annotation (containerd or CRI-O) and
+# "io.katacontainers.pkg.oci.container_type" designate the expected type -
+# either "sandbox" or "container". Then validate other annotations accordingly.
 allow_by_container_types(p_oci, i_oci, s_name, s_namespace) if {
-    print("allow_by_container_types: checking io.kubernetes.cri.container-type")
+    print("allow_by_container_types: checking container-type")
 
-    c_type := "io.kubernetes.cri.container-type"
-
-    p_cri_type := p_oci.Annotations[c_type]
-    i_cri_type := i_oci.Annotations[c_type]
+    p_cri_type := p_oci.Annotations[C_TYPE_KEY]
+    i_cri_type := get_input_anno(i_oci, C_TYPE_KEY, C_TYPE_KEY_CRIO)
     print("allow_by_container_types: p_cri_type =", p_cri_type, "i_cri_type =", i_cri_type)
     p_cri_type == i_cri_type
 
@@ -375,44 +396,54 @@ allow_by_container_type(i_cri_type, p_oci, i_oci, s_name, s_namespace) if {
     print("allow_by_container_type 2: true")
 }
 
-# "io.kubernetes.cri.container-name" annotation
+# Container name: sandbox has none; container must match (containerd or CRI-O key).
 allow_sandbox_container_name(p_oci, i_oci) if {
     print("allow_sandbox_container_name: start")
-
-    container_annotation_missing(p_oci, i_oci, "io.kubernetes.cri.container-name")
-
+    container_annotation_missing_cri_crio(p_oci, i_oci, CONTAINER_NAME_KEY, CONTAINER_NAME_KEY_CRIO)
     print("allow_sandbox_container_name: true")
 }
 
 allow_container_name(p_oci, i_oci) if {
     print("allow_container_name: start")
-
-    allow_container_annotation(p_oci, i_oci, "io.kubernetes.cri.container-name")
-
+    allow_container_annotation_cri_crio(p_oci, i_oci, CONTAINER_NAME_KEY, CONTAINER_NAME_KEY_CRIO)
     print("allow_container_name: true")
 }
 
 container_annotation_missing(p_oci, i_oci, key) if {
     print("container_annotation_missing:", key)
-
     not p_oci.Annotations[key]
     not i_oci.Annotations[key]
-
     print("container_annotation_missing: true")
 }
 
+# Both policy and input lack the annotation (input checked for both CRI and CRI-O keys).
+container_annotation_missing_cri_crio(p_oci, i_oci, cri_key, crio_key) if {
+    print("container_annotation_missing_cri_crio:", cri_key)
+    not p_oci.Annotations[cri_key]
+    not i_oci.Annotations[cri_key]
+    not i_oci.Annotations[crio_key]
+    print("container_annotation_missing_cri_crio: true")
+}
+
 allow_container_annotation(p_oci, i_oci, key) if {
     print("allow_container_annotation: key =", key)
-
     p_value := p_oci.Annotations[key]
     i_value := i_oci.Annotations[key]
     print("allow_container_annotation: p_value =", p_value, "i_value =", i_value)
-
     p_value == i_value
-
     print("allow_container_annotation: true")
 }
 
+# Policy uses CRI key; input may have CRI or CRI-O key.
+allow_container_annotation_cri_crio(p_oci, i_oci, cri_key, crio_key) if {
+    print("allow_container_annotation_cri_crio: cri_key =", cri_key)
+    p_value := p_oci.Annotations[cri_key]
+    i_value := get_input_anno(i_oci, cri_key, crio_key)
+    print("allow_container_annotation_cri_crio: p_value =", p_value, "i_value =", i_value)
+    p_value == i_value
+    print("allow_container_annotation_cri_crio: true")
+}
+
 # "nerdctl/network-namespace" annotation
 allow_sandbox_net_namespace(p_oci, i_oci) if {
     print("allow_sandbox_net_namespace: start")
@@ -439,18 +470,16 @@ allow_net_namespace(p_oci, i_oci) if {
     print("allow_net_namespace: true")
 }
 
-# "io.kubernetes.cri.sandbox-log-directory" annotation
+# Sandbox log directory (containerd or CRI-O: cri-o uses LogPath)
 allow_sandbox_log_directory(p_oci, i_oci, s_name, s_namespace) if {
     print("allow_sandbox_log_directory: start")
 
-    key := "io.kubernetes.cri.sandbox-log-directory"
-
-    p_dir := p_oci.Annotations[key]
+    p_dir := p_oci.Annotations[SANDBOX_LOG_DIR_KEY]
     regex1 := replace(p_dir, "$(sandbox-name)", s_name)
     regex2 := replace(regex1, "$(sandbox-namespace)", s_namespace)
     print("allow_sandbox_log_directory: regex2 =", regex2)
 
-    i_dir := i_oci.Annotations[key]
+    i_dir := get_input_anno(i_oci, SANDBOX_LOG_DIR_KEY, SANDBOX_LOG_DIR_KEY_CRIO)
     print("allow_sandbox_log_directory: i_dir =", i_dir)
 
     regex.match(regex2, i_dir)
@@ -460,12 +489,9 @@ allow_sandbox_log_directory(p_oci, i_oci, s_name, s_namespace) if {
 
 allow_log_directory(p_oci, i_oci) if {
     print("allow_log_directory: start")
-
-    key := "io.kubernetes.cri.sandbox-log-directory"
-
-    not p_oci.Annotations[key]
-    not i_oci.Annotations[key]
-
+    not p_oci.Annotations[SANDBOX_LOG_DIR_KEY]
+    not i_oci.Annotations[SANDBOX_LOG_DIR_KEY]
+    not i_oci.Annotations[SANDBOX_LOG_DIR_KEY_CRIO]
     print("allow_log_directory: true")
 }
 
@@ -776,22 +802,25 @@ allow_linux_sysctl(p_linux, i_linux) if {
     print("allow_linux_sysctl 2: true")
 }
 
-# Check the consistency of the input "io.katacontainers.pkg.oci.bundle_path"
-# and io.kubernetes.cri.sandbox-id" values with other fields.
+# Check sandbox_id and derive bundle_id from guest root path (CRI-agnostic: works for containerd and CRI-O).
+# Bundle path on the host is runtime-specific; root path in the guest is stable, so we extract bundle_id from it.
 allow_by_bundle_or_sandbox_id(p_oci, i_oci, p_storages, i_storages) if {
     print("allow_by_bundle_or_sandbox_id: start")
 
-    bundle_path := i_oci.Annotations["io.katacontainers.pkg.oci.bundle_path"]
-    bundle_id := replace(bundle_path, "/run/containerd/io.containerd.runtime.v2.task/k8s.io/", "")
-
-    key := "io.kubernetes.cri.sandbox-id"
-
-    p_regex := p_oci.Annotations[key]
-    sandbox_id := i_oci.Annotations[key]
-
+    p_regex := p_oci.Annotations[SANDBOX_ID_KEY]
+    sandbox_id := get_input_anno(i_oci, SANDBOX_ID_KEY, SANDBOX_ID_KEY_CRIO)
     print("allow_by_bundle_or_sandbox_id: sandbox_id =", sandbox_id, "regex =", p_regex)
     regex.match(p_regex, sandbox_id)
 
+    # Derive bundle_id from guest root path (e.g. /run/kata-containers/<bundle_id>/rootfs).
+    # Match 64-char hex (real runtimes) or any single path segment (e.g. test data: bundle-id, gpu-container, dummy).
+    i_root := i_oci.Root.Path
+    p_root_pattern1 := p_oci.Root.Path
+    p_root_pattern2 := replace(p_root_pattern1, "$(root_path)", policy_data.common.root_path)
+    p_root_pattern3 := replace(p_root_pattern2, "$(bundle-id)", "([0-9a-f]{64}|[^/]+)")
+    print("allow_by_bundle_or_sandbox_id: i_root =", i_root, "regex =", p_root_pattern3)
+    bundle_id := regex.find_all_string_submatch_n(p_root_pattern3, i_root, 1)[0][1]
+
     allow_root_path(p_oci, i_oci, bundle_id)
 
     # Match each input mount with a Policy mount.
@@ -1133,6 +1162,20 @@ check_mount(p_mount, i_mount, bundle_id, sandbox_id) if {
 
     print("check_mount 2: true")
 }
+check_mount(p_mount, i_mount, bundle_id, sandbox_id) if {
+    # This check passes if the policy container has RW, the input container has
+    # RO and the volume type is sysfs, working around different handling of
+    # privileged containers after containerd 2.0.4.
+    i_mount.type_ == "sysfs"
+    p_mount.type_ == i_mount.type_
+    p_mount.destination == i_mount.destination
+    p_mount.source == i_mount.source
+
+    i_options := {x | x = i_mount.options[_]} | {"rw"}
+    p_options := {x | x = p_mount.options[_]} | {"ro"}
+    p_options == i_options
+    print("check_mount 3: true")
+}
 
 mount_source_allows(p_mount, i_mount, bundle_id, sandbox_id) if {
     regex1 := p_mount.source

src/tools/genpolicy/src/policy.rs

@@ -675,6 +675,9 @@ impl AgentPolicy {
         );
 
         let is_privileged = yaml_container.is_privileged();
+        let needs_privileged_mounts = is_privileged
+            || (is_pause_container && resource.get_containers().iter().any(|c| c.is_privileged()));
+
         let process = self.get_container_process(
             resource,
             yaml_container,
@@ -684,7 +687,7 @@ impl AgentPolicy {
             is_privileged,
         );
 
-        let mut mounts = containerd::get_mounts(is_pause_container, is_privileged);
+        let mut mounts = containerd::get_mounts(is_pause_container, needs_privileged_mounts);
         mount_and_storage::get_policy_mounts(
             &self.config.settings,
             &mut mounts,

src/tools/kata-ctl/Cargo.lock

@@ -3349,9 +3349,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv"
-version = "0.7.44"
+version = "0.7.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0"
+checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1"
 dependencies = [
  "bitvec",
  "bytecheck",
@@ -3367,9 +3367,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv_derive"
-version = "0.7.44"
+version = "0.7.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65"
+checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5"
 dependencies = [
  "proc-macro2",
  "quote",

tests/cmd/check-spelling/data/acronyms.txt

@@ -52,7 +52,10 @@ mem/B	# For terms like "virtio-mem"
 memdisk/B
 MDEV/AB
 NEMU/AB
+NFD/AB		# Node Feature Discovery
 NIC/AB
+nodeSelector/B	# Kubernetes RuntimeClass scheduling field
+nodeSelectors/B
 nv/AB		# NVIDIA abbreviation (lowercase)
 NVDIMM/AB
 OCI/AB
@@ -74,15 +77,20 @@ QEMU/AB
 RBAC/AB
 RDMA/AB
 RNG/AB
+RuntimeClass/B	# Kubernetes resource (node.k8s.io)
+RuntimeClasses/B
 SaaS/B  # Software as a Service
 SCSI/AB
 SDK/AB
 seccomp # secure computing mode
 SHA/AB
+SEL/AB		# IBM Secure Execution for Linux
 SPDX/AB
 SRIOV/AB
+SEV-SNP/B	# AMD Secure Encrypted Virtualization - Secure Nested Paging
 SVG/AB
 TBD/AB
+TEE/AB		# Trusted Execution Environment
 TOC/AB
 TOML/AB
 TTY/AB

tests/cmd/check-spelling/kata-dictionary.dic

@@ -1,4 +1,4 @@
-409
+417
 ACPI/AB
 ACS/AB
 API/AB
@@ -93,6 +93,7 @@ Mellanox/B
 Minikube/B
 MonitorTest/A
 NEMU/AB
+NFD/AB
 NIC/AB
 NVDIMM/AB
 NVIDIA/A
@@ -134,10 +135,14 @@ RBAC/AB
 RDMA/AB
 RHEL/B
 RNG/AB
+RuntimeClass/B
+RuntimeClasses/B
 Rustlang/B
 SCSI/AB
 SDK/AB
+SEL/AB
 SELinux/B
+SEV-SNP/B
 SHA/AB
 SLES/B
 SPDX/AB
@@ -153,6 +158,7 @@ Submodule/A
 Sysbench/B
 TBD/AB
 TDX
+TEE/AB
 TOC/AB
 TOML/AB
 TTY/AB
@@ -306,6 +312,8 @@ nack/AB
 namespace/ABCD
 netlink
 netns/AB
+nodeSelector/B
+nodeSelectors/B
 nv/AB
 nvidia/A
 onwards

tests/common.bash

@@ -810,16 +810,17 @@ function install_nydus_snapshotter() {
 	rm -f "${tarball_name}"
 }
 
-# version: the CRI-O version to be installe
+# version: the CRI-O version to be installed (major.minor, e.g. 1.35)
+# Repo: https://github.com/cri-o/packaging (OpenSUSE Build Service, not pkgs.k8s.io)
 function install_crio() {
 	local version=${1}
 
 	sudo mkdir -p /etc/apt/keyrings
 	sudo mkdir -p /etc/apt/sources.list.d
 
-	curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/v${version}/deb/Release.key | \
+	curl -fsSL https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v${version}/deb/Release.key | \
 		sudo gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg
-	echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v${version}/deb/ /" | \
+	echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v${version}/deb/ /" | \
 		sudo tee /etc/apt/sources.list.d/cri-o.list
 
 	sudo apt update

tests/gha-run-k8s-common.sh

@@ -95,6 +95,7 @@ function create_cluster() {
 	local short_sha
 	local tags
 	local rg
+	local aks_create
 
 	# First ensure it didn't fail to get cleaned up from a previous run.
 	delete_cluster "${test_type}" || true
@@ -117,19 +118,16 @@ function create_cluster() {
 	# Required by e.g. AKS App Routing for KBS installation.
 	az extension add --name aks-preview
 
-	# Adding a double quote on the last line ends up causing issues
-	# ine the cbl-mariner installation.  Because of that, let's just
-	# disable the warning for this specific case.
-	# shellcheck disable=SC2046
-	az aks create \
-		-g "${rg}" \
-		--node-resource-group "node-${rg}" \
-		-n "$(_print_cluster_name "${test_type}")" \
-		-s "$(_print_instance_type)" \
-		--node-count 1 \
-		--generate-ssh-keys \
-		--tags "${tags[@]}" \
-		$([[ "${KATA_HOST_OS}" = "cbl-mariner" ]] && echo "--os-sku AzureLinux --workload-runtime KataVmIsolation")
+	# Create the cluster.
+	aks_create=(az aks create
+		-g "${rg}"
+		--node-resource-group "node-${rg}"
+		-n "$(_print_cluster_name "${test_type}")"
+		-s "$(_print_instance_type)"
+		--node-count 1
+		--generate-ssh-keys
+		--tags "${tags[@]}")
+	"${aks_create[@]}"
 }
 
 function install_bats() {
@@ -397,8 +395,33 @@ EOF
    	sudo apt-get -y install kubeadm kubelet kubectl --allow-downgrades
    	sudo apt-mark hold kubeadm kubelet kubectl
 
-	# Deploy k8s using kubeadm
-   	sudo kubeadm init --pod-network-cidr=10.244.0.0/16
+	# Deploy k8s using kubeadm with CreateContainerRequest (CRI) timeout set to 600s,
+	# mainly for CoCo (Confidential Containers) tests (attestation, policy, image pull, VM start).
+	local cri_socket
+	case "${CONTAINER_ENGINE:-containerd}" in
+		crio) cri_socket="/var/run/crio/crio.sock" ;;
+		containerd) cri_socket="/run/containerd/containerd.sock" ;;
+		*) cri_socket="/run/containerd/containerd.sock" ;;
+	esac
+	local kubeadm_config
+	kubeadm_config="$(mktemp --tmpdir kubeadm-config.XXXXXX.yaml)"
+	cat <<EOF | tee "${kubeadm_config}"
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: InitConfiguration
+nodeRegistration:
+  criSocket: "${cri_socket}"
+---
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: ClusterConfiguration
+networking:
+  podSubnet: "10.244.0.0/16"
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+runtimeRequestTimeout: "600s"
+EOF
+	sudo kubeadm init --config "${kubeadm_config}"
+	rm -f "${kubeadm_config}"
 	mkdir -p $HOME/.kube
 	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
 	sudo chown $(id -u):$(id -g) $HOME/.kube/config
@@ -410,8 +433,29 @@ EOF
 	kubectl taint nodes --all node-role.kubernetes.io/control-plane-
 }
 
-# container_engine: containerd (only containerd is supported for now, support for crio is welcome)
-# container_engine_version: major.minor (and then we'll install the latest patch release matching that major.minor)
+# Try to install CRI-O for the given k8s-matching version (major.minor); if the repo/package
+# is not available yet (k8s released before CRI-O), try previous minor (x.y-1).
+function try_install_crio_for_k8s() {
+	local version="${1}"
+	local major minor
+	major="${version%%.*}"
+	minor="${version##*.}"
+
+	if install_crio "${version}"; then
+		return 0
+	fi
+	if [[ "${minor}" -gt 0 ]]; then
+		minor=$((minor - 1))
+		echo "CRI-O v${version} not available yet, trying v${major}.${minor}"
+		install_crio "${major}.${minor}"
+	else
+		echo "CRI-O v${version} failed and no fallback (minor would be < 0)" >&2
+		return 1
+	fi
+}
+
+# container_engine: containerd or crio
+# container_engine_version: for containerd: major.minor or lts/active; for crio: major.minor (e.g. 1.31) or active
 function deploy_vanilla_k8s() {
 	container_engine="${1}"
 	container_engine_version="${2}"
@@ -419,6 +463,22 @@ function deploy_vanilla_k8s() {
 	[[ -z "${container_engine}" ]] && die "container_engine is required"
 	[[ -z "${container_engine_version}" ]] && die "container_engine_version is required"
 
+	# Export so do_deploy_k8s can pick the right CRI socket
+	export CONTAINER_ENGINE="${container_engine}"
+
+	# Resolve lts/active to the actual version from versions.yaml (e.g. v1.7, v2.1)
+		case "${container_engine_version}" in
+		lts|active)
+			if [[ "${container_engine}" == "containerd" ]]; then
+				container_engine_version=$(get_from_kata_deps ".externals.containerd.${container_engine_version}")
+			else
+				# CRI-O version matches k8s: use latest k8s stable major.minor (e.g. 1.31)
+				container_engine_version=$(curl -Ls https://dl.k8s.io/release/stable.txt | sed -e 's/^v//' | cut -d. -f-2)
+			fi
+			;;
+		*) ;;
+	esac
+
 	install_system_dependencies "runc"
 	load_k8s_needed_modules
 	set_k8s_network_parameters
@@ -429,10 +489,43 @@ function deploy_vanilla_k8s() {
 			sudo mkdir -p /etc/containerd
 			containerd config default | sed -e 's/SystemdCgroup = false/SystemdCgroup = true/' | sudo tee /etc/containerd/config.toml
 			;;
+		crio)
+			# CRI-O version is major.minor (e.g. 1.31) for download.opensuse.org/isv:cri-o:stable
+			# If k8s was released before CRI-O, try previous minor (x.y-1)
+			try_install_crio_for_k8s "${container_engine_version}"
+			;;
 		*) die "${container_engine} is not a container engine supported by this script" ;;
 	esac
 	sudo systemctl daemon-reload && sudo systemctl restart "${container_engine}"
-	do_deploy_k8s
+	local max_retries=5 retry_wait_sec=180 attempt=1
+	while true; do
+		local errexit_set=0
+		local deploy_status
+
+		# Detect if errexit (-e) is currently set and temporarily disable it
+		if [[ $- == *e* ]]; then
+			errexit_set=1
+			set +e
+		fi
+
+		do_deploy_k8s
+		deploy_status=$?
+
+		# Restore errexit state if it was previously enabled
+		if [[ ${errexit_set} -eq 1 ]]; then
+			set -e
+		fi
+		if [[ ${deploy_status} -eq 0 ]]; then
+			return 0
+		fi
+		if [[ ${attempt} -ge ${max_retries} ]]; then
+			>&2 echo "do_deploy_k8s failed after ${max_retries} attempts"
+			return 1
+		fi
+		>&2 echo "do_deploy_k8s attempt ${attempt}/${max_retries} failed, waiting ${retry_wait_sec}s before retry"
+		sleep "${retry_wait_sec}"
+		attempt=$((attempt + 1))
+	done
 }
 
 function deploy_k8s() {

tests/integration/kubernetes/gha-run.sh

@@ -36,6 +36,7 @@ export PULL_TYPE="${PULL_TYPE:-default}"
 export TEST_CLUSTER_NAMESPACE="${TEST_CLUSTER_NAMESPACE:-kata-containers-k8s-tests}"
 export GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}"
 export TARGET_ARCH="${TARGET_ARCH:-x86_64}"
+export RUNS_ON_AKS="${RUNS_ON_AKS:-false}"
 
 function configure_devmapper() {
 	sudo mkdir -p /var/lib/containerd/devmapper
@@ -175,7 +176,7 @@ function deploy_kata() {
 
 	ANNOTATIONS="default_vcpus"
 	if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
-		ANNOTATIONS="image kernel default_vcpus disable_image_nvdimm cc_init_data"
+		ANNOTATIONS="image kernel default_vcpus cc_init_data"
 	fi
 	if [[ "${KATA_HYPERVISOR}" = "qemu" ]]; then
 		ANNOTATIONS="image initrd kernel default_vcpus"
@@ -555,18 +556,22 @@ function main() {
 	export KATA_HOST_OS="${KATA_HOST_OS:-}"
 	export K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-}"
 
-	AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-}"
+	if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
+		# Temporary workaround for missing cloud-hypervisor/cloud-hypervisor@bf6f0f8, the fix for a bug
+		# exposed by the large ttrpc replies intentionally produced by the Kata CI Policy tests.
+		AUTO_GENERATE_POLICY="no"
+	else
+		AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-}"
 
-	# Auto-generate policy on some Host types, if the caller didn't specify an AUTO_GENERATE_POLICY value.
-	if [[ -z "${AUTO_GENERATE_POLICY}" ]]; then
-		if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
-			AUTO_GENERATE_POLICY="yes"
-		elif [[ "${KATA_HYPERVISOR}" = qemu-coco-dev* && \
-		        "${TARGET_ARCH}" = "x86_64" && \
-		        "${PULL_TYPE}" != "experimental-force-guest-pull" ]]; then
-			AUTO_GENERATE_POLICY="yes"
-		elif [[ "${KATA_HYPERVISOR}" = qemu-nvidia-gpu-* ]]; then
-			AUTO_GENERATE_POLICY="yes"
+		# Auto-generate policy on some Host types, if the caller didn't specify an AUTO_GENERATE_POLICY value.
+		if [[ -z "${AUTO_GENERATE_POLICY}" ]]; then
+			if [[ "${KATA_HYPERVISOR}" = qemu-coco-dev* && \
+					"${TARGET_ARCH}" = "x86_64" && \
+					"${PULL_TYPE}" != "experimental-force-guest-pull" ]]; then
+				AUTO_GENERATE_POLICY="yes"
+			elif [[ "${KATA_HYPERVISOR}" = qemu-nvidia-gpu-* ]]; then
+				AUTO_GENERATE_POLICY="yes"
+			fi
 		fi
 	fi
 

tests/integration/kubernetes/k8s-block-volume.bats

@@ -10,8 +10,6 @@ load "${BATS_TEST_DIRNAME}/../../common.bash"
 load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 setup() {
-	[ "$(uname -m)" == "s390x" ] && [ "${KATA_HYPERVISOR}" == "qemu-runtime-rs" ] && skip "See: https://github.com/kata-containers/kata-containers/pull/12105#issuecomment-3551916090"
-	[ "${KATA_HYPERVISOR}" == "qemu-se-runtime-rs" ]  && skip "See: https://github.com/kata-containers/kata-containers/pull/12105#issuecomment-3551916090"
 	( [ "${KATA_HYPERVISOR}" == "fc" ] || [ "${KATA_HYPERVISOR}" == "stratovirt" ] ) && skip "See: https://github.com/kata-containers/kata-containers/issues/10873"
 
 	setup_common || die "setup_common failed"
@@ -93,8 +91,6 @@ setup() {
 }
 
 teardown() {
-	[ "$(uname -m)" == "s390x" ] && [ "${KATA_HYPERVISOR}" == "qemu-runtime-rs" ] && skip "See: https://github.com/kata-containers/kata-containers/pull/12105#issuecomment-3551916090"
-	[ "${KATA_HYPERVISOR}" == "qemu-se-runtime-rs" ]  && skip "See: https://github.com/kata-containers/kata-containers/pull/12105#issuecomment-3551916090"
 	( [ "${KATA_HYPERVISOR}" == "fc" ] || [ "${KATA_HYPERVISOR}" == "stratovirt" ] ) && skip "See: https://github.com/kata-containers/kata-containers/issues/10873"
 
 	 # Debugging information

tests/integration/kubernetes/k8s-configmap.bats

@@ -10,47 +10,88 @@ load "${BATS_TEST_DIRNAME}/../../common.bash"
 load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 setup() {
+	config_name="test-configmap"
+	pod_env_name="config-env-test-pod"
+	pod_volume_name="configmap-volume-test-pod"
+
 	setup_common || die "setup_common failed"
 	policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
 
 	cmd="env"
 	exec_command=(sh -c "${cmd}")
 	add_exec_to_policy_settings "${policy_settings_dir}" "${exec_command[@]}"
+
+	# Add policy for volume mount test
+	check_config_cmd="cat /etc/config/data-1"
+	check_config_exec_command=(sh -c "${check_config_cmd}")
+	add_exec_to_policy_settings "${policy_settings_dir}" "${check_config_exec_command[@]}"
+
 	add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
 
 	configmap_yaml_file="${pod_config_dir}/configmap.yaml"
 	pod_yaml_file="${pod_config_dir}/pod-configmap.yaml"
+	pod_volume_yaml_file="${pod_config_dir}/pod-configmap-volume.yaml"
 
 	auto_generate_policy "${policy_settings_dir}" "${pod_yaml_file}" "${configmap_yaml_file}"
+	auto_generate_policy "${policy_settings_dir}" "${pod_volume_yaml_file}" "${configmap_yaml_file}"
 }
 
 @test "ConfigMap for a pod" {
-	config_name="test-configmap"
-	pod_name="config-env-test-pod"
-
 	# Create ConfigMap
 	kubectl create -f "${configmap_yaml_file}"
 
 	# View the values of the keys
-	kubectl get configmaps $config_name -o yaml | grep -q "data-"
+	kubectl get configmaps "${config_name}" -o yaml | grep -q "data-"
 
 	# Create a pod that consumes the ConfigMap
 	kubectl create -f "${pod_yaml_file}"
 
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_env_name}"
 
 	# Check env
-	grep_pod_exec_output "${pod_name}" "KUBE_CONFIG_1=value-1" "${exec_command[@]}"
-	grep_pod_exec_output "${pod_name}" "KUBE_CONFIG_2=value-2" "${exec_command[@]}"
+	grep_pod_exec_output "${pod_env_name}" "KUBE_CONFIG_1=value-1" "${exec_command[@]}"
+	grep_pod_exec_output "${pod_env_name}" "KUBE_CONFIG_2=value-2" "${exec_command[@]}"
+}
+
+@test "ConfigMap propagation to volume-mounted pod" {
+	original_value="value-1"
+	updated_value="updated-value-1"
+
+	# Create ConfigMap
+	kubectl create -f "${configmap_yaml_file}"
+
+	# Create a pod that consumes the ConfigMap via volume mount
+	kubectl create -f "${pod_volume_yaml_file}"
+	kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_volume_name}"
+
+	# Verify initial value from volume
+	grep_pod_exec_output "${pod_volume_name}" "${original_value}" "${check_config_exec_command[@]}"
+
+	# Update ConfigMap to test propagation
+	kubectl patch configmap "${config_name}" -p "{\"data\":{\"data-1\":\"${updated_value}\"}}"
+
+	# Wait for propagation (kubelet sync period ~60s, but allow extra time for slow clusters)
+	info "Waiting for ConfigMap propagation to volume-mounted pod"
+	propagation_wait_time=180
+
+	# Define check function for waitForProcess
+	check_configmap_propagated() {
+		pod_exec "${pod_volume_name}" "${check_config_exec_command[@]}" | grep -q "${updated_value}"
+	}
+
+	if waitForProcess "${propagation_wait_time}" "${sleep_time}" check_configmap_propagated; then
+		info "ConfigMap successfully propagated to volume"
+	else
+		info "ConfigMap propagation test failed after ${propagation_wait_time} seconds"
+		return 1
+	fi
 }
 
 teardown() {
-	# Debugging information
-	kubectl describe "pod/$pod_name"
-
-	kubectl delete pod "$pod_name"
-	kubectl delete configmap "$config_name"
+	kubectl delete pod "${pod_env_name}" --ignore-not-found=true
+	kubectl delete pod "${pod_volume_name}" --ignore-not-found=true
+	kubectl delete configmap "${config_name}" --ignore-not-found=true
 
 	delete_tmp_policy_settings_dir "${policy_settings_dir}"
 	teardown_common "${node}" "${node_start_time:-}"

tests/integration/kubernetes/k8s-credentials-secrets.bats

@@ -13,21 +13,31 @@ setup() {
 	[ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}"
 	[ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}"
 
+	secret_name="test-secret"
+	pod_name="secret-test-pod"
+	second_pod_name="secret-envars-test-pod"
+
 	setup_common || die "setup_common failed"
 
 	# Add policy to pod-secret.yaml.
 	pod_yaml_file="${pod_config_dir}/pod-secret.yaml"
-	set_node "$pod_yaml_file" "$node"
+	set_node "${pod_yaml_file}" "${node}"
 	pod_cmd="ls /tmp/secret-volume"
 	pod_exec_command=(sh -c "${pod_cmd}")
+
+	# Also add policy for reading secret content (for propagation test)
+	check_secret_cmd="cat /tmp/secret-volume/username"
+	check_secret_exec_command=(sh -c "${check_secret_cmd}")
+
 	pod_policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
 	add_exec_to_policy_settings "${pod_policy_settings_dir}" "${pod_exec_command[@]}"
+	add_exec_to_policy_settings "${pod_policy_settings_dir}" "${check_secret_exec_command[@]}"
 	add_requests_to_policy_settings "${pod_policy_settings_dir}" "ReadStreamRequest"
-	auto_generate_policy "${pod_policy_settings_dir}" "${pod_yaml_file}"
+	auto_generate_policy "${pod_policy_settings_dir}" "${pod_yaml_file}" "${pod_config_dir}/inject_secret.yaml"
 
 	# Add policy to pod-secret-env.yaml.
 	pod_env_yaml_file="${pod_config_dir}/pod-secret-env.yaml"
-	set_node "$pod_env_yaml_file" "$node"
+	set_node "${pod_env_yaml_file}" "${node}"
 	pod_env_cmd="printenv"
 	pod_env_exec_command=(sh -c "${pod_env_cmd}")
 	pod_env_policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
@@ -37,10 +47,6 @@ setup() {
 }
 
 @test "Credentials using secrets" {
-	secret_name="test-secret"
-	pod_name="secret-test-pod"
-	second_pod_name="secret-envars-test-pod"
-
 	# Create the secret
 	kubectl create -f "${pod_config_dir}/inject_secret.yaml"
 
@@ -51,7 +57,7 @@ setup() {
 	kubectl create -f "${pod_yaml_file}"
 
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}"
 
 	# List the files
 	pod_exec "${pod_name}" "${pod_exec_command[@]}" | grep -w "password"
@@ -61,18 +67,54 @@ setup() {
 	kubectl create -f "${pod_env_yaml_file}"
 
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$second_pod_name"
+	kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${second_pod_name}"
 
 	# Display environment variables
 	pod_exec "${second_pod_name}" "${pod_env_exec_command[@]}" | grep -w "SECRET_USERNAME"
 	pod_exec "${second_pod_name}" "${pod_env_exec_command[@]}" | grep -w "SECRET_PASSWORD"
 }
 
+@test "Secret propagation to volume-mounted pod" {
+	original_username="my-app"
+	updated_username="updated-username"
+
+	# Create the secret
+	kubectl create -f "${pod_config_dir}/inject_secret.yaml"
+
+	# Create a pod that has access to the secret through a volume
+	kubectl create -f "${pod_yaml_file}"
+	kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}"
+
+	# Verify initial secret value
+	grep_pod_exec_output "${pod_name}" "${original_username}" "${check_secret_exec_command[@]}"
+
+	# Update Secret to test propagation
+	kubectl patch secret "${secret_name}" -p "{\"stringData\":{\"username\":\"${updated_username}\"}}"
+
+	# Wait for propagation (kubelet sync period ~60s, but allow extra time for slow clusters)
+	info "Waiting for Secret propagation to volume-mounted pod"
+	propagation_wait_time=180
+
+	# Define check function for waitForProcess
+	check_secret_propagated() {
+		pod_exec "${pod_name}" "${check_secret_exec_command[@]}" | grep -q "${updated_username}"
+	}
+
+	if waitForProcess "${propagation_wait_time}" "${sleep_time}" check_secret_propagated; then
+		info "Secret successfully propagated to volume"
+	else
+		info "Secret propagation test failed after ${propagation_wait_time} seconds"
+		return 1
+	fi
+}
+
 teardown() {
 	[ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}"
 	[ "${KATA_HYPERVISOR}" == "fc" ] && skip "test not working see: ${fc_limitations}"
 
-	kubectl delete secret "$secret_name"
+	kubectl delete pod "${pod_name}" --ignore-not-found=true
+	kubectl delete pod "${second_pod_name}" --ignore-not-found=true
+	kubectl delete secret "${secret_name}" --ignore-not-found=true
 
 	delete_tmp_policy_settings_dir "${pod_policy_settings_dir}"
 	delete_tmp_policy_settings_dir "${pod_env_policy_settings_dir}"

tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats

@@ -10,14 +10,15 @@ load "${BATS_TEST_DIRNAME}/confidential_common.sh"
 export KBS="${KBS:-false}"
 export SNAPSHOTTER="${SNAPSHOTTER:-}"
 export EXPERIMENTAL_FORCE_GUEST_PULL="${EXPERIMENTAL_FORCE_GUEST_PULL:-}"
+export PULL_TYPE="${PULL_TYPE:-}"
 
 setup() {
     if ! is_confidential_runtime_class; then
         skip "Test not supported for ${KATA_HYPERVISOR}."
     fi
 
-    if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ]; then
-        skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test"
+    if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ] && [ "${PULL_TYPE}" != "guest-pull" ]; then
+        skip "Either SNAPSHOTTER=nydus, EXPERIMENTAL_FORCE_GUEST_PULL, or PULL_TYPE=guest-pull must be set for this test"
     fi
 
     setup_common || die "setup_common failed"
@@ -174,8 +175,8 @@ teardown() {
         skip "Test not supported for ${KATA_HYPERVISOR}."
     fi
 
-    if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ]; then
-        skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test"
+    if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ] && [ "${PULL_TYPE}" != "guest-pull" ]; then
+        skip "Either SNAPSHOTTER=nydus, EXPERIMENTAL_FORCE_GUEST_PULL, or PULL_TYPE=guest-pull must be set for this test"
     fi
 
     confidential_teardown_common "${node}" "${node_start_time:-}"

tests/integration/kubernetes/k8s-guest-pull-image-signature.bats

@@ -11,14 +11,15 @@ load "${BATS_TEST_DIRNAME}/confidential_common.sh"
 export KBS="${KBS:-false}"
 export SNAPSHOTTER="${SNAPSHOTTER:-}"
 export EXPERIMENTAL_FORCE_GUEST_PULL="${EXPERIMENTAL_FORCE_GUEST_PULL:-}"
+export PULL_TYPE="${PULL_TYPE:-}"
 
 setup() {
     if ! is_confidential_runtime_class; then
         skip "Test not supported for ${KATA_HYPERVISOR}."
     fi
 
-    if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ]; then
-        skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test"
+    if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ] && [ "${PULL_TYPE}" != "guest-pull" ]; then
+        skip "Either SNAPSHOTTER=nydus, EXPERIMENTAL_FORCE_GUEST_PULL, or PULL_TYPE=guest-pull must be set for this test"
     fi
 
     tag_suffix=""
@@ -243,8 +244,8 @@ teardown() {
         skip "Test not supported for ${KATA_HYPERVISOR}."
     fi
 
-    if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ]; then
-        skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test"
+    if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ] && [ "${PULL_TYPE}" != "guest-pull" ]; then
+        skip "Either SNAPSHOTTER=nydus, EXPERIMENTAL_FORCE_GUEST_PULL, or PULL_TYPE=guest-pull must be set for this test"
     fi
 
     teardown_common "${node}" "${node_start_time:-}"

tests/integration/kubernetes/k8s-guest-pull-image.bats

@@ -8,12 +8,18 @@
 load "${BATS_TEST_DIRNAME}/lib.sh"
 load "${BATS_TEST_DIRNAME}/confidential_common.sh"
 
+export SNAPSHOTTER="${SNAPSHOTTER:-}"
+export EXPERIMENTAL_FORCE_GUEST_PULL="${EXPERIMENTAL_FORCE_GUEST_PULL:-}"
+export PULL_TYPE="${PULL_TYPE:-}"
+
 setup() {
     if ! is_confidential_runtime_class; then
         skip "Test not supported for ${KATA_HYPERVISOR}."
     fi
 
-    [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one"
+    if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ] && [ "${PULL_TYPE}" != "guest-pull" ]; then
+        skip "Either SNAPSHOTTER=nydus, EXPERIMENTAL_FORCE_GUEST_PULL, or PULL_TYPE=guest-pull must be set for this test"
+    fi
 
     setup_common || die "setup_common failed"
     unencrypted_image="quay.io/prometheus/busybox:latest"
@@ -87,9 +93,6 @@ setup() {
 }
 
 @test "Test we can pull an image inside the guest using trusted storage" {
-	[ "$(uname -m)" == "s390x" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838"
-    [ "${KATA_HYPERVISOR}" == "qemu-snp" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838"
-    [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838"
     # The image pulled in the guest will be downloaded and unpacked in the `/run/kata-containers/image` directory.
     # The tests will use `cryptsetup` to encrypt a block device and mount it at `/run/kata-containers/image`.
 
@@ -107,14 +110,18 @@ setup() {
     pod_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${pod_config_template}").XXX")
     IMAGE="$image_pulled_time_less_than_default_time" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config"
 
-
-    # Set CreateContainerRequest timeout for qemu-coco-dev
-    if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]]; then
-        create_container_timeout=300
-        set_metadata_annotation "$pod_config" \
-            "io.katacontainers.config.runtime.create_container_timeout" \
-            "${create_container_timeout}"
+    # Set CreateContainerRequest timeout in the annotation to allow for enough time for guest-pull where
+    # the container remains in 'creating' state until the pull completes. Usually pulling this and the large image in
+    # below test takes 30-60 seconds, but we occasionally observe spikes on all our bare-metal runners.
+    create_container_timeout=300
+    # On AKS, so far, these spikes have not been observed. Issue 10299, as referenced in other parts of this test, tells us
+    # that we cannot modify the runtimeRequestTimeout on AKS. We hence set the timeout to the 120s default value.
+    if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && [ "${KBS_INGRESS}" = "aks" ]; then
+        create_container_timeout=120
     fi
+    set_metadata_annotation "$pod_config" \
+        "io.katacontainers.config.runtime.create_container_timeout" \
+        "${create_container_timeout}"
 
     # Set annotation to pull image in guest
     set_metadata_annotation "${pod_config}" \
@@ -126,16 +133,14 @@ setup() {
     cat $pod_config
 
     add_allow_all_policy_to_yaml "$pod_config"
-    local wait_time=120
-    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && wait_time=300
+    local wait_time=300
+    if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && [ "${KBS_INGRESS}" = "aks" ]; then
+        wait_time=120
+    fi
     k8s_create_pod "$pod_config" "$wait_time"
 }
 
 @test "Test we cannot pull a large image that pull time exceeds createcontainer timeout inside the guest" {
-	[ "$(uname -m)" == "s390x" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838"
-    [ "${KATA_HYPERVISOR}" == "qemu-snp" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838"
-    [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838"
-
     storage_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${storage_config_template}").XXX")
     local_device=$(create_loop_device)
     LOCAL_DEVICE="$local_device" NODE_NAME="$node" envsubst < "$storage_config_template" > "$storage_config"
@@ -181,10 +186,6 @@ setup() {
 }
 
 @test "Test we can pull a large image inside the guest with large createcontainer timeout" {
-	[ "$(uname -m)" == "s390x" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838"
-    [ "${KATA_HYPERVISOR}" == "qemu-snp" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838"
-    [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10838"
-
     if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && [ "${KBS_INGRESS}" = "aks" ]; then
         skip "skip this specific one due to issue https://github.com/kata-containers/kata-containers/issues/10299"
     fi
@@ -203,8 +204,8 @@ setup() {
     IMAGE="$large_image" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config"
 
     # Set CreateContainerRequest timeout in the annotation to pull large image in guest
-    create_container_timeout=120
-    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && create_container_timeout=600
+    # Bare-metal CI runners' kubelets are configured with an equivalent runtimeRequestTimeout of 600s
+    create_container_timeout=600
     set_metadata_annotation "$pod_config" \
         "io.katacontainers.config.runtime.create_container_timeout" \
         "${create_container_timeout}"
@@ -219,8 +220,7 @@ setup() {
     cat $pod_config
 
     add_allow_all_policy_to_yaml "$pod_config"
-    local wait_time=120
-    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && wait_time=600
+    local wait_time=600
     k8s_create_pod "$pod_config" "$wait_time"
 }
 
@@ -229,7 +229,9 @@ teardown() {
         skip "Test not supported for ${KATA_HYPERVISOR}."
     fi
 
-    [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one"
+    if [ "${SNAPSHOTTER}" != "nydus" ] && [ -z "${EXPERIMENTAL_FORCE_GUEST_PULL}" ] && [ "${PULL_TYPE}" != "guest-pull" ]; then
+        skip "Either SNAPSHOTTER=nydus, EXPERIMENTAL_FORCE_GUEST_PULL, or PULL_TYPE=guest-pull must be set for this test"
+    fi
 
     teardown_common "${node}" "${node_start_time:-}"
     kubectl delete --ignore-not-found pvc trusted-pvc

tests/integration/kubernetes/k8s-nvidia-nim.bats

@@ -113,6 +113,27 @@ setup_langchain_flow() {
     [[ "$(pip show beautifulsoup4 2>/dev/null | awk '/^Version:/{print $2}')" = "4.13.4" ]] || pip install beautifulsoup4==4.13.4
 }
 
+# Create Docker config for genpolicy so it can authenticate to nvcr.io when
+# pulling image manifests (avoids "UnauthorizedError" from genpolicy's registry pull).
+# Genpolicy (src/tools/genpolicy) uses docker_credential::get_credential() in
+# src/tools/genpolicy/src/registry.rs build_auth(). The docker_credential crate
+# reads config from DOCKER_CONFIG (directory) + "/config.json", so we set
+# DOCKER_CONFIG to a directory containing config.json with nvcr.io auth.
+setup_genpolicy_registry_auth() {
+	if [[ -z "${NGC_API_KEY:-}" ]]; then
+		return
+	fi
+	local auth_dir
+	auth_dir="${BATS_SUITE_TMPDIR}/.docker-genpolicy"
+	mkdir -p "${auth_dir}"
+	# Docker config format: auths -> registry -> auth (base64 of "user:password")
+	echo -n "{\"auths\":{\"nvcr.io\":{\"username\":\"\$oauthtoken\",\"password\":\"${NGC_API_KEY}\",\"auth\":\"$(echo -n "\$oauthtoken:${NGC_API_KEY}" | base64 -w0)\"}}}" \
+		> "${auth_dir}/config.json"
+	export DOCKER_CONFIG="${auth_dir}"
+	# REGISTRY_AUTH_FILE (containers-auth.json format) is the same structure for auths
+	export REGISTRY_AUTH_FILE="${auth_dir}/config.json"
+}
+
 # Create initdata TOML file for genpolicy with CDH configuration.
 # This file is used by genpolicy via --initdata-path. Genpolicy will add the
 # generated policy.rego to it and set it as the cc_init_data annotation.
@@ -222,6 +243,9 @@ setup_file() {
     add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
 
     if [ "${TEE}" = "true" ]; then
+        # So genpolicy can pull nvcr.io image manifests when generating policy (avoids UnauthorizedError).
+        setup_genpolicy_registry_auth
+
         setup_kbs_credentials
         # Overwrite the empty default-initdata.toml with our CDH configuration.
         # This must happen AFTER create_tmp_policy_settings_dir() copies the empty

tests/integration/kubernetes/k8s-openvpn.bats

@@ -34,17 +34,10 @@ setup() {
     client_secret_template_yaml="${pod_config_dir}/openvpn/openvpn-client-secret.yaml.in"
     client_secret_instance_yaml="${pod_config_dir}/openvpn/openvpn-client-secret-instance.yaml"
 
-    # See issue https://github.com/kata-containers/kata-containers/issues/11162 and
-    # other references to this issue in the genpolicy source folder.
-    if [[ "${SNAPSHOTTER:-}" == "nydus" ]]; then
-        add_allow_all_policy_to_yaml "$server_pod_yaml"
-        add_allow_all_policy_to_yaml "$client_pod_yaml"
-    else
-        policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
-        add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
-        auto_generate_policy "${policy_settings_dir}" "$server_pod_yaml" "$server_configmap_yaml" "--config-file $server_secret_template_yaml"
-        auto_generate_policy "${policy_settings_dir}" "$client_pod_yaml" "$client_configmap_yaml" "--config-file $client_secret_template_yaml"
-    fi
+    policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
+    add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
+    auto_generate_policy "${policy_settings_dir}" "$server_pod_yaml" "$server_configmap_yaml" "--config-file $server_secret_template_yaml"
+    auto_generate_policy "${policy_settings_dir}" "$client_pod_yaml" "$client_configmap_yaml" "--config-file $client_secret_template_yaml"
 }
 
 @test "Pods establishing a VPN connection using openvpn" {
@@ -92,9 +85,7 @@ teardown() {
     echo "=== OpenVPN Client Pod Logs ==="
     kubectl logs "$client_pod_name" || true
 
-    if [[ "${SNAPSHOTTER:-}" != "nydus" ]]; then
-        delete_tmp_policy_settings_dir "${policy_settings_dir}"
-    fi
+    delete_tmp_policy_settings_dir "${policy_settings_dir}"
     teardown_common "${node}" "${node_start_time:-}"
 
     # teardown cleans up pods, but not other resources

tests/integration/kubernetes/k8s-policy-pvc.bats

@@ -11,7 +11,7 @@ load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 setup() {
 	auto_generate_policy_enabled || skip "Auto-generated policy tests are disabled."
-	( [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] || [ "${KATA_HYPERVISOR}" == "qemu-snp" ] ) && skip "https://github.com/kata-containers/kata-containers/issues/9846"
+	[[ "${RUNS_ON_AKS}" == "true" ]] || skip "https://github.com/kata-containers/kata-containers/issues/9846"
 	setup_common || die "setup_common failed"
 	pod_name="policy-pod-pvc"
 	pvc_name="policy-dev"
@@ -58,7 +58,7 @@ test_pod_policy_error() {
 
 teardown() {
 	auto_generate_policy_enabled || skip "Auto-generated policy tests are disabled."
-	( [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] || [ "${KATA_HYPERVISOR}" == "qemu-snp" ] ) && skip "https://github.com/kata-containers/kata-containers/issues/9846"
+	[[ "${RUNS_ON_AKS}" == "true" ]] || skip "https://github.com/kata-containers/kata-containers/issues/9846"
 
 	# Debugging information. Don't print the "Message:" line because it contains a truncated policy log.
 	kubectl describe pod "${pod_name}" | grep -v "Message:"

tests/integration/kubernetes/k8s-sandbox-cgroup.bats

@@ -0,0 +1,39 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2026 Chiranjeevi Uddanti
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/../../common.bash"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+setup() {
+	pod_name="sandbox-cgroup-pod"
+
+	setup_common || die "setup_common failed"
+
+	yaml_file="${pod_config_dir}/pod-sandbox-cgroup.yaml"
+	set_node "$yaml_file" "$node"
+
+	# Add policy to yaml
+	policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
+
+	add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
+	auto_generate_policy "${policy_settings_dir}" "${yaml_file}"
+}
+
+# Regression test for https://github.com/kata-containers/kata-containers/issues/12479
+@test "Pod with sandbox_cgroup_only=false starts successfully" {
+	# Create pod
+	kubectl create -f "${yaml_file}"
+
+	# Wait for pod to be ready
+	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+}
+
+teardown() {
+	delete_tmp_policy_settings_dir "${policy_settings_dir}"
+	teardown_common "${node}" "${node_start_time:-}"
+}

tests/integration/kubernetes/k8s-sandbox-vcpus-allocation.bats

@@ -12,7 +12,7 @@ load "${BATS_TEST_DIRNAME}/tests_common.sh"
 setup() {
 	[ "$(uname -m)" == "aarch64" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10928"
 	[[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && skip "Requires CPU hotplug which disabled by static_sandbox_resource_mgmt"
-	[[ "${KATA_HYPERVISOR}" == qemu-tdx ]] && skip "See: https://github.com/kata-containers/kata-containers/issues/12492"
+	[[ "${KATA_HYPERVISOR}" == "qemu-tdx" ]] && skip "See: https://github.com/kata-containers/kata-containers/issues/12492"
 
 	setup_common || die "setup_common failed"
 
@@ -53,6 +53,7 @@ setup() {
 teardown() {
 	[ "$(uname -m)" == "aarch64" ] && skip "See: https://github.com/kata-containers/kata-containers/issues/10928"
 	[[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && skip "Requires CPU hotplug which disabled by static_sandbox_resource_mgmt"
+	[[ "${KATA_HYPERVISOR}" == "qemu-tdx" ]] && skip "See: https://github.com/kata-containers/kata-containers/issues/12492"
 
 	for pod in "${pods[@]}"; do
 		kubectl logs ${pod}

tests/integration/kubernetes/lib.sh

@@ -194,8 +194,15 @@ assert_pod_fail() {
 		echo "Waiting for a container to fail"
 		sleep "${sleep_time}"
 		elapsed_time=$((elapsed_time+sleep_time))
-		if [[ $(kubectl get pod "${pod_name}" \
-			-o jsonpath='{.status.containerStatuses[0].state.waiting.reason}') = *BackOff* ]]; then
+		waiting_reason=$(kubectl get pod "${pod_name}" \
+			-o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || true)
+		terminated_reason=$(kubectl get pod "${pod_name}" \
+			-o jsonpath='{.status.containerStatuses[0].state.terminated.reason}' 2>/dev/null || true)
+		# BackOff/CrashLoopBackOff = container repeatedly failed; RunContainerError = e.g. image pull in guest failed
+		if [[ "${waiting_reason}" == *BackOff* ]] || [[ "${waiting_reason}" == *RunContainerError* ]]; then
+			return 0
+		fi
+		if [[ "${terminated_reason}" == "StartError" ]] || [[ "${terminated_reason}" == "Error" ]]; then
 			return 0
 		fi
 		if [[ "${elapsed_time}" -gt "${duration}" ]]; then

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -88,6 +88,7 @@ else
 		"k8s-privileged.bats" \
 		"k8s-projected-volume.bats" \
 		"k8s-replication.bats" \
+		"k8s-sandbox-cgroup.bats" \
 		"k8s-seccomp.bats" \
 		"k8s-sysctls.bats" \
 		"k8s-security-context.bats" \

tests/integration/kubernetes/runtimeclass_workloads/openvpn/openvpn-client-pod.yaml

@@ -11,6 +11,13 @@ metadata:
   labels:
     app: openvpn-client
 spec:
+  # Explicit user/group/supplementary groups to support nydus guest-pull.
+  # See issue https://github.com/kata-containers/kata-containers/issues/11162 and
+  # other references to this issue in the genpolicy source folder.
+  securityContext:
+    runAsUser: 0
+    runAsGroup: 0
+    supplementalGroups: [1, 2, 3, 4, 6, 10, 11, 20, 26, 27]
   containers:
   - name: openvpn-client
     image: quay.io/kata-containers/alpine:3.22.1-openvpn

tests/integration/kubernetes/runtimeclass_workloads/openvpn/openvpn-server-pod.yaml

@@ -11,6 +11,13 @@ metadata:
   labels:
     app: openvpn-server
 spec:
+  # Explicit user/group/supplementary groups to support nydus guest-pull.
+  # See issue https://github.com/kata-containers/kata-containers/issues/11162 and
+  # other references to this issue in the genpolicy source folder.
+  securityContext:
+    runAsUser: 0
+    runAsGroup: 0
+    supplementalGroups: [1, 2, 3, 4, 6, 10, 11, 20, 26, 27]
   containers:
   - name: openvpn-server
     image: quay.io/kata-containers/alpine:3.22.1-openvpn

tests/integration/kubernetes/runtimeclass_workloads/pod-configmap-volume.yaml

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2026 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: v1
+kind: Pod
+metadata:
+  name: configmap-volume-test-pod
+spec:
+  terminationGracePeriodSeconds: 0
+  runtimeClassName: kata
+  containers:
+  - name: test-container
+    image: quay.io/prometheus/busybox:latest
+    command: ["sh", "-c", "while true; do sleep 10; done"]
+    volumeMounts:
+    - name: config-volume
+      mountPath: /etc/config
+      readOnly: true
+  volumes:
+  - name: config-volume
+    configMap:
+      name: test-configmap
+  restartPolicy: Never

tests/integration/kubernetes/runtimeclass_workloads/pod-sandbox-cgroup.yaml

@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2026 Chiranjeevi Uddanti
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sandbox-cgroup-pod
+  annotations:
+    io.katacontainers.config.runtime.sandbox_cgroup_only: "false"
+spec:
+  runtimeClassName: kata
+  restartPolicy: Never
+  containers:
+  - image: quay.io/prometheus/busybox:latest
+    name: sandbox-cgroup-test

tests/integration/kubernetes/setup.sh

@@ -14,6 +14,7 @@ export AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-no}"
 export KATA_HOST_OS="${KATA_HOST_OS:-}"
 export KATA_HYPERVISOR="${KATA_HYPERVISOR:-}"
 export PULL_TYPE="${PULL_TYPE:-default}"
+export RUNS_ON_AKS="${RUNS_ON_AKS:-false}"
 
 declare -r kubernetes_dir=$(dirname "$(readlink -f "$0")")
 declare -r runtimeclass_workloads_work_dir="${kubernetes_dir}/runtimeclass_workloads_work"
@@ -99,23 +100,26 @@ add_annotations_to_yaml() {
 	esac
 }
 
+add_cbl_mariner_annotation_to_yaml() {
+	local -r yaml_file="$1"
+
+	local -r mariner_annotation_image="io.katacontainers.config.hypervisor.image"
+	local -r mariner_image_path="/opt/kata/share/kata-containers/kata-containers-mariner.img"
+	add_annotations_to_yaml "${yaml_file}" "${mariner_annotation_image}" "${mariner_image_path}"
+}
+
 add_cbl_mariner_specific_annotations() {
 	if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
-		info "Add kernel and image path and annotations for cbl-mariner"
-		local mariner_annotation_kernel="io.katacontainers.config.hypervisor.kernel"
-		local mariner_kernel_path="/usr/share/cloud-hypervisor/vmlinux.bin"
-
-		local mariner_annotation_image="io.katacontainers.config.hypervisor.image"
-		local mariner_image_path="/opt/kata/share/kata-containers/kata-containers-mariner.img"
-
-		local mariner_annotation_disable_image_nvdimm="io.katacontainers.config.hypervisor.disable_image_nvdimm"
-		local mariner_disable_image_nvdimm=true
+		info "Adding annotations for cbl-mariner"
 
 		for K8S_TEST_YAML in runtimeclass_workloads_work/*.yaml
 		do
-			add_annotations_to_yaml "${K8S_TEST_YAML}" "${mariner_annotation_kernel}" "${mariner_kernel_path}"
-			add_annotations_to_yaml "${K8S_TEST_YAML}" "${mariner_annotation_image}" "${mariner_image_path}"
-			add_annotations_to_yaml "${K8S_TEST_YAML}" "${mariner_annotation_disable_image_nvdimm}" "${mariner_disable_image_nvdimm}"
+			add_cbl_mariner_annotation_to_yaml "${K8S_TEST_YAML}"
+		done
+
+		for K8S_TEST_YAML in runtimeclass_workloads_work/openvpn/*.yaml
+		do
+			add_cbl_mariner_annotation_to_yaml "${K8S_TEST_YAML}"
 		done
 	fi
 }

tests/integration/kubernetes/tests_common.sh

@@ -39,6 +39,7 @@ AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-}"
 GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-}"
 KATA_HYPERVISOR="${KATA_HYPERVISOR:-}"
 KATA_HOST_OS="${KATA_HOST_OS:-}"
+RUNS_ON_AKS="${RUNS_ON_AKS:-false}"
 
 # Common setup for tests.
 #
@@ -98,13 +99,11 @@ is_nvidia_gpu_platform() {
 }
 
 is_aks_cluster() {
-	case "${KATA_HYPERVISOR}" in
-		"qemu-tdx"|"qemu-snp"|qemu-nvidia-gpu*)
-			return 1
-			;;
-		*)
-			return 0
-	esac
+	if [[ "${RUNS_ON_AKS}" = "true" ]]; then
+		return 0
+	fi
+
+	return 1
 }
 
 adapt_common_policy_settings_for_non_coco() {
@@ -172,6 +171,24 @@ adapt_common_policy_settings_for_nvidia_gpu() {
 	jq '.kata_config.oci_version = "1.2.1"' "${settings_dir}/genpolicy-settings.json" > temp.json && mv temp.json "${settings_dir}/genpolicy-settings.json"
 }
 
+# Adapt OCI version in policy settings to match containerd version.
+# containerd 2.2.x (active) vendors v1.3.0.
+adapt_common_policy_settings_for_containerd_version() {
+	local settings_dir=${1}
+
+	info "Adapting common policy settings for containerd's latest release"
+	jq '.kata_config.oci_version = "1.3.0"' "${settings_dir}/genpolicy-settings.json" > temp.json && mv temp.json "${settings_dir}/genpolicy-settings.json"
+}
+
+# When using experimental-force-guest-pull, genpolicy must not use guest_pull (we pull via oci-distribution for policy generation).
+adapt_common_policy_settings_for_experimental_force_guest_pull() {
+	local settings_dir=$1
+
+	info "Adapting common policy settings for experimental-force-guest-pull: disable guest_pull"
+	jq '.cluster_config.guest_pull = false' "${settings_dir}/genpolicy-settings.json" > temp.json
+	mv temp.json "${settings_dir}/genpolicy-settings.json"
+}
+
 # adapt common policy settings for various platforms
 adapt_common_policy_settings() {
 	local settings_dir=$1
@@ -179,6 +196,8 @@ adapt_common_policy_settings() {
 	is_coco_platform || adapt_common_policy_settings_for_non_coco "${settings_dir}"
 	is_aks_cluster && adapt_common_policy_settings_for_aks "${settings_dir}"
 	is_nvidia_gpu_platform && adapt_common_policy_settings_for_nvidia_gpu "${settings_dir}"
+	[[ -n "${CONTAINER_ENGINE_VERSION:-}" ]] && adapt_common_policy_settings_for_containerd_version "${settings_dir}"
+	[[ "${PULL_TYPE:-}" == "experimental-force-guest-pull" ]] && adapt_common_policy_settings_for_experimental_force_guest_pull "${settings_dir}"
 
 	case "${KATA_HOST_OS}" in
 		"cbl-mariner")

tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh

@@ -45,8 +45,8 @@ install_nvidia_fabricmanager() {
 		return
 	}
 	echo "chroot: Install NVIDIA fabricmanager"
-	eval "${APT_INSTALL}" nvidia-fabricmanager libnvidia-nscq
-	apt-mark hold nvidia-fabricmanager libnvidia-nscq
+	eval "${APT_INSTALL}" nvidia-fabricmanager libnvidia-nscq nvlsm
+	apt-mark hold nvidia-fabricmanager libnvidia-nscq nvlsm
 }
 
 install_userspace_components() {

tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh

@@ -145,8 +145,8 @@ chisseled_nvswitch() {
 
 	mkdir -p usr/share/nvidia/nvswitch
 
-	cp -a "${stage_one}"/usr/bin/nv-fabricmanager 	bin/.
-	cp -a "${stage_one}"/usr/share/nvidia/nvswitch usr/share/nvidia/.
+	cp -a "${stage_one}"/usr/bin/nv-fabricmanager	bin/.
+	cp -a "${stage_one}"/usr/share/nvidia/nvswitch	usr/share/nvidia/.
 
 	libdir=usr/lib/"${machine_arch}"-linux-gnu
 
@@ -156,6 +156,14 @@ chisseled_nvswitch() {
 	# if the specified log file can't be opened or the path is empty.
 	# LOG_FILE_NAME=/var/log/fabricmanager.log -> setting to empty for stderr -> kmsg
 	sed -i 's|^LOG_FILE_NAME=.*|LOG_FILE_NAME=|' usr/share/nvidia/nvswitch/fabricmanager.cfg
+
+	# NVLINK SubnetManager dependencies
+	local nvlsm=usr/share/nvidia/nvlsm
+	mkdir -p "${nvlsm}"
+
+	cp -a "${stage_one}"/opt/nvidia/nvlsm/lib/libgrpc_mgr.so	lib/.
+	cp -a "${stage_one}"/opt/nvidia/nvlsm/sbin/nvlsm			sbin/.
+	cp -a "${stage_one}/${nvlsm}"/*.conf						"${nvlsm}"/.
 }
 
 chisseled_dcgm() {

tools/packaging/kata-deploy/binary/src/artifacts/install.rs

@@ -194,10 +194,9 @@ fn install_custom_runtime_configs(config: &Config) -> Result<()> {
         // Copy base config to the handler directory
         // Custom runtime drop-ins will overlay on top of this
         let base_config_filename = format!("configuration-{}.toml", runtime.base_config);
-        let original_config = format!(
-            "/host/{}/share/defaults/kata-containers/{}",
-            config.dest_dir, base_config_filename
-        );
+        let config_base =
+            utils::get_kata_containers_original_config_path(&runtime.base_config, &config.dest_dir);
+        let original_config = format!("/host{}/{}", config_base, base_config_filename);
         let dest_config = format!("{}/{}", handler_dir, base_config_filename);
 
         if Path::new(&original_config).exists() {
@@ -292,15 +291,45 @@ fn remove_custom_runtime_configs(config: &Config) -> Result<()> {
 
 /// Note: The src parameter is kept to allow for unit testing with temporary directories,
 /// even though in production it always uses /opt/kata-artifacts/opt/kata
+///
+/// Symlinks in the source tree are preserved at the destination (recreated as symlinks
+/// instead of copying the target file). Absolute targets under the source root are
+/// rewritten to the destination root so they remain valid.
 fn copy_artifacts(src: &str, dst: &str) -> Result<()> {
-    for entry in WalkDir::new(src) {
+    let src_path = Path::new(src);
+    for entry in WalkDir::new(src).follow_links(false) {
         let entry = entry?;
-        let src_path = entry.path();
-        let relative_path = src_path.strip_prefix(src)?;
+        let src_path_entry = entry.path();
+        let relative_path = src_path_entry.strip_prefix(src)?;
         let dst_path = Path::new(dst).join(relative_path);
 
         if entry.file_type().is_dir() {
             fs::create_dir_all(&dst_path)?;
+        } else if entry.file_type().is_symlink() {
+            // Preserve symlinks: create a symlink at destination instead of copying the target
+            let link_target = fs::read_link(src_path_entry)
+                .with_context(|| format!("Failed to read symlink: {:?}", src_path_entry))?;
+            let new_target: std::path::PathBuf = if link_target.is_absolute() {
+                // Rewrite absolute targets that point inside the source tree
+                if let Ok(rel) = link_target.strip_prefix(src_path) {
+                    Path::new(dst).join(rel)
+                } else {
+                    link_target.into()
+                }
+            } else {
+                link_target.into()
+            };
+
+            if let Some(parent) = dst_path.parent() {
+                fs::create_dir_all(parent)?;
+            }
+            match fs::remove_file(&dst_path) {
+                Ok(()) => {}
+                Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+                Err(e) => return Err(e.into()),
+            }
+            std::os::unix::fs::symlink(&new_target, &dst_path)
+                .with_context(|| format!("Failed to create symlink {:?} -> {:?}", dst_path, new_target))?;
         } else {
             if let Some(parent) = dst_path.parent() {
                 fs::create_dir_all(parent)?;
@@ -318,7 +347,7 @@ fn copy_artifacts(src: &str, dst: &str) -> Result<()> {
                 Err(e) => return Err(e.into()), // Other errors should be propagated
             }
 
-            fs::copy(src_path, &dst_path)?;
+            fs::copy(src_path_entry, &dst_path)?;
         }
     }
     Ok(())
@@ -889,65 +918,54 @@ async fn configure_mariner(config: &Config) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rstest::rstest;
 
-    #[test]
-    fn test_get_hypervisor_name_qemu_variants() {
-        // Test all QEMU variants
-        assert_eq!(get_hypervisor_name("qemu").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-tdx").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-snp").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-se").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-coco-dev").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-cca").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-nvidia-gpu").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-nvidia-gpu-tdx").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-nvidia-gpu-snp").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-runtime-rs").unwrap(), "qemu");
-        assert_eq!(
-            get_hypervisor_name("qemu-coco-dev-runtime-rs").unwrap(),
-            "qemu"
-        );
-        assert_eq!(get_hypervisor_name("qemu-se-runtime-rs").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-snp-runtime-rs").unwrap(), "qemu");
-        assert_eq!(get_hypervisor_name("qemu-tdx-runtime-rs").unwrap(), "qemu");
+    #[rstest]
+    #[case("qemu", "qemu")]
+    #[case("qemu-tdx", "qemu")]
+    #[case("qemu-snp", "qemu")]
+    #[case("qemu-se", "qemu")]
+    #[case("qemu-coco-dev", "qemu")]
+    #[case("qemu-cca", "qemu")]
+    #[case("qemu-nvidia-gpu", "qemu")]
+    #[case("qemu-nvidia-gpu-tdx", "qemu")]
+    #[case("qemu-nvidia-gpu-snp", "qemu")]
+    #[case("qemu-runtime-rs", "qemu")]
+    #[case("qemu-coco-dev-runtime-rs", "qemu")]
+    #[case("qemu-se-runtime-rs", "qemu")]
+    #[case("qemu-snp-runtime-rs", "qemu")]
+    #[case("qemu-tdx-runtime-rs", "qemu")]
+    fn test_get_hypervisor_name_qemu_variants(#[case] shim: &str, #[case] expected: &str) {
+        assert_eq!(get_hypervisor_name(shim).unwrap(), expected);
     }
 
-    #[test]
-    fn test_get_hypervisor_name_other_hypervisors() {
-        // Test other hypervisors
-        assert_eq!(get_hypervisor_name("clh").unwrap(), "clh");
-        assert_eq!(
-            get_hypervisor_name("cloud-hypervisor").unwrap(),
-            "cloud-hypervisor"
-        );
-        assert_eq!(get_hypervisor_name("dragonball").unwrap(), "dragonball");
-        assert_eq!(get_hypervisor_name("fc").unwrap(), "firecracker");
-        assert_eq!(get_hypervisor_name("firecracker").unwrap(), "firecracker");
-        assert_eq!(get_hypervisor_name("remote").unwrap(), "remote");
+    #[rstest]
+    #[case("clh", "clh")]
+    #[case("cloud-hypervisor", "cloud-hypervisor")]
+    #[case("dragonball", "dragonball")]
+    #[case("fc", "firecracker")]
+    #[case("firecracker", "firecracker")]
+    #[case("remote", "remote")]
+    fn test_get_hypervisor_name_other_hypervisors(#[case] shim: &str, #[case] expected: &str) {
+        assert_eq!(get_hypervisor_name(shim).unwrap(), expected);
     }
 
-    #[test]
-    fn test_get_hypervisor_name_unknown() {
-        // Test unknown shim returns error with clear message
-        let result = get_hypervisor_name("unknown-shim");
+    #[rstest]
+    #[case("")]
+    #[case("unknown-shim")]
+    #[case("custom")]
+    fn test_get_hypervisor_name_unknown(#[case] shim: &str) {
+        let result = get_hypervisor_name(shim);
         assert!(result.is_err(), "Unknown shim should return an error");
         let err_msg = result.unwrap_err().to_string();
         assert!(
-            err_msg.contains("Unknown shim 'unknown-shim'"),
+            err_msg.contains(&format!("Unknown shim '{}'", shim)),
             "Error message should mention the unknown shim"
         );
         assert!(
             err_msg.contains("Valid shims are:"),
             "Error message should list valid shims"
         );
-
-        let result = get_hypervisor_name("custom");
-        assert!(result.is_err(), "Custom shim should return an error");
-        let err_msg = result.unwrap_err().to_string();
-        assert!(
-            err_msg.contains("Unknown shim 'custom'"),
-            "Error message should mention the custom shim"
-        );
     }
 
     #[test]
@@ -1024,10 +1042,36 @@ mod tests {
     }
 
     #[test]
-    fn test_get_hypervisor_name_empty() {
-        let result = get_hypervisor_name("");
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("Unknown shim"));
+    fn test_copy_artifacts_preserves_symlinks() {
+        let src_dir = tempfile::tempdir().unwrap();
+        let dst_dir = tempfile::tempdir().unwrap();
+
+        // Create a real file and a symlink pointing to it
+        let real_file = src_dir.path().join("real-file.txt");
+        fs::write(&real_file, "actual content").unwrap();
+        let link_path = src_dir.path().join("link-to-real");
+        std::os::unix::fs::symlink(&real_file, &link_path).unwrap();
+
+        copy_artifacts(
+            src_dir.path().to_str().unwrap(),
+            dst_dir.path().to_str().unwrap(),
+        )
+        .unwrap();
+
+        let dst_link = dst_dir.path().join("link-to-real");
+        let dst_real = dst_dir.path().join("real-file.txt");
+        assert!(dst_real.exists(), "real file should be copied");
+        assert!(dst_link.is_symlink(), "destination should be a symlink");
+        assert_eq!(
+            fs::read_link(&dst_link).unwrap(),
+            dst_real,
+            "symlink should point to the real file in the same tree"
+        );
+        assert_eq!(
+            fs::read_to_string(&dst_link).unwrap(),
+            "actual content",
+            "following the symlink should yield the real content"
+        );
     }
+
 }

tools/packaging/kata-deploy/binary/src/config.rs

@@ -6,6 +6,65 @@
 use anyhow::{Context, Result};
 use log::info;
 use std::env;
+use std::fs;
+
+use crate::k8s;
+
+/// K3s/RKE2 containerd config template filenames (under the mounted containerd dir).
+/// V3 is for containerd 2.x; V2 is for containerd 1.x.
+pub const K3S_RKE2_CONTAINERD_V3_TMPL: &str = "/etc/containerd/config-v3.toml.tmpl";
+pub const K3S_RKE2_CONTAINERD_V2_TMPL: &str = "/etc/containerd/config.toml.tmpl";
+
+/// Resolves whether to use containerd config v3 (true) or v2 (false) for K3s/RKE2.
+/// 1. Tries config.toml (containerd config file): if it exists and contains "version = 3" or "version = 2", use that.
+/// 2. Else falls back to the node's containerRuntimeVersion (e.g. "containerd://2.1.5-k3s1").
+/// 3. If neither is available, returns an error.
+pub fn k3s_rke2_resolve_use_v3(
+    config_file_path: &str,
+    container_runtime_version: Option<&str>,
+) -> Result<bool> {
+    use crate::runtime::manager;
+
+    // 1. Try config.toml (generated config that may already exist on the node)
+    if let Ok(content) = fs::read_to_string(config_file_path) {
+        if content.contains("version = 3") {
+            return Ok(true);
+        }
+        if content.contains("version = 2") {
+            return Ok(false);
+        }
+    }
+
+    // 2. Fall back to node's container runtime version
+    if let Some(version) = container_runtime_version {
+        return Ok(manager::containerd_version_is_2_or_newer(version));
+    }
+
+    // 3. Neither source available
+    Err(anyhow::anyhow!(
+        "K3s/RKE2: cannot determine containerd config version (v2 vs v3). \
+         Need version from {config_file_path} (version = 2/3) or node containerRuntimeVersion."
+    ))
+}
+
+/// Returns the K3s/RKE2 containerd template path. Use v3 for containerd 2.x, v2 for 1.x.
+pub fn k3s_rke2_containerd_template_path(use_v3: bool) -> &'static str {
+    if use_v3 {
+        K3S_RKE2_CONTAINERD_V3_TMPL
+    } else {
+        K3S_RKE2_CONTAINERD_V2_TMPL
+    }
+}
+
+/// Returns the containerd CRI plugin ID for K3s/RKE2 (section key we write under).
+/// Config v3 uses "io.containerd.cri.v1.runtime", v2 uses "io.containerd.grpc.v1.cri".
+pub fn k3s_rke2_containerd_plugin_id(use_v3: bool) -> &'static str {
+    if use_v3 {
+        "\"io.containerd.cri.v1.runtime\""
+    } else {
+        "\"io.containerd.grpc.v1.cri\""
+    }
+}
 
 /// Default Kata Containers installation directory.
 /// This is where Kata artifacts are installed by default.
@@ -25,6 +84,8 @@ pub struct ContainerdPaths {
     pub drop_in_file: String,
     /// Whether drop-in files can be used (based on containerd version)
     pub use_drop_in: bool,
+    /// For K3s/RKE2: CRI plugin ID to use (derived from containerd version). Others: None (read from file).
+    pub plugin_id: Option<String>,
 }
 
 /// Custom runtime configuration parsed from ConfigMap
@@ -443,6 +504,7 @@ impl Config {
                 imports_file: None, // k0s auto-loads from containerd.d/, imports not needed
                 drop_in_file: "/etc/containerd/containerd.d/kata-deploy.toml".to_string(),
                 use_drop_in,
+                plugin_id: None,
             },
             "microk8s" => ContainerdPaths {
                 // microk8s uses containerd-template.toml instead of config.toml
@@ -451,22 +513,41 @@ impl Config {
                 imports_file: Some("/etc/containerd/containerd-template.toml".to_string()),
                 drop_in_file: self.containerd_drop_in_conf_file.clone(),
                 use_drop_in,
+                plugin_id: None,
             },
-            "k3s" | "k3s-agent" | "rke2-agent" | "rke2-server" => ContainerdPaths {
-                // k3s/rke2 generates config.toml from config.toml.tmpl on each restart
-                // We must modify the template file so our changes persist
-                config_file: "/etc/containerd/config.toml.tmpl".to_string(),
-                backup_file: "/etc/containerd/config.toml.tmpl.bak".to_string(),
-                imports_file: Some("/etc/containerd/config.toml.tmpl".to_string()),
-                drop_in_file: self.containerd_drop_in_conf_file.clone(),
-                use_drop_in,
-            },
+            "k3s" | "k3s-agent" | "rke2-agent" | "rke2-server" => {
+                // K3s/RKE2 generate config.toml from a template on each restart; we modify
+                // the template so our changes persist. Which template is chosen by containerd version
+                // (see k3s_rke2_resolve_use_v3). Refs: docs.k3s.io/advanced#configuring-containerd
+                let container_runtime_version = k8s::get_node_field(
+                    self,
+                    ".status.nodeInfo.containerRuntimeVersion",
+                )
+                .await
+                .ok();
+                let use_v3 = k3s_rke2_resolve_use_v3(
+                    &self.containerd_conf_file,
+                    container_runtime_version.as_deref(),
+                )?;
+                let config_file =
+                    k3s_rke2_containerd_template_path(use_v3).to_string();
+                let backup_file = format!("{config_file}.bak");
+                ContainerdPaths {
+                    config_file: config_file.clone(),
+                    backup_file,
+                    imports_file: Some(config_file),
+                    drop_in_file: self.containerd_drop_in_conf_file.clone(),
+                    use_drop_in,
+                    plugin_id: Some(k3s_rke2_containerd_plugin_id(use_v3).to_string()),
+                }
+            }
             _ => ContainerdPaths {
                 config_file: self.containerd_conf_file.clone(),
                 backup_file: self.containerd_conf_file_backup.clone(),
                 imports_file: Some(self.containerd_conf_file.clone()),
                 drop_in_file: self.containerd_drop_in_conf_file.clone(),
                 use_drop_in,
+                plugin_id: None,
             },
         };
 

tools/packaging/kata-deploy/binary/src/main.rs

@@ -187,7 +187,7 @@ async fn install(config: &config::Config, runtime: &str) -> Result<()> {
         None => {}
     }
 
-    runtime::containerd::setup_containerd_config_files(runtime, config)?;
+    runtime::containerd::setup_containerd_config_files(runtime, config).await?;
 
     artifacts::install_artifacts(config).await?;
 

tools/packaging/kata-deploy/binary/src/runtime/containerd.rs

@@ -25,19 +25,33 @@ struct ContainerdRuntimeParams {
     snapshotter: Option<String>,
 }
 
+/// Plugin ID for CRI runtime in containerd config v3 (version = 3).
+const CONTAINERD_V3_RUNTIME_PLUGIN_ID: &str = "\"io.containerd.cri.v1.runtime\"";
+/// Plugin ID for CRI in containerd config v2 (version = 2).
+const CONTAINERD_V2_CRI_PLUGIN_ID: &str = "\"io.containerd.grpc.v1.cri\"";
+/// Legacy plugin key when config has no version (pre-v2).
+const CONTAINERD_LEGACY_CRI_PLUGIN_ID: &str = "cri";
+/// Plugin ID for CRI images in containerd config v3 (version = 3).
+const CONTAINERD_CRI_IMAGES_PLUGIN_ID: &str = "\"io.containerd.cri.v1.images\"";
+
 fn get_containerd_pluginid(config_file: &str) -> Result<&'static str> {
     let content = fs::read_to_string(config_file)
         .with_context(|| format!("Failed to read containerd config file: {}", config_file))?;
 
     if content.contains("version = 3") {
-        Ok("\"io.containerd.cri.v1.runtime\"")
+        Ok(CONTAINERD_V3_RUNTIME_PLUGIN_ID)
     } else if content.contains("version = 2") {
-        Ok("\"io.containerd.grpc.v1.cri\"")
+        Ok(CONTAINERD_V2_CRI_PLUGIN_ID)
     } else {
-        Ok("cri")
+        Ok(CONTAINERD_LEGACY_CRI_PLUGIN_ID)
     }
 }
 
+/// True when the containerd config is v3 (version = 3), i.e. we use the split CRI plugins.
+fn is_containerd_v3_config(pluginid: &str) -> bool {
+    pluginid == CONTAINERD_V3_RUNTIME_PLUGIN_ID
+}
+
 fn get_containerd_output_path(paths: &ContainerdPaths) -> PathBuf {
     if paths.use_drop_in {
         if paths.drop_in_file.starts_with("/etc/containerd/") {
@@ -95,6 +109,26 @@ fn write_containerd_runtime_config(
             &format!("{runtime_table}.snapshotter"),
             snapshotter,
         )?;
+        // In containerd config v3 the CRI plugin is split into runtime and images,
+        // and setting the snapshotter only on the runtime plugin is not enough for image
+        // pull/prepare.
+        //
+        // The images plugin must have runtime_platforms.<runtime>.snapshotter so it
+        // uses the correct snapshotter per runtime (e.g. nydus, erofs).
+        //
+        // A PR on the containerd side is open so we can rely on the runtime plugin
+        // snapshotter alone: https://github.com/containerd/containerd/pull/12836
+        if is_containerd_v3_config(pluginid) {
+            toml_utils::set_toml_value(
+                config_file,
+                &format!(
+                    ".plugins.{}.runtime_platforms.\"{}\".snapshotter",
+                    CONTAINERD_CRI_IMAGES_PLUGIN_ID,
+                    params.runtime_name
+                ),
+                snapshotter,
+            )?;
+        }
     }
 
     Ok(())
@@ -116,7 +150,10 @@ pub async fn configure_containerd_runtime(
 
     let paths = config.get_containerd_paths(runtime).await?;
     let configuration_file = get_containerd_output_path(&paths);
-    let pluginid = get_containerd_pluginid(&paths.config_file)?;
+    let pluginid = match paths.plugin_id.as_deref() {
+        Some(plugin_id) => plugin_id,
+        None => get_containerd_pluginid(&paths.config_file)?,
+    };
 
     log::info!(
         "configure_containerd_runtime: Writing to {:?}, pluginid={}",
@@ -187,7 +224,10 @@ pub async fn configure_custom_containerd_runtime(
 
     let paths = config.get_containerd_paths(runtime).await?;
     let configuration_file = get_containerd_output_path(&paths);
-    let pluginid = get_containerd_pluginid(&paths.config_file)?;
+    let pluginid = match paths.plugin_id.as_deref() {
+        Some(plugin_id) => plugin_id,
+        None => get_containerd_pluginid(&paths.config_file)?,
+    };
 
     log::info!(
         "configure_custom_containerd_runtime: Writing to {:?}, pluginid={}",
@@ -225,7 +265,9 @@ pub async fn configure_custom_containerd_runtime(
         snapshotter,
     };
 
-    write_containerd_runtime_config(&configuration_file, pluginid, &params)
+    write_containerd_runtime_config(&configuration_file, pluginid, &params)?;
+
+    Ok(())
 }
 
 pub async fn configure_containerd(config: &Config, runtime: &str) -> Result<()> {
@@ -347,13 +389,24 @@ pub async fn cleanup_containerd(config: &Config, runtime: &str) -> Result<()> {
     Ok(())
 }
 
-/// Setup containerd config files based on runtime type
-pub fn setup_containerd_config_files(runtime: &str, config: &Config) -> Result<()> {
+/// Setup containerd config files based on runtime type.
+/// For K3s/RKE2, resolves which template (v2 or v3) to use from the node's containerd version,
+/// then creates only that template file.
+pub async fn setup_containerd_config_files(runtime: &str, config: &Config) -> Result<()> {
+    const K3S_RKE2_BASE_TMPL: &str = "{{ template \"base\" . }}\n";
+
     match runtime {
         "k3s" | "k3s-agent" | "rke2-agent" | "rke2-server" => {
-            let tmpl_file = format!("{}.tmpl", config.containerd_conf_file);
-            if !Path::new(&tmpl_file).exists() && Path::new(&config.containerd_conf_file).exists() {
-                fs::copy(&config.containerd_conf_file, &tmpl_file)?;
+            // K3s/RKE2: create only the chosen template (v2 or v3). See docs.k3s.io/advanced#configuring-containerd
+            let paths = config.get_containerd_paths(runtime).await?;
+            let path = &paths.config_file;
+            if !Path::new(path).exists() {
+                if let Some(parent) = Path::new(path).parent() {
+                    fs::create_dir_all(parent)
+                        .with_context(|| format!("Failed to create containerd config dir: {parent:?}"))?;
+                }
+                fs::write(path, K3S_RKE2_BASE_TMPL)
+                    .with_context(|| format!("Failed to write K3s/RKE2 template: {path}"))?;
             }
         }
         "k0s-worker" | "k0s-controller" => {
@@ -516,102 +569,167 @@ pub fn snapshotter_handler_mapping_validation_check(config: &Config) -> Result<(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::utils::toml as toml_utils;
+    use rstest::rstest;
+    use std::path::Path;
+    use tempfile::NamedTempFile;
 
-    #[test]
-    fn test_check_containerd_snapshotter_version_support_1_6_with_mapping() {
-        // Version 1.6 with snapshotter mapping should fail
-        let result = check_containerd_snapshotter_version_support("containerd://1.6.28", true);
-        assert!(result.is_err());
-        assert!(result
-            .unwrap_err()
-            .to_string()
-            .contains("kata-deploy only supports snapshotter configuration with containerd 1.7 or newer"));
+    fn make_params(
+        runtime_name: &str,
+        snapshotter: Option<&str>,
+    ) -> ContainerdRuntimeParams {
+        ContainerdRuntimeParams {
+            runtime_name: runtime_name.to_string(),
+            runtime_path: "\"/opt/kata/bin/kata-runtime\"".to_string(),
+            config_path: "\"/opt/kata/share/defaults/kata-containers/configuration-qemu.toml\""
+                .to_string(),
+            pod_annotations: "[\"io.katacontainers.*\"]",
+            snapshotter: snapshotter.map(|s| s.to_string()),
+        }
     }
 
-    #[test]
-    fn test_check_containerd_snapshotter_version_support_1_6_without_mapping() {
-        // Version 1.6 without snapshotter mapping should pass (no mapping means no check needed)
-        let result = check_containerd_snapshotter_version_support("containerd://1.6.28", false);
-        assert!(result.is_ok());
-    }
+    /// CRI images runtime_platforms snapshotter is set only for v3 config when a snapshotter is configured.
+    #[rstest]
+    #[case(CONTAINERD_V3_RUNTIME_PLUGIN_ID, Some("\"nydus\""), "kata-qemu", true)]
+    #[case(CONTAINERD_V2_CRI_PLUGIN_ID, Some("\"nydus\""), "kata-qemu", false)]
+    #[case(CONTAINERD_V3_RUNTIME_PLUGIN_ID, None, "kata-qemu", false)]
+    #[case(CONTAINERD_V3_RUNTIME_PLUGIN_ID, Some("\"erofs\""), "kata-clh", true)]
+    fn test_write_containerd_runtime_config_cri_images_runtime_platforms_snapshotter(
+        #[case] pluginid: &str,
+        #[case] snapshotter: Option<&str>,
+        #[case] runtime_name: &str,
+        #[case] expect_runtime_platforms_set: bool,
+    ) {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path();
+        std::fs::write(path, "").unwrap();
 
-    #[test]
-    fn test_check_containerd_snapshotter_version_support_1_7_with_mapping() {
-        // Version 1.7 with snapshotter mapping should pass
-        let result = check_containerd_snapshotter_version_support("containerd://1.7.15", true);
-        assert!(result.is_ok());
-    }
+        let params = make_params(runtime_name, snapshotter);
+        write_containerd_runtime_config(path, pluginid, &params).unwrap();
 
-    #[test]
-    fn test_check_containerd_snapshotter_version_support_2_0_with_mapping() {
-        // Version 2.0 with snapshotter mapping should pass
-        let result = check_containerd_snapshotter_version_support("containerd://2.0.0", true);
-        assert!(result.is_ok());
-    }
+        let images_snapshotter_path = format!(
+            ".plugins.\"io.containerd.cri.v1.images\".runtime_platforms.\"{}\".snapshotter",
+            runtime_name
+        );
+        let result = toml_utils::get_toml_value(Path::new(path), &images_snapshotter_path);
 
-    #[test]
-    fn test_check_containerd_snapshotter_version_support_without_prefix() {
-        // Version without containerd:// prefix should still work
-        let result = check_containerd_snapshotter_version_support("1.6.28", true);
-        assert!(result.is_err());
-    }
-
-    #[test]
-    fn test_check_containerd_snapshotter_version_support_1_6_variants() {
-        // Test various 1.6.x versions
-        assert!(check_containerd_snapshotter_version_support("containerd://1.6.0", true).is_err());
-        assert!(check_containerd_snapshotter_version_support("containerd://1.6.28", true).is_err());
-        assert!(check_containerd_snapshotter_version_support("containerd://1.6.999", true).is_err());
-    }
-
-    #[test]
-    fn test_check_containerd_snapshotter_version_support_1_7_variants() {
-        // Test various 1.7+ versions should pass
-        assert!(check_containerd_snapshotter_version_support("containerd://1.7.0", true).is_ok());
-        assert!(check_containerd_snapshotter_version_support("containerd://1.7.15", true).is_ok());
-        assert!(check_containerd_snapshotter_version_support("containerd://1.8.0", true).is_ok());
-    }
-
-    #[test]
-    fn test_check_containerd_erofs_version_support() {
-        // Versions that should pass (2.2.0+)
-        let passing_versions = [
-            "containerd://2.2.0",
-            "containerd://2.2.0-rc.1",
-            "containerd://2.2.1",
-            "containerd://2.3.0",
-            "containerd://3.0.0",
-            "containerd://2.3.0-beta.0",
-            "2.2.0", // without prefix
-        ];
-        for version in passing_versions {
+        if expect_runtime_platforms_set {
+            let value = result.unwrap_or_else(|e| {
+                panic!(
+                    "expected CRI images runtime_platforms.{} snapshotter to be set: {}",
+                    runtime_name, e
+                )
+            });
+            assert_eq!(
+                value,
+                snapshotter.unwrap().trim_matches('"'),
+                "runtime_platforms snapshotter value"
+            );
+        } else {
             assert!(
-                check_containerd_erofs_version_support(version).is_ok(),
-                "Expected {} to pass",
-                version
+                result.is_err(),
+                "expected CRI images runtime_platforms.{} snapshotter not to be set for pluginid={:?} snapshotter={:?}",
+                runtime_name,
+                pluginid,
+                snapshotter
             );
         }
+    }
 
-        // Versions that should fail (< 2.2.0)
-        let failing_versions = [
-            ("containerd://2.1.0", "containerd must be 2.2.0 or newer"),
-            ("containerd://2.1.5-rc.1", "containerd must be 2.2.0 or newer"),
-            ("containerd://2.0.0", "containerd must be 2.2.0 or newer"),
-            ("containerd://1.7.0", "containerd must be 2.2.0 or newer"),
-            ("containerd://1.6.28", "containerd must be 2.2.0 or newer"),
-            ("2.1.0", "containerd must be 2.2.0 or newer"), // without prefix
-            ("invalid", "Invalid containerd version format"),
-            ("containerd://abc.2.0", "Failed to parse major version"),
-        ];
-        for (version, expected_error) in failing_versions {
-            let result = check_containerd_erofs_version_support(version);
-            assert!(result.is_err(), "Expected {} to fail", version);
-            assert!(
-                result.unwrap_err().to_string().contains(expected_error),
-                "Expected error for {} to contain '{}'",
-                version,
-                expected_error
-            );
+    /// Written containerd config (e.g. drop-in) must not start with blank lines when written to an initially empty file.
+    #[rstest]
+    #[case(CONTAINERD_V3_RUNTIME_PLUGIN_ID)]
+    #[case(CONTAINERD_V2_CRI_PLUGIN_ID)]
+    fn test_write_containerd_runtime_config_empty_file_no_leading_newlines(
+        #[case] pluginid: &str,
+    ) {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path();
+        std::fs::write(path, "").unwrap();
+
+        let params = make_params("kata-qemu", Some("\"nydus\""));
+        write_containerd_runtime_config(path, pluginid, &params).unwrap();
+
+        let content = std::fs::read_to_string(path).unwrap();
+        assert!(
+            !content.starts_with('\n'),
+            "containerd config must not start with newline(s), got {} leading newlines (pluginid={})",
+            content.chars().take_while(|&c| c == '\n').count(),
+            pluginid
+        );
+        assert!(
+            content.trim_start().starts_with('['),
+            "config should start with a TOML table"
+        );
+    }
+
+    #[rstest]
+    #[case("containerd://1.6.28", true, false, Some("kata-deploy only supports snapshotter configuration with containerd 1.7 or newer"))]
+    #[case("containerd://1.6.28", false, true, None)]
+    #[case("containerd://1.6.0", true, false, None)]
+    #[case("containerd://1.6.999", true, false, None)]
+    #[case("containerd://1.7.0", true, true, None)]
+    #[case("containerd://1.7.15", true, true, None)]
+    #[case("containerd://1.8.0", true, true, None)]
+    #[case("containerd://2.0.0", true, true, None)]
+    #[case("1.6.28", true, false, None)]
+    fn test_check_containerd_snapshotter_version_support(
+        #[case] version: &str,
+        #[case] has_mapping: bool,
+        #[case] expect_ok: bool,
+        #[case] expected_error_substring: Option<&str>,
+    ) {
+        let result = check_containerd_snapshotter_version_support(version, has_mapping);
+        if expect_ok {
+            assert!(result.is_ok(), "expected ok for version={} has_mapping={}", version, has_mapping);
+        } else {
+            assert!(result.is_err(), "expected err for version={} has_mapping={}", version, has_mapping);
+            if let Some(sub) = expected_error_substring {
+                assert!(
+                    result.unwrap_err().to_string().contains(sub),
+                    "error should contain {:?}",
+                    sub
+                );
+            }
         }
     }
+
+    #[rstest]
+    #[case("containerd://2.2.0")]
+    #[case("containerd://2.2.0-rc.1")]
+    #[case("containerd://2.2.1")]
+    #[case("containerd://2.3.0")]
+    #[case("containerd://3.0.0")]
+    #[case("containerd://2.3.0-beta.0")]
+    #[case("2.2.0")]
+    fn test_check_containerd_erofs_version_support_passing(#[case] version: &str) {
+        assert!(
+            check_containerd_erofs_version_support(version).is_ok(),
+            "Expected {} to pass",
+            version
+        );
+    }
+
+    #[rstest]
+    #[case("containerd://2.1.0", "containerd must be 2.2.0 or newer")]
+    #[case("containerd://2.1.5-rc.1", "containerd must be 2.2.0 or newer")]
+    #[case("containerd://2.0.0", "containerd must be 2.2.0 or newer")]
+    #[case("containerd://1.7.0", "containerd must be 2.2.0 or newer")]
+    #[case("containerd://1.6.28", "containerd must be 2.2.0 or newer")]
+    #[case("2.1.0", "containerd must be 2.2.0 or newer")]
+    #[case("invalid", "Invalid containerd version format")]
+    #[case("containerd://abc.2.0", "Failed to parse major version")]
+    fn test_check_containerd_erofs_version_support_failing(
+        #[case] version: &str,
+        #[case] expected_error: &str,
+    ) {
+        let result = check_containerd_erofs_version_support(version);
+        assert!(result.is_err(), "Expected {} to fail", version);
+        assert!(
+            result.unwrap_err().to_string().contains(expected_error),
+            "Expected error for {} to contain '{}'",
+            version,
+            expected_error
+        );
+    }
 }

tools/packaging/kata-deploy/binary/src/runtime/manager.rs

@@ -25,8 +25,18 @@ const CONTAINERD_BASED_RUNTIMES: &[&str] = &[
     "microk8s",
 ];
 
-/// Runtimes that don't support containerd drop-in configuration files
-const RUNTIMES_WITHOUT_CONTAINERD_DROP_IN_SUPPORT: &[&str] = &["crio"];
+/// Runtimes that don't support containerd drop-in configuration files.
+///
+/// K3s and RKE2 generate the final config from a template (config.toml.tmpl or
+/// config-v3.toml.tmpl); they do not merge drop-in files when rendering, so we
+/// must write the Kata runtime block directly into the template.
+const RUNTIMES_WITHOUT_CONTAINERD_DROP_IN_SUPPORT: &[&str] = &[
+    "crio",
+    "k3s",
+    "k3s-agent",
+    "rke2-agent",
+    "rke2-server",
+];
 
 fn is_containerd_based(runtime: &str) -> bool {
     CONTAINERD_BASED_RUNTIMES.contains(&runtime)
@@ -80,25 +90,33 @@ pub async fn get_container_runtime(config: &Config) -> Result<String> {
     Ok(runtime)
 }
 
-/// Check if a containerd version string supports drop-in files
-/// Returns Ok(()) if version >= 2.0, Err otherwise
-fn check_containerd_version_supports_drop_in(runtime_version: &str) -> Result<()> {
-    let version_re = Regex::new(r"containerd://(\d+)\.(\d+)")?;
+/// Returns true if containerRuntimeVersion (e.g. "containerd://2.1.5-k3s1") indicates
+/// containerd 2.x or newer, false for 1.x or unparseable. Used for drop-in support
+/// and for K3s/RKE2 template selection (config-v3.toml.tmpl vs config.toml.tmpl).
+pub fn containerd_version_is_2_or_newer(runtime_version: &str) -> bool {
+    let version_re = match Regex::new(r"containerd://(\d+)\.(\d+)") {
+        Ok(r) => r,
+        Err(_) => return false,
+    };
     if let Some(caps) = version_re.captures(runtime_version) {
-        let major: u32 = caps.get(1).unwrap().as_str().parse()?;
-        if major >= 2 {
-            return Ok(());
+        if let Ok(major) = caps.get(1).unwrap().as_str().parse::<u32>() {
+            return major >= 2;
         }
-        return Err(anyhow::anyhow!(
-            "containerd version {}.x does not support drop-in files (requires >= 2.0)",
-            major
-        ));
     }
-    // If version string is malformed/unparseable, conservatively assume no support
-    Err(anyhow::anyhow!(
-        "Unable to parse containerd version from '{}', assuming no drop-in support",
-        runtime_version
-    ))
+    false
+}
+
+/// Check if a containerd version string supports drop-in files.
+/// Wrapper around containerd_version_is_2_or_newer for call sites that need Result.
+fn check_containerd_version_supports_drop_in(runtime_version: &str) -> Result<()> {
+    if containerd_version_is_2_or_newer(runtime_version) {
+        Ok(())
+    } else {
+        Err(anyhow::anyhow!(
+            "containerd version does not support drop-in files (requires >= 2.0), got '{}'",
+            runtime_version
+        ))
+    }
 }
 
 pub async fn is_containerd_capable_of_using_drop_in_files(
@@ -166,69 +184,53 @@ pub async fn cleanup_cri_runtime(config: &Config, runtime: &str) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rstest::rstest;
 
-    /// Helper function to test version check with expected error result
-    fn assert_version_check_error(version: &str) {
+    // --- containerd_version_is_2_or_newer ---
+
+    #[rstest]
+    #[case("containerd://2.0.0", true)]
+    #[case("containerd://2.1.5", true)]
+    #[case("containerd://2.1.5-k3s1", true)]
+    #[case("containerd://2.2.0", true)]
+    #[case("containerd://2.3.1", true)]
+    #[case("containerd://2.0.0-rc.1", true)]
+    #[case("containerd://1.6.28", false)]
+    #[case("containerd://1.7.15", false)]
+    #[case("containerd://1.7.0", false)]
+    #[case("containerd://", false)]
+    #[case("1.7.0", false)]
+    #[case("not-a-version", false)]
+    fn test_containerd_version_is_2_or_newer(#[case] version: &str, #[case] expected: bool) {
+        assert_eq!(
+            containerd_version_is_2_or_newer(version),
+            expected,
+            "version: {}",
+            version
+        );
+    }
+
+    // --- check_containerd_version_supports_drop_in (Result wrapper) ---
+
+    #[rstest]
+    #[case("containerd://2.0.0", true)]
+    #[case("containerd://2.1.5-k3s1", true)]
+    #[case("containerd://1.7.15", false)]
+    #[case("containerd://1.6.28", false)]
+    #[case("containerd://", false)]
+    #[case("1.7.0", false)]
+    #[case("not-a-version", false)]
+    fn test_check_containerd_version_supports_drop_in(
+        #[case] version: &str,
+        #[case] expected_ok: bool,
+    ) {
         let result = check_containerd_version_supports_drop_in(version);
-        assert!(result.is_err(), "Expected error for version: {}", version);
-    }
-
-    /// Helper function to test version check with expected success result
-    fn assert_version_check_ok(version: &str) {
-        let result = check_containerd_version_supports_drop_in(version);
-        assert!(result.is_ok(), "Expected success for version: {}", version);
-    }
-
-    #[test]
-    fn test_containerd_version_1_6_returns_error() {
-        assert_version_check_error("containerd://1.6.28");
-    }
-
-    #[test]
-    fn test_containerd_version_1_7_returns_error() {
-        assert_version_check_error("containerd://1.7.15");
-    }
-
-    #[test]
-    fn test_containerd_version_2_0_returns_ok() {
-        assert_version_check_ok("containerd://2.0.0");
-    }
-
-    #[test]
-    fn test_containerd_version_2_1_returns_ok() {
-        assert_version_check_ok("containerd://2.1.5");
-    }
-
-    #[test]
-    fn test_containerd_version_2_2_returns_ok() {
-        assert_version_check_ok("containerd://2.2.0");
-    }
-
-    #[test]
-    fn test_containerd_version_2_3_returns_ok() {
-        assert_version_check_ok("containerd://2.3.1");
-    }
-
-    #[test]
-    fn test_containerd_version_with_prerelease() {
-        assert_version_check_ok("containerd://2.0.0-rc.1");
-    }
-
-    #[test]
-    fn test_containerd_version_invalid_format() {
-        // Missing version number - conservatively assume no support
-        assert_version_check_error("containerd://");
-    }
-
-    #[test]
-    fn test_containerd_version_no_protocol() {
-        // No protocol prefix - conservatively assume no support
-        assert_version_check_error("1.7.0");
-    }
-
-    #[test]
-    fn test_containerd_version_malformed() {
-        // Malformed version - conservatively assume no support
-        assert_version_check_error("not-a-version");
+        assert_eq!(
+            result.is_ok(),
+            expected_ok,
+            "version: {}, result: {:?}",
+            version,
+            result
+        );
     }
 }

tools/packaging/kata-deploy/binary/src/utils/toml.rs

@@ -50,12 +50,49 @@ fn parse_toml_path(path: &str) -> Result<Vec<String>> {
     Ok(parts)
 }
 
+/// K3s/RKE2 containerd config templates start with a Go template directive that
+/// is not valid TOML. This helper strips it before parsing and returns it so we
+/// can prepend it back when writing.
+fn split_non_toml_header(content: &str) -> (&str, &str) {
+    if content.starts_with("{{") {
+        if let Some(pos) = content.find('\n') {
+            return (&content[..=pos], &content[pos + 1..]);
+        }
+        return (content, "");
+    }
+    ("", content)
+}
+
+/// Write a TOML file with an optional non-TOML header (e.g. K3s template line).
+/// Ensures the header ends with a newline before the TOML body.
+/// Trims leading newlines from the serialized document to avoid many blank lines
+/// when the file was initially empty (e.g. containerd drop-in).
+fn write_toml_with_header(
+    file_path: &Path,
+    header: &str,
+    doc: &DocumentMut,
+) -> Result<()> {
+    let normalized_header = if header.is_empty() {
+        String::new()
+    } else if header.ends_with('\n') {
+        header.to_string()
+    } else {
+        format!("{header}\n")
+    };
+    let body = doc.to_string();
+    let body_trimmed = body.trim_start_matches('\n');
+    std::fs::write(file_path, format!("{}{}", normalized_header, body_trimmed))
+        .with_context(|| format!("Failed to write TOML file: {file_path:?}"))?;
+    Ok(())
+}
+
 /// Set a TOML value at a given path (e.g., ".plugins.cri.containerd.runtimes.kata.runtime_type")
 pub fn set_toml_value(file_path: &Path, path: &str, value: &str) -> Result<()> {
     let content = std::fs::read_to_string(file_path)
         .with_context(|| format!("Failed to read TOML file: {file_path:?}"))?;
 
-    let mut doc = content
+    let (header, toml_content) = split_non_toml_header(&content);
+    let mut doc = toml_content
         .parse::<DocumentMut>()
         .context("Failed to parse TOML")?;
 
@@ -83,8 +120,7 @@ pub fn set_toml_value(file_path: &Path, path: &str, value: &str) -> Result<()> {
         }
     }
 
-    std::fs::write(file_path, doc.to_string())
-        .with_context(|| format!("Failed to write TOML file: {file_path:?}"))?;
+    write_toml_with_header(file_path, header, &doc)?;
 
     Ok(())
 }
@@ -94,7 +130,8 @@ pub fn get_toml_value(file_path: &Path, path: &str) -> Result<String> {
     let content = std::fs::read_to_string(file_path)
         .with_context(|| format!("Failed to read TOML file: {file_path:?}"))?;
 
-    let doc = content
+    let (_header, toml_content) = split_non_toml_header(&content);
+    let doc = toml_content
         .parse::<DocumentMut>()
         .context("Failed to parse TOML")?;
 
@@ -161,7 +198,8 @@ pub fn append_to_toml_array(file_path: &Path, path: &str, value: &str) -> Result
     let content = std::fs::read_to_string(file_path)
         .with_context(|| format!("Failed to read TOML file: {file_path:?}"))?;
 
-    let mut doc = content
+    let (header, toml_content) = split_non_toml_header(&content);
+    let mut doc = toml_content
         .parse::<DocumentMut>()
         .context("Failed to parse TOML")?;
 
@@ -214,8 +252,7 @@ pub fn append_to_toml_array(file_path: &Path, path: &str, value: &str) -> Result
         }
     }
 
-    std::fs::write(file_path, doc.to_string())
-        .with_context(|| format!("Failed to write TOML file: {file_path:?}"))?;
+    write_toml_with_header(file_path, header, &doc)?;
 
     Ok(())
 }
@@ -225,7 +262,8 @@ pub fn remove_from_toml_array(file_path: &Path, path: &str, value: &str) -> Resu
     let content = std::fs::read_to_string(file_path)
         .with_context(|| format!("Failed to read TOML file: {file_path:?}"))?;
 
-    let mut doc = content
+    let (header, toml_content) = split_non_toml_header(&content);
+    let mut doc = toml_content
         .parse::<DocumentMut>()
         .context("Failed to parse TOML")?;
 
@@ -265,8 +303,7 @@ pub fn remove_from_toml_array(file_path: &Path, path: &str, value: &str) -> Resu
         }
     }
 
-    std::fs::write(file_path, doc.to_string())
-        .with_context(|| format!("Failed to write TOML file: {file_path:?}"))?;
+    write_toml_with_header(file_path, header, &doc)?;
 
     Ok(())
 }
@@ -276,7 +313,8 @@ pub fn get_toml_array(file_path: &Path, path: &str) -> Result<Vec<String>> {
     let content = std::fs::read_to_string(file_path)
         .with_context(|| format!("Failed to read TOML file: {file_path:?}"))?;
 
-    let doc = content
+    let (_header, toml_content) = split_non_toml_header(&content);
+    let doc = toml_content
         .parse::<DocumentMut>()
         .context("Failed to parse TOML")?;
 
@@ -319,7 +357,8 @@ pub fn set_toml_array(file_path: &Path, path: &str, values: &[String]) -> Result
     let content = std::fs::read_to_string(file_path)
         .with_context(|| format!("Failed to read TOML file: {file_path:?}"))?;
 
-    let mut doc = content
+    let (header, toml_content) = split_non_toml_header(&content);
+    let mut doc = toml_content
         .parse::<DocumentMut>()
         .context("Failed to parse TOML")?;
 
@@ -348,8 +387,7 @@ pub fn set_toml_array(file_path: &Path, path: &str, values: &[String]) -> Result
         }
     }
 
-    std::fs::write(file_path, doc.to_string())
-        .with_context(|| format!("Failed to write TOML file: {file_path:?}"))?;
+    write_toml_with_header(file_path, header, &doc)?;
 
     Ok(())
 }
@@ -394,8 +432,173 @@ fn parse_toml_value(value: &str) -> Item {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rstest::rstest;
     use tempfile::NamedTempFile;
 
+    // --- split_non_toml_header ---
+
+    #[rstest]
+    #[case("", "", "")]
+    #[case("key = \"value\"\n", "", "key = \"value\"\n")]
+    #[case("[plugins]\nfoo = 1\n", "", "[plugins]\nfoo = 1\n")]
+    #[case(
+        "{{ template \"base\" . }}\n",
+        "{{ template \"base\" . }}\n",
+        ""
+    )]
+    #[case(
+        "{{ template \"base\" . }}\n[plugins]\nfoo = 1\n",
+        "{{ template \"base\" . }}\n",
+        "[plugins]\nfoo = 1\n"
+    )]
+    #[case(
+        "{{ template \"base\" . }}\n\n[debug]\nlevel = \"debug\"\n",
+        "{{ template \"base\" . }}\n",
+        "\n[debug]\nlevel = \"debug\"\n"
+    )]
+    // No trailing newline after the template line
+    #[case("{{ template \"base\" . }}", "{{ template \"base\" . }}", "")]
+    fn test_split_non_toml_header(
+        #[case] input: &str,
+        #[case] expected_header: &str,
+        #[case] expected_toml: &str,
+    ) {
+        let (header, toml) = split_non_toml_header(input);
+        assert_eq!(header, expected_header, "header mismatch for input: {:?}", input);
+        assert_eq!(toml, expected_toml, "toml mismatch for input: {:?}", input);
+    }
+
+    // --- TOML operations on files with K3s/RKE2 base template header ---
+
+    #[rstest]
+    fn test_set_toml_value_with_k3s_header() {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path();
+        std::fs::write(path, "{{ template \"base\" . }}\n").unwrap();
+
+        set_toml_value(
+            path,
+            ".plugins.\"io.containerd.cri.v1.runtime\".containerd.runtimes.kata-qemu.runtime_type",
+            "\"io.containerd.kata-qemu.v2\"",
+        )
+        .unwrap();
+
+        let content = std::fs::read_to_string(path).unwrap();
+        assert!(content.starts_with("{{ template \"base\" . }}\n"), "header must be preserved");
+        assert!(content.contains("runtime_type"), "value must be written");
+
+        let value = get_toml_value(
+            path,
+            ".plugins.\"io.containerd.cri.v1.runtime\".containerd.runtimes.kata-qemu.runtime_type",
+        )
+        .unwrap();
+        assert_eq!(value, "io.containerd.kata-qemu.v2");
+    }
+
+    #[rstest]
+    fn test_set_toml_value_with_k3s_header_idempotent() {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path();
+        std::fs::write(path, "{{ template \"base\" . }}\n").unwrap();
+
+        for _ in 0..3 {
+            set_toml_value(path, ".debug.level", "\"debug\"").unwrap();
+        }
+
+        let content = std::fs::read_to_string(path).unwrap();
+        assert_eq!(
+            content.matches("{{ template \"base\" . }}").count(),
+            1,
+            "header must appear exactly once"
+        );
+        let value = get_toml_value(path, ".debug.level").unwrap();
+        assert_eq!(value, "debug");
+    }
+
+    #[rstest]
+    fn test_append_to_toml_array_with_k3s_header() {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path();
+        std::fs::write(path, "{{ template \"base\" . }}\nimports = []\n").unwrap();
+
+        append_to_toml_array(path, ".imports", "\"/etc/containerd/conf.d/kata.toml\"").unwrap();
+
+        let content = std::fs::read_to_string(path).unwrap();
+        assert!(content.starts_with("{{ template \"base\" . }}\n"));
+        assert!(content.contains("/etc/containerd/conf.d/kata.toml"));
+    }
+
+    #[rstest]
+    fn test_remove_from_toml_array_with_k3s_header() {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path();
+        std::fs::write(
+            path,
+            "{{ template \"base\" . }}\nimports = [\"/path/a\", \"/path/b\"]\n",
+        )
+        .unwrap();
+
+        remove_from_toml_array(path, ".imports", "\"/path/a\"").unwrap();
+
+        let content = std::fs::read_to_string(path).unwrap();
+        assert!(content.starts_with("{{ template \"base\" . }}\n"));
+        assert!(!content.contains("/path/a"));
+        assert!(content.contains("/path/b"));
+    }
+
+    #[rstest]
+    fn test_set_toml_array_with_k3s_header() {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path();
+        std::fs::write(path, "{{ template \"base\" . }}\n").unwrap();
+
+        set_toml_array(
+            path,
+            ".imports",
+            &["/path/one".to_string(), "/path/two".to_string()],
+        )
+        .unwrap();
+
+        let content = std::fs::read_to_string(path).unwrap();
+        assert!(content.starts_with("{{ template \"base\" . }}\n"));
+        let values = get_toml_array(path, ".imports").unwrap();
+        assert_eq!(values, vec!["/path/one", "/path/two"]);
+    }
+
+    #[rstest]
+    fn test_multiple_runtimes_with_k3s_header() {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path();
+        std::fs::write(path, "{{ template \"base\" . }}\n").unwrap();
+
+        let pluginid = "\"io.containerd.cri.v1.runtime\"";
+        for shim in &["kata-qemu", "kata-clh"] {
+            let table = format!(".plugins.{pluginid}.containerd.runtimes.{shim}");
+            set_toml_value(
+                path,
+                &format!("{table}.runtime_type"),
+                &format!("\"io.containerd.{shim}.v2\""),
+            )
+            .unwrap();
+            set_toml_value(path, &format!("{table}.privileged_without_host_devices"), "true")
+                .unwrap();
+        }
+
+        let content = std::fs::read_to_string(path).unwrap();
+        assert!(content.starts_with("{{ template \"base\" . }}\n"));
+        assert!(content.contains("kata-qemu"));
+        assert!(content.contains("kata-clh"));
+
+        for shim in &["kata-qemu", "kata-clh"] {
+            let rt = get_toml_value(
+                path,
+                &format!(".plugins.{pluginid}.containerd.runtimes.{shim}.runtime_type"),
+            )
+            .unwrap();
+            assert_eq!(rt, format!("io.containerd.{shim}.v2"));
+        }
+    }
+
     #[test]
     fn test_set_toml_value() {
         let file = NamedTempFile::new().unwrap();
@@ -412,6 +615,37 @@ mod tests {
         assert!(content.contains("runtime_type"));
     }
 
+    #[rstest]
+    #[case("", "")]
+    #[case("{{ template \"base\" . }}\n", "{{ template \"base\" . }}\n")]
+    fn test_set_toml_value_empty_file_no_leading_newlines(
+        #[case] initial_content: &str,
+        #[case] expected_prefix: &str,
+    ) {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path();
+        std::fs::write(path, initial_content).unwrap();
+
+        set_toml_value(
+            path,
+            ".plugins.\"io.containerd.cri.v1.runtime\".containerd.runtimes.kata-qemu.runtime_type",
+            "\"io.containerd.kata-qemu.v2\"",
+        )
+        .unwrap();
+        let content = std::fs::read_to_string(path).unwrap();
+        assert!(content.starts_with(expected_prefix), "header/prefix must be preserved");
+        let body_start = content.strip_prefix(expected_prefix).unwrap();
+        assert!(
+            !body_start.starts_with('\n'),
+            "written TOML body must not start with newline(s) after header, got {} leading newlines",
+            body_start.chars().take_while(|&c| c == '\n').count()
+        );
+        assert!(
+            body_start.trim_start().starts_with('['),
+            "body should start with a TOML table"
+        );
+    }
+
     #[test]
     fn test_get_toml_value() {
         let file = NamedTempFile::new().unwrap();
@@ -544,24 +778,22 @@ mod tests {
         assert_eq!(agent_debug, "false");
     }
 
-    #[test]
-    fn test_toml_value_types() {
+    #[rstest]
+    #[case("test.string_value", "test_string", "test_string")]
+    #[case("test.bool_value", "true", "true")]
+    #[case("test.int_value", "42", "42")]
+    fn test_toml_value_types(
+        #[case] path: &str,
+        #[case] value: &str,
+        #[case] expected: &str,
+    ) {
         let file = NamedTempFile::new().unwrap();
-        let path = file.path();
-        std::fs::write(path, "").unwrap();
+        let file_path = file.path();
+        std::fs::write(file_path, "").unwrap();
 
-        // Test different value types
-        set_toml_value(path, "test.string_value", "test_string").unwrap();
-        set_toml_value(path, "test.bool_value", "true").unwrap();
-        set_toml_value(path, "test.int_value", "42").unwrap();
-
-        let string_val = get_toml_value(path, "test.string_value").unwrap();
-        let bool_val = get_toml_value(path, "test.bool_value").unwrap();
-        let int_val = get_toml_value(path, "test.int_value").unwrap();
-
-        assert_eq!(string_val, "test_string");
-        assert_eq!(bool_val, "true");
-        assert_eq!(int_val, "42");
+        set_toml_value(file_path, path, value).unwrap();
+        let got = get_toml_value(file_path, path).unwrap();
+        assert_eq!(got, expected);
     }
 
     #[test]
@@ -1073,17 +1305,20 @@ kernel_params = "console=hvc0"
             .contains("Failed to read TOML file"));
     }
 
-    #[test]
-    fn test_get_toml_value_invalid_toml() {
+    #[rstest]
+    #[case("get")]
+    #[case("set")]
+    fn test_invalid_toml(#[case] op: &str) {
         let temp_file = NamedTempFile::new().unwrap();
         let temp_path = temp_file.path();
-
-        // Write invalid TOML
         std::fs::write(temp_path, "this is not [ valid toml {").unwrap();
 
-        let result = get_toml_value(temp_path, "some.path");
-        assert!(result.is_err(), "Should fail parsing invalid TOML");
-        // Just verify it's an error, don't check specific message
+        let result = match op {
+            "get" => get_toml_value(temp_path, "some.path").map(drop),
+            "set" => set_toml_value(temp_path, "some.path", "\"value\""),
+            _ => panic!("unknown op"),
+        };
+        assert!(result.is_err(), "Should fail parsing invalid TOML (op={})", op);
     }
 
     #[test]
@@ -1108,18 +1343,6 @@ kernel_params = "console=hvc0"
         assert!(result.is_err());
     }
 
-    #[test]
-    fn test_set_toml_value_invalid_toml() {
-        let temp_file = NamedTempFile::new().unwrap();
-        let temp_path = temp_file.path();
-
-        // Write invalid TOML
-        std::fs::write(temp_path, "this is not [ valid toml {").unwrap();
-
-        let result = set_toml_value(temp_path, "some.path", "\"value\"");
-        assert!(result.is_err(), "Should fail parsing invalid TOML");
-    }
-
     #[test]
     fn test_append_to_toml_array_nonexistent_file() {
         let result = append_to_toml_array(
@@ -1130,30 +1353,25 @@ kernel_params = "console=hvc0"
         assert!(result.is_err());
     }
 
-    #[test]
-    fn test_append_to_toml_array_not_an_array() {
+    #[rstest]
+    #[case("append")]
+    #[case("get")]
+    fn test_toml_array_not_an_array(#[case] op: &str) {
         let temp_file = NamedTempFile::new().unwrap();
         let temp_path = temp_file.path();
-
-        // Write TOML with a string, not an array
         std::fs::write(temp_path, "[section]\nkey = \"value\"").unwrap();
 
-        let result = append_to_toml_array(temp_path, "section.key", "\"item\"");
-        assert!(result.is_err());
-        assert!(result.unwrap_err().to_string().contains("not an array"));
-    }
-
-    #[test]
-    fn test_get_toml_array_not_an_array() {
-        let temp_file = NamedTempFile::new().unwrap();
-        let temp_path = temp_file.path();
-
-        // Write TOML with a string, not an array
-        std::fs::write(temp_path, "[section]\nkey = \"value\"").unwrap();
-
-        let result = get_toml_array(temp_path, "section.key");
-        assert!(result.is_err());
-        assert!(result.unwrap_err().to_string().contains("not an array"));
+        let result = match op {
+            "append" => append_to_toml_array(temp_path, "section.key", "\"item\"").map(drop),
+            "get" => get_toml_array(temp_path, "section.key").map(drop),
+            _ => panic!("unknown op"),
+        };
+        assert!(result.is_err(), "op={}", op);
+        assert!(
+            result.unwrap_err().to_string().contains("not an array"),
+            "op={}",
+            op
+        );
     }
 
     #[test]

tools/packaging/kata-deploy/helm-chart/README.md

@@ -229,6 +229,7 @@ shims:
     agent:
       httpsProxy: ""
       noProxy: ""
+    # Optional: set runtimeClass.nodeSelector to pin TEE to specific nodes (always applied). If unset, NFD TEE labels are auto-injected when NFD is detected.
 
 # Default shim per architecture
 defaultShim:
@@ -311,8 +312,8 @@ helm install kata-deploy oci://ghcr.io/kata-containers/kata-deploy-charts/kata-d
 Includes:
 - `qemu-snp` - AMD SEV-SNP (amd64)
 - `qemu-tdx` - Intel TDX (amd64)
-- `qemu-se` - IBM Secure Execution (s390x)
-- `qemu-se-runtime-rs` - IBM Secure Execution Rust runtime (s390x)
+- `qemu-se` - IBM Secure Execution for Linux (SEL) (s390x)
+- `qemu-se-runtime-rs` - IBM Secure Execution for Linux (SEL) Rust runtime (s390x)
 - `qemu-cca` - Arm Confidential Compute Architecture (arm64)
 - `qemu-coco-dev` - Confidential Containers development (amd64, s390x)
 - `qemu-coco-dev-runtime-rs` - Confidential Containers development Rust runtime (amd64, s390x)
@@ -334,6 +335,27 @@ Includes:
 
 **Note**: These example files are located in the chart directory. When installing from the OCI registry, you'll need to download them separately or clone the repository to access them.
 
+### RuntimeClass Node Selectors for TEE Shims
+
+**Manual configuration:** Any `nodeSelector` you set under `shims.<shim>.runtimeClass.nodeSelector`
+is **always applied** to that shim's RuntimeClass, whether or not NFD is present. Use this when
+you want to pin TEE workloads to specific nodes (e.g. without NFD, or with custom labels).
+
+**Auto-inject when NFD is present:** If you do *not* set a `runtimeClass.nodeSelector` for a
+TEE shim, the chart can **automatically inject** NFD-based labels when NFD is detected in the
+cluster (deployed by this chart with `node-feature-discovery.enabled=true` or found externally):
+- AMD SEV-SNP shims: `amd.feature.node.kubernetes.io/snp: "true"`
+- Intel TDX shims: `intel.feature.node.kubernetes.io/tdx: "true"`
+- IBM Secure Execution for Linux (SEL) shims (s390x): `feature.node.kubernetes.io/cpu-security.se.enabled: "true"`
+
+The chart uses Helm's `lookup` function to detect NFD (by looking for the
+`node-feature-discovery-worker` DaemonSet). Auto-inject only runs when NFD is detected and
+no manual `runtimeClass.nodeSelector` is set for that shim.
+
+**Note**: NFD detection requires cluster access. During `helm template` (dry-run without a
+cluster), external NFD is not seen, so auto-injected labels are not added. Manual
+`runtimeClass.nodeSelector` values are still applied in all cases.
+
 ## `RuntimeClass` Management
 
 **NEW**: Starting with Kata Containers v3.23.0, `runtimeClasses` are managed by

tools/packaging/kata-deploy/helm-chart/kata-deploy/Chart.yaml

@@ -15,13 +15,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: "3.26.0"
+version: "3.27.0"
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "3.26.0"
+appVersion: "3.27.0"
 
 dependencies:
   - name: node-feature-discovery