versions: bump containerd active version to 2.2

SSIA

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>

.github/dependabot.yml

@@ -12,7 +12,6 @@ updates:
       - "/src/tools/agent-ctl"
       - "/src/tools/genpolicy"
       - "/src/tools/kata-ctl"
-      - "/src/tools/runk"
       - "/src/tools/trace-forwarder"
     schedule:
       interval: "daily"

.github/workflows/basic-ci-amd64.yaml

@@ -26,8 +26,6 @@ jobs:
       matrix:
         containerd_version: ['active']
         vmm: ['dragonball', 'cloud-hypervisor', 'qemu-runtime-rs']
-    # TODO: enable me when https://github.com/containerd/containerd/issues/11640 is fixed
-    if: false
     runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: ${{ matrix.containerd_version }}
@@ -163,42 +161,6 @@ jobs:
         timeout-minutes: 10
         run: bash tests/integration/nydus/gha-run.sh run
 
-  run-runk:
-    name: run-runk
-    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
-    if: false
-    runs-on: ubuntu-22.04
-    env:
-      CONTAINERD_VERSION: lts
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          ref: ${{ inputs.commit-hash }}
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Rebase atop of the latest target branch
-        run: |
-          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
-        env:
-          TARGET_BRANCH: ${{ inputs.target-branch }}
-
-      - name: Install dependencies
-        run: bash tests/integration/runk/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Install kata
-        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
-
-      - name: Run runk tests
-        timeout-minutes: 10
-        run: bash tests/integration/runk/gha-run.sh run
-
   run-tracing:
     name: run-tracing
     strategy:

.github/workflows/basic-ci-s390x.yaml

@@ -26,8 +26,6 @@ jobs:
       matrix:
         containerd_version: ['active']
         vmm: ['qemu-runtime-rs']
-    # TODO: enable me when https://github.com/containerd/containerd/issues/11640 is fixed
-    if: false
     runs-on: s390x-large
     env:
       CONTAINERD_VERSION: ${{ matrix.containerd_version }}
@@ -48,7 +46,7 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Install dependencies
-        run: bash tests/integration/cri-containerd/gha-run.sh
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
         env:
           GH_TOKEN: ${{ github.token }}
 

.github/workflows/ci-nightly-rust.yaml

@@ -1,36 +0,0 @@
-name: Kata Containers Nightly CI (Rust)
-on:
-  schedule:
-    - cron: '0 1 * * *' # Run at 1 AM UTC (1 hour after script-based nightly)
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-permissions: {}
-
-jobs:
-  kata-containers-ci-on-push-rust:
-    permissions:
-      contents: read
-      packages: write
-      id-token: write
-      attestations: write
-    uses: ./.github/workflows/ci.yaml
-    with:
-      commit-hash: ${{ github.sha }}
-      pr-number: "nightly-rust"
-      tag: ${{ github.sha }}-nightly-rust
-      target-branch: ${{ github.ref_name }}
-      build-type: "rust" # Use Rust-based build
-    secrets:
-      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
-      AZ_APPID: ${{ secrets.AZ_APPID }}
-      AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
-      AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-      CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
-      ITA_KEY: ${{ secrets.ITA_KEY }}
-      QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-      NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
-      KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }}
-

.github/workflows/ci.yaml

@@ -19,11 +19,6 @@ on:
         required: false
         type: string
         default: no
-      build-type:
-        description: The build type for kata-deploy. Use 'rust' for Rust-based build, empty or omit for script-based (default).
-        required: false
-        type: string
-        default: ""
     secrets:
       AUTHENTICATED_IMAGE_PASSWORD:
         required: true
@@ -77,7 +72,6 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
       runner: ubuntu-22.04
       arch: amd64
-      build-type: ${{ inputs.build-type }}
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
@@ -110,7 +104,6 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
       runner: ubuntu-24.04-arm
       arch: arm64
-      build-type: ${{ inputs.build-type }}
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
@@ -156,7 +149,6 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
       runner: ubuntu-24.04-s390x
       arch: s390x
-      build-type: ${{ inputs.build-type }}
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
@@ -175,7 +167,6 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
       runner: ubuntu-24.04-ppc64le
       arch: ppc64le
-      build-type: ${{ inputs.build-type }}
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
@@ -297,7 +288,7 @@ jobs:
       tarball-suffix: -${{ inputs.tag }}
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ inputs.tag }}-amd64${{ inputs.build-type == 'rust' && '-rust' || '' }}
+      tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
@@ -313,7 +304,7 @@ jobs:
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ inputs.tag }}-arm64${{ inputs.build-type == 'rust' && '-rust' || '' }}
+      tag: ${{ inputs.tag }}-arm64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
@@ -326,7 +317,7 @@ jobs:
       tarball-suffix: -${{ inputs.tag }}
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ inputs.tag }}-amd64${{ inputs.build-type == 'rust' && '-rust' || '' }}
+      tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
@@ -348,7 +339,7 @@ jobs:
       tarball-suffix: -${{ inputs.tag }}
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ inputs.tag }}-amd64${{ inputs.build-type == 'rust' && '-rust' || '' }}
+      tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
@@ -366,7 +357,7 @@ jobs:
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ inputs.tag }}-s390x${{ inputs.build-type == 'rust' && '-rust' || '' }}
+      tag: ${{ inputs.tag }}-s390x
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
@@ -380,7 +371,7 @@ jobs:
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ inputs.tag }}-ppc64le${{ inputs.build-type == 'rust' && '-rust' || '' }}
+      tag: ${{ inputs.tag }}-ppc64le
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
@@ -392,7 +383,7 @@ jobs:
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ inputs.tag }}-amd64${{ inputs.build-type == 'rust' && '-rust' || '' }}
+      tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}

.github/workflows/payload-after-push.yaml

@@ -82,7 +82,6 @@ jobs:
       target-branch: ${{ github.ref_name }}
       runner: ubuntu-22.04
       arch: amd64
-      build-type: "" # Use script-based build (default)
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
@@ -100,7 +99,6 @@ jobs:
       target-branch: ${{ github.ref_name }}
       runner: ubuntu-24.04-arm
       arch: arm64
-      build-type: "" # Use script-based build (default)
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
@@ -118,7 +116,6 @@ jobs:
       target-branch: ${{ github.ref_name }}
       runner: s390x
       arch: s390x
-      build-type: "" # Use script-based build (default)
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
@@ -136,7 +133,6 @@ jobs:
       target-branch: ${{ github.ref_name }}
       runner: ubuntu-24.04-ppc64le
       arch: ppc64le
-      build-type: "" # Use script-based build (default)
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 

.github/workflows/publish-kata-deploy-payload.yaml

@@ -30,11 +30,6 @@ on:
         description: The arch of the tarball.
         required: true
         type: string
-      build-type:
-        description: The build type for kata-deploy. Use 'rust' for Rust-based build, empty or omit for script-based (default).
-        required: false
-        type: string
-        default: ""
     secrets:
       QUAY_DEPLOYER_PASSWORD:
         required: true
@@ -106,10 +101,8 @@ jobs:
           REGISTRY: ${{ inputs.registry }}
           REPO: ${{ inputs.repo }}
           TAG: ${{ inputs.tag }}
-          BUILD_TYPE: ${{ inputs.build-type }}
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
           "$(pwd)/kata-static.tar.zst" \
           "${REGISTRY}/${REPO}" \
-          "${TAG}" \
-          "${BUILD_TYPE}"
+          "${TAG}"

.github/workflows/run-k8s-tests-on-arm64.yaml

@@ -32,6 +32,7 @@ jobs:
       matrix:
         vmm:
           - qemu
+          - qemu-runtime-rs
         k8s:
           - kubeadm
     runs-on: arm64-k8s
@@ -65,7 +66,7 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh install-bats
 
       - name: Run tests
-        timeout-minutes: 60
+        timeout-minutes: 30
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Report tests

.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml

@@ -126,5 +126,6 @@ jobs:
 
       - name: Delete CoCo KBS
         if: always() && matrix.environment.name != 'nvidia-gpu'
+        timeout-minutes: 10
         run: |
           bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -137,10 +137,12 @@ jobs:
 
       - name: Delete kata-deploy
         if: always()
+        timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-zvsi
 
       - name: Delete CoCo KBS
         if: always()
+        timeout-minutes: 10
         run: |
           if [ "${KBS}" == "true" ]; then
             bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs

.github/workflows/run-kata-coco-tests.yaml

@@ -120,10 +120,12 @@ jobs:
 
       - name: Delete kata-deploy
         if: always()
+        timeout-minutes: 15
         run: bash tests/integration/kubernetes/gha-run.sh cleanup
 
       - name: Delete CoCo KBS
         if: always()
+        timeout-minutes: 10
         run: |
           [[ "${KATA_HYPERVISOR}" == "qemu-tdx" ]] && echo "ITA_KEY=${GH_ITA_KEY}" >> "${GITHUB_ENV}"
           bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs

.github/workflows/run-kata-deploy-tests.yaml

@@ -87,4 +87,4 @@ jobs:
 
       - name: Report tests
         if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh report-tests
+        run: bash tests/functional/kata-deploy/gha-run.sh report-tests

.github/workflows/run-runk-tests.yaml

@@ -1,54 +0,0 @@
-name: CI | Run runk tests
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      commit-hash:
-        required: false
-        type: string
-      target-branch:
-        required: false
-        type: string
-        default: ""
-
-permissions: {}
-
-jobs:
-  run-runk:
-    name: run-runk
-    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
-    if: false
-    runs-on: ubuntu-22.04
-    env:
-      CONTAINERD_VERSION: lts
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          ref: ${{ inputs.commit-hash }}
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Rebase atop of the latest target branch
-        run: |
-          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
-        env:
-          TARGET_BRANCH: ${{ inputs.target-branch }}
-
-      - name: Install dependencies
-        run: bash tests/integration/runk/gha-run.sh install-dependencies
-        env:
-          GH_TOKEN: ${{ github.token }}
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Install kata
-        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
-
-      - name: Run runk tests
-        run: bash tests/integration/runk/gha-run.sh run

.github/workflows/stale.yaml

@@ -6,14 +6,21 @@ on:
 
 permissions: {}
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   stale:
     name: stale
     runs-on: ubuntu-22.04
+    permissions:
+      actions: write # Needed to manage caches for state persistence across runs
+      pull-requests: write # Needed to add/remove labels, post comments, or close PRs
     steps:
       - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0
         with:
-          stale-pr-message: 'This PR has been opened without with no activity for 180 days. Comment on the issue otherwise it will be closed in 7 days'
+          stale-pr-message: 'This PR has been opened without activity for 180 days. Please comment on the issue or it will be closed in 7 days.'
           days-before-pr-stale: 180
           days-before-pr-close: 7
           days-before-issue-stale: -1

.github/workflows/zizmor.yaml

@@ -21,7 +21,7 @@ jobs:
           persist-credentials: false
 
       - name: Run zizmor
-        uses: zizmorcore/zizmor-action@e673c3917a1aef3c65c972347ed84ccd013ecda4 # v0.2.0
+        uses: zizmorcore/zizmor-action@135698455da5c3b3e55f73f4419e481ab68cdd95 # v0.4.1
         with:
           advanced-security: false
           annotations: true

Cargo.lock

@@ -4005,6 +4005,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "common",
+ "containerd-shim-protos",
  "go-flag",
  "logging",
  "nix 0.26.4",

Makefile

@@ -18,7 +18,6 @@ TOOLS =
 TOOLS += agent-ctl
 TOOLS += kata-ctl
 TOOLS += log-parser
-TOOLS += runk
 TOOLS += trace-forwarder
 
 STANDARD_TARGETS = build check clean install static-checks-build test vendor

README.md

@@ -139,7 +139,6 @@ The table below lists the remaining parts of the project:
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
 | [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
-| [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](.github/workflows) | CI | Continuous Integration configuration files and scripts. |
 | [`ocp-ci`](ci/openshift-ci/README.md) | CI | Continuous Integration configuration for the OpenShift pipelines. |
 | [`katacontainers.io`](https://github.com/kata-containers/www.katacontainers.io) | Source for the [`katacontainers.io`](https://www.katacontainers.io) site. |

VERSION

@@ -1 +1 @@
-3.24.0
+3.26.0

ci/openshift-ci/cleanup.sh

@@ -46,16 +46,12 @@ fi
 [[ ${SELINUX_PERMISSIVE} == "yes" ]] && oc delete -f "${deployments_dir}/machineconfig_selinux.yaml.in"
 
 # Delete kata-containers
-pushd "${katacontainers_repo_dir}/tools/packaging/kata-deploy" || { echo "Failed to push to ${katacontainers_repo_dir}/tools/packaging/kata-deploy"; exit 125; }
-oc delete -f kata-deploy/base/kata-deploy.yaml
+helm uninstall kata-deploy --wait --namespace kube-system
 oc -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-oc apply -f kata-cleanup/base/kata-cleanup.yaml
 echo "Wait for all related pods to be gone"
 ( repeats=1; for _ in $(seq 1 600); do
   oc get pods -l name="kubelet-kata-cleanup" --no-headers=true -n kube-system 2>&1 | grep "No resources found" -q && ((repeats++)) || repeats=1
   [[ "${repeats}" -gt 5 ]] && echo kata-cleanup finished && break
   sleep 1
 done) || { echo "There are still some kata-cleanup related pods after 600 iterations"; oc get all -n kube-system; exit 1; }
-oc delete -f kata-cleanup/base/kata-cleanup.yaml
-oc delete -f kata-rbac/base/kata-rbac.yaml
 oc delete -f runtimeclasses/kata-runtimeClasses.yaml

ci/openshift-ci/cluster/install_kata.sh

@@ -51,13 +51,13 @@ apply_kata_deploy() {
 
 	oc label --overwrite ns kube-system pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/warn=baseline pod-security.kubernetes.io/audit=baseline
 	local version chart
-	version=$(curl -sSL https://api.github.com/repos/kata-containers/kata-containers/releases/latest | jq .tag_name | tr -d '"')
+	version='0.0.0-dev'
 	chart="oci://ghcr.io/kata-containers/kata-deploy-charts/kata-deploy"
 
 	# Ensure any potential leftover is cleaned up ... and this secret usually is not in case of previous failures
 	oc delete secret sh.helm.release.v1.kata-deploy.v1 -n kube-system || true
 
-	echo "Installing kata using helm ${chart} ${version}"
+	echo "Installing kata using helm ${chart} ${version} (sha printed in helm output)"
 	helm install kata-deploy --wait --namespace kube-system --set "image.reference=${KATA_DEPLOY_IMAGE%%:*},image.tag=${KATA_DEPLOY_IMAGE##*:}" "${chart}" --version "${version}"
 }
 

ci/openshift-ci/peer-pods-azure.sh

@@ -157,6 +157,16 @@ if [[ -z "${CAA_IMAGE}" ]]; then
 fi
 
 # Get latest PP image
+#
+# You can list the CI images by:
+#     az sig image-version list-community --location "eastus" --public-gallery-name "cocopodvm-d0e4f35f-5530-4b9c-8596-112487cdea85" --gallery-image-definition "podvm_image0" --output table
+# or the release images by:
+#     az sig image-version list-community --location "eastus" --public-gallery-name "cococommunity-42d8482d-92cd-415b-b332-7648bd978eff" --gallery-image-definition "peerpod-podvm-fedora" --output table
+# or the release debug images by:
+#     az sig image-version list-community --location "eastus" --public-gallery-name "cococommunity-42d8482d-92cd-415b-b332-7648bd978eff" --gallery-image-definition "peerpod-podvm-fedora-debug" --output table
+#
+# Note there are other flavours of the released images, you can list them by:
+#     az sig image-definition list-community --location "eastus" --public-gallery-name "cococommunity-42d8482d-92cd-415b-b332-7648bd978eff" --output table
 if [[ -z "${PP_IMAGE_ID}" ]]; then
 	SUCCESS_TIME=$(curl -s \
 	  -H "Accept: application/vnd.github+json" \

docs/Developer-Guide.md

@@ -125,7 +125,7 @@ If you want to enable SELinux in Permissive mode, add `enforcing=0` to the kerne
 Enable full debug as follows:
 
 ```bash
-$ sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml
+$ sudo sed -i -E 's/^(\s*enable_debug\s*=\s*)false/\1true/' /etc/kata-containers/configuration.toml
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug initcall_debug"/g' /etc/kata-containers/configuration.toml
 ```
 

docs/design/architecture/storage.md

@@ -51,6 +51,7 @@ containers started after the VM has been launched.
 Users can check to see if the container uses the `devicemapper` block
 device as its rootfs by calling `mount(8)` within the container. If
 the `devicemapper` block device is used, the root filesystem (`/`)
-will be mounted from `/dev/vda`. Users can disable direct mounting of
-the underlying block device through the runtime
-[configuration](README.md#configuration).
+will be mounted from `/dev/vda`. Users can enable direct mounting of
+the underlying block device by setting the runtime
+[configuration](README.md#configuration) flag `disable_block_device_use` to
+`false`.

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -50,7 +50,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.default_max_vcpus` | uint32| the maximum number of vCPUs allocated for the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.default_memory` | uint32| the memory assigned for a VM by the hypervisor in `MiB` |
 | `io.katacontainers.config.hypervisor.default_vcpus` | float32| the default vCPUs assigned for a VM by the hypervisor |
-| `io.katacontainers.config.hypervisor.disable_block_device_use` | `boolean` | disallow a block device from being used |
+| `io.katacontainers.config.hypervisor.disable_block_device_use` | `boolean` | disable hotplugging host block devices to guest VMs for container rootfs |
 | `io.katacontainers.config.hypervisor.disable_image_nvdimm` | `boolean` | specify if a `nvdimm` device should be used as rootfs for the guest (QEMU) |
 | `io.katacontainers.config.hypervisor.disable_vhost_net` | `boolean` | specify if `vhost-net` is not available on the host |
 | `io.katacontainers.config.hypervisor.enable_hugepages` | `boolean` | if the memory should be `pre-allocated` from huge pages |

docs/install/minikube-installation-guide.md

@@ -103,48 +103,8 @@ $ minikube ssh "grep -c -E 'vmx|svm' /proc/cpuinfo"
 
 ## Installing Kata Containers
 
-You can now install the Kata Containers runtime components. You will need a local copy of some Kata
-Containers components to help with this, and then use `kubectl` on the host (that Minikube has already
-configured for you) to deploy them:
-
-```sh
-$ git clone https://github.com/kata-containers/kata-containers.git
-$ cd kata-containers/tools/packaging/kata-deploy
-$ kubectl apply -f kata-rbac/base/kata-rbac.yaml
-$ kubectl apply -f kata-deploy/base/kata-deploy.yaml
-```
-
-This installs the Kata Containers components into `/opt/kata` inside the Minikube node. It can take
-a few minutes for the operation to complete. You can check the installation has worked by checking
-the status of the `kata-deploy` pod, which will be executing
-[this script](../../tools/packaging/kata-deploy/scripts/kata-deploy.sh),
-and will be executing a `sleep infinity` once it has successfully completed its work.
-You can accomplish this by running the following:
-
-```sh
-$ podname=$(kubectl -n kube-system get pods -o=name | grep -F kata-deploy | sed 's?pod/??')
-$ kubectl -n kube-system exec ${podname} -- ps -ef | grep -F infinity
-```
-
-> *NOTE:* This check only works for single node clusters, which is the default for Minikube.
-> For multi-node clusters, the check would need to be adapted to check `kata-deploy` had
-> completed on all nodes.
-
-## Enabling Kata Containers
-
-Now you have installed the Kata Containers components in the Minikube node. Next, you need to configure
-Kubernetes `RuntimeClass` to know when to use Kata Containers to run a pod.
-
-### Register the runtime
-
-Now register the `kata qemu` runtime with that class. This should result in no errors:
-
-```sh
-$ cd kata-containers/tools/packaging/kata-deploy/runtimeclasses
-$ kubectl apply -f kata-runtimeClasses.yaml
-```
-
-The Kata Containers installation process should be complete and enabled in the Minikube cluster.
+You can now install the Kata Containers runtime components
+[following the official instructions](../../tools/packaging/kata-deploy/helm-chart).
 
 ## Testing Kata Containers
 

src/agent/Cargo.lock

@@ -4305,6 +4305,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
 name = "test-utils"
 version = "0.1.0"
 dependencies = [
+ "libc",
  "nix 0.26.4",
 ]
 

src/agent/rustjail/src/container.rs

@@ -1588,9 +1588,11 @@ async fn join_namespaces(
         cm.apply(p.pid)?;
     }
 
-    if p.init && res.is_some() {
-        info!(logger, "set properties to cgroups!");
-        cm.set(res.unwrap(), false)?;
+    if p.init {
+        if let Some(resource) = res {
+            info!(logger, "set properties to cgroups!");
+            cm.set(resource, false)?;
+        }
     }
 
     info!(logger, "notify child to continue");

src/agent/rustjail/src/mount.rs

@@ -752,15 +752,6 @@ fn parse_mount(m: &Mount) -> (MsFlags, MsFlags, String) {
     (flags, pgflags, data.join(","))
 }
 
-// This function constructs a canonicalized path by combining the `rootfs` and `unsafe_path` elements.
-// The resulting path is guaranteed to be ("below" / "in a directory under") the `rootfs` directory.
-//
-// Parameters:
-//
-// - `rootfs` is the absolute path to the root of the containers root filesystem directory.
-// - `unsafe_path` is path inside a container. It is unsafe since it may try to "escape" from the containers
-//    rootfs by using one or more "../" path elements or is its a symlink to path.
-
 fn mount_from(
     cfd_log: RawFd,
     m: &Mount,

src/dragonball/src/api/v1/vmm_action.rs

@@ -10,7 +10,7 @@ use std::fs::File;
 use std::sync::{Arc, Mutex};
 
 use crossbeam_channel::{Receiver, Sender, TryRecvError};
-use log::{debug, error, info, warn};
+use log::{debug, info, warn};
 use std::sync::mpsc;
 use tracing::instrument;
 

src/dragonball/src/device_manager/mod.rs

@@ -24,7 +24,6 @@ use dbs_legacy_devices::ConsoleHandler;
 use dbs_pci::CAPABILITY_BAR_SIZE;
 use dbs_utils::epoll_manager::EpollManager;
 use kvm_ioctls::VmFd;
-use log::error;
 use virtio_queue::QueueSync;
 
 #[cfg(feature = "dbs-virtio-devices")]

src/libs/kata-types/src/annotations/mod.rs

@@ -149,6 +149,9 @@ pub const KATA_ANNO_CFG_HYPERVISOR_KERNEL_HASH: &str =
 /// A sandbox annotation for passing additional guest kernel parameters.
 pub const KATA_ANNO_CFG_HYPERVISOR_KERNEL_PARAMS: &str =
     "io.katacontainers.config.hypervisor.kernel_params";
+/// A sandbox annotation for passing guest dm-verity parameters.
+pub const KATA_ANNO_CFG_HYPERVISOR_KERNEL_VERITY_PARAMS: &str =
+    "io.katacontainers.config.hypervisor.kernel_verity_params";
 /// A sandbox annotation for passing a container guest image path.
 pub const KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH: &str = "io.katacontainers.config.hypervisor.image";
 /// A sandbox annotation for passing a container guest image SHA-512 hash value.
@@ -630,6 +633,9 @@ impl Annotation {
                     KATA_ANNO_CFG_HYPERVISOR_KERNEL_PARAMS => {
                         hv.boot_info.replace_kernel_params(value);
                     }
+                    KATA_ANNO_CFG_HYPERVISOR_KERNEL_VERITY_PARAMS => {
+                        hv.boot_info.replace_kernel_verity_params(value)?;
+                    }
                     KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH => {
                         hv.boot_info.validate_boot_path(value)?;
                         hv.boot_info.image = value.to_string();

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -76,6 +76,134 @@ const VIRTIO_FS_INLINE: &str = "inline-virtio-fs";
 const MAX_BRIDGE_SIZE: u32 = 5;
 
 const KERNEL_PARAM_DELIMITER: &str = " ";
+/// Block size (in bytes) used by dm-verity block size validation.
+pub const VERITY_BLOCK_SIZE_BYTES: u64 = 512;
+/// Parsed kernel dm-verity parameters.
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub struct KernelVerityParams {
+    /// Root hash value.
+    pub root_hash: String,
+    /// Salt used to generate verity hash tree.
+    pub salt: String,
+    /// Number of data blocks in the verity mapping.
+    pub data_blocks: u64,
+    /// Data block size in bytes.
+    pub data_block_size: u64,
+    /// Hash block size in bytes.
+    pub hash_block_size: u64,
+}
+
+/// Parse and validate kernel dm-verity parameters.
+pub fn parse_kernel_verity_params(params: &str) -> Result<Option<KernelVerityParams>> {
+    if params.trim().is_empty() {
+        return Ok(None);
+    }
+
+    let mut values = HashMap::new();
+    for field in params.split(',') {
+        let field = field.trim();
+        if field.is_empty() {
+            continue;
+        }
+        let mut parts = field.splitn(2, '=');
+        let key = parts.next().unwrap_or("");
+        let value = parts.next().ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("Invalid kernel_verity_params entry: {field}"),
+            )
+        })?;
+        if key.is_empty() {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("Invalid kernel_verity_params entry: {field}"),
+            ));
+        }
+        values.insert(key.to_string(), value.to_string());
+    }
+
+    let root_hash = values
+        .get("root_hash")
+        .ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::InvalidData,
+                "Missing kernel_verity_params root_hash",
+            )
+        })?
+        .to_string();
+
+    let salt = values.get("salt").cloned().unwrap_or_default();
+
+    let parse_uint_field = |name: &str| -> Result<u64> {
+        match values.get(name) {
+            Some(value) if !value.is_empty() => value.parse::<u64>().map_err(|e| {
+                io::Error::new(
+                    io::ErrorKind::InvalidData,
+                    format!("Invalid kernel_verity_params {} '{}': {}", name, value, e),
+                )
+            }),
+            _ => Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("Missing kernel_verity_params {name}"),
+            )),
+        }
+    };
+
+    let data_blocks = parse_uint_field("data_blocks")?;
+    let data_block_size = parse_uint_field("data_block_size")?;
+    let hash_block_size = parse_uint_field("hash_block_size")?;
+
+    if salt.is_empty() {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            "Missing kernel_verity_params salt",
+        ));
+    }
+    if data_blocks == 0 {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            "Invalid kernel_verity_params data_blocks: must be non-zero",
+        ));
+    }
+    if data_block_size == 0 {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            "Invalid kernel_verity_params data_block_size: must be non-zero",
+        ));
+    }
+    if hash_block_size == 0 {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            "Invalid kernel_verity_params hash_block_size: must be non-zero",
+        ));
+    }
+    if data_block_size % VERITY_BLOCK_SIZE_BYTES != 0 {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            format!(
+                "Invalid kernel_verity_params data_block_size: must be multiple of {}",
+                VERITY_BLOCK_SIZE_BYTES
+            ),
+        ));
+    }
+    if hash_block_size % VERITY_BLOCK_SIZE_BYTES != 0 {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            format!(
+                "Invalid kernel_verity_params hash_block_size: must be multiple of {}",
+                VERITY_BLOCK_SIZE_BYTES
+            ),
+        ));
+    }
+
+    Ok(Some(KernelVerityParams {
+        root_hash,
+        salt,
+        data_blocks,
+        data_block_size,
+        hash_block_size,
+    }))
+}
 
 lazy_static! {
     static ref HYPERVISOR_PLUGINS: Mutex<HashMap<String, Arc<dyn ConfigPlugin>>> =
@@ -294,6 +422,10 @@ pub struct BootInfo {
     #[serde(default)]
     pub kernel_params: String,
 
+    /// Guest kernel dm-verity parameters.
+    #[serde(default)]
+    pub kernel_verity_params: String,
+
     /// Path to initrd file on host.
     #[serde(default)]
     pub initrd: String,
@@ -441,6 +573,17 @@ impl BootInfo {
         self.kernel_params = all_params.join(KERNEL_PARAM_DELIMITER);
     }
 
+    /// Replace kernel dm-verity parameters after validation.
+    pub fn replace_kernel_verity_params(&mut self, new_params: &str) -> Result<()> {
+        if new_params.trim().is_empty() {
+            return Ok(());
+        }
+
+        parse_kernel_verity_params(new_params)?;
+        self.kernel_verity_params = new_params.to_string();
+        Ok(())
+    }
+
     /// Validate guest kernel image annotation.
     pub fn validate_boot_path(&self, path: &str) -> Result<()> {
         validate_path!(path, "path {} is invalid{}")?;
@@ -770,10 +913,11 @@ impl MachineInfo {
 }
 
 /// Huge page type for VM RAM backend
-#[derive(Clone, Debug, Deserialize_enum_str, Serialize_enum_str, PartialEq, Eq)]
+#[derive(Clone, Debug, Deserialize_enum_str, Serialize_enum_str, PartialEq, Eq, Default)]
 pub enum HugePageType {
     /// Memory allocated using hugetlbfs backend
     #[serde(rename = "hugetlbfs")]
+    #[default]
     Hugetlbfs,
 
     /// Memory allocated using transparent huge pages
@@ -781,12 +925,6 @@ pub enum HugePageType {
     THP,
 }
 
-impl Default for HugePageType {
-    fn default() -> Self {
-        Self::Hugetlbfs
-    }
-}
-
 /// Virtual machine memory configuration information.
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct MemoryInfo {

src/libs/kata-types/src/config/mod.rs

@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::fs;
 use std::io::{self, Result};
 use std::path::{Path, PathBuf};
@@ -206,8 +206,8 @@ impl TomlConfig {
     }
 
     /// Get agent-specfic kernel parameters for further Hypervisor config revision
-    pub fn get_agent_kernel_params(&self) -> Result<HashMap<String, String>> {
-        let mut kv = HashMap::new();
+    pub fn get_agent_kernel_params(&self) -> Result<BTreeMap<String, String>> {
+        let mut kv = BTreeMap::new();
         if let Some(cfg) = self.agent.get(&self.runtime.agent_name) {
             if cfg.debug {
                 kv.insert(LOG_LEVEL_OPTION.to_string(), LOG_LEVEL_DEBUG.to_string());

src/libs/kata-types/src/initdata.rs

@@ -366,8 +366,8 @@ key = "value"
 
         let result = add_hypervisor_initdata_overrides(&encoded);
         // This might fail depending on whether algorithm is required
-        if result.is_err() {
-            assert!(result.unwrap_err().to_string().contains("parse initdata"));
+        if let Err(error) = result {
+            assert!(error.to_string().contains("parse initdata"));
         }
     }
 
@@ -386,8 +386,8 @@ key = "value"
 
         let result = add_hypervisor_initdata_overrides(&encoded);
         // This might fail depending on whether version is required
-        if result.is_err() {
-            assert!(result.unwrap_err().to_string().contains("parse initdata"));
+        if let Err(error) = result {
+            assert!(error.to_string().contains("parse initdata"));
         }
     }
 
@@ -488,7 +488,7 @@ key = "value"
         let valid_toml = r#"
             version = "0.1.0"
             algorithm = "sha384"
-            
+
             [data]
             valid_key = "valid_value"
         "#;
@@ -497,7 +497,7 @@ key = "value"
         // Invalid TOML (missing version)
         let invalid_toml = r#"
             algorithm = "sha256"
-            
+
             [data]
             key = "value"
         "#;

src/libs/test-utils/src/lib.rs

@@ -136,8 +136,6 @@ macro_rules! skip_loop_by_user {
 
 #[cfg(test)]
 mod tests {
-    use super::{skip_if_kvm_unaccessable, skip_if_not_root, skip_if_root};
-
     #[test]
     fn test_skip_if_not_root() {
         skip_if_not_root!();

src/runtime-rs/Cargo.toml

@@ -22,6 +22,7 @@ cloud-hypervisor = ["runtimes/cloud-hypervisor"]
 
 [dependencies]
 anyhow = { workspace = true }
+containerd-shim-protos = { workspace = true }
 go-flag = { workspace = true }
 nix = { workspace = true }
 tokio = { workspace = true, features = ["rt", "rt-multi-thread"] }

src/runtime-rs/Makefile

@@ -130,8 +130,33 @@ FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD)
 FCVALIDJAILERPATHS = [\"$(FCJAILERPATH)\"]
 
 PKGLIBEXECDIR := $(LIBEXECDIR)/$(PROJECT_DIR)
+
+# EDK2 firmware names per architecture
+ifeq ($(ARCH), aarch64)
+    EDK2_NAME := aavmf
+endif
+
+# Set firmware paths from QEMUFW/QEMUFWVOL if defined
 FIRMWAREPATH :=
 FIRMWAREVOLUMEPATH :=
+ifneq (,$(QEMUCMD))
+    ifneq (,$(QEMUFW))
+        FIRMWAREPATH := $(PREFIXDEPS)/share/$(EDK2_NAME)/$(QEMUFW)
+    endif
+    ifneq (,$(QEMUFWVOL))
+        FIRMWAREVOLUMEPATH := $(PREFIXDEPS)/share/$(EDK2_NAME)/$(QEMUFWVOL)
+    endif
+endif
+
+KERNELVERITYPARAMS ?= ""
+
+# TDX
+DEFSHAREDFS_QEMU_TDX_VIRTIOFS := none
+FIRMWARETDXPATH := $(PREFIXDEPS)/share/ovmf/OVMF.inteltdx.fd
+
+# SEV-SNP
+FIRMWARE_SNP_PATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd
+FIRMWARE_VOLUME_SNP_PATH :=
 
 ##VAR DEFVCPUS=<number> Default number of vCPUs
 DEFVCPUS := 1
@@ -150,8 +175,8 @@ DEFMEMSLOTS := 10
 DEFMAXMEMSZ := 0
 ##VAR DEFBRIDGES=<number> Default number of bridges
 DEFBRIDGES := 1
-DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"default_vcpus\", \"default_memory\"]
-DEFENABLEANNOTATIONS_COCO := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"default_vcpus\", \"default_memory\", \"cc_init_data\"]
+DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"kernel_verity_params\", \"default_vcpus\", \"default_memory\"]
+DEFENABLEANNOTATIONS_COCO := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"kernel_verity_params\", \"default_vcpus\", \"default_memory\", \"cc_init_data\"]
 DEFDISABLEGUESTSECCOMP := true
 DEFDISABLEGUESTEMPTYDIR := false
 ##VAR DEFAULTEXPFEATURES=[features] Default experimental features enabled
@@ -176,6 +201,7 @@ DEFVIRTIOFSQUEUESIZE ?= 1024
 # Make sure you quote args.
 DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"-o\", \"announce_submounts\"]
 DEFENABLEIOTHREADS := false
+DEFINDEPIOTHREADS := 0
 DEFENABLEVHOSTUSERSTORE := false
 DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user
 DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"]
@@ -192,6 +218,8 @@ QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT := 4050
 DEFCREATECONTAINERTIMEOUT ?= 30
 DEFCREATECONTAINERTIMEOUT_COCO ?= 60
 DEFSTATICRESOURCEMGMT_COCO = true
+DEFDISABLEIMAGENVDIMM ?= false
+DEFPODRESOURCEAPISOCK := ""
 
 SED = sed
 CLI_DIR = cmd
@@ -292,6 +320,30 @@ ifneq (,$(QEMUCMD))
 
     CONFIGS += $(CONFIG_QEMU)
 
+    CONFIG_FILE_QEMU_TDX = configuration-qemu-tdx-runtime-rs.toml
+    CONFIG_QEMU_TDX = config/$(CONFIG_FILE_QEMU_TDX)
+    CONFIG_QEMU_TDX_IN = $(CONFIG_QEMU_TDX).in
+
+    CONFIG_PATH_QEMU_TDX = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_TDX))
+    CONFIG_PATHS += $(CONFIG_PATH_QEMU_TDX)
+
+    SYSCONFIG_QEMU_TDX = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_TDX))
+    SYSCONFIG_PATHS += $(SYSCONFIG_QEMU_TDX)
+
+    CONFIGS += $(CONFIG_QEMU_TDX)
+
+    CONFIG_FILE_QEMU_SNP = configuration-qemu-snp-runtime-rs.toml
+    CONFIG_QEMU_SNP = config/$(CONFIG_FILE_QEMU_SNP)
+    CONFIG_QEMU_SNP_IN = $(CONFIG_QEMU_SNP).in
+
+    CONFIG_PATH_QEMU_SNP = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SNP))
+    CONFIG_PATHS += $(CONFIG_PATH_QEMU_SNP)
+
+    SYSCONFIG_QEMU_SNP = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SNP))
+    SYSCONFIG_PATHS += $(SYSCONFIG_QEMU_SNP)
+
+    CONFIGS += $(CONFIG_QEMU_SNP)
+
     CONFIG_FILE_QEMU_SE = configuration-qemu-se-runtime-rs.toml
     CONFIG_QEMU_SE = config/$(CONFIG_FILE_QEMU_SE)
     CONFIG_QEMU_SE_IN = $(CONFIG_QEMU_SE).in
@@ -336,6 +388,11 @@ ifneq (,$(QEMUCMD))
 ifeq ($(ARCH), s390x)
     VMROOTFSDRIVER_QEMU := virtio-blk-ccw
     DEFBLOCKSTORAGEDRIVER_QEMU := virtio-blk-ccw
+else ifeq ($(ARCH), aarch64)
+    # NVDIMM/virtio-pmem has issues on arm64 (cache coherency problems with DAX),
+    # so we use virtio-blk-pci instead.
+    VMROOTFSDRIVER_QEMU := virtio-blk-pci
+    DEFBLOCKSTORAGEDRIVER_QEMU := virtio-scsi
 else
     VMROOTFSDRIVER_QEMU := virtio-pmem
     DEFBLOCKSTORAGEDRIVER_QEMU := virtio-scsi
@@ -469,6 +526,7 @@ USER_VARS += MACHINEACCELERATORS
 USER_VARS += CPUFEATURES
 USER_VARS += DEFMACHINETYPE_CLH
 USER_VARS += KERNELPARAMS
+USER_VARS += KERNELVERITYPARAMS
 USER_VARS += KERNELPARAMS_DB
 USER_VARS += KERNELPARAMS_FC
 USER_VARS += LIBEXECDIR
@@ -522,6 +580,7 @@ USER_VARS += DEFVIRTIOFSEXTRAARGS
 USER_VARS += DEFENABLEANNOTATIONS
 USER_VARS += DEFENABLEANNOTATIONS_COCO
 USER_VARS += DEFENABLEIOTHREADS
+USER_VARS += DEFINDEPIOTHREADS
 USER_VARS += DEFSECCOMPSANDBOXPARAM
 USER_VARS += DEFGUESTSELINUXLABEL
 USER_VARS += DEFENABLEVHOSTUSERSTORE
@@ -542,6 +601,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_FC
 USER_VARS += DEFSTATICRESOURCEMGMT_CLH
 USER_VARS += DEFSTATICRESOURCEMGMT_QEMU
 USER_VARS += DEFSTATICRESOURCEMGMT_COCO
+USER_VARS += DEFDISABLEIMAGENVDIMM
 USER_VARS += DEFBINDMOUNTS
 USER_VARS += DEFVFIOMODE
 USER_VARS += DEFVFIOMODE_SE
@@ -562,6 +622,13 @@ USER_VARS += DEFFORCEGUESTPULL
 USER_VARS += QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT
 USER_VARS += DEFCREATECONTAINERTIMEOUT
 USER_VARS += DEFCREATECONTAINERTIMEOUT_COCO
+USER_VARS += QEMUTDXEXPERIMENTALCMD
+USER_VARS += FIRMWARE_SNP_PATH
+USER_VARS += FIRMWARE_VOLUME_SNP_PATH
+USER_VARS += KERNELTDXPARAMS
+USER_VARS += DEFSHAREDFS_QEMU_TDX_VIRTIOFS
+USER_VARS += FIRMWARETDXPATH
+USER_VARS += DEFPODRESOURCEAPISOCK
 
 SOURCES := \
   $(shell find . 2>&1 | grep -E '.*\.rs$$') \
@@ -599,6 +666,8 @@ GENERATED_VARS = \
 		VERSION \
 		CONFIG_DB_IN \
 		CONFIG_FC_IN \
+		CONFIG_QEMU_TDX_IN \
+		CONFIG_QEMU_SNP_IN \
 		$(USER_VARS)
 
 

src/runtime-rs/arch/aarch64-options.mk

@@ -4,12 +4,16 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-MACHINETYPE :=
+# ARM 64 settings
+
+MACHINETYPE := virt
 KERNELPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1
-MACHINEACCELERATORS :=
+MACHINEACCELERATORS := usb=off,gic-version=host
 CPUFEATURES := pmu=off
 
 QEMUCMD := qemu-system-aarch64
+QEMUFW := AAVMF_CODE.fd
+QEMUFWVOL := AAVMF_VARS.fd
 
 # dragonball binary name
 DBCMD := dragonball

src/runtime-rs/config/configuration-cloud-hypervisor.toml.in

@@ -19,7 +19,7 @@ image = "@IMAGEPATH@"
 #   - xfs
 #   - erofs
 rootfs_type = @DEFROOTFSTYPE@
- 
+
 # Block storage driver to be used for the VM rootfs is backed
 # by a block device.
 vm_rootfs_driver = "@VMROOTFSDRIVER_CLH@"
@@ -41,7 +41,7 @@ valid_hypervisor_paths = @CLHVALIDHYPERVISORPATHS@
 
 # List of valid annotations values for ctlpath
 # The default if not set is empty (all annotations rejected.)
-# Your distribution recommends: 
+# Your distribution recommends:
 valid_ctlpaths = []
 
 # Optional space-separated list of options to pass to the guest kernel.

src/runtime-rs/config/configuration-dragonball.toml.in

@@ -23,7 +23,7 @@ image = "@IMAGEPATH@"
 #   - erofs
 rootfs_type = @DEFROOTFSTYPE@
 
- 
+
 # Block storage driver to be used for the VM rootfs is backed
 # by a block device. This is virtio-blk-pci, virtio-blk-mmio or nvdimm
 vm_rootfs_driver = "@VMROOTFSDRIVER_DB@"
@@ -41,7 +41,7 @@ valid_hypervisor_paths = @DBVALIDHYPERVISORPATHS@
 
 # List of valid annotations values for ctlpath
 # The default if not set is empty (all annotations rejected.)
-# Your distribution recommends: 
+# Your distribution recommends:
 valid_ctlpaths = []
 
 # Optional space-separated list of options to pass to the guest kernel.

src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in

@@ -373,16 +373,16 @@ disable_image_nvdimm = false
 # Default false
 hotplug_vfio_on_root_bus = false
 
-# Enable hot-plugging of VFIO devices to a bridge-port, 
-# root-port or switch-port. 
+# Enable hot-plugging of VFIO devices to a bridge-port,
+# root-port or switch-port.
 # The default setting is  "no-port"
 hot_plug_vfio = "no-port"
 
 # In a confidential compute environment hot-plugging can compromise
-# security. 
-# Enable cold-plugging of VFIO devices to a bridge-port, 
-# root-port or switch-port. 
-# The default setting is  "no-port", which means disabled. 
+# security.
+# Enable cold-plugging of VFIO devices to a bridge-port,
+# root-port or switch-port.
+# The default setting is  "no-port", which means disabled.
 cold_plug_vfio = "no-port"
 
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.

src/runtime-rs/config/configuration-qemu-snp-runtime-rs.toml.in

@@ -0,0 +1,770 @@
+# Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
+# Copyright (c) 2024 IBM Corp.
+# Copyright (c) 2025-2026 Ant Group
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_QEMU_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.qemu]
+path = "@QEMUPATH@"
+kernel = "@KERNELPATH_COCO@"
+initrd = "@INITRDCONFIDENTIALPATH@"
+# image = "@IMAGECONFIDENTIALPATH@"
+machine_type = "@MACHINETYPE@"
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+confidential_guest = true
+
+# Enable AMD SEV-SNP confidential guests
+# In case of using confidential guests on AMD hardware that supports SEV-SNP,
+# the following enables SEV-SNP guests. Default true
+sev_snp_guest = true
+
+# SNP 'ID Block' and 'ID Authentication Information Structure'.
+# If one of snp_id_block or snp_id_auth is specified, the other must be specified, too.
+# Notice that the default SNP policy of QEMU (0x30000) is used by Kata, if not explicitly
+# set via 'snp_guest_policy' option. The IDBlock contains the guest policy as field, and
+# it must match the value from 'snp_guest_policy' or, if unset, the QEMU default policy.
+#
+# 96-byte, base64-encoded blob to provide the ‘ID Block’ structure for the
+# SNP_LAUNCH_FINISH command defined in the SEV-SNP firmware ABI (QEMU default: all-zero)
+snp_id_block = ""
+# 4096-byte, base64-encoded blob to provide the ‘ID Authentication Information Structure’
+# for the SNP_LAUNCH_FINISH command defined in the SEV-SNP firmware ABI (QEMU default: all-zero)
+snp_id_auth = ""
+
+# SNP Guest Policy, the ‘POLICY’ parameter to the SNP_LAUNCH_START command.
+# If unset, the QEMU default policy (0x30000) will be used.
+# Notice that the guest policy is enforced at VM launch, and your pod VMs
+# won't start at all if the policy denys it. This will be indicated by a
+# 'SNP_LAUNCH_START' error.
+snp_guest_policy = 196608
+
+# rootfs filesystem type:
+#   - ext4 (default)
+#   - xfs
+#   - erofs
+rootfs_type = @DEFROOTFSTYPE@
+
+# Block storage driver to be used for the VM rootfs is backed
+# by a block device. This is virtio-blk-pci, virtio-blk-mmio or nvdimm
+vm_rootfs_driver = "virtio-blk-pci"
+
+# Enable running QEMU VMM as a non-root user.
+# By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
+# a non-root random user. See documentation for the limitations of this mode.
+rootless = false
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+enable_annotations = @DEFENABLEANNOTATIONS_COCO@
+
+# List of valid annotations values for the hypervisor
+# Each member of the list is a path pattern as described by glob(3).
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWARE_SNP_PATH@"
+
+# Path to the firmware volume.
+# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables
+# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables
+# can be customized per each user while UEFI code is kept same.
+firmware_volume = "@FIRMWARE_VOLUME_SNP_PATH@"
+
+# Machine accelerators
+# comma-separated list of machine accelerators to pass to the hypervisor.
+# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
+machine_accelerators = "@MACHINEACCELERATORS@"
+
+# Qemu seccomp sandbox feature
+# comma-separated list of seccomp sandbox features to control the syscall access.
+# For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"`
+# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox
+# Another note: enabling this feature may reduce performance, you may enable
+# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html
+# Recommended value when enabling: "on,obsolete=deny,spawn=deny,resourcecontrol=deny"
+seccompsandbox = "@DEFSECCOMPSANDBOXPARAM@"
+
+# CPU features
+# comma-separated list of cpu features to pass to the cpu
+# For example, `cpu_features = "pmu=off,vmx=off"
+cpu_features = "@CPUFEATURES@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = @DEFVCPUS_QEMU@
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+default_maxvcpus = @DEFMAXVCPUS_QEMU@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+memory_slots = @DEFMEMSLOTS@
+
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM device.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+memory_offset = 0
+
+# Specifies virtio-mem will be enabled or not.
+# Please note that this option should be used with the command
+# "echo 1 > /proc/sys/vm/overcommit_memory".
+# Default false
+enable_virtio_mem = false
+
+# Disable block device from being used for a container's rootfs.
+# In case of a storage driver like devicemapper where a container's
+# root file system is backed by a block device, the block device is passed
+# directly to the hypervisor for performance reasons.
+# This flag prevents the block device from being passed to the hypervisor,
+# virtio-fs is used instead to pass the rootfs.
+disable_block_device_use = @DEFDISABLEBLOCK@
+
+# Shared file system type:
+#   - virtio-fs (default)
+#   - virtio-fs-nydus
+#   - none
+shared_fs = "none"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# List of valid annotations values for the virtiofs daemon
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@
+valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Default size of virtqueues
+virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@
+
+# Extra args for virtiofsd daemon
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+# Examples:
+#   Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"]
+#
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - never
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - metadata
+#    Metadata and pathname lookup are cached in guest and never expire.
+#    Data is never cached in guest.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@"
+
+# aio is the I/O mechanism used by qemu
+# Options:
+#
+#   - threads
+#     Pthread based disk I/O.
+#
+#   - native
+#     Native Linux I/O.
+#
+#   - io_uring
+#     Linux io_uring API. This provides the fastest I/O operations on Linux, requires kernel>5.1 and
+#     qemu >=5.0.
+block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
+
+# Specifies cache-related options will be set to block devices or not.
+# Default false
+block_device_cache_set = false
+
+# Specifies cache-related options for block devices.
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
+# Default false
+block_device_cache_direct = false
+
+# Specifies cache-related options for block devices.
+# Denotes whether flush requests for the device are ignored.
+# Default false
+block_device_cache_noflush = false
+
+# Enable iothreads (data-plane) to be used. This causes IO to be
+# handled in a separate IO thread. This is currently only implemented
+# for SCSI.
+#
+enable_iothreads = @DEFENABLEIOTHREADS@
+
+# Independent IOThreads enables IO to be processed in a separate thread, it is
+# for QEMU hotplug device attach to iothread, like virtio-blk.
+indep_iothreads = @DEFINDEPIOTHREADS@
+
+# Enable pre allocation of VM RAM, default false
+# Enabling this will result in lower container density
+# as all of the memory will be allocated and locked
+# This is useful when you want to reserve all the memory
+# upfront or in the cases where you want memory latencies
+# to be very predictable
+# Default false
+enable_mem_prealloc = false
+
+# Reclaim guest freed memory.
+# Enabling this will result in the VM balloon device having f_reporting=on set.
+# Then the hypervisor will use it to reclaim guest freed memory.
+# This is useful for reducing the amount of memory used by a VM.
+# Enabling this feature may sometimes reduce the speed of memory access in
+# the VM.
+#
+# Default false
+reclaim_guest_freed_memory = false
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+# This is useful when you want to use vhost-user network
+# stacks within the container. This will automatically
+# result in memory pre allocation
+enable_hugepages = false
+
+# Enable vhost-user storage device, default false
+# Enabling this will result in some Linux reserved block type
+# major range 240-254 being chosen to represent vhost-user devices.
+enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
+
+# The base directory specifically used for vhost-user devices.
+# Its sub-path "block" is used for block devices; "block/sockets" is
+# where we expect vhost-user sockets to live; "block/devices" is where
+# simulated block device nodes for vhost-user devices to live.
+vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
+
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: intel_iommu=on,iommu=pt
+enable_iommu = false
+
+# Enable IOMMU_PLATFORM, default false
+# Enabling this will result in the VM device having iommu_platform=on set
+enable_iommu_platform = false
+
+# List of valid annotations values for the vhost user store path
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@
+valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@
+
+# The timeout for reconnecting on non-server spdk sockets when the remote end goes away.
+# qemu will delay this many seconds and then attempt to reconnect.
+# Zero disables reconnecting, and the default is zero.
+vhost_user_reconnect_timeout_sec = 0
+
+# Enable file based guest memory support. The default is an empty string which
+# will disable this feature. In the case of virtio-fs, this is enabled
+# automatically and '/dev/shm' is used as the backing folder.
+# This option will be ignored if VM templating is enabled.
+file_mem_backend = ""
+
+# List of valid annotations values for the file_mem_backend annotation
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDFILEMEMBACKENDS@
+valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
+
+# -pflash can add image file to VM. The arguments of it should be in format
+# of ["/path/to/flash0.img", "/path/to/flash1.img"]
+pflashes = []
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. And Debug also enable the hmp socket.
+#
+# Default false
+enable_debug = false
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+#
+disable_nesting_checks = true
+
+# If false and nvdimm is supported, use nvdimm device to plug guest image.
+# Otherwise virtio-block device is used.
+#
+# nvdimm is not supported when `confidential_guest = true`.
+disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM@
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# Default 0
+pcie_root_port = 0
+
+# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
+# security (vhost-net runs ring0) for network I/O performance.
+disable_vhost_net = false
+
+# This option allows to add an extra HMP or QMP socket when `enable_debug = true`
+#
+# WARNING: Anyone with access to the extra socket can take full control of
+# Qemu. This is for debugging purpose only and must *NEVER* be used in
+# production.
+#
+# Valid values are :
+# - "hmp"
+# - "qmp"
+# - "qmp-pretty" (same as "qmp" with pretty json formatting)
+#
+# If set to the empty string "", no extra monitor socket is added. This is
+# the default.
+#extra_monitor_socket = "hmp"
+
+#
+# Default entropy source.
+# The path to a host source of entropy (including a real hardware RNG)
+# /dev/urandom and /dev/random are two main options.
+# Be aware that /dev/random is a blocking source of entropy.  If the host
+# runs out of entropy, the VMs boot time will increase leading to get startup
+# timeouts.
+# The source of entropy /dev/urandom is non-blocking and provides a
+# generally acceptable source of entropy. It should work well for pretty much
+# all practical purposes.
+entropy_source = "@DEFENTROPYSOURCE@"
+
+# List of valid annotations values for entropy_source
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+# Recommended value when enabling: "/usr/share/oci/hooks"
+guest_hook_path = ""
+
+#
+# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic.
+# Default 0-sized value means unlimited rate.
+rx_rate_limiter_max_rate = 0
+# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block)
+# to discipline traffic.
+# Default 0-sized value means unlimited rate.
+tx_rate_limiter_max_rate = 0
+
+# Set where to save the guest memory dump file.
+# If set, when GUEST_PANICKED event occurred,
+# guest memeory will be dumped to host filesystem under guest_memory_dump_path,
+# This directory will be created automatically if it does not exist.
+#
+# The dumped file(also called vmcore) can be processed with crash or gdb.
+#
+# WARNING:
+#   Dump guest's memory can take very long depending on the amount of guest memory
+#   and use much disk space.
+# Recommended value when enabling: "/var/crash/kata"
+guest_memory_dump_path = ""
+
+# If enable paging.
+# Basically, if you want to use "gdb" rather than "crash",
+# or need the guest-virtual addresses in the ELF vmcore,
+# then you should enable paging.
+#
+# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details
+guest_memory_dump_paging = false
+
+# Enable swap in the guest. Default false.
+# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device
+# if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness")
+# is bigger than 0.
+# The size of the swap device should be
+# swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes.
+# If swap_in_bytes is not set, the size should be memory_limit_in_bytes.
+# If swap_in_bytes and memory_limit_in_bytes is not set, the size should
+# be default_memory.
+enable_guest_swap = false
+
+# use legacy serial for guest console if available and implemented for architecture. Default false
+use_legacy_serial = false
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux = @DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+disable_guest_selinux = @DEFDISABLEGUESTSELINUX@
+
+
+[factory]
+# VM templating support. Once enabled, new VMs are created from template
+# using vm cloning. They will share the same initial kernel, initramfs and
+# agent memory by mapping it readonly. It helps speeding up new container
+# creation and saves a lot of memory if there are many kata containers running
+# on the same host.
+#
+# When disabled, new VMs are created from scratch.
+#
+# Note: Requires "initrd=" to be set ("image=" is not supported).
+#
+# Default false
+enable_template = false
+
+# Specifies the path of template.
+#
+# Default "/run/vc/vm/template"
+template_path = "/run/vc/vm/template"
+
+# The number of caches of VMCache:
+# unspecified or == 0   --> VMCache is disabled
+# > 0                   --> will be set to the specified number
+#
+# VMCache is a function that creates VMs as caches before using it.
+# It helps speed up new container creation.
+# The function consists of a server and some clients communicating
+# through Unix socket.  The protocol is gRPC in protocols/cache/cache.proto.
+# The VMCache server will create some VMs and cache them by factory cache.
+# It will convert the VM to gRPC format and transport it when gets
+# requestion from clients.
+# Factory grpccache is the VMCache client.  It will request gRPC format
+# VM and convert it back to a VM.  If VMCache function is enabled,
+# kata-runtime will request VM from factory grpccache when it creates
+# a new sandbox.
+#
+# Default 0
+vm_cache_number = 0
+
+# Specify the address of the Unix socket that is used by VMCache.
+#
+# Default /var/run/kata-containers/cache.sock
+vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+enable_debug = false
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+enable_tracing = false
+
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules = []
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+debug_console_enabled = false
+
+# Agent dial timeout in millisecond.
+# (default: 10)
+dial_timeout_ms = 10
+
+# Agent reconnect timeout in millisecond.
+# Retry times = reconnect_timeout_ms / dial_timeout_ms (default: 300)
+# If you find pod cannot connect to the agent when starting, please
+# consider increasing this value to increase the retry times.
+# You'd better not change the value of dial_timeout_ms, unless you have an
+# idea of what you are doing.
+# (default: 3000)
+reconnect_timeout_ms = 3000
+
+# Create Container Request Timeout
+# This timeout value is used to set the maximum duration for the agent to process a CreateContainerRequest.
+# It's also used to ensure that workloads, especially those involving large image pulls within the guest,
+# have sufficient time to complete.
+#
+# Effective Timeout Determination:
+# The effective timeout for a CreateContainerRequest is determined by taking the minimum of the following two values:
+# - create_container_timeout: The timeout value configured for creating containers (default: 30,000 milliseconds).
+# - runtime-request-timeout: The timeout value specified in the Kubelet configuration described as the link below:
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout)
+# Defaults to @DEFCREATECONTAINERTIMEOUT_COCO@ second(s)
+create_container_timeout = @DEFCREATECONTAINERTIMEOUT_COCO@
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+enable_debug = false
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_QEMU@"
+
+name="@RUNTIMENAME@"
+hypervisor_name="@HYPERVISOR_QEMU@"
+agent_name="@PROJECT_TYPE@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp = @DEFDISABLEGUESTSECCOMP@
+
+# vCPUs pinning settings
+# if enabled, each vCPU thread will be scheduled to a fixed CPU
+# qualified condition: num(vCPU threads) == num(CPUs in sandbox's CPUSet)
+enable_vcpus_pinning = false
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+guest_selinux_label = "@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+enable_tracing = false
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+disable_new_netns = false
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only = @DEFSANDBOXCGROUPONLY_QEMU@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt = @DEFSTATICRESOURCEMGMT_COCO@
+
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts = @DEFBINDMOUNTS@
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode = "@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# for example:
+#       experimental=["force_guest_pull"]
+# which is for enable force_guest_pull mode in CoCo scenarios.
+# (default: [])
+experimental = @DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+enable_pprof = false
+
+# Base directory of directly attachable network config.
+# Network devices for VM-based containers are allowed to be placed in the
+# host netns to eliminate as many hops as possible, which is what we
+# called a "Directly Attachable Network". The config, set by special CNI
+# plugins, is used to tell the Kata containers what devices are attached
+# to the hypervisor.
+# (default: /run/kata-containers/dans)
+dan_conf = "@DEFDANCONF@"
+
+# pod_resource_api_sock specifies the unix socket for the Kubelet's
+# PodResource API endpoint. If empty, kubernetes based cold plug
+# will not be attempted. In order for this feature to work, the
+# KubeletPodResourcesGet featureGate must be enabled in Kubelet,
+# if using Kubelet older than 1.34.
+#
+# The pod resource API's socket is relative to the Kubelet's root-dir,
+# which is defined by the cluster admin, and its location is:
+# ${KubeletRootDir}/pod-resources/kubelet.sock
+#
+# cold_plug_vfio(see hypervisor config) acts as a feature gate:
+#      cold_plug_vfio = no_port (default) => no cold plug
+#      cold_plug_vfio != no_port AND pod_resource_api_sock = "" => need
+#              explicit CDI annotation for cold plug (applies mainly
+#              to non-k8s cases)
+#      cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet
+#              based cold plug.
+pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK@"

src/runtime-rs/config/configuration-qemu-tdx-runtime-rs.toml.in

@@ -0,0 +1,751 @@
+# Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
+# Copyright (c) 2025-2026 Ant Group
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_QEMU_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.qemu]
+path = "@QEMUPATH@"
+kernel = "@KERNELPATH_COCO@"
+image = "@IMAGECONFIDENTIALPATH@"
+# initrd = "@INITRDPATH@"
+machine_type = "@MACHINETYPE@"
+tdx_quote_generation_service_socket_port = @QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT@
+
+# rootfs filesystem type:
+#   - ext4 (default)
+#   - xfs
+#   - erofs
+rootfs_type = @DEFROOTFSTYPE@
+
+# Block storage driver to be used for the VM rootfs is backed
+# by a block device. This is virtio-blk-pci, virtio-blk-mmio or nvdimm
+vm_rootfs_driver = "virtio-blk-pci"
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+confidential_guest = true
+
+# Enable running QEMU VMM as a non-root user.
+# By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
+# a non-root random user. See documentation for the limitations of this mode.
+rootless = false
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+enable_annotations = @DEFENABLEANNOTATIONS_COCO@
+
+# List of valid annotations values for the hypervisor
+# Each member of the list is a path pattern as described by glob(3).
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELTDXPARAMS@"
+
+# Optional dm-verity parameters (comma-separated key=value list):
+# root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
+# These are used by the runtime to assemble dm-verity kernel params.
+kernel_verity_params = "@KERNELVERITYPARAMS@"
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWARETDXPATH@"
+
+# Path to the firmware volume.
+# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables
+# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables
+# can be customized per each user while UEFI code is kept same.
+firmware_volume = "@FIRMWAREVOLUMEPATH@"
+
+# Machine accelerators
+# comma-separated list of machine accelerators to pass to the hypervisor.
+# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
+machine_accelerators = "@MACHINEACCELERATORS@"
+
+# Qemu seccomp sandbox feature
+# comma-separated list of seccomp sandbox features to control the syscall access.
+# For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"`
+# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox
+# Another note: enabling this feature may reduce performance, you may enable
+# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html
+# Recommended value when enabling: "on,obsolete=deny,spawn=deny,resourcecontrol=deny"
+seccompsandbox = "@DEFSECCOMPSANDBOXPARAM@"
+
+# CPU features
+# comma-separated list of cpu features to pass to the cpu
+# For example, `cpu_features = "pmu=off,vmx=off"
+cpu_features = "@CPUFEATURES@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+memory_slots = @DEFMEMSLOTS@
+
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM device.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+memory_offset = 0
+
+# Specifies virtio-mem will be enabled or not.
+# Please note that this option should be used with the command
+# "echo 1 > /proc/sys/vm/overcommit_memory".
+# Default false
+enable_virtio_mem = false
+
+# Disable block device from being used for a container's rootfs.
+# In case of a storage driver like devicemapper where a container's
+# root file system is backed by a block device, the block device is passed
+# directly to the hypervisor for performance reasons.
+# This flag prevents the block device from being passed to the hypervisor,
+# virtio-fs is used instead to pass the rootfs.
+disable_block_device_use = @DEFDISABLEBLOCK@
+
+# Shared file system type:
+#   - virtio-fs (default)
+#   - virtio-fs-nydus
+#   - none
+shared_fs = "@DEFSHAREDFS_QEMU_TDX_VIRTIOFS@"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# List of valid annotations values for the virtiofs daemon
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@
+valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Default size of virtqueues
+virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@
+
+# Extra args for virtiofsd daemon
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+# Examples:
+#   Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"]
+#
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - never
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - metadata
+#    Metadata and pathname lookup are cached in guest and never expire.
+#    Data is never cached in guest.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@"
+
+# aio is the I/O mechanism used by qemu
+# Options:
+#
+#   - threads
+#     Pthread based disk I/O.
+#
+#   - native
+#     Native Linux I/O.
+#
+#   - io_uring
+#     Linux io_uring API. This provides the fastest I/O operations on Linux, requires kernel>5.1 and
+#     qemu >=5.0.
+block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
+
+# Specifies cache-related options will be set to block devices or not.
+# Default false
+block_device_cache_set = false
+
+# Specifies cache-related options for block devices.
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
+# Default false
+block_device_cache_direct = false
+
+# Specifies cache-related options for block devices.
+# Denotes whether flush requests for the device are ignored.
+# Default false
+block_device_cache_noflush = false
+
+# Enable iothreads (data-plane) to be used. This causes IO to be
+# handled in a separate IO thread. This is currently implemented
+# for virtio-scsi and virtio-blk.
+#
+enable_iothreads = @DEFENABLEIOTHREADS@
+
+# Independent IOThreads enables IO to be processed in a separate thread, it is
+# for QEMU hotplug device attach to iothread, like virtio-blk.
+indep_iothreads = @DEFINDEPIOTHREADS@
+
+# Enable pre allocation of VM RAM, default false
+# Enabling this will result in lower container density
+# as all of the memory will be allocated and locked
+# This is useful when you want to reserve all the memory
+# upfront or in the cases where you want memory latencies
+# to be very predictable
+# Default false
+enable_mem_prealloc = false
+
+# Reclaim guest freed memory.
+# Enabling this will result in the VM balloon device having f_reporting=on set.
+# Then the hypervisor will use it to reclaim guest freed memory.
+# This is useful for reducing the amount of memory used by a VM.
+# Enabling this feature may sometimes reduce the speed of memory access in
+# the VM.
+#
+# Default false
+reclaim_guest_freed_memory = false
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+# This is useful when you want to use vhost-user network
+# stacks within the container. This will automatically
+# result in memory pre allocation
+enable_hugepages = false
+
+# Enable vhost-user storage device, default false
+# Enabling this will result in some Linux reserved block type
+# major range 240-254 being chosen to represent vhost-user devices.
+enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
+
+# The base directory specifically used for vhost-user devices.
+# Its sub-path "block" is used for block devices; "block/sockets" is
+# where we expect vhost-user sockets to live; "block/devices" is where
+# simulated block device nodes for vhost-user devices to live.
+vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
+
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: intel_iommu=on,iommu=pt
+enable_iommu = false
+
+# Enable IOMMU_PLATFORM, default false
+# Enabling this will result in the VM device having iommu_platform=on set
+enable_iommu_platform = false
+
+# List of valid annotations values for the vhost user store path
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@
+valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@
+
+# The timeout for reconnecting on non-server spdk sockets when the remote end goes away.
+# qemu will delay this many seconds and then attempt to reconnect.
+# Zero disables reconnecting, and the default is zero.
+vhost_user_reconnect_timeout_sec = 0
+
+# Enable file based guest memory support. The default is an empty string which
+# will disable this feature. In the case of virtio-fs, this is enabled
+# automatically and '/dev/shm' is used as the backing folder.
+# This option will be ignored if VM templating is enabled.
+file_mem_backend = "@DEFFILEMEMBACKEND@"
+
+# List of valid annotations values for the file_mem_backend annotation
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDFILEMEMBACKENDS@
+valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
+
+# -pflash can add image file to VM. The arguments of it should be in format
+# of ["/path/to/flash0.img", "/path/to/flash1.img"]
+pflashes = []
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. And Debug also enable the hmp socket.
+#
+# Default false
+enable_debug = false
+
+# This option allows to add an extra HMP or QMP socket when `enable_debug = true`
+#
+# WARNING: Anyone with access to the extra socket can take full control of
+# Qemu. This is for debugging purpose only and must *NEVER* be used in
+# production.
+#
+# Valid values are :
+# - "hmp"
+# - "qmp"
+# - "qmp-pretty" (same as "qmp" with pretty json formatting)
+#
+# If set to the empty string "", no extra monitor socket is added. This is
+# the default.
+extra_monitor_socket = ""
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+#
+disable_nesting_checks = false
+
+# If false and nvdimm is supported, use nvdimm device to plug guest image.
+# Otherwise virtio-block device is used.
+#
+# nvdimm is not supported when `confidential_guest = true`.
+disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM@
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# Default 0
+pcie_root_port = 0
+
+# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
+# security (vhost-net runs ring0) for network I/O performance.
+disable_vhost_net = false
+
+#
+# Default entropy source.
+# The path to a host source of entropy (including a real hardware RNG)
+# /dev/urandom and /dev/random are two main options.
+# Be aware that /dev/random is a blocking source of entropy.  If the host
+# runs out of entropy, the VMs boot time will increase leading to get startup
+# timeouts.
+# The source of entropy /dev/urandom is non-blocking and provides a
+# generally acceptable source of entropy. It should work well for pretty much
+# all practical purposes.
+entropy_source = "@DEFENTROPYSOURCE@"
+
+
+# List of valid annotations values for entropy_source
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+# Recommended value when enabling: "/usr/share/oci/hooks"
+guest_hook_path = ""
+#
+# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic.
+# Default 0-sized value means unlimited rate.
+rx_rate_limiter_max_rate = 0
+# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block)
+# to discipline traffic.
+# Default 0-sized value means unlimited rate.
+tx_rate_limiter_max_rate = 0
+
+# Set where to save the guest memory dump file.
+# If set, when GUEST_PANICKED event occurred,
+# guest memeory will be dumped to host filesystem under guest_memory_dump_path,
+# This directory will be created automatically if it does not exist.
+#
+# The dumped file(also called vmcore) can be processed with crash or gdb.
+#
+# WARNING:
+#   Dump guest's memory can take very long depending on the amount of guest memory
+#   and use much disk space.
+# Recommended value when enabling: "/var/crash/kata"
+guest_memory_dump_path = ""
+
+# If enable paging.
+# Basically, if you want to use "gdb" rather than "crash",
+# or need the guest-virtual addresses in the ELF vmcore,
+# then you should enable paging.
+#
+# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details
+guest_memory_dump_paging = false
+
+# Enable swap in the guest. Default false.
+# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device
+# if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness")
+# is bigger than 0.
+# The size of the swap device should be
+# swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes.
+# If swap_in_bytes is not set, the size should be memory_limit_in_bytes.
+# If swap_in_bytes and memory_limit_in_bytes is not set, the size should
+# be default_memory.
+enable_guest_swap = false
+
+# use legacy serial for guest console if available and implemented for architecture. Default false
+use_legacy_serial = false
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux = @DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+disable_guest_selinux = @DEFDISABLEGUESTSELINUX@
+
+
+[factory]
+# VM templating support. Once enabled, new VMs are created from template
+# using vm cloning. They will share the same initial kernel, initramfs and
+# agent memory by mapping it readonly. It helps speeding up new container
+# creation and saves a lot of memory if there are many kata containers running
+# on the same host.
+#
+# When disabled, new VMs are created from scratch.
+#
+# Note: Requires "initrd=" to be set ("image=" is not supported).
+#
+# Default false
+enable_template = false
+
+# Specifies the path of template.
+#
+# Default "/run/vc/vm/template"
+template_path = "/run/vc/vm/template"
+
+# The number of caches of VMCache:
+# unspecified or == 0   --> VMCache is disabled
+# > 0                   --> will be set to the specified number
+#
+# VMCache is a function that creates VMs as caches before using it.
+# It helps speed up new container creation.
+# The function consists of a server and some clients communicating
+# through Unix socket.  The protocol is gRPC in protocols/cache/cache.proto.
+# The VMCache server will create some VMs and cache them by factory cache.
+# It will convert the VM to gRPC format and transport it when gets
+# requestion from clients.
+# Factory grpccache is the VMCache client.  It will request gRPC format
+# VM and convert it back to a VM.  If VMCache function is enabled,
+# kata-runtime will request VM from factory grpccache when it creates
+# a new sandbox.
+#
+# Default 0
+vm_cache_number = 0
+
+# Specify the address of the Unix socket that is used by VMCache.
+#
+# Default /var/run/kata-containers/cache.sock
+vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+enable_debug = false
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+enable_tracing = false
+
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules = []
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+debug_console_enabled = false
+
+# Agent dial timeout in millisecond.
+# (default: 10)
+dial_timeout_ms = 10
+
+# Agent reconnect timeout in millisecond.
+# Retry times = reconnect_timeout_ms / dial_timeout_ms (default: 300)
+# If you find pod cannot connect to the agent when starting, please
+# consider increasing this value to increase the retry times.
+# You'd better not change the value of dial_timeout_ms, unless you have an
+# idea of what you are doing.
+# (default: 3000)
+reconnect_timeout_ms = 3000
+
+# Create Container Request Timeout
+# This timeout value is used to set the maximum duration for the agent to process a CreateContainerRequest.
+# It's also used to ensure that workloads, especially those involving large image pulls within the guest,
+# have sufficient time to complete.
+#
+# Effective Timeout Determination:
+# The effective timeout for a CreateContainerRequest is determined by taking the minimum of the following two values:
+# - create_container_timeout: The timeout value configured for creating containers (default: 30,000 milliseconds).
+# - runtime-request-timeout: The timeout value specified in the Kubelet configuration described as the link below:
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout)
+# Defaults to @DEFCREATECONTAINERTIMEOUT_COCO@ second(s)
+create_container_timeout = @DEFCREATECONTAINERTIMEOUT_COCO@
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+enable_debug = false
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model = "@DEFNETWORKMODEL_QEMU@"
+
+name="@RUNTIMENAME@"
+hypervisor_name="@HYPERVISOR_QEMU@"
+agent_name="@PROJECT_TYPE@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp = @DEFDISABLEGUESTSECCOMP@
+
+# vCPUs pinning settings
+# if enabled, each vCPU thread will be scheduled to a fixed CPU
+# qualified condition: num(vCPU threads) == num(CPUs in sandbox's CPUSet)
+enable_vcpus_pinning = false
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+# Example value when enabling: "system_u:system_r:container_t"
+guest_selinux_label = "@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+enable_tracing = false
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+disable_new_netns = false
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only = @DEFSANDBOXCGROUPONLY_QEMU@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt = @DEFSTATICRESOURCEMGMT_COCO@
+
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts = @DEFBINDMOUNTS@
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode = "@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# for example:
+#       experimental=["force_guest_pull"]
+# which is for enable force_guest_pull mode in CoCo scenarios.
+# (default: [])
+experimental = @DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+enable_pprof = false
+
+# Base directory of directly attachable network config.
+# Network devices for VM-based containers are allowed to be placed in the
+# host netns to eliminate as many hops as possible, which is what we
+# called a "Directly Attachable Network". The config, set by special CNI
+# plugins, is used to tell the Kata containers what devices are attached
+# to the hypervisor.
+# (default: /run/kata-containers/dans)
+dan_conf = "@DEFDANCONF@"
+
+# pod_resource_api_sock specifies the unix socket for the Kubelet's
+# PodResource API endpoint. If empty, kubernetes based cold plug
+# will not be attempted. In order for this feature to work, the
+# KubeletPodResourcesGet featureGate must be enabled in Kubelet,
+# if using Kubelet older than 1.34.
+#
+# The pod resource API's socket is relative to the Kubelet's root-dir,
+# which is defined by the cluster admin, and its location is:
+# ${KubeletRootDir}/pod-resources/kubelet.sock
+#
+# cold_plug_vfio(see hypervisor config) acts as a feature gate:
+#      cold_plug_vfio = no_port (default) => no cold plug
+#      cold_plug_vfio != no_port AND pod_resource_api_sock = "" => need
+#              explicit CDI annotation for cold plug (applies mainly
+#              to non-k8s cases)
+#      cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet
+#              based cold plug.
+pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK@"

src/runtime-rs/config/configuration-rs-fc.toml.in

@@ -304,7 +304,7 @@ debug_console_enabled = false
 
 # Agent connection dialing timeout value in seconds
 # (default: 45)
-dial_timeout = 45 
+dial_timeout = 45
 
 # Confidential Data Hub API timeout value in seconds
 # (default: 50)

src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs

@@ -151,7 +151,11 @@ impl CloudHypervisorInner {
         #[cfg(target_arch = "aarch64")]
         let console_param_debug = KernelParams::from_string("console=ttyAMA0,115200n8");
 
-        let mut rootfs_param = KernelParams::new_rootfs_kernel_params(rootfs_driver, rootfs_type)?;
+        let mut rootfs_params = KernelParams::new_rootfs_kernel_params(
+            &cfg.boot_info.kernel_verity_params,
+            rootfs_driver,
+            rootfs_type,
+        )?;
 
         let mut console_params = if enable_debug {
             if confidential_guest {
@@ -165,8 +169,7 @@ impl CloudHypervisorInner {
 
         params.append(&mut console_params);
 
-        // Add the rootfs device
-        params.append(&mut rootfs_param);
+        params.append(&mut rootfs_params);
 
         // Now add some additional options required for CH
         let extra_options = [

src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs

@@ -144,13 +144,14 @@ impl DragonballInner {
         let mut kernel_params = KernelParams::new(self.config.debug_info.enable_debug);
 
         if self.config.boot_info.initrd.is_empty() {
-            // get rootfs driver
+            // When booting from the image, add rootfs and verity parameters here.
             let rootfs_driver = self.config.blockdev_info.block_device_driver.clone();
-
-            kernel_params.append(&mut KernelParams::new_rootfs_kernel_params(
+            let mut rootfs_params = KernelParams::new_rootfs_kernel_params(
+                &self.config.boot_info.kernel_verity_params,
                 &rootfs_driver,
                 &self.config.boot_info.rootfs_type,
-            )?);
+            )?;
+            kernel_params.append(&mut rootfs_params);
         }
 
         kernel_params.append(&mut KernelParams::from_string(

src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs

@@ -86,12 +86,12 @@ impl FcInner {
         let mut kernel_params = KernelParams::new(self.config.debug_info.enable_debug);
         kernel_params.push(Param::new("pci", "off"));
         kernel_params.push(Param::new("iommu", "off"));
-        let rootfs_driver = self.config.blockdev_info.block_device_driver.clone();
-
-        kernel_params.append(&mut KernelParams::new_rootfs_kernel_params(
-            &rootfs_driver,
+        let mut rootfs_params = KernelParams::new_rootfs_kernel_params(
+            &self.config.boot_info.kernel_verity_params,
+            &self.config.blockdev_info.block_device_driver,
             &self.config.boot_info.rootfs_type,
-        )?);
+        )?;
+        kernel_params.append(&mut rootfs_params);
         kernel_params.append(&mut KernelParams::from_string(
             &self.config.boot_info.kernel_params,
         ));

src/runtime-rs/crates/hypervisor/src/kernel_param.rs

@@ -11,6 +11,7 @@ use crate::{
     VM_ROOTFS_ROOT_BLK, VM_ROOTFS_ROOT_PMEM,
 };
 use kata_types::config::LOG_VPORT_OPTION;
+use kata_types::config::hypervisor::{parse_kernel_verity_params, VERITY_BLOCK_SIZE_BYTES};
 use kata_types::fs::{
     VM_ROOTFS_FILESYSTEM_EROFS, VM_ROOTFS_FILESYSTEM_EXT4, VM_ROOTFS_FILESYSTEM_XFS,
 };
@@ -20,7 +21,76 @@ use kata_types::fs::{
 const VSOCK_LOGS_PORT: &str = "1025";
 
 const KERNEL_KV_DELIMITER: &str = "=";
-const KERNEL_PARAM_DELIMITER: &str = " ";
+const KERNEL_PARAM_DELIMITER: char = ' ';
+
+// Split kernel params on spaces, but keep quoted substrings intact.
+// Example: dm-mod.create="dm-verity,,,ro,0 736328 verity 1 /dev/vda1 /dev/vda2 ...".
+fn split_kernel_params(params_string: &str) -> Vec<String> {
+    let mut params = Vec::new();
+    let mut current = String::new();
+    let mut in_quote = false;
+
+    for c in params_string.chars() {
+        if c == '"' {
+            in_quote = !in_quote;
+            current.push(c);
+            continue;
+        }
+
+        if c == KERNEL_PARAM_DELIMITER && !in_quote {
+            let trimmed = current.trim();
+            if !trimmed.is_empty() {
+                params.push(trimmed.to_string());
+            }
+            current.clear();
+            continue;
+        }
+
+        current.push(c);
+    }
+
+    let trimmed = current.trim();
+    if !trimmed.is_empty() {
+        params.push(trimmed.to_string());
+    }
+
+    params
+}
+
+struct KernelVerityConfig {
+    root_hash: String,
+    salt: String,
+    data_blocks: u64,
+    data_block_size: u64,
+    hash_block_size: u64,
+}
+
+fn new_kernel_verity_params(params_string: &str) -> Result<Option<KernelVerityConfig>> {
+    let cfg = parse_kernel_verity_params(params_string)
+        .map_err(|err| anyhow!(err.to_string()))?;
+
+    Ok(cfg.map(|params| KernelVerityConfig {
+        root_hash: params.root_hash,
+        salt: params.salt,
+        data_blocks: params.data_blocks,
+        data_block_size: params.data_block_size,
+        hash_block_size: params.hash_block_size,
+    }))
+}
+
+fn kernel_verity_root_flags(rootfs_type: &str) -> Result<String> {
+    let normalized = if rootfs_type.is_empty() {
+        VM_ROOTFS_FILESYSTEM_EXT4
+    } else {
+        rootfs_type
+    };
+
+    match normalized {
+        VM_ROOTFS_FILESYSTEM_EXT4 => Ok("data=ordered,errors=remount-ro ro".to_string()),
+        VM_ROOTFS_FILESYSTEM_XFS | VM_ROOTFS_FILESYSTEM_EROFS => Ok("ro".to_string()),
+        _ => Err(anyhow!("Unsupported rootfs type {}", rootfs_type)),
+    }
+}
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct Param {
@@ -71,46 +141,28 @@ impl KernelParams {
         Self { params }
     }
 
-    pub(crate) fn new_rootfs_kernel_params(rootfs_driver: &str, rootfs_type: &str) -> Result<Self> {
+    pub(crate) fn new_rootfs_kernel_params(
+        kernel_verity_params: &str,
+        rootfs_driver: &str,
+        rootfs_type: &str,
+    ) -> Result<Self> {
         let mut params = vec![];
 
-        // DAX is disabled on aarch64 due to kernel panic in dax_disassociate_entry
-        // with virtio-pmem on kernel 6.18.x
-        #[cfg(target_arch = "aarch64")]
-        let use_dax = false;
-        #[cfg(not(target_arch = "aarch64"))]
-        let use_dax = true;
-
         match rootfs_driver {
             VM_ROOTFS_DRIVER_PMEM => {
                 params.push(Param::new("root", VM_ROOTFS_ROOT_PMEM));
                 match rootfs_type {
                     VM_ROOTFS_FILESYSTEM_EXT4 => {
-                        if use_dax {
-                            params.push(Param::new(
-                                "rootflags",
-                                "dax,data=ordered,errors=remount-ro ro",
-                            ));
-                        } else {
-                            params.push(Param::new(
-                                "rootflags",
-                                "data=ordered,errors=remount-ro ro",
-                            ));
-                        }
+                        params.push(Param::new(
+                            "rootflags",
+                            "dax,data=ordered,errors=remount-ro ro",
+                        ));
                     }
                     VM_ROOTFS_FILESYSTEM_XFS => {
-                        if use_dax {
-                            params.push(Param::new("rootflags", "dax ro"));
-                        } else {
-                            params.push(Param::new("rootflags", "ro"));
-                        }
+                        params.push(Param::new("rootflags", "dax ro"));
                     }
                     VM_ROOTFS_FILESYSTEM_EROFS => {
-                        if use_dax {
-                            params.push(Param::new("rootflags", "dax ro"));
-                        } else {
-                            params.push(Param::new("rootflags", "ro"));
-                        }
+                        params.push(Param::new("rootflags", "dax ro"));
                     }
                     _ => {
                         return Err(anyhow!("Unsupported rootfs type {}", rootfs_type));
@@ -141,7 +193,52 @@ impl KernelParams {
 
         params.push(Param::new("rootfstype", rootfs_type));
 
-        Ok(Self { params })
+        let mut params = Self { params };
+        let cfg = match new_kernel_verity_params(kernel_verity_params)? {
+            Some(cfg) => cfg,
+            None => return Ok(params),
+        };
+
+        let (root_device, hash_device) = match rootfs_driver {
+            VM_ROOTFS_DRIVER_PMEM => ("/dev/pmem0p1", "/dev/pmem0p2"),
+            VM_ROOTFS_DRIVER_BLK | VM_ROOTFS_DRIVER_BLK_CCW | VM_ROOTFS_DRIVER_MMIO => {
+                ("/dev/vda1", "/dev/vda2")
+            }
+            _ => return Err(anyhow!("Unsupported rootfs driver {}", rootfs_driver)),
+        };
+
+        let data_sectors = (cfg.data_block_size / VERITY_BLOCK_SIZE_BYTES) * cfg.data_blocks;
+        let root_flags = kernel_verity_root_flags(rootfs_type)?;
+
+        let dm_cmd = format!(
+            "dm-verity,,,ro,0 {} verity 1 {} {} {} {} {} 0 sha256 {} {}",
+            data_sectors,
+            root_device,
+            hash_device,
+            cfg.data_block_size,
+            cfg.hash_block_size,
+            cfg.data_blocks,
+            cfg.root_hash,
+            cfg.salt
+        );
+
+        params.remove_all_by_key("root".to_string());
+        params.remove_all_by_key("rootflags".to_string());
+        params.remove_all_by_key("rootfstype".to_string());
+
+        params.push(Param {
+            key: "dm-mod.create".to_string(),
+            value: format!("\"{}\"", dm_cmd),
+        });
+        params.push(Param::new("root", "/dev/dm-0"));
+        params.push(Param::new("rootflags", &root_flags));
+        if rootfs_type.is_empty() {
+            params.push(Param::new("rootfstype", VM_ROOTFS_FILESYSTEM_EXT4));
+        } else {
+            params.push(Param::new("rootfstype", rootfs_type));
+        }
+
+        Ok(params)
     }
 
     pub(crate) fn append(&mut self, params: &mut KernelParams) {
@@ -160,7 +257,7 @@ impl KernelParams {
     pub(crate) fn from_string(params_string: &str) -> Self {
         let mut params = vec![];
 
-        let parameters_vec: Vec<&str> = params_string.split(KERNEL_PARAM_DELIMITER).collect();
+        let parameters_vec = split_kernel_params(params_string);
 
         for param in parameters_vec.iter() {
             if param.is_empty() {
@@ -192,7 +289,7 @@ impl KernelParams {
             parameters.push(param.to_string()?);
         }
 
-        Ok(parameters.join(KERNEL_PARAM_DELIMITER))
+        Ok(parameters.join(&KERNEL_PARAM_DELIMITER.to_string()))
     }
 }
 
@@ -255,22 +352,6 @@ mod tests {
 
     #[test]
     fn test_rootfs_kernel_params() {
-        // DAX is disabled on aarch64
-        #[cfg(target_arch = "aarch64")]
-        let ext4_pmem_rootflags = "data=ordered,errors=remount-ro ro";
-        #[cfg(not(target_arch = "aarch64"))]
-        let ext4_pmem_rootflags = "dax,data=ordered,errors=remount-ro ro";
-
-        #[cfg(target_arch = "aarch64")]
-        let xfs_pmem_rootflags = "ro";
-        #[cfg(not(target_arch = "aarch64"))]
-        let xfs_pmem_rootflags = "dax ro";
-
-        #[cfg(target_arch = "aarch64")]
-        let erofs_pmem_rootflags = "ro";
-        #[cfg(not(target_arch = "aarch64"))]
-        let erofs_pmem_rootflags = "dax ro";
-
         let tests = &[
             // EXT4
             TestData {
@@ -279,7 +360,7 @@ mod tests {
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_PMEM),
-                        Param::new("rootflags", ext4_pmem_rootflags),
+                        Param::new("rootflags", "dax,data=ordered,errors=remount-ro ro"),
                         Param::new("rootfstype", VM_ROOTFS_FILESYSTEM_EXT4),
                     ]
                     .to_vec(),
@@ -306,7 +387,7 @@ mod tests {
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_PMEM),
-                        Param::new("rootflags", xfs_pmem_rootflags),
+                        Param::new("rootflags", "dax ro"),
                         Param::new("rootfstype", VM_ROOTFS_FILESYSTEM_XFS),
                     ]
                     .to_vec(),
@@ -333,7 +414,7 @@ mod tests {
                 expect_params: KernelParams {
                     params: [
                         Param::new("root", VM_ROOTFS_ROOT_PMEM),
-                        Param::new("rootflags", erofs_pmem_rootflags),
+                        Param::new("rootflags", "dax ro"),
                         Param::new("rootfstype", VM_ROOTFS_FILESYSTEM_EROFS),
                     ]
                     .to_vec(),
@@ -385,7 +466,8 @@ mod tests {
 
         for (i, t) in tests.iter().enumerate() {
             let msg = format!("test[{i}]: {t:?}");
-            let result = KernelParams::new_rootfs_kernel_params(t.rootfs_driver, t.rootfs_type);
+            let result =
+                KernelParams::new_rootfs_kernel_params("", t.rootfs_driver, t.rootfs_type);
             let msg = format!("{msg}, result: {result:?}");
             if t.result.is_ok() {
                 assert!(result.is_ok(), "{}", msg);
@@ -397,4 +479,55 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_kernel_verity_params() -> Result<()> {
+        let params = KernelParams::new_rootfs_kernel_params(
+            "root_hash=abc,salt=def,data_blocks=1,data_block_size=4096,hash_block_size=4096",
+            VM_ROOTFS_DRIVER_BLK,
+            VM_ROOTFS_FILESYSTEM_EXT4,
+        )?;
+        let params_string = params.to_string()?;
+        assert!(params_string.contains("dm-mod.create="));
+        assert!(params_string.contains("root=/dev/dm-0"));
+        assert!(params_string.contains("rootfstype=ext4"));
+
+        let err = KernelParams::new_rootfs_kernel_params(
+            "root_hash=abc,data_blocks=1,data_block_size=4096,hash_block_size=4096",
+            VM_ROOTFS_DRIVER_BLK,
+            VM_ROOTFS_FILESYSTEM_EXT4,
+        )
+        .err()
+        .expect("expected missing salt error");
+        assert!(format!("{err}").contains("Missing kernel_verity_params salt"));
+
+        let err = KernelParams::new_rootfs_kernel_params(
+            "root_hash=abc,salt=def,data_block_size=4096,hash_block_size=4096",
+            VM_ROOTFS_DRIVER_BLK,
+            VM_ROOTFS_FILESYSTEM_EXT4,
+        )
+        .err()
+        .expect("expected missing data_blocks error");
+        assert!(format!("{err}").contains("Missing kernel_verity_params data_blocks"));
+
+        let err = KernelParams::new_rootfs_kernel_params(
+            "root_hash=abc,salt=def,data_blocks=foo,data_block_size=4096,hash_block_size=4096",
+            VM_ROOTFS_DRIVER_BLK,
+            VM_ROOTFS_FILESYSTEM_EXT4,
+        )
+        .err()
+        .expect("expected invalid data_blocks error");
+        assert!(format!("{err}").contains("Invalid kernel_verity_params data_blocks"));
+
+        let err = KernelParams::new_rootfs_kernel_params(
+            "root_hash=abc,salt=def,data_blocks=1,data_block_size=4096,hash_block_size=4096,badfield",
+            VM_ROOTFS_DRIVER_BLK,
+            VM_ROOTFS_FILESYSTEM_EXT4,
+        )
+        .err()
+        .expect("expected invalid entry error");
+        assert!(format!("{err}").contains("Invalid kernel_verity_params entry"));
+
+        Ok(())
+    }
 }

src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs

@@ -179,16 +179,13 @@ impl Kernel {
         let mut kernel_params = KernelParams::new(config.debug_info.enable_debug);
 
         if config.boot_info.initrd.is_empty() {
-            // QemuConfig::validate() has already made sure that if initrd is
-            // empty, image cannot be so we don't need to re-check that here
-
-            kernel_params.append(
-                &mut KernelParams::new_rootfs_kernel_params(
-                    &config.boot_info.vm_rootfs_driver,
-                    &config.boot_info.rootfs_type,
-                )
-                .context("adding rootfs params failed")?,
-            );
+            let mut rootfs_params = KernelParams::new_rootfs_kernel_params(
+                &config.boot_info.kernel_verity_params,
+                &config.boot_info.vm_rootfs_driver,
+                &config.boot_info.rootfs_type,
+            )
+            .context("adding rootfs/verity params failed")?;
+            kernel_params.append(&mut rootfs_params);
         }
 
         kernel_params.append(&mut KernelParams::from_string(
@@ -2296,6 +2293,14 @@ impl<'a> QemuCmdLine<'a> {
     }
 
     fn add_iommu(&mut self) {
+        // vIOMMU (Intel IOMMU) is not supported on the "virt" machine type (arm64)
+        if self.machine.r#type == "virt" {
+            self.kernel
+                .params
+                .append(&mut KernelParams::from_string("iommu.passthrough=0"));
+            return;
+        }
+
         let dev_iommu = DeviceIntelIommu::new();
         self.devices.push(Box::new(dev_iommu));
 

src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs

@@ -28,8 +28,13 @@ use std::str::FromStr;
 use std::time::Duration;
 
 use qapi_spec::Dictionary;
+use std::thread;
+use std::time::Instant;
+
 /// default qmp connection read timeout
 const DEFAULT_QMP_READ_TIMEOUT: u64 = 250;
+const DEFAULT_QMP_CONNECT_DEADLINE_MS: u64 = 5000;
+const DEFAULT_QMP_RETRY_SLEEP_MS: u64 = 50;
 
 pub struct Qmp {
     qmp: qapi::Qmp<qapi::Stream<BufReader<UnixStream>, UnixStream>>,
@@ -58,29 +63,43 @@ impl Debug for Qmp {
 
 impl Qmp {
     pub fn new(qmp_sock_path: &str) -> Result<Self> {
-        let stream = UnixStream::connect(qmp_sock_path)?;
+        let try_new_once_fn = || -> Result<Qmp> {
+            let stream = UnixStream::connect(qmp_sock_path)?;
 
-        // Set the read timeout to protect runtime-rs from blocking forever
-        // trying to set up QMP connection if qemu fails to launch.  The exact
-        // value is a matter of judegement.  Setting it too long would risk
-        // being ineffective since container runtime would timeout first anyway
-        // (containerd's task creation timeout is 2 s by default).  OTOH
-        // setting it too short would risk interfering with a normal launch,
-        // perhaps just seeing some delay due to a heavily loaded host.
-        stream.set_read_timeout(Some(Duration::from_millis(DEFAULT_QMP_READ_TIMEOUT)))?;
+            stream
+                .set_read_timeout(Some(Duration::from_millis(DEFAULT_QMP_READ_TIMEOUT)))
+                .context("set qmp read timeout")?;
 
-        let mut qmp = Qmp {
-            qmp: qapi::Qmp::new(qapi::Stream::new(
-                BufReader::new(stream.try_clone()?),
-                stream,
-            )),
-            guest_memory_block_size: 0,
+            let mut qmp = Qmp {
+                qmp: qapi::Qmp::new(qapi::Stream::new(
+                    BufReader::new(stream.try_clone()?),
+                    stream,
+                )),
+                guest_memory_block_size: 0,
+            };
+
+            let info = qmp.qmp.handshake().context("qmp handshake failed")?;
+            info!(sl!(), "QMP initialized: {:#?}", info);
+
+            Ok(qmp)
         };
 
-        let info = qmp.qmp.handshake()?;
-        info!(sl!(), "QMP initialized: {:#?}", info);
+        let deadline = Instant::now() + Duration::from_millis(DEFAULT_QMP_CONNECT_DEADLINE_MS);
+        let mut last_err: Option<anyhow::Error> = None;
 
-        Ok(qmp)
+        while Instant::now() < deadline {
+            match try_new_once_fn() {
+                Ok(qmp) => return Ok(qmp),
+                Err(e) => {
+                    debug!(sl!(), "QMP not ready yet: {}", e);
+                    last_err = Some(e);
+                    thread::sleep(Duration::from_millis(DEFAULT_QMP_RETRY_SLEEP_MS));
+                }
+            }
+        }
+
+        Err(last_err.unwrap_or_else(|| anyhow!("QMP init timed out")))
+            .with_context(|| format!("timed out waiting for QMP ready: {}", qmp_sock_path))
     }
 
     pub fn set_ignore_shared_memory_capability(&mut self) -> Result<()> {

src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/shim_io.rs

@@ -6,39 +6,54 @@
 
 use std::{
     io,
-    os::{
-        fd::IntoRawFd,
-        unix::{
-            fs::OpenOptionsExt,
-            io::{FromRawFd, RawFd},
-            net::UnixStream as StdUnixStream,
-            prelude::AsRawFd,
-        },
+    os::unix::{
+        fs::{FileTypeExt, OpenOptionsExt},
+        io::RawFd,
+        prelude::AsRawFd,
     },
     pin::Pin,
     task::{Context as TaskContext, Poll},
 };
 
-use anyhow::{anyhow, Context, Result};
+use anyhow::{Context, Result};
 use tokio::{
-    fs::OpenOptions,
+    fs::{File, OpenOptions},
     io::{AsyncRead, AsyncWrite},
-    net::UnixStream as AsyncUnixStream,
 };
 use url::Url;
 
-fn open_fifo_write(path: &str) -> Result<AsyncUnixStream> {
+/// Clear O_NONBLOCK for an fd (turn it into blocking mode).
+fn set_flag_with_blocking(fd: RawFd) {
+    let flag = unsafe { libc::fcntl(fd, libc::F_GETFL) };
+    if flag < 0 {
+        error!(sl!(), "failed to fcntl(F_GETFL) fd {} ret {}", fd, flag);
+        return;
+    }
+
+    let ret = unsafe { libc::fcntl(fd, libc::F_SETFL, flag & !libc::O_NONBLOCK) };
+    if ret < 0 {
+        error!(sl!(), "failed to fcntl(F_SETFL) fd {} ret {}", fd, ret);
+    }
+}
+
+fn open_fifo_write(path: &str) -> Result<File> {
     let std_file = std::fs::OpenOptions::new()
         .write(true)
         // It's not for non-block openning FIFO but for non-block stream which
         // will be add into tokio runtime.
         .custom_flags(libc::O_NONBLOCK)
         .open(path)
-        .with_context(|| format!("open {path} with write"))?;
-    let fd = std_file.into_raw_fd();
-    let std_stream = unsafe { StdUnixStream::from_raw_fd(fd) };
+        .with_context(|| format!("open fifo for write: {path}"))?;
 
-    AsyncUnixStream::from_std(std_stream).map_err(|e| anyhow!(e))
+    // Debug
+    let meta = std_file.metadata()?;
+    if !meta.file_type().is_fifo() {
+        debug!(sl!(), "[DEBUG]{} is not a fifo (type mismatch)", path);
+    }
+
+    set_flag_with_blocking(std_file.as_raw_fd());
+
+    Ok(File::from_std(std_file))
 }
 
 pub struct ShimIo {
@@ -58,14 +73,6 @@ impl ShimIo {
             "new shim io stdin {:?} stdout {:?} stderr {:?}", stdin, stdout, stderr
         );
 
-        let set_flag_with_blocking = |fd: RawFd| {
-            let flag = unsafe { libc::fcntl(fd, libc::F_GETFL) };
-            let ret = unsafe { libc::fcntl(fd, libc::F_SETFL, flag & !libc::O_NONBLOCK) };
-            if ret < 0 {
-                error!(sl!(), "failed to set fcntl for fd {} error {}", fd, ret);
-            }
-        };
-
         let stdin_fd: Option<Box<dyn AsyncRead + Send + Unpin>> = if let Some(stdin) = stdin {
             info!(sl!(), "open stdin {:?}", &stdin);
 
@@ -98,9 +105,7 @@ impl ShimIo {
                 None => None,
                 Some(out) => match Url::parse(out.as_str()) {
                     Err(url::ParseError::RelativeUrlWithoutBase) => {
-                        let out = "fifo://".to_owned() + out.as_str();
-                        let u = Url::parse(out.as_str()).unwrap();
-                        Some(u)
+                        Url::parse(&format!("fifo://{}", out)).ok()
                     }
                     Err(err) => {
                         warn!(sl!(), "unable to parse stdout uri: {}", err);
@@ -111,26 +116,25 @@ impl ShimIo {
             }
         };
 
-        let stdout_url = get_url(stdout);
         let get_fd = |url: &Option<Url>| -> Option<Box<dyn AsyncWrite + Send + Unpin>> {
             info!(sl!(), "get fd for {:?}", &url);
             if let Some(url) = url {
                 if url.scheme() == "fifo" {
                     let path = url.path();
                     match open_fifo_write(path) {
-                        Ok(s) => {
-                            return Some(Box::new(ShimIoWrite::Stream(s)));
-                        }
-                        Err(err) => {
-                            error!(sl!(), "failed to open file {} error {:?}", url.path(), err);
-                        }
+                        Ok(f) => return Some(Box::new(ShimIoWrite::File(f))),
+                        Err(err) => error!(sl!(), "failed to open fifo {} error {:?}", path, err),
                     }
+                } else {
+                    warn!(sl!(), "unsupported io scheme {}", url.scheme());
                 }
             }
             None
         };
 
+        let stdout_url = get_url(stdout);
         let stderr_url = get_url(stderr);
+
         Ok(Self {
             stdin: stdin_fd,
             stdout: get_fd(&stdout_url),
@@ -141,7 +145,7 @@ impl ShimIo {
 
 #[derive(Debug)]
 enum ShimIoWrite {
-    Stream(AsyncUnixStream),
+    File(File),
     // TODO: support other type
 }
 
@@ -151,20 +155,20 @@ impl AsyncWrite for ShimIoWrite {
         cx: &mut TaskContext<'_>,
         buf: &[u8],
     ) -> Poll<io::Result<usize>> {
-        match *self {
-            ShimIoWrite::Stream(ref mut s) => Pin::new(s).poll_write(cx, buf),
+        match &mut *self {
+            ShimIoWrite::File(f) => Pin::new(f).poll_write(cx, buf),
         }
     }
 
     fn poll_flush(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<io::Result<()>> {
-        match *self {
-            ShimIoWrite::Stream(ref mut s) => Pin::new(s).poll_flush(cx),
+        match &mut *self {
+            ShimIoWrite::File(f) => Pin::new(f).poll_flush(cx),
         }
     }
 
     fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<io::Result<()>> {
-        match *self {
-            ShimIoWrite::Stream(ref mut s) => Pin::new(s).poll_shutdown(cx),
+        match &mut *self {
+            ShimIoWrite::File(f) => Pin::new(f).poll_shutdown(cx),
         }
     }
 }

src/runtime-rs/crates/shim/src/bin/main.rs

@@ -6,10 +6,15 @@
 
 use std::{
     ffi::{OsStr, OsString},
+    io::Write,
     path::PathBuf,
 };
 
 use anyhow::{anyhow, Context, Result};
+use containerd_shim_protos::{
+    protobuf::Message,
+    types::introspection::{RuntimeInfo, RuntimeVersion},
+};
 use nix::{
     mount::{mount, MsFlags},
     sched::{self, CloneFlags},
@@ -29,11 +34,13 @@ enum Action {
     Delete(Args),
     Help,
     Version,
+    Info,
 }
 
 fn parse_args(args: &[OsString]) -> Result<Action> {
     let mut help = false;
     let mut version = false;
+    let mut info = false;
     let mut shim_args = Args::default();
 
     // Crate `go_flag` is used to keep compatible with go/flag package.
@@ -46,6 +53,7 @@ fn parse_args(args: &[OsString]) -> Result<Action> {
         flags.add_flag("publish-binary", &mut shim_args.publish_binary);
         flags.add_flag("help", &mut help);
         flags.add_flag("version", &mut version);
+        flags.add_flag("info", &mut info);
     })
     .context(Error::ParseArgument(format!("{args:?}")))?;
 
@@ -53,6 +61,8 @@ fn parse_args(args: &[OsString]) -> Result<Action> {
         Ok(Action::Help)
     } else if version {
         Ok(Action::Version)
+    } else if info {
+        Ok(Action::Info)
     } else if rest_args.is_empty() {
         Ok(Action::Run(shim_args))
     } else if rest_args[0] == "start" {
@@ -83,6 +93,8 @@ fn show_help(cmd: &OsStr) {
         enable debug output in logs
   -id string
         id of the task
+  -info
+        output the runtime info as protobuf (for containerd v2.0+)
   -namespace string
         namespace that owns the shim
   -publish-binary string
@@ -114,6 +126,25 @@ fn show_version(err: Option<anyhow::Error>) {
     }
 }
 
+fn show_info() -> Result<()> {
+    let mut version = RuntimeVersion::new();
+    version.version = config::RUNTIME_VERSION.to_string();
+    version.revision = config::RUNTIME_GIT_COMMIT.to_string();
+
+    let mut info = RuntimeInfo::new();
+    info.name = config::CONTAINERD_RUNTIME_NAME.to_string();
+    info.version = Some(version).into();
+
+    let data = info
+        .write_to_bytes()
+        .context("failed to marshal RuntimeInfo")?;
+    std::io::stdout()
+        .write_all(&data)
+        .context("failed to write RuntimeInfo to stdout")?;
+
+    Ok(())
+}
+
 fn get_tokio_runtime() -> Result<tokio::runtime::Runtime> {
     let worker_threads = std::env::var(ENV_TOKIO_RUNTIME_WORKER_THREADS)
         .unwrap_or_default()
@@ -155,6 +186,7 @@ fn real_main() -> Result<()> {
         }
         Action::Help => show_help(&args[0]),
         Action::Version => show_version(None),
+        Action::Info => show_info().context("show info")?,
     }
     Ok(())
 }

src/runtime/Makefile

@@ -152,9 +152,9 @@ FIRMWARETDVFVOLUMEPATH :=
 
 FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd
 
-ROOTMEASURECONFIG ?= ""
-KERNELTDXPARAMS += $(ROOTMEASURECONFIG)
-KERNELQEMUCOCODEVPARAMS += $(ROOTMEASURECONFIG)
+KERNELVERITYPARAMS ?= ""
+KERNELVERITYPARAMS_NV ?= ""
+KERNELVERITYPARAMS_CONFIDENTIAL_NV ?= ""
 
 # Name of default configuration file the runtime will use.
 CONFIG_FILE = configuration.toml
@@ -174,10 +174,6 @@ HYPERVISORS := $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISO
 QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD)
 QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"]
 
-#QEMUTDXPATH := $(QEMUBINDIR)/$(QEMUTDXCMD)
-QEMUTDXPATH := PLACEHOLDER_FOR_DISTRO_QEMU_WITH_TDX_SUPPORT
-QEMUTDXVALIDHYPERVISORPATHS := [\"$(QEMUTDXPATH)\"]
-
 QEMUTDXEXPERIMENTALPATH := $(QEMUBINDIR)/$(QEMUTDXEXPERIMENTALCMD)
 QEMUTDXEXPERIMENTALVALIDHYPERVISORPATHS := [\"$(QEMUTDXEXPERIMENTALPATH)\"]
 
@@ -221,8 +217,8 @@ DEFMEMSLOTS := 10
 DEFMAXMEMSZ := 0
 #Default number of bridges
 DEFBRIDGES := 1
-DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\"]
-DEFENABLEANNOTATIONS_COCO := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"default_vcpus\", \"default_memory\", \"cc_init_data\"]
+DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"kernel_verity_params\"]
+DEFENABLEANNOTATIONS_COCO := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"kernel_verity_params\", \"default_vcpus\", \"default_memory\", \"cc_init_data\"]
 DEFDISABLEGUESTSECCOMP := true
 DEFDISABLEGUESTEMPTYDIR := false
 #Default experimental features enabled
@@ -250,7 +246,7 @@ DEFSECCOMPSANDBOXPARAM :=
 DEFENTROPYSOURCE := /dev/urandom
 DEFVALIDENTROPYSOURCES := [\"/dev/urandom\",\"/dev/random\",\"\"]
 
-DEFDISABLEBLOCK := false
+DEFDISABLEBLOCK := true
 DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs
 DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
 # Please keep DEFSHAREDFS_QEMU_COCO_DEV_VIRTIOFS in sync with TDX/SNP
@@ -659,6 +655,8 @@ USER_VARS += DEFAULTMEMORY_NV
 USER_VARS += DEFAULTVFIOPORT_NV
 USER_VARS += DEFAULTPCIEROOTPORT_NV
 USER_VARS += KERNELPARAMS_NV
+USER_VARS += KERNELVERITYPARAMS_NV
+USER_VARS += KERNELVERITYPARAMS_CONFIDENTIAL_NV
 USER_VARS += DEFAULTTIMEOUT_NV
 USER_VARS += DEFSANDBOXCGROUPONLY_NV
 USER_VARS += DEFROOTFSTYPE
@@ -685,6 +683,7 @@ USER_VARS += TDXCPUFEATURES
 USER_VARS += DEFMACHINETYPE_CLH
 USER_VARS += DEFMACHINETYPE_STRATOVIRT
 USER_VARS += KERNELPARAMS
+USER_VARS += KERNELVERITYPARAMS
 USER_VARS += KERNELTDXPARAMS
 USER_VARS += KERNELQEMUCOCODEVPARAMS
 USER_VARS += LIBEXECDIR
@@ -702,18 +701,15 @@ USER_VARS += PROJECT_TYPE
 USER_VARS += PROJECT_URL
 USER_VARS += QEMUBINDIR
 USER_VARS += QEMUCMD
-USER_VARS += QEMUTDXCMD
 USER_VARS += QEMUTDXEXPERIMENTALCMD
 USER_VARS += QEMUCCAEXPERIMENTALCMD
 USER_VARS += QEMUSNPCMD
 USER_VARS += QEMUPATH
-USER_VARS += QEMUTDXPATH
 USER_VARS += QEMUTDXEXPERIMENTALPATH
 USER_VARS += QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT
 USER_VARS += QEMUSNPPATH
 USER_VARS += QEMUCCAEXPERIMENTALPATH
 USER_VARS += QEMUVALIDHYPERVISORPATHS
-USER_VARS += QEMUTDXVALIDHYPERVISORPATHS
 USER_VARS += QEMUTDXEXPERIMENTALVALIDHYPERVISORPATHS
 USER_VARS += QEMUCCAVALIDHYPERVISORPATHS
 USER_VARS += QEMUCCAEXPERIMENTALVALIDHYPERVISORPATHS

src/runtime/cmd/containerd-shim-kata-v2/main.go

@@ -9,7 +9,9 @@ import (
 	"fmt"
 	"os"
 
+	containerdtypes "github.com/containerd/containerd/api/types"
 	shimapi "github.com/containerd/containerd/runtime/v2/shim"
+	"google.golang.org/protobuf/proto"
 
 	shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
@@ -21,6 +23,25 @@ func shimConfig(config *shimapi.Config) {
 	config.NoSubreaper = true
 }
 
+func handleInfoFlag() {
+	info := &containerdtypes.RuntimeInfo{
+		Name: types.DefaultKataRuntimeName,
+		Version: &containerdtypes.RuntimeVersion{
+			Version:  katautils.VERSION,
+			Revision: katautils.COMMIT,
+		},
+	}
+
+	data, err := proto.Marshal(info)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to marshal RuntimeInfo: %v\n", err)
+		os.Exit(1)
+	}
+
+	os.Stdout.Write(data)
+	os.Exit(0)
+}
+
 func main() {
 
 	if len(os.Args) == 2 && os.Args[1] == "--version" {
@@ -28,5 +49,9 @@ func main() {
 		os.Exit(0)
 	}
 
+	if len(os.Args) == 2 && os.Args[1] == "-info" {
+		handleInfoFlag()
+	}
+
 	shimapi.Run(types.DefaultKataRuntimeName, shim.New, shimConfig)
 }

src/runtime/config/configuration-clh.toml.in

@@ -109,6 +109,20 @@ memory_slots = @DEFMEMSLOTS@
 # > amount of physical RAM      --> will be set to the actual amount of physical RAM
 default_maxmemory = @DEFMAXMEMSZ@
 
+# Disable hotplugging host block devices to guest VMs for container rootfs.
+# In case of a storage driver like devicemapper where a container's
+# root file system is backed by a block device, the block device is passed
+# directly to the hypervisor for performance reasons.
+# This flag prevents the block device from being passed to the hypervisor,
+# virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
+disable_block_device_use = @DEFDISABLEBLOCK@
+
 # Shared file system type:
 #   - virtio-fs (default)
 #   - virtio-fs-nydus
@@ -237,9 +251,9 @@ guest_hook_path = ""
 # and we strongly advise users to refer the Cloud Hypervisor official
 # documentation for a better understanding of its internals:
 # https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/docs/io_throttling.md
-# 
+#
 # Bandwidth rate limiter options
-# 
+#
 # net_rate_limiter_bw_max_rate controls network I/O bandwidth (size in bits/sec
 # for SB/VM).
 # The same value is used for inbound and outbound bandwidth.
@@ -273,9 +287,9 @@ net_rate_limiter_ops_one_time_burst = 0
 # and we strongly advise users to refer the Cloud Hypervisor official
 # documentation for a better understanding of its internals:
 # https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/docs/io_throttling.md
-# 
+#
 # Bandwidth rate limiter options
-# 
+#
 # disk_rate_limiter_bw_max_rate controls disk I/O bandwidth (size in bits/sec
 # for SB/VM).
 # The same value is used for inbound and outbound bandwidth.
@@ -462,9 +476,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 

src/runtime/config/configuration-fc.toml.in

@@ -367,9 +367,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 

src/runtime/config/configuration-qemu-cca.toml.in

@@ -159,12 +159,18 @@ memory_offset = 0
 # Default false
 enable_virtio_mem = false
 
-# Disable block device from being used for a container's rootfs.
+# Disable hotplugging host block devices to guest VMs for container rootfs.
 # In case of a storage driver like devicemapper where a container's
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
 # virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -630,9 +636,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 

src/runtime/config/configuration-qemu-coco-dev.toml.in

@@ -52,6 +52,11 @@ valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
 # container and look for 'default-kernel-parameters' log entries.
 kernel_params = "@KERNELQEMUCOCODEVPARAMS@"
 
+# Optional dm-verity parameters (comma-separated key=value list):
+# root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
+# These are used by the runtime to assemble dm-verity kernel params.
+kernel_verity_params = "@KERNELVERITYPARAMS@"
+
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
 firmware = "@FIRMWAREPATH@"
@@ -145,12 +150,18 @@ memory_offset = 0
 # Default false
 enable_virtio_mem = false
 
-# Disable block device from being used for a container's rootfs.
+# Disable hotplugging host block devices to guest VMs for container rootfs.
 # In case of a storage driver like devicemapper where a container's
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
 # virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -356,17 +367,17 @@ msize_9p = @DEFMSIZE9P@
 # nvdimm is not supported when `confidential_guest = true`.
 disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM@
 
-# Enable hot-plugging of VFIO devices to a bridge-port, 
-# root-port or switch-port. 
+# Enable hot-plugging of VFIO devices to a bridge-port,
+# root-port or switch-port.
 # The default setting is  "no-port"
-hot_plug_vfio = "no-port" 
+hot_plug_vfio = "no-port"
 
 # In a confidential compute environment hot-plugging can compromise
-# security. 
-# Enable cold-plugging of VFIO devices to a bridge-port, 
-# root-port or switch-port. 
-# The default setting is  "no-port", which means disabled. 
-cold_plug_vfio = "no-port" 
+# security.
+# Enable cold-plugging of VFIO devices to a bridge-port,
+# root-port or switch-port.
+# The default setting is  "no-port", which means disabled.
+cold_plug_vfio = "no-port"
 
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
@@ -688,9 +699,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 

src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in

@@ -15,7 +15,7 @@
 [hypervisor.qemu]
 path = "@QEMUSNPPATH@"
 kernel = "@KERNELPATH_CONFIDENTIAL_NV@"
-initrd = "@INITRDPATH_CONFIDENTIAL_NV@"
+image = "@IMAGEPATH_CONFIDENTIAL_NV@"
 
 machine_type = "@MACHINETYPE@"
 
@@ -34,7 +34,7 @@ rootfs_type = @DEFROOTFSTYPE@
 #
 # Known limitations:
 # * Does not work by design:
-#   - CPU Hotplug 
+#   - CPU Hotplug
 #   - Memory Hotplug
 #   - NVDIMM devices
 #
@@ -75,7 +75,7 @@ snp_id_auth = ""
 
 # SNP Guest Policy, the ‘POLICY’ parameter to the SNP_LAUNCH_START command.
 # If unset, the QEMU default policy (0x30000) will be used.
-# Notice that the guest policy is enforced at VM launch, and your pod VMs 
+# Notice that the guest policy is enforced at VM launch, and your pod VMs
 # won't start at all if the policy denys it. This will be indicated by a
 # 'SNP_LAUNCH_START' error.
 snp_guest_policy = 196608
@@ -92,6 +92,11 @@ snp_guest_policy = 196608
 # container and look for 'default-kernel-parameters' log entries.
 kernel_params = "@KERNELPARAMS_NV@"
 
+# Optional dm-verity parameters (comma-separated key=value list):
+# root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
+# These are used by the runtime to assemble dm-verity kernel params.
+kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@"
+
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
 firmware = "@FIRMWARESNPPATH@"
@@ -185,12 +190,18 @@ memory_offset = 0
 # Default false
 enable_virtio_mem = false
 
-# Disable block device from being used for a container's rootfs.
+# Disable hotplugging host block devices to guest VMs for container rootfs.
 # In case of a storage driver like devicemapper where a container's
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
 # virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -388,10 +399,10 @@ disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM_NV@
 pcie_root_port = 0
 
 # In a confidential compute environment hot-plugging can compromise
-# security. 
-# Enable cold-plugging of VFIO devices to a bridge-port, 
-# root-port or switch-port. 
-# The default setting is  "no-port", which means disabled. 
+# security.
+# Enable cold-plugging of VFIO devices to a bridge-port,
+# root-port or switch-port.
+# The default setting is  "no-port", which means disabled.
 cold_plug_vfio = "@DEFAULTVFIOPORT_NV@"
 
 # If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
@@ -704,9 +715,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFAULTTIMEOUT_NV@
 

src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in

@@ -14,7 +14,7 @@
 [hypervisor.qemu]
 path = "@QEMUTDXEXPERIMENTALPATH@"
 kernel = "@KERNELPATH_CONFIDENTIAL_NV@"
-initrd = "@INITRDPATH_CONFIDENTIAL_NV@"
+image = "@IMAGEPATH_CONFIDENTIAL_NV@"
 
 machine_type = "@MACHINETYPE@"
 tdx_quote_generation_service_socket_port = @QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT@
@@ -34,7 +34,7 @@ rootfs_type = @DEFROOTFSTYPE@
 #
 # Known limitations:
 # * Does not work by design:
-#   - CPU Hotplug 
+#   - CPU Hotplug
 #   - Memory Hotplug
 #   - NVDIMM devices
 #
@@ -69,6 +69,11 @@ valid_hypervisor_paths = @QEMUTDXEXPERIMENTALVALIDHYPERVISORPATHS@
 # container and look for 'default-kernel-parameters' log entries.
 kernel_params = "@KERNELPARAMS_NV@"
 
+# Optional dm-verity parameters (comma-separated key=value list):
+# root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
+# These are used by the runtime to assemble dm-verity kernel params.
+kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@"
+
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
 firmware = "@FIRMWARETDVFPATH@"
@@ -162,12 +167,18 @@ memory_offset = 0
 # Default false
 enable_virtio_mem = false
 
-# Disable block device from being used for a container's rootfs.
+# Disable hotplugging host block devices to guest VMs for container rootfs.
 # In case of a storage driver like devicemapper where a container's
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
 # virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -365,10 +376,10 @@ disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM_NV@
 pcie_root_port = 0
 
 # In a confidential compute environment hot-plugging can compromise
-# security. 
-# Enable cold-plugging of VFIO devices to a bridge-port, 
-# root-port or switch-port. 
-# The default setting is  "no-port", which means disabled. 
+# security.
+# Enable cold-plugging of VFIO devices to a bridge-port,
+# root-port or switch-port.
+# The default setting is  "no-port", which means disabled.
 cold_plug_vfio = "@DEFAULTVFIOPORT_NV@"
 
 # If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
@@ -681,9 +692,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFAULTTIMEOUT_NV@
 

src/runtime/config/configuration-qemu-nvidia-gpu.toml.in

@@ -14,7 +14,7 @@
 [hypervisor.qemu]
 path = "@QEMUPATH@"
 kernel = "@KERNELPATH_NV@"
-initrd = "@INITRDPATH_NV@"
+image = "@IMAGEPATH_NV@"
 machine_type = "@MACHINETYPE@"
 
 # rootfs filesystem type:
@@ -51,6 +51,11 @@ valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
 # container and look for 'default-kernel-parameters' log entries.
 kernel_params = "@KERNELPARAMS_NV@"
 
+# Optional dm-verity parameters (comma-separated key=value list):
+# root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
+# These are used by the runtime to assemble dm-verity kernel params.
+kernel_verity_params = "@KERNELVERITYPARAMS_NV@"
+
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
 firmware = "@FIRMWAREPATH@"
@@ -144,12 +149,18 @@ memory_offset = 0
 # Default false
 enable_virtio_mem = false
 
-# Disable block device from being used for a container's rootfs.
+# Disable hotplugging host block devices to guest VMs for container rootfs.
 # In case of a storage driver like devicemapper where a container's
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
 # virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -355,16 +366,16 @@ msize_9p = @DEFMSIZE9P@
 # nvdimm is not supported when `confidential_guest = true`.
 disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM_NV@
 
-# Enable hot-plugging of VFIO devices to a bridge-port, 
-# root-port or switch-port. 
+# Enable hot-plugging of VFIO devices to a bridge-port,
+# root-port or switch-port.
 # The default setting is  "no-port"
 hot_plug_vfio = "no-port"
 
 # In a confidential compute environment hot-plugging can compromise
-# security. 
-# Enable cold-plugging of VFIO devices to a bridge-port, 
-# root-port or switch-port. 
-# The default setting is  "no-port", which means disabled. 
+# security.
+# Enable cold-plugging of VFIO devices to a bridge-port,
+# root-port or switch-port.
+# The default setting is  "no-port", which means disabled.
 cold_plug_vfio = "@DEFAULTVFIOPORT_NV@"
 
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
@@ -683,9 +694,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFAULTTIMEOUT_NV@
 

src/runtime/config/configuration-qemu-se.toml.in

@@ -25,7 +25,7 @@ machine_type = "@MACHINETYPE@"
 #
 # Known limitations:
 # * Does not work by design:
-#   - CPU Hotplug 
+#   - CPU Hotplug
 #   - Memory Hotplug
 #   - NVDIMM devices
 #
@@ -153,12 +153,18 @@ memory_offset = 0
 # Default false
 enable_virtio_mem = false
 
-# Disable block device from being used for a container's rootfs.
+# Disable hotplugging host block devices to guest VMs for container rootfs.
 # In case of a storage driver like devicemapper where a container's
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
 # virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -343,7 +349,7 @@ msize_9p = @DEFMSIZE9P@
 # nvdimm is not supported when `confidential_guest = true`.
 disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM@
 
-# Enable hot-plugging of VFIO devices to a bridge-port, 
+# Enable hot-plugging of VFIO devices to a bridge-port,
 # root-port or switch-port.
 # The default setting is "no-port"
 hot_plug_vfio = "no-port"
@@ -671,9 +677,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 

src/runtime/config/configuration-qemu-snp.toml.in

@@ -33,7 +33,7 @@ rootfs_type = @DEFROOTFSTYPE@
 #
 # Known limitations:
 # * Does not work by design:
-#   - CPU Hotplug 
+#   - CPU Hotplug
 #   - Memory Hotplug
 #   - NVDIMM devices
 #
@@ -74,7 +74,7 @@ snp_id_auth = ""
 
 # SNP Guest Policy, the ‘POLICY’ parameter to the SNP_LAUNCH_START command.
 # If unset, the QEMU default policy (0x30000) will be used.
-# Notice that the guest policy is enforced at VM launch, and your pod VMs 
+# Notice that the guest policy is enforced at VM launch, and your pod VMs
 # won't start at all if the policy denys it. This will be indicated by a
 # 'SNP_LAUNCH_START' error.
 snp_guest_policy = 196608
@@ -184,12 +184,18 @@ memory_offset = 0
 # Default false
 enable_virtio_mem = false
 
-# Disable block device from being used for a container's rootfs.
+# Disable hotplugging host block devices to guest VMs for container rootfs.
 # In case of a storage driver like devicemapper where a container's
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
 # virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -696,9 +702,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 

src/runtime/config/configuration-qemu-tdx.toml.in

@@ -12,7 +12,7 @@
 # XXX:   Type: @PROJECT_TYPE@
 
 [hypervisor.qemu]
-path = "@QEMUTDXPATH@"
+path = "@QEMUPATH@"
 kernel = "@KERNELCONFIDENTIALPATH@"
 image = "@IMAGECONFIDENTIALPATH@"
 machine_type = "@MACHINETYPE@"
@@ -33,7 +33,7 @@ rootfs_type = @DEFROOTFSTYPE@
 #
 # Known limitations:
 # * Does not work by design:
-#   - CPU Hotplug 
+#   - CPU Hotplug
 #   - Memory Hotplug
 #   - NVDIMM devices
 #
@@ -54,7 +54,7 @@ enable_annotations = @DEFENABLEANNOTATIONS_COCO@
 # Each member of the list is a path pattern as described by glob(3).
 # The default if not set is empty (all annotations rejected.)
 # Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@
-valid_hypervisor_paths = @QEMUTDXVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
 
 # Optional space-separated list of options to pass to the guest kernel.
 # For example, use `kernel_params = "vsyscall=emulate"` if you are having
@@ -68,6 +68,11 @@ valid_hypervisor_paths = @QEMUTDXVALIDHYPERVISORPATHS@
 # container and look for 'default-kernel-parameters' log entries.
 kernel_params = "@KERNELTDXPARAMS@"
 
+# Optional dm-verity parameters (comma-separated key=value list):
+# root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
+# These are used by the runtime to assemble dm-verity kernel params.
+kernel_verity_params = "@KERNELVERITYPARAMS@"
+
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
 firmware = "@FIRMWARETDVFPATH@"
@@ -161,12 +166,18 @@ memory_offset = 0
 # Default false
 enable_virtio_mem = false
 
-# Disable block device from being used for a container's rootfs.
+# Disable hotplugging host block devices to guest VMs for container rootfs.
 # In case of a storage driver like devicemapper where a container's
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
 # virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -673,9 +684,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 

src/runtime/config/configuration-qemu.toml.in

@@ -144,12 +144,18 @@ memory_offset = 0
 # Default false
 enable_virtio_mem = false
 
-# Disable block device from being used for a container's rootfs.
+# Disable hotplugging host block devices to guest VMs for container rootfs.
 # In case of a storage driver like devicemapper where a container's
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
 # virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -355,17 +361,17 @@ msize_9p = @DEFMSIZE9P@
 # nvdimm is not supported when `confidential_guest = true`.
 disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM@
 
-# Enable hot-plugging of VFIO devices to a bridge-port, 
-# root-port or switch-port. 
+# Enable hot-plugging of VFIO devices to a bridge-port,
+# root-port or switch-port.
 # The default setting is  "no-port"
 hot_plug_vfio = "no-port"
 
 # In a confidential compute environment hot-plugging can compromise
-# security. 
-# Enable cold-plugging of VFIO devices to a bridge-port, 
-# root-port or switch-port. 
-# The default setting is  "no-port", which means disabled. 
-cold_plug_vfio = "no-port" 
+# security.
+# Enable cold-plugging of VFIO devices to a bridge-port,
+# root-port or switch-port.
+# The default setting is  "no-port", which means disabled.
+cold_plug_vfio = "no-port"
 
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
@@ -687,9 +693,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 

src/runtime/config/configuration-stratovirt.toml.in

@@ -103,12 +103,18 @@ default_maxmemory = @DEFMAXMEMSZ@
 # Default 0
 memory_offset = 0
 
-# Disable block device from being used for a container's rootfs.
+# Disable hotplugging host block devices to guest VMs for container rootfs.
 # In case of a storage driver like devicemapper where a container's
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
 # virtio-fs is used instead to pass the rootfs.
+# WARNING:
+#   Don't set this flag to false if you don't understand well the behavior of
+#   your container runtime and image snapshotter. Some snapshotters might use
+#   container image storage devices that are not meant to be hotplugged into a
+#   guest VM - e.g., because they contain files used by the host or by other
+#   guests.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -404,9 +410,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 

src/runtime/go.mod

@@ -1,7 +1,7 @@
 module github.com/kata-containers/kata-containers/src/runtime
 
 // Keep in sync with version in versions.yaml
-go 1.24.11
+go 1.24.12
 
 // WARNING: Do NOT use `replace` directives as those break dependabot:
 // https://github.com/kata-containers/kata-containers/issues/11020
@@ -49,7 +49,7 @@ require (
 	github.com/safchain/ethtool v0.6.2
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.11.1
-	github.com/urfave/cli v1.22.15
+	github.com/urfave/cli v1.22.17
 	github.com/vishvananda/netlink v1.3.1
 	github.com/vishvananda/netns v0.0.5
 	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965
@@ -85,7 +85,7 @@ require (
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/containerd/platforms v0.2.1 // indirect
 	github.com/containernetworking/cni v1.3.0 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
 	github.com/cyphar/filepath-securejoin v0.6.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.6.0 // indirect

src/runtime/go.sum

@@ -8,7 +8,6 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h
 github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA=
 github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
 github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
@@ -70,9 +69,8 @@ github.com/containernetworking/plugins v1.9.0 h1:Mg3SXBdRGkdXyFC4lcwr6u2ZB2SDeL6
 github.com/containernetworking/plugins v1.9.0/go.mod h1:JG3BxoJifxxHBhG3hFyxyhid7JgRVBu/wtooGEvWf1c=
 github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo=
 github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
-github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
+github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/cri-o/cri-o v1.34.0 h1:ux2URwAyENy5e5hD9Z95tshdfy98eqatZk0fxx3rhuk=
 github.com/cri-o/cri-o v1.34.0/go.mod h1:kP40HG+1EW5CDNHjqQBFhb6dehT5dCBKcmtO5RZAm6k=
 github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is=
@@ -289,13 +287,13 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/urfave/cli v1.22.15 h1:nuqt+pdC/KqswQKhETJjo7pvn/k4xMUxgW6liI7XpnM=
-github.com/urfave/cli v1.22.15/go.mod h1:wSan1hmo5zeyLGBjRJbzRTNk8gwoYa2B9n4q9dmRIc0=
+github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ=
+github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo=
 github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
 github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
 github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=

src/runtime/pkg/containerd-shim-v2/device_cold_plug.go

@@ -19,8 +19,13 @@ import (
 )
 
 const (
+	// containerd CRI annotations
 	nameAnnotation      = "io.kubernetes.cri.sandbox-name"
 	namespaceAnnotation = "io.kubernetes.cri.sandbox-namespace"
+
+	// CRI-O annotations
+	crioNameAnnotation      = "io.kubernetes.cri-o.KubeName"
+	crioNamespaceAnnotation = "io.kubernetes.cri-o.Namespace"
 )
 
 // coldPlugDevices handles cold plug of CDI devices into the sandbox
@@ -78,8 +83,7 @@ func coldPlugWithAPI(ctx context.Context, s *service, ociSpec *specs.Spec) error
 // the Kubelet does not pass the device information via CRI during
 // Sandbox creation.
 func getDeviceSpec(ctx context.Context, socket string, ann map[string]string) ([]string, error) {
-	podName := ann[nameAnnotation]
-	podNs := ann[namespaceAnnotation]
+	podName, podNs := getPodIdentifiers(ann)
 
 	// create dialer for unix socket
 	dialer := func(ctx context.Context, target string) (net.Conn, error) {
@@ -111,7 +115,7 @@ func getDeviceSpec(ctx context.Context, socket string, ann map[string]string) ([
 	}
 	resp, err := client.Get(ctx, prr)
 	if err != nil {
-		return nil, fmt.Errorf("cold plug: GetPodResources failed: %w", err)
+		return nil, fmt.Errorf("cold plug: GetPodResources failed for pod(%s) in namespace(%s): %w", podName, podNs, err)
 	}
 	podRes := resp.PodResources
 	if podRes == nil {
@@ -141,6 +145,24 @@ func formatCDIDevIDs(specName string, devIDs []string) []string {
 	return result
 }
 
-func debugPodID(ann map[string]string) string {
-	return fmt.Sprintf("%s/%s", ann[namespaceAnnotation], ann[nameAnnotation])
+// getPodIdentifiers returns the pod name and namespace from annotations.
+// It first checks containerd CRI annotations, then falls back to CRI-O annotations.
+func getPodIdentifiers(ann map[string]string) (podName, podNamespace string) {
+	podName = ann[nameAnnotation]
+	podNamespace = ann[namespaceAnnotation]
+
+	// Fall back to CRI-O annotations if containerd annotations are empty
+	if podName == "" {
+		podName = ann[crioNameAnnotation]
+	}
+	if podNamespace == "" {
+		podNamespace = ann[crioNamespaceAnnotation]
+	}
+
+	return podName, podNamespace
+}
+
+func debugPodID(ann map[string]string) string {
+	podName, podNamespace := getPodIdentifiers(ann)
+	return fmt.Sprintf("%s/%s", podNamespace, podName)
 }

src/runtime/pkg/katautils/config.go

@@ -93,6 +93,7 @@ type hypervisor struct {
 	MachineAccelerators            string                    `toml:"machine_accelerators"`
 	CPUFeatures                    string                    `toml:"cpu_features"`
 	KernelParams                   string                    `toml:"kernel_params"`
+	KernelVerityParams             string                    `toml:"kernel_verity_params"`
 	MachineType                    string                    `toml:"machine_type"`
 	QgsPort                        uint32                    `toml:"tdx_quote_generation_service_socket_port"`
 	BlockDeviceDriver              string                    `toml:"block_device_driver"`
@@ -387,6 +388,10 @@ func (h hypervisor) kernelParams() string {
 	return h.KernelParams
 }
 
+func (h hypervisor) kernelVerityParams() string {
+	return h.KernelVerityParams
+}
+
 func (h hypervisor) machineType() string {
 	if h.MachineType == "" {
 		return defaultMachineType
@@ -814,6 +819,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		RootfsType:            rootfsType,
 		FirmwarePath:          firmware,
 		KernelParams:          vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
+		KernelVerityParams:    h.kernelVerityParams(),
 		NumVCPUsF:             h.defaultVCPUs(),
 		DefaultMaxVCPUs:       h.defaultMaxVCPUs(),
 		MemorySize:            h.defaultMemSz(),
@@ -948,6 +954,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		MachineAccelerators:      machineAccelerators,
 		CPUFeatures:              cpuFeatures,
 		KernelParams:             vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
+		KernelVerityParams:       h.kernelVerityParams(),
 		HypervisorMachineType:    machineType,
 		QgsPort:                  h.qgsPort(),
 		NumVCPUsF:                h.defaultVCPUs(),
@@ -1088,6 +1095,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		FirmwarePath:                   firmware,
 		MachineAccelerators:            machineAccelerators,
 		KernelParams:                   vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
+		KernelVerityParams:             h.kernelVerityParams(),
 		HypervisorMachineType:          machineType,
 		NumVCPUsF:                      h.defaultVCPUs(),
 		DefaultMaxVCPUs:                h.defaultMaxVCPUs(),
@@ -1165,16 +1173,17 @@ func newDragonballHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	kernelParams := h.kernelParams()
 
 	return vc.HypervisorConfig{
-		KernelPath:      kernel,
-		ImagePath:       image,
-		RootfsType:      rootfsType,
-		KernelParams:    vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
-		NumVCPUsF:       h.defaultVCPUs(),
-		DefaultMaxVCPUs: h.defaultMaxVCPUs(),
-		MemorySize:      h.defaultMemSz(),
-		MemSlots:        h.defaultMemSlots(),
-		EntropySource:   h.GetEntropySource(),
-		Debug:           h.Debug,
+		KernelPath:         kernel,
+		ImagePath:          image,
+		RootfsType:         rootfsType,
+		KernelParams:       vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
+		KernelVerityParams: h.kernelVerityParams(),
+		NumVCPUsF:          h.defaultVCPUs(),
+		DefaultMaxVCPUs:    h.defaultMaxVCPUs(),
+		MemorySize:         h.defaultMemSz(),
+		MemSlots:           h.defaultMemSlots(),
+		EntropySource:      h.GetEntropySource(),
+		Debug:              h.Debug,
 	}, nil
 }
 
@@ -1249,6 +1258,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		ImagePath:             image,
 		RootfsType:            rootfsType,
 		KernelParams:          vc.DeserializeParams(strings.Fields(kernelParams)),
+		KernelVerityParams:    h.kernelVerityParams(),
 		HypervisorMachineType: machineType,
 		NumVCPUsF:             h.defaultVCPUs(),
 		DefaultMaxVCPUs:       h.defaultMaxVCPUs(),

src/runtime/pkg/oci/utils.go

@@ -636,6 +636,15 @@ func addHypervisorPathOverrides(ocispec specs.Spec, config *vc.SandboxConfig, ru
 		}
 	}
 
+	if value, ok := ocispec.Annotations[vcAnnotations.KernelVerityParams]; ok {
+		if value != "" {
+			if _, err := vc.ParseKernelVerityParams(value); err != nil {
+				return fmt.Errorf("invalid kernel_verity_params in annotation: %w", err)
+			}
+			config.HypervisorConfig.KernelVerityParams = value
+		}
+	}
+
 	return nil
 }
 

src/runtime/vendor/github.com/cpuguy83/go-md2man/v2/md2man/md2man.go

@@ -1,3 +1,4 @@
+// Package md2man aims in converting markdown into roff (man pages).
 package md2man
 
 import (

src/runtime/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go

@@ -47,13 +47,13 @@ const (
 	tableStart        = "\n.TS\nallbox;\n"
 	tableEnd          = ".TE\n"
 	tableCellStart    = "T{\n"
-	tableCellEnd      = "\nT}\n"
+	tableCellEnd      = "\nT}"
 	tablePreprocessor = `'\" t`
 )
 
 // NewRoffRenderer creates a new blackfriday Renderer for generating roff documents
 // from markdown
-func NewRoffRenderer() *roffRenderer { // nolint: golint
+func NewRoffRenderer() *roffRenderer {
 	return &roffRenderer{}
 }
 
@@ -316,9 +316,8 @@ func (r *roffRenderer) handleTableCell(w io.Writer, node *blackfriday.Node, ente
 		} else if nodeLiteralSize(node) > 30 {
 			end = tableCellEnd
 		}
-		if node.Next == nil && end != tableCellEnd {
-			// Last cell: need to carriage return if we are at the end of the
-			// header row and content isn't wrapped in a "tablecell"
+		if node.Next == nil {
+			// Last cell: need to carriage return if we are at the end of the header row.
 			end += crTag
 		}
 		out(w, end)
@@ -356,7 +355,7 @@ func countColumns(node *blackfriday.Node) int {
 }
 
 func out(w io.Writer, output string) {
-	io.WriteString(w, output) // nolint: errcheck
+	io.WriteString(w, output) //nolint:errcheck
 }
 
 func escapeSpecialChars(w io.Writer, text []byte) {
@@ -395,7 +394,7 @@ func escapeSpecialCharsLine(w io.Writer, text []byte) {
 			i++
 		}
 		if i > org {
-			w.Write(text[org:i]) // nolint: errcheck
+			w.Write(text[org:i]) //nolint:errcheck
 		}
 
 		// escape a character
@@ -403,7 +402,7 @@ func escapeSpecialCharsLine(w io.Writer, text []byte) {
 			break
 		}
 
-		w.Write([]byte{'\\', text[i]}) // nolint: errcheck
+		w.Write([]byte{'\\', text[i]}) //nolint:errcheck
 	}
 }
 

src/runtime/vendor/modules.txt

@@ -257,7 +257,7 @@ github.com/containernetworking/plugins/pkg/testutils
 # github.com/coreos/go-systemd/v22 v22.6.0
 ## explicit; go 1.23
 github.com/coreos/go-systemd/v22/dbus
-# github.com/cpuguy83/go-md2man/v2 v2.0.6
+# github.com/cpuguy83/go-md2man/v2 v2.0.7
 ## explicit; go 1.12
 github.com/cpuguy83/go-md2man/v2/md2man
 # github.com/cri-o/cri-o v1.34.0
@@ -526,7 +526,7 @@ github.com/stretchr/testify/assert/yaml
 # github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
 ## explicit
 github.com/syndtr/gocapability/capability
-# github.com/urfave/cli v1.22.15
+# github.com/urfave/cli v1.22.17
 ## explicit; go 1.11
 github.com/urfave/cli
 # github.com/vishvananda/netlink v1.3.1

src/runtime/virtcontainers/clh.go

@@ -466,8 +466,8 @@ func (clh *cloudHypervisor) enableProtection() error {
 	}
 }
 
-func getNonUserDefinedKernelParams(rootfstype string, disableNvdimm bool, dax bool, debug bool, confidential bool, iommu bool) ([]Param, error) {
-	params, err := GetKernelRootParams(rootfstype, disableNvdimm, dax)
+func getNonUserDefinedKernelParams(rootfstype string, disableNvdimm bool, dax bool, debug bool, confidential bool, iommu bool, kernelVerityParams string) ([]Param, error) {
+	params, err := GetKernelRootParams(rootfstype, disableNvdimm, dax, kernelVerityParams)
 	if err != nil {
 		return []Param{}, err
 	}
@@ -585,11 +585,9 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 	clh.vmconfig.Cpus = chclient.NewCpusConfig(int32(clh.config.NumVCPUs()), int32(clh.config.DefaultMaxVCPUs))
 
 	disableNvdimm := (clh.config.DisableImageNvdimm || clh.config.ConfidentialGuest)
-	// DAX is disabled on aarch64 due to kernel panic in dax_disassociate_entry
-	// with virtio-pmem on kernel 6.18.x
-	enableDax := !disableNvdimm && runtime.GOARCH != "arm64"
+	enableDax := !disableNvdimm
 
-	params, err := getNonUserDefinedKernelParams(hypervisorConfig.RootfsType, disableNvdimm, enableDax, clh.config.Debug, clh.config.ConfidentialGuest, clh.config.IOMMU)
+	params, err := getNonUserDefinedKernelParams(hypervisorConfig.RootfsType, disableNvdimm, enableDax, clh.config.Debug, clh.config.ConfidentialGuest, clh.config.IOMMU, hypervisorConfig.KernelVerityParams)
 	if err != nil {
 		return err
 	}

src/runtime/virtcontainers/fc.go

@@ -699,7 +699,12 @@ func (fc *firecracker) fcInitConfiguration(ctx context.Context) error {
 		return err
 	}
 
-	params, err := GetKernelRootParams(fc.config.RootfsType, true, false)
+	params, err := GetKernelRootParams(
+		fc.config.RootfsType,
+		true,
+		false,
+		fc.config.KernelVerityParams,
+	)
 	if err != nil {
 		return err
 	}

src/runtime/virtcontainers/hypervisor.go

@@ -16,6 +16,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -126,18 +127,56 @@ const (
 	EROFS RootfsType = "erofs"
 )
 
-func GetKernelRootParams(rootfstype string, disableNvdimm bool, dax bool) ([]Param, error) {
-	var kernelRootParams []Param
+func GetKernelRootParams(rootfstype string, disableNvdimm bool, dax bool, kernelVerityParams string) ([]Param, error) {
+	cfg, err := ParseKernelVerityParams(kernelVerityParams)
+	if err != nil {
+		return []Param{}, err
+	}
 
 	// EXT4 filesystem is used by default.
 	if rootfstype == "" {
 		rootfstype = string(EXT4)
 	}
 
+	if cfg != nil {
+		rootDevice := "/dev/pmem0p1"
+		hashDevice := "/dev/pmem0p2"
+		if disableNvdimm {
+			rootDevice = "/dev/vda1"
+			hashDevice = "/dev/vda2"
+		}
+
+		dataSectors := (cfg.dataBlockSize / 512) * cfg.dataBlocks
+		verityCmd := fmt.Sprintf(
+			"dm-verity,,,ro,0 %d verity 1 %s %s %d %d %d 0 sha256 %s %s",
+			dataSectors,
+			rootDevice,
+			hashDevice,
+			cfg.dataBlockSize,
+			cfg.hashBlockSize,
+			cfg.dataBlocks,
+			cfg.rootHash,
+			cfg.salt,
+		)
+
+		rootFlags, err := kernelVerityRootFlags(rootfstype)
+		if err != nil {
+			return []Param{}, err
+		}
+
+		return []Param{
+			{Key: "dm-mod.create", Value: fmt.Sprintf("\"%s\"", verityCmd)},
+			{Key: "root", Value: "/dev/dm-0"},
+			{Key: "rootflags", Value: rootFlags},
+			{Key: "rootfstype", Value: rootfstype},
+		}, nil
+	}
+
 	if disableNvdimm && dax {
 		return []Param{}, fmt.Errorf("Virtio-Blk does not support DAX")
 	}
 
+	kernelRootParams := []Param{}
 	if disableNvdimm {
 		// Virtio-Blk
 		kernelRootParams = append(kernelRootParams, Param{"root", string(VirtioBlk)})
@@ -171,10 +210,116 @@ func GetKernelRootParams(rootfstype string, disableNvdimm bool, dax bool) ([]Par
 	}
 
 	kernelRootParams = append(kernelRootParams, Param{"rootfstype", rootfstype})
-
 	return kernelRootParams, nil
 }
 
+const (
+	verityBlockSizeBytes = 512
+)
+
+type kernelVerityConfig struct {
+	rootHash      string
+	salt          string
+	dataBlocks    uint64
+	dataBlockSize uint64
+	hashBlockSize uint64
+}
+
+func ParseKernelVerityParams(params string) (*kernelVerityConfig, error) {
+	if strings.TrimSpace(params) == "" {
+		return nil, nil
+	}
+
+	values := map[string]string{}
+	for _, field := range strings.Split(params, ",") {
+		field = strings.TrimSpace(field)
+		if field == "" {
+			continue
+		}
+		parts := strings.SplitN(field, "=", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid kernel_verity_params entry: %q", field)
+		}
+		values[parts[0]] = parts[1]
+	}
+
+	cfg := &kernelVerityConfig{
+		rootHash: values["root_hash"],
+		salt:     values["salt"],
+	}
+	if cfg.rootHash == "" {
+		return nil, fmt.Errorf("missing kernel_verity_params root_hash")
+	}
+
+	parseUintField := func(name string) (uint64, error) {
+		value, ok := values[name]
+		if !ok || value == "" {
+			return 0, fmt.Errorf("missing kernel_verity_params %s", name)
+		}
+		parsed, err := strconv.ParseUint(value, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("invalid kernel_verity_params %s %q: %w", name, value, err)
+		}
+		return parsed, nil
+	}
+
+	dataBlocks, err := parseUintField("data_blocks")
+	if err != nil {
+		return nil, err
+	}
+	dataBlockSize, err := parseUintField("data_block_size")
+	if err != nil {
+		return nil, err
+	}
+	hashBlockSize, err := parseUintField("hash_block_size")
+	if err != nil {
+		return nil, err
+	}
+
+	if cfg.salt == "" {
+		return nil, fmt.Errorf("missing kernel_verity_params salt")
+	}
+	if dataBlocks == 0 {
+		return nil, fmt.Errorf("invalid kernel_verity_params data_blocks: must be non-zero")
+	}
+	if dataBlockSize == 0 {
+		return nil, fmt.Errorf("invalid kernel_verity_params data_block_size: must be non-zero")
+	}
+	if hashBlockSize == 0 {
+		return nil, fmt.Errorf("invalid kernel_verity_params hash_block_size: must be non-zero")
+	}
+	if dataBlockSize%verityBlockSizeBytes != 0 {
+		return nil, fmt.Errorf("invalid kernel_verity_params data_block_size: must be multiple of %d", verityBlockSizeBytes)
+	}
+	if hashBlockSize%verityBlockSizeBytes != 0 {
+		return nil, fmt.Errorf("invalid kernel_verity_params hash_block_size: must be multiple of %d", verityBlockSizeBytes)
+	}
+
+	cfg.dataBlocks = dataBlocks
+	cfg.dataBlockSize = dataBlockSize
+	cfg.hashBlockSize = hashBlockSize
+
+	return cfg, nil
+}
+
+func kernelVerityRootFlags(rootfstype string) (string, error) {
+	// EXT4 filesystem is used by default.
+	if rootfstype == "" {
+		rootfstype = string(EXT4)
+	}
+
+	switch RootfsType(rootfstype) {
+	case EROFS:
+		return "ro", nil
+	case XFS:
+		return "ro", nil
+	case EXT4:
+		return "data=ordered,errors=remount-ro ro", nil
+	default:
+		return "", fmt.Errorf("unsupported rootfs type")
+	}
+}
+
 // DeviceType describes a virtualized device type.
 type DeviceType int
 
@@ -483,6 +628,9 @@ type HypervisorConfig struct {
 	// KernelParams are additional guest kernel parameters.
 	KernelParams []Param
 
+	// KernelVerityParams are additional guest dm-verity parameters.
+	KernelVerityParams string
+
 	// HypervisorParams are additional hypervisor parameters.
 	HypervisorParams []Param
 

src/runtime/virtcontainers/hypervisor_test.go

@@ -22,6 +22,7 @@ func TestGetKernelRootParams(t *testing.T) {
 		expected      []Param
 		disableNvdimm bool
 		dax           bool
+		verityParams  string
 		error         bool
 	}{
 		// EXT4
@@ -34,6 +35,7 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: false,
 			dax:           false,
+			verityParams:  "",
 			error:         false,
 		},
 		{
@@ -45,6 +47,7 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: false,
 			dax:           true,
+			verityParams:  "",
 			error:         false,
 		},
 		{
@@ -56,6 +59,7 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: true,
 			dax:           false,
+			verityParams:  "",
 			error:         false,
 		},
 
@@ -69,6 +73,7 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: false,
 			dax:           false,
+			verityParams:  "",
 			error:         false,
 		},
 		{
@@ -80,6 +85,7 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: false,
 			dax:           true,
+			verityParams:  "",
 			error:         false,
 		},
 		{
@@ -91,6 +97,7 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: true,
 			dax:           false,
+			verityParams:  "",
 			error:         false,
 		},
 
@@ -104,6 +111,7 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: false,
 			dax:           false,
+			verityParams:  "",
 			error:         false,
 		},
 		{
@@ -115,6 +123,7 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: false,
 			dax:           true,
+			verityParams:  "",
 			error:         false,
 		},
 		{
@@ -126,6 +135,7 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: true,
 			dax:           false,
+			verityParams:  "",
 			error:         false,
 		},
 
@@ -139,6 +149,7 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: false,
 			dax:           false,
+			verityParams:  "",
 			error:         true,
 		},
 
@@ -152,12 +163,61 @@ func TestGetKernelRootParams(t *testing.T) {
 			},
 			disableNvdimm: true,
 			dax:           true,
+			verityParams:  "",
+			error:         true,
+		},
+		{
+			rootfstype: string(EXT4),
+			expected: []Param{
+				{
+					Key:   "dm-mod.create",
+					Value: "\"dm-verity,,,ro,0 8 verity 1 /dev/vda1 /dev/vda2 4096 4096 1 0 sha256 abc def\"",
+				},
+				{Key: "root", Value: "/dev/dm-0"},
+				{Key: "rootflags", Value: "data=ordered,errors=remount-ro ro"},
+				{Key: "rootfstype", Value: string(EXT4)},
+			},
+			disableNvdimm: true,
+			dax:           false,
+			verityParams:  "root_hash=abc,salt=def,data_blocks=1,data_block_size=4096,hash_block_size=4096",
+			error:         false,
+		},
+		{
+			rootfstype:    string(EXT4),
+			expected:      []Param{},
+			disableNvdimm: false,
+			dax:           false,
+			verityParams:  "root_hash=abc,data_blocks=1,data_block_size=4096,hash_block_size=4096",
+			error:         true,
+		},
+		{
+			rootfstype:    string(EXT4),
+			expected:      []Param{},
+			disableNvdimm: false,
+			dax:           false,
+			verityParams:  "root_hash=abc,salt=def,data_block_size=4096,hash_block_size=4096",
+			error:         true,
+		},
+		{
+			rootfstype:    string(EXT4),
+			expected:      []Param{},
+			disableNvdimm: false,
+			dax:           false,
+			verityParams:  "root_hash=abc,salt=def,data_blocks=foo,data_block_size=4096,hash_block_size=4096",
+			error:         true,
+		},
+		{
+			rootfstype:    string(EXT4),
+			expected:      []Param{},
+			disableNvdimm: false,
+			dax:           false,
+			verityParams:  "root_hash=abc,salt=def,data_blocks=1,data_block_size=4096,hash_block_size=4096,badfield",
 			error:         true,
 		},
 	}
 
 	for _, t := range tests {
-		kernelRootParams, err := GetKernelRootParams(t.rootfstype, t.disableNvdimm, t.dax)
+		kernelRootParams, err := GetKernelRootParams(t.rootfstype, t.disableNvdimm, t.dax, t.verityParams)
 		if t.error {
 			assert.Error(err)
 			continue

src/runtime/virtcontainers/pkg/annotations/annotations.go

@@ -84,6 +84,9 @@ const (
 	// KernelParams is a sandbox annotation for passing additional guest kernel parameters.
 	KernelParams = kataAnnotHypervisorPrefix + "kernel_params"
 
+	// KernelVerityParams is a sandbox annotation for passing guest dm-verity parameters.
+	KernelVerityParams = kataAnnotHypervisorPrefix + "kernel_verity_params"
+
 	// MachineType is a sandbox annotation to specify the type of machine being emulated by the hypervisor.
 	MachineType = kataAnnotHypervisorPrefix + "machine_type"
 

src/runtime/virtcontainers/qemu.go

@@ -861,6 +861,10 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig
 				return fmt.Errorf("Cannot get VFIO device from IOMMUFD with device: %v err: %v", dev, err)
 			}
 		} else {
+			if q.config.ConfidentialGuest {
+				return fmt.Errorf("ConfidentialGuest needs IOMMUFD - cannot use %s", dev.HostPath)
+			}
+
 			vfioDevices, err = drivers.GetAllVFIODevicesFromIOMMUGroup(dev)
 			if err != nil {
 				return fmt.Errorf("Cannot get all VFIO devices from IOMMU group with device: %v err: %v", dev, err)

src/runtime/virtcontainers/qemu_arch_base.go

@@ -773,7 +773,12 @@ func (q *qemuArchBase) setEndpointDevicePath(endpoint Endpoint, bridgeAddr int,
 
 func (q *qemuArchBase) handleImagePath(config HypervisorConfig) error {
 	if config.ImagePath != "" {
-		kernelRootParams, err := GetKernelRootParams(config.RootfsType, q.disableNvdimm, false)
+		kernelRootParams, err := GetKernelRootParams(
+			config.RootfsType,
+			q.disableNvdimm,
+			false,
+			config.KernelVerityParams,
+		)
 		if err != nil {
 			return err
 		}
@@ -781,7 +786,12 @@ func (q *qemuArchBase) handleImagePath(config HypervisorConfig) error {
 			q.qemuMachine.Options = strings.Join([]string{
 				q.qemuMachine.Options, qemuNvdimmOption,
 			}, ",")
-			kernelRootParams, err = GetKernelRootParams(config.RootfsType, q.disableNvdimm, q.dax)
+			kernelRootParams, err = GetKernelRootParams(
+				config.RootfsType,
+				q.disableNvdimm,
+				q.dax,
+				config.KernelVerityParams,
+			)
 			if err != nil {
 				return err
 			}

src/runtime/virtcontainers/qemu_arm64.go

@@ -69,11 +69,9 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) {
 			kernelParamsDebug:    kernelParamsDebug,
 			kernelParams:         kernelParams,
 			disableNvdimm:        config.DisableImageNvdimm,
-			// DAX is disabled on aarch64 due to kernel panic in dax_disassociate_entry
-			// with virtio-pmem on kernel 6.18.x
-			dax:          false,
-			protection:   noneProtection,
-			legacySerial: config.LegacySerial,
+			dax:                  true,
+			protection:           noneProtection,
+			legacySerial:         config.LegacySerial,
 		},
 		measurementAlgo: config.MeasurementAlgo,
 	}

src/runtime/virtcontainers/qemu_s390x.go

@@ -83,7 +83,12 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) {
 	}
 
 	if config.ImagePath != "" {
-		kernelParams, err := GetKernelRootParams(config.RootfsType, true, false)
+		kernelParams, err := GetKernelRootParams(
+			config.RootfsType,
+			true,
+			false,
+			config.KernelVerityParams,
+		)
 		if err != nil {
 			return nil, err
 		}

src/runtime/virtcontainers/sandbox.go

@@ -1415,6 +1415,13 @@ func (s *Sandbox) startVM(ctx context.Context, prestartHookFunc func(context.Con
 		if err != nil {
 			return err
 		}
+		// If we want the network, scan the netns again to update the network
+		// configuration after the prestart hooks have run.
+		if !s.config.NetworkConfig.DisableNewNetwork {
+			if _, err := s.network.AddEndpoints(ctx, s, nil, false); err != nil {
+				return err
+			}
+		}
 	}
 
 	if err := s.network.Run(ctx, func() error {

src/runtime/virtcontainers/stratovirt.go

@@ -337,7 +337,12 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str
 	var kernelParams []Param
 
 	if initrdPath == "" {
-		params, err := GetKernelRootParams(s.config.RootfsType, true, false)
+		params, err := GetKernelRootParams(
+			s.config.RootfsType,
+			true,
+			false,
+			s.config.KernelVerityParams,
+		)
 		if err != nil {
 			return "", err
 		}

src/tools/csi-kata-directvolume/go.mod

@@ -1,7 +1,7 @@
 module kata-containers/csi-kata-directvolume
 
 // Keep in sync with version in versions.yaml
-go 1.24.11
+go 1.24.12
 
 // WARNING: Do NOT use `replace` directives as those break dependabot:
 // https://github.com/kata-containers/kata-containers/issues/11020

src/tools/kata-ctl/Cargo.lock

@@ -3024,9 +3024,9 @@ dependencies = [
 
 [[package]]
 name = "qapi"
-version = "0.14.0"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6412bdd014ebee03ddbbe79ac03a0b622cce4d80ba45254f6357c847f06fa38"
+checksum = "7b047adab56acc4948d4b9b58693c1f33fd13efef2d6bb5f0f66a47436ceada8"
 dependencies = [
  "bytes",
  "futures",
@@ -3061,9 +3061,9 @@ dependencies = [
 
 [[package]]
 name = "qapi-qmp"
-version = "0.14.0"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8b944db7e544d2fa97595e9a000a6ba5c62c426fa185e7e00aabe4b5640b538"
+checksum = "45303cac879d89361cad0287ae15f9ae1e7799b904b474152414aeece39b9875"
 dependencies = [
  "qapi-codegen",
  "qapi-spec",

src/tools/kata-ctl/src/args.rs

@@ -81,6 +81,7 @@ pub enum Commands {
 #[error("Argument is not valid")]
 pub struct CheckArgument {
     #[clap(subcommand)]
+    #[allow(unused_assignments)]
     pub command: CheckSubCommand,
 }
 

src/tools/kata-ctl/src/check.rs

@@ -486,11 +486,11 @@ mod tests {
         let releases = get_kata_all_releases_by_url(KATA_GITHUB_RELEASE_URL);
         // sometime in GitHub action accessing to github.com API may fail
         // we can skip this test to prevent the whole test fail.
-        if releases.is_err() {
+        if let Err(error) = releases {
             warn!(
                 sl!(),
                 "get kata version failed({:?}), this maybe a temporary error, just skip the test.",
-                releases.unwrap_err()
+                error
             );
             return;
         }

src/tools/log-parser/go.mod

@@ -1,7 +1,7 @@
 module github.com/kata-containers/kata-containers/src/tools/log-parser
 
 // Keep in sync with version in versions.yaml
-go 1.24.11
+go 1.24.12
 
 require (
 	github.com/BurntSushi/toml v1.1.0

src/tools/runk/.gitignore

@@ -1 +0,0 @@
-/vendor/

src/tools/runk/Cargo.toml

@@ -1,38 +0,0 @@
-[package]
-name = "runk"
-version = "0.0.1"
-authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
-description = "runk: Kata OCI container runtime based on Kata agent"
-license = "Apache-2.0"
-edition = "2018"
-
-[dependencies]
-libcontainer = { path = "./libcontainer" }
-rustjail = { path = "../../agent/rustjail", features = [
-    "standard-oci-runtime",
-] }
-runtime-spec = { path = "../../libs/runtime-spec" }
-oci-spec = { version = "0.8.1", features = ["runtime"] }
-logging = { path = "../../libs/logging" }
-liboci-cli = "0.5.3"
-clap = { version = "4.5.40", features = ["derive", "cargo"] }
-libc = "0.2.108"
-nix = "0.23.0"
-anyhow = "1.0.52"
-slog = "2.7.0"
-chrono = { version = "0.4.19", features = ["serde"] }
-slog-async = "2.7.0"
-tokio = { version = "1.44.2", features = ["full"] }
-serde = { version = "1.0.133", features = ["derive"] }
-serde_json = "1.0.74"
-uzers = "0.12.1"
-tabwriter = "1.2.1"
-
-[features]
-seccomp = ["rustjail/seccomp"]
-
-[dev-dependencies]
-tempfile = "3.19.1"
-
-[workspace]
-members = ["libcontainer"]

src/tools/runk/Makefile

@@ -1,67 +0,0 @@
-# Copyright 2021-2022 Sony Group Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-# LIBC=musl|gnu (default: gnu)
-LIBC ?= gnu
-
-include ../../../utils.mk
-
-TARGET = runk
-TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
-AGENT_SOURCE_PATH = ../../agent
-
-EXTRA_RUSTFEATURES :=
-
-# Define if runk enables seccomp support (default: yes)
-SECCOMP := yes
-
-# BINDIR is a directory for installing executable programs
-BINDIR := /usr/local/bin
-
-ifeq ($(SECCOMP),yes)
-    override EXTRA_RUSTFEATURES += seccomp
-endif
-
-ifneq ($(EXTRA_RUSTFEATURES),)
-    override EXTRA_RUSTFEATURES := --features "$(EXTRA_RUSTFEATURES)"
-endif
-
-.DEFAULT_GOAL := default
-default: build
-
-build:
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
-
-static-checks-build:
-	@echo "INFO: static-checks-build do nothing.."
-
-install:
-	install -D $(TARGET_PATH) $(BINDIR)/$(TARGET)
-
-clean:
-	cargo clean
-
-vendor:
-	cargo vendor
-
-test: test-runk test-agent
-
-test-runk:
-	cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
-
-test-agent:
-	make test -C $(AGENT_SOURCE_PATH) STANDARD_OCI_RUNTIME=yes
-
-check: standard_rust_check
-
-.PHONY: \
-	build \
-	install \
-	clean \
-	clippy \
-	format \
-	vendor \
-	test \
-	check \

src/tools/runk/README.md

@@ -1,352 +0,0 @@
-# runk
-
-## Overview
-
-> **Warnings:**
-> `runk` is currently an experimental tool.
-> Only continue if you are using a non-critical system.
-
-`runk` is a standard OCI container runtime written in Rust based on a modified version of
-the [Kata Container agent](https://github.com/kata-containers/kata-containers/tree/main/src/agent), `kata-agent`.
-
-`runk` conforms to the [OCI Container Runtime specifications](https://github.com/opencontainers/runtime-spec).
-
-Unlike the [Kata Container runtime](https://github.com/kata-containers/kata-containers/tree/main/src/agent#features),
-`kata-runtime`, `runk` spawns and runs containers on the host machine directly.
-The user can run `runk` in the same way as the existing container runtimes such as `runc`,
-the most used implementation of the OCI runtime specs.
-
-## Why does `runk` exist?
-
-The `kata-agent` is a process running inside a virtual machine (VM) as a supervisor for managing containers
-and processes running within those containers.
-In other words, the `kata-agent` is a kind of "low-level" container runtime inside VM because the agent
-spawns and runs containers according to the OCI runtime specs.
-However, the `kata-agent` does not have the OCI Command-Line Interface (CLI) that is defined in the
-[runtime spec](https://github.com/opencontainers/runtime-spec/blob/main/runtime.md).
-The `kata-runtime` provides the CLI part of the Kata Containers runtime component,
-but the `kata-runtime` is a container runtime for creating hardware-virtualized containers running on the host.
-
-`runk` is a Rust-based standard OCI container runtime that manages normal containers,
-not hardware-virtualized containers.
-`runk` aims to become one of the alternatives to existing OCI compliant container runtimes.
-The `kata-agent` has most of the [features](https://github.com/kata-containers/kata-containers/tree/main/src/agent#features)
-needed for the container runtime and delivers high performance with a low memory footprint owing to the
-implementation by Rust language.
-Therefore, `runk` leverages the mechanism of the `kata-agent` to avoid reinventing the wheel.
-
-## Performance
-
-`runk` is faster than `runc` and has a lower memory footprint.
-
-This table shows the average of the elapsed time and the memory footprint (maximum resident set size)
-for running sequentially 100 containers, the containers run `/bin/true` using `run` command with
-[detached mode](https://github.com/opencontainers/runc/blob/main/docs/terminals.md#detached)
-on 12 CPU cores (`3.8 GHz AMD Ryzen 9 3900X`) and 32 GiB of RAM.
-`runk` always runs containers with detached mode currently.
-
-Evaluation Results:
-
-|                       | `runk` (v0.0.1) | `runc` (v1.0.3) | `crun` (v1.4.2) |
-|-----------------------|---------------|---------------|---------------|
-| time [ms]           | 39.83         | 50.39         | 38.41         |
-| memory footprint [MB] | 4.013         | 10.78         | 1.738         |
-
-## Status of `runk`
-
-We drafted the initial code here, and any contributions to `runk` and [`kata-agent`](https://github.com/kata-containers/kata-containers/tree/main/src/agent)
-are welcome.
-
-Regarding features compared to `runc`, see the `Status of runk` section in the [issue](https://github.com/kata-containers/kata-containers/issues/2784).
-
-## Building
-
-In order to enable seccomp support, you need to install the `libseccomp` library on
-your platform.
-
-> e.g. `libseccomp-dev` for Ubuntu, or `libseccomp-devel` for CentOS
-
-You can build `runk`:
-
-```bash
-$ cd runk
-$ make
-```
-
-If you want to build a statically linked binary of `runk`, set the environment
-variables for the [`libseccomp` crate](https://github.com/libseccomp-rs/libseccomp-rs) and
-set the `LIBC` to `musl`:
-
-```bash
-$ export LIBSECCOMP_LINK_TYPE=static
-$ export LIBSECCOMP_LIB_PATH="the path of the directory containing libseccomp.a"
-$ export LIBC=musl
-$ make
-```
-
-> **Note**:
->
-> - If the compilation fails when `runk` tries to link the `libseccomp` library statically
->   against `musl`, you will need to build the `libseccomp` manually with `-U_FORTIFY_SOURCE`.
->   For the details, see [our script](https://github.com/kata-containers/kata-containers/blob/main/ci/install_libseccomp.sh)
->   to install the `libseccomp` for the agent.
-> - On `ppc64le` and `s390x`, `glibc` should be used even if `LIBC=musl` is specified.
-> - If you do not want to enable seccomp support, run `make SECCOMP=no`.
-
-To install `runk` into default directory for executable program (`/usr/local/bin`):
-
-```bash
-$ sudo -E make install
-```
-
-## Using `runk` directly
-
-Please note that `runk` is a low level tool not developed with an end user in mind.
-It is mostly employed by other higher-level container software like `containerd`.
-
-If you still want to use `runk` directly, here's how.
-
-### Prerequisites
-
-It is necessary to create an OCI bundle to use the tool. The simplest method is:
-
-``` bash
-$ bundle_dir="bundle"
-$ rootfs_dir="$bundle_dir/rootfs"
-$ image="busybox"
-$ mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runk spec)
-$ sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xf -
-```
-
-> **Note:**
-> If you use the unmodified `runk spec` template, this should give a `sh` session inside the container.
-> However, if you use `runk` directly and run a container with the unmodified template,
-> `runk` cannot launch the `sh` session because `runk` does not support terminal handling yet.
-> You need to edit the process field in the `config.json` should look like this below
-> with `"terminal": false` and `"args": ["sleep", "10"]`.
-
-```json
-"process": {
-    "terminal": false,
-    "user": {
-        "uid": 0,
-        "gid": 0
-    },
-    "args": [
-        "sleep",
-        "10"
-    ],
-    "env": [
-        "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
-        "TERM=xterm"
-    ],
-    "cwd": "/",
-    [...]
-}
-```
-
-If you want to launch the `sh` session inside the container, you need to run `runk` from `containerd`.
-
-Please refer to the [Using `runk` from containerd](#using-runk-from-containerd) section
-
-### Running a container
-
-Now you can go through the [lifecycle operations](https://github.com/opencontainers/runtime-spec/blob/main/runtime.md)
-in your shell.
-You need to run `runk` as `root` because `runk` does not have the rootless feature which is the ability
-to run containers without root privileges.
-
-```bash
-$ cd $bundle_dir
-
-# Create a container
-$ sudo runk create test
-
-# View the container is created and in the "created" state
-$ sudo runk state test
-
-# Start the process inside the container
-$ sudo runk start test
-
-# After 10 seconds view that the container has exited and is now in the "stopped" state
-$ sudo runk state test
-
-# Now delete the container
-$ sudo runk delete test
-```
-
-## Using `runk` from `Docker`
-
-`runk` can run containers using [`Docker`](https://github.com/docker).
-
-First, install `Docker` from package by following the
-[`Docker` installation instructions](https://docs.docker.com/engine/install/).
-
-### Running a container with `Docker` command line
-
-Start the docker daemon:
-
-```bash
-$ sudo dockerd --experimental --add-runtime="runk=/usr/local/bin/runk"
-```
-
-> **Note:**
-> Before starting the `dockerd`, you need to stop the normal docker daemon
-> running on your environment (i.e., `systemctl stop docker`).
-
-Launch a container in a different terminal:
-
-```bash
-$ sudo docker run -it --rm --runtime runk busybox sh
-/ #
-```
-
-## Using `runk` from `Podman`
-
-`runk` can run containers using [`Podman`](https://github.com/containers/podman).
-
-First, install `Podman` from source code or package by following the
-[`Podman` installation instructions](https://podman.io/getting-started/installation).
-
-### Running a container with `Podman` command line
-
-```bash
-$ sudo podman --runtime /usr/local/bin/runk run -it --rm busybox sh
-/ #
-```
-
-> **Note:**
-> `runk` does not support some commands except
-> [OCI standard operations](https://github.com/opencontainers/runtime-spec/blob/main/runtime.md#operations)
-> yet, so those commands do not work in `Docker/Podman`. Regarding commands currently
-> implemented in `runk`, see the [Status of `runk`](#status-of-runk) section.
-
-## Using `runk` from `containerd`
-
-`runk` can run containers with the containerd runtime handler support on `containerd`.
-
-### Prerequisites for `runk` with containerd
-
-* `containerd` v1.2.4 or above
-* `cri-tools`
-
-> **Note:**
-> [`cri-tools`](https://github.com/kubernetes-sigs/cri-tools) is a set of tools for CRI
-> used for development and testing.
-
-Install `cri-tools` from source code:
-
-```bash
-$ go get github.com/kubernetes-sigs/cri-tools
-$ pushd $GOPATH/src/github.com/kubernetes-sigs/cri-tools
-$ make
-$ sudo -E make install
-$ popd
-```
-
-Write the `crictl` configuration file:
-
-``` bash
-$ cat <<EOF | sudo tee /etc/crictl.yaml
-runtime-endpoint: unix:///run/containerd/containerd.sock
-EOF
-```
-
-### Configure `containerd` to use `runk`
-
-Update `/etc/containerd/config.toml`:
-
-```bash
-$ cat <<EOF | sudo tee /etc/containerd/config.toml
-version = 2
-[plugins."io.containerd.runtime.v1.linux"]
-  shim_debug = true
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-  runtime_type = "io.containerd.runc.v2"
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runk]
-  runtime_type = "io.containerd.runc.v2"
-  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runk.options]
-    BinaryName = "/usr/local/bin/runk"
-EOF
-```
-
-Restart `containerd`:
-
-```bash
-$ sudo systemctl restart containerd
-```
-
-### Running a container with `crictl` command line
-
-You can run containers in `runk` via containerd's CRI.
-
-Pull the `busybox` image:
-
-``` bash
-$ sudo crictl pull busybox
-```
-
-Create the sandbox configuration:
-
-``` bash
-$ cat <<EOF | tee sandbox.json
-{
-    "metadata": {
-        "name": "busybox-sandbox",
-        "namespace": "default",
-        "attempt": 1,
-        "uid": "hdishd83djaidwnduwk28bcsb"
-    },
-    "log_directory": "/tmp",
-    "linux": {
-    }
-}
-EOF
-```
-
-Create the container configuration:
-
-``` bash
-$ cat <<EOF | tee container.json
-{
-    "metadata": {
-        "name": "busybox"
-    },
-    "image": {
-        "image": "docker.io/busybox"
-    },
-    "command": [
-        "sh"
-    ],
-    "envs": [
-        {
-            "key": "PATH",
-            "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-        },
-        {
-            "key": "TERM",
-            "value": "xterm"
-        }
-    ],
-    "log_path": "busybox.0.log",
-    "stdin": true,
-    "stdin_once": true,
-    "tty": true
-}
-EOF
-```
-
-With the `crictl` command line of `cri-tools`, you can specify runtime class with `-r` or `--runtime` flag.
-
-Launch a sandbox and container using the `crictl`:
-
-```bash
-# Run a container inside a sandbox
-$ sudo crictl run -r runk container.json sandbox.json
-f492eee753887ba3dfbba9022028975380739aba1269df431d097b73b23c3871
-
-# Attach to the running container
-$ sudo crictl attach --stdin --tty f492eee753887ba3dfbba9022028975380739aba1269df431d097b73b23c3871
-/ #
-```
-

src/tools/runk/libcontainer/Cargo.toml

@@ -1,32 +0,0 @@
-[package]
-name = "libcontainer"
-version = "0.0.1"
-authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
-description = "Library for runk container"
-license = "Apache-2.0"
-edition = "2018"
-
-[dependencies]
-rustjail = { path = "../../../agent/rustjail", features = [
-    "standard-oci-runtime",
-] }
-runtime-spec = { path = "../../../libs/runtime-spec" }
-oci-spec = { version = "0.8.1", features = ["runtime"] }
-kata-sys-util = { path = "../../../libs/kata-sys-util" }
-logging = { path = "../../../libs/logging" }
-derive_builder = "0.10.2"
-libc = "0.2.108"
-nix = "0.23.0"
-anyhow = "1.0.52"
-slog = "2.7.0"
-chrono = { version = "0.4.19", features = ["serde"] }
-serde = { version = "1.0.133", features = ["derive"] }
-serde_json = "1.0.74"
-scopeguard = "1.1.0"
-cgroups = { package = "cgroups-rs", git = "https://github.com/kata-containers/cgroups-rs", rev = "v0.3.5" }
-procfs = "0.14.0"
-
-[dev-dependencies]
-tempfile = "3.19.1"
-test-utils = { path = "../../../libs/test-utils" }
-protocols = { path = "../../../libs/protocols" }

src/tools/runk/libcontainer/src/activated_builder.rs

@@ -1,336 +0,0 @@
-// Copyright 2021-2022 Sony Group Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use crate::container::{load_linux_container, Container, ContainerLauncher};
-use crate::status::Status;
-use crate::utils::validate_spec;
-use anyhow::{anyhow, Result};
-use derive_builder::Builder;
-use oci::{Process as OCIProcess, Spec};
-use oci_spec::runtime as oci;
-use runtime_spec::ContainerState;
-use rustjail::container::update_namespaces;
-use slog::{debug, Logger};
-use std::fs::File;
-use std::path::{Path, PathBuf};
-
-/// Used for exec command. It will prepare the options for joining an existing container.
-#[derive(Default, Builder, Debug, Clone)]
-#[builder(build_fn(validate = "Self::validate"))]
-pub struct ActivatedContainer {
-    pub id: String,
-    pub root: PathBuf,
-    pub console_socket: Option<PathBuf>,
-    pub pid_file: Option<PathBuf>,
-    pub tty: bool,
-    pub cwd: Option<PathBuf>,
-    pub env: Vec<(String, String)>,
-    pub no_new_privs: bool,
-    pub args: Vec<String>,
-    pub process: Option<PathBuf>,
-}
-
-impl ActivatedContainerBuilder {
-    /// pre-validate before building ActivatedContainer
-    fn validate(&self) -> Result<(), String> {
-        // ensure container exists
-        let id = self.id.as_ref().unwrap();
-        let root = self.root.as_ref().unwrap();
-        let status_path = Status::get_dir_path(root, id);
-        if !status_path.exists() {
-            return Err(format!(
-                "container {} does not exist at path {:?}",
-                id, root
-            ));
-        }
-
-        // ensure argv will not be empty in process exec phase later
-        let process = self.process.as_ref().unwrap();
-        let args = self.args.as_ref().unwrap();
-        if process.is_none() && args.is_empty() {
-            return Err("process and args cannot be all empty".to_string());
-        }
-        Ok(())
-    }
-}
-
-impl ActivatedContainer {
-    /// Create ContainerLauncher that can be used to spawn a process in an existing container.
-    /// This reads the spec from status file of an existing container and adapts it with given process file
-    /// or other options like args, env, etc. It also changes the namespace in spec to join the container.
-    pub fn create_launcher(self, logger: &Logger) -> Result<ContainerLauncher> {
-        debug!(
-            logger,
-            "enter ActivatedContainer::create_launcher {:?}", self
-        );
-        let mut container = Container::load(&self.root, &self.id)?;
-
-        // If state is Created or Running, we can execute the process.
-        if container.state != ContainerState::Created && container.state != ContainerState::Running
-        {
-            return Err(anyhow!(
-                "cannot exec in a stopped or paused container, state: {:?}",
-                container.state
-            ));
-        }
-
-        let spec = container
-            .status
-            .config
-            .spec
-            .as_mut()
-            .ok_or_else(|| anyhow!("spec config was not present"))?;
-        self.adapt_exec_spec(spec, container.status.pid, logger)?;
-        debug!(logger, "adapted spec: {:?}", spec);
-        validate_spec(spec, &self.console_socket)?;
-
-        debug!(
-            logger,
-            "load LinuxContainer with config: {:?}", &container.status.config
-        );
-        let runner = load_linux_container(&container.status, self.console_socket, logger)?;
-
-        Ok(ContainerLauncher::new(
-            &self.id,
-            &container.status.bundle,
-            &self.root,
-            false,
-            runner,
-            self.pid_file,
-        ))
-    }
-
-    /// Adapt spec to execute a new process which will join the container.
-    fn adapt_exec_spec(&self, spec: &mut Spec, pid: i32, logger: &Logger) -> Result<()> {
-        // If with --process, load process from file.
-        // Otherwise, update process with args and other options.
-        if let Some(process_path) = self.process.as_ref() {
-            spec.set_process(Some(Self::get_process(process_path)?));
-        } else if let Some(process) = spec.process_mut().as_mut() {
-            self.update_process(process)?;
-        } else {
-            return Err(anyhow!("process is empty in spec"));
-        };
-        // Exec process will join the container's namespaces
-        update_namespaces(logger, spec, pid)?;
-        Ok(())
-    }
-
-    /// Update process with args and other options.
-    fn update_process(&self, process: &mut OCIProcess) -> Result<()> {
-        process.set_args(Some(self.args.clone()));
-        process.set_no_new_privileges(Some(self.no_new_privs));
-        process.set_terminal(Some(self.tty));
-        if let Some(cwd) = self.cwd.as_ref() {
-            process.set_cwd(cwd.as_path().to_path_buf());
-        }
-        if let Some(process_env) = process.env_mut() {
-            process_env.extend(self.env.iter().map(|kv| format!("{}={}", kv.0, kv.1)));
-        }
-        Ok(())
-    }
-
-    /// Read and parse OCI Process from path
-    fn get_process(process_path: &Path) -> Result<OCIProcess> {
-        let f = File::open(process_path)?;
-        Ok(serde_json::from_reader(f)?)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::status::Status;
-    use crate::utils::test_utils::*;
-    use nix::unistd::getpid;
-    use oci_spec::runtime::{LinuxBuilder, LinuxNamespaceBuilder, ProcessBuilder, User};
-    use rustjail::container::TYPETONAME;
-    use scopeguard::defer;
-    use slog::o;
-    use std::{
-        fs::{create_dir_all, File},
-        path::PathBuf,
-    };
-    use tempfile::tempdir;
-    use test_utils::skip_if_not_root;
-
-    fn create_activated_dirs(root: &Path, id: &str, bundle: &Path) {
-        Status::create_dir(root, id).unwrap();
-        create_dir_all(bundle.join(TEST_ROOTFS_PATH)).unwrap();
-    }
-
-    #[test]
-    fn test_activated_container_validate() {
-        let root = tempdir().unwrap();
-        let id = TEST_CONTAINER_ID.to_string();
-        Status::create_dir(root.path(), &id).unwrap();
-        let result = ActivatedContainerBuilder::default()
-            .id(id)
-            .root(root.into_path())
-            .console_socket(None)
-            .pid_file(None)
-            .tty(false)
-            .cwd(None)
-            .env(Vec::new())
-            .no_new_privs(false)
-            .process(None)
-            .args(vec!["sleep".to_string(), "10".to_string()])
-            .build();
-        assert!(result.is_ok());
-    }
-
-    #[test]
-    fn test_activated_container_create() {
-        // create cgroup directory needs root permission
-        skip_if_not_root!();
-        let logger = slog::Logger::root(slog::Discard, o!());
-        let bundle_dir = tempdir().unwrap();
-        let root = tempdir().unwrap();
-        // Since tests are executed concurrently, container_id must be unique in tests with cgroup.
-        // Or the cgroup directory may be removed by other tests in advance.
-        let id = "test_activated_container_create".to_string();
-        create_activated_dirs(root.path(), &id, bundle_dir.path());
-        let pid = getpid().as_raw();
-
-        let mut spec = create_dummy_spec();
-        spec.root_mut()
-            .as_mut()
-            .unwrap()
-            .set_path(bundle_dir.path().join(TEST_ROOTFS_PATH));
-
-        let status = create_custom_dummy_status(&id, pid, root.path(), &spec);
-        status.save().unwrap();
-
-        // create empty cgroup directory to avoid is_pause failing
-        let cgroup = create_dummy_cgroup(Path::new(id.as_str()));
-        defer!(cgroup.delete().unwrap());
-
-        let result = ActivatedContainerBuilder::default()
-            .id(id)
-            .root(root.into_path())
-            .console_socket(Some(PathBuf::from(TEST_CONSOLE_SOCKET_PATH)))
-            .pid_file(Some(PathBuf::from(TEST_PID_FILE_PATH)))
-            .tty(true)
-            .cwd(Some(PathBuf::from(TEST_BUNDLE_PATH)))
-            .env(vec![
-                ("K1".to_string(), "V1".to_string()),
-                ("K2".to_string(), "V2".to_string()),
-            ])
-            .no_new_privs(true)
-            .process(None)
-            .args(vec!["sleep".to_string(), "10".to_string()])
-            .build()
-            .unwrap();
-
-        let linux = LinuxBuilder::default()
-            .namespaces(
-                TYPETONAME
-                    .iter()
-                    .filter(|&(_, &name)| name != "user")
-                    .map(|ns| {
-                        LinuxNamespaceBuilder::default()
-                            .typ(ns.0.clone())
-                            .path(PathBuf::from(&format!("/proc/{}/ns/{}", pid, ns.1)))
-                            .build()
-                            .unwrap()
-                    })
-                    .collect::<Vec<_>>(),
-            )
-            .build()
-            .unwrap();
-
-        spec.set_linux(Some(linux));
-        let process = ProcessBuilder::default()
-            .terminal(result.tty)
-            .user(User::default())
-            .args(result.args.clone())
-            .cwd(result.cwd.clone().unwrap().to_string_lossy().to_string())
-            .env(vec![
-                "PATH=/bin:/usr/bin".to_string(),
-                "K1=V1".to_string(),
-                "K2=V2".to_string(),
-            ])
-            .no_new_privileges(result.no_new_privs)
-            .build()
-            .unwrap();
-
-        spec.set_process(Some(process));
-        let launcher = result.clone().create_launcher(&logger).unwrap();
-        assert!(!launcher.init);
-        assert_eq!(launcher.runner.config.spec.unwrap(), spec);
-        assert_eq!(
-            launcher.runner.console_socket,
-            result.console_socket.unwrap()
-        );
-        assert_eq!(launcher.pid_file, result.pid_file);
-    }
-
-    #[test]
-    fn test_activated_container_create_with_process() {
-        // create cgroup directory needs root permission
-        skip_if_not_root!();
-        let bundle_dir = tempdir().unwrap();
-        let process_file = bundle_dir.path().join(TEST_PROCESS_FILE_NAME);
-
-        let mut process_template = OCIProcess::default();
-        process_template.set_args(Some(vec!["sleep".to_string(), "10".to_string()]));
-        process_template.set_cwd(PathBuf::from("/"));
-
-        let file = File::create(process_file.clone()).unwrap();
-        serde_json::to_writer(&file, &process_template).unwrap();
-
-        let logger = slog::Logger::root(slog::Discard, o!());
-        let root = tempdir().unwrap();
-        // Since tests are executed concurrently, container_id must be unique in tests with cgroup.
-        // Or the cgroup directory may be removed by other tests in advance.
-        let id = "test_activated_container_create_with_process".to_string();
-        let pid = getpid().as_raw();
-        let mut spec = create_dummy_spec();
-        spec.root_mut()
-            .as_mut()
-            .unwrap()
-            .set_path(bundle_dir.path().join(TEST_ROOTFS_PATH));
-        create_activated_dirs(root.path(), &id, bundle_dir.path());
-
-        let status = create_custom_dummy_status(&id, pid, root.path(), &spec);
-        status.save().unwrap();
-        // create empty cgroup directory to avoid is_pause failing
-        let cgroup = create_dummy_cgroup(Path::new(id.as_str()));
-        defer!(cgroup.delete().unwrap());
-
-        let launcher = ActivatedContainerBuilder::default()
-            .id(id)
-            .root(root.into_path())
-            .console_socket(Some(PathBuf::from(TEST_CONSOLE_SOCKET_PATH)))
-            .pid_file(None)
-            .tty(true)
-            .cwd(Some(PathBuf::from(TEST_BUNDLE_PATH)))
-            .env(vec![
-                ("K1".to_string(), "V1".to_string()),
-                ("K2".to_string(), "V2".to_string()),
-            ])
-            .no_new_privs(true)
-            .process(Some(process_file))
-            .args(vec!["sleep".to_string(), "10".to_string()])
-            .build()
-            .unwrap()
-            .create_launcher(&logger)
-            .unwrap();
-
-        assert!(!launcher.init);
-
-        assert_eq!(
-            launcher
-                .runner
-                .config
-                .spec
-                .unwrap()
-                .process()
-                .clone()
-                .unwrap(),
-            process_template
-        );
-    }
-}

src/tools/runk/libcontainer/src/cgroup.rs

@@ -1,77 +0,0 @@
-// Copyright 2021-2022 Sony Group Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use anyhow::anyhow;
-use anyhow::Result;
-use cgroups;
-use cgroups::freezer::{FreezerController, FreezerState};
-use std::{thread, time};
-
-// Try to remove the provided cgroups path five times with increasing delay between tries.
-// If after all there are not removed cgroups, an appropriate error will be returned.
-pub fn remove_cgroup_dir(cgroup: &cgroups::Cgroup) -> Result<()> {
-    let mut retries = 5;
-    let mut delay = time::Duration::from_millis(10);
-    while retries != 0 {
-        if retries != 5 {
-            delay *= 2;
-            thread::sleep(delay);
-        }
-
-        if cgroup.delete().is_ok() {
-            return Ok(());
-        }
-
-        retries -= 1;
-    }
-
-    Err(anyhow!("failed to remove cgroups paths"))
-}
-
-// Make sure we get a stable freezer state, so retry if the cgroup is still undergoing freezing.
-pub fn get_freezer_state(freezer: &FreezerController) -> Result<FreezerState> {
-    let mut retries = 10;
-    while retries != 0 {
-        let state = freezer.state()?;
-        match state {
-            FreezerState::Thawed => return Ok(FreezerState::Thawed),
-            FreezerState::Frozen => return Ok(FreezerState::Frozen),
-            FreezerState::Freezing => {
-                // sleep for 10 ms, wait for the cgroup to finish freezing
-                thread::sleep(time::Duration::from_millis(10));
-                retries -= 1;
-            }
-        }
-    }
-    Ok(FreezerState::Freezing)
-}
-
-// check whether freezer state is frozen
-pub fn is_paused(cgroup: &cgroups::Cgroup) -> Result<bool> {
-    let freezer_controller: &FreezerController = cgroup
-        .controller_of()
-        .ok_or_else(|| anyhow!("failed to get freezer controller"))?;
-    let freezer_state = get_freezer_state(freezer_controller)?;
-    match freezer_state {
-        FreezerState::Frozen => Ok(true),
-        _ => Ok(false),
-    }
-}
-
-pub fn freeze(cgroup: &cgroups::Cgroup, state: FreezerState) -> Result<()> {
-    let freezer_controller: &FreezerController = cgroup
-        .controller_of()
-        .ok_or_else(|| anyhow!("failed to get freezer controller"))?;
-    match state {
-        FreezerState::Frozen => {
-            freezer_controller.freeze()?;
-        }
-        FreezerState::Thawed => {
-            freezer_controller.thaw()?;
-        }
-        _ => return Err(anyhow!("invalid freezer state")),
-    }
-    Ok(())
-}

src/tools/runk/libcontainer/src/container.rs

@@ -1,437 +0,0 @@
-// Copyright 2021-2022 Sony Group Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use crate::cgroup::{freeze, remove_cgroup_dir};
-use crate::status::{self, get_current_container_state, Status};
-use anyhow::{anyhow, Result};
-use cgroups;
-use cgroups::freezer::FreezerState;
-use cgroups::hierarchies::is_cgroup2_unified_mode;
-use nix::sys::signal::kill;
-use nix::{
-    sys::signal::Signal,
-    sys::signal::SIGKILL,
-    unistd::{chdir, unlink, Pid},
-};
-use procfs;
-use runtime_spec::{ContainerState, State as OCIState};
-use rustjail::cgroups::fs::Manager as CgroupManager;
-use rustjail::{
-    container::{BaseContainer, LinuxContainer, EXEC_FIFO_FILENAME},
-    process::{Process, ProcessOperations},
-    specconv::CreateOpts,
-};
-use scopeguard::defer;
-use slog::{debug, info, Logger};
-use std::{
-    env::current_dir,
-    fs,
-    path::{Path, PathBuf},
-};
-
-use kata_sys_util::hooks::HookStates;
-
-pub const CONFIG_FILE_NAME: &str = "config.json";
-
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub enum ContainerAction {
-    Create,
-    Start,
-    Run,
-}
-
-#[derive(Debug)]
-pub struct Container {
-    pub status: Status,
-    pub state: ContainerState,
-    pub cgroup: cgroups::Cgroup,
-}
-
-// Container represents a container that is created by the container runtime.
-impl Container {
-    pub fn load(state_root: &Path, id: &str) -> Result<Self> {
-        let status = Status::load(state_root, id)?;
-        let spec = status
-            .config
-            .spec
-            .as_ref()
-            .ok_or_else(|| anyhow!("spec config was not present"))?;
-        let linux = spec
-            .linux()
-            .as_ref()
-            .ok_or_else(|| anyhow!("linux config was not present"))?;
-        let cpath = if linux.cgroups_path().is_none() {
-            id.to_string()
-        } else {
-            linux
-                .cgroups_path()
-                .clone()
-                .unwrap_or_default()
-                .display()
-                .to_string()
-                .trim_start_matches('/')
-                .to_string()
-        };
-        let cgroup = cgroups::Cgroup::load(cgroups::hierarchies::auto(), cpath);
-        let state = get_current_container_state(&status, &cgroup)?;
-        Ok(Self {
-            status,
-            state,
-            cgroup,
-        })
-    }
-
-    pub fn processes(&self) -> Result<Vec<Pid>> {
-        let pids = self.cgroup.tasks();
-        let result = pids.iter().map(|x| Pid::from_raw(x.pid as i32)).collect();
-        Ok(result)
-    }
-
-    pub fn kill(&self, signal: Signal, all: bool) -> Result<()> {
-        if all {
-            let pids = self.processes()?;
-            for pid in pids {
-                if !status::is_process_running(pid)? {
-                    continue;
-                }
-                kill(pid, signal)?;
-            }
-        } else {
-            // If --all option is not specified and the container is stopped,
-            // kill operation generates an error in accordance with the OCI runtime spec.
-            if self.state == ContainerState::Stopped {
-                return Err(anyhow!(
-                    "container {} can't be killed because it is {:?}",
-                    self.status.id,
-                    self.state
-                )
-                // This error message mustn't be chagned because the containerd integration tests
-                // expect that OCI container runtimes return the message.
-                // Ref. https://github.com/containerd/containerd/blob/release/1.7/pkg/process/utils.go#L135
-                .context("container not running"));
-            }
-
-            let pid = Pid::from_raw(self.status.pid);
-            if status::is_process_running(pid)? {
-                kill(pid, signal)?;
-            }
-        }
-        // For cgroup v1, killing a process in a frozen cgroup does nothing until it's thawed.
-        // Only thaw the cgroup for SIGKILL.
-        // Ref: https://github.com/opencontainers/runc/pull/3217
-        if !is_cgroup2_unified_mode() && self.state == ContainerState::Paused && signal == SIGKILL {
-            freeze(&self.cgroup, FreezerState::Thawed)?;
-        }
-        Ok(())
-    }
-
-    pub async fn delete(&self, force: bool, logger: &Logger) -> Result<()> {
-        let status = &self.status;
-        let spec = status
-            .config
-            .spec
-            .as_ref()
-            .ok_or_else(|| anyhow!("spec config was not present in the status"))?;
-
-        let oci_state = OCIState {
-            version: status.oci_version.clone(),
-            id: status.id.clone(),
-            status: self.state,
-            pid: status.pid,
-            bundle: status
-                .bundle
-                .to_str()
-                .ok_or_else(|| anyhow!("invalid bundle path"))?
-                .to_string(),
-            annotations: spec.annotations().clone().unwrap_or_default(),
-        };
-
-        if let Some(hooks) = spec.hooks().as_ref() {
-            info!(&logger, "Poststop Hooks");
-            let mut poststop_hookstates = HookStates::new();
-            poststop_hookstates.execute_hooks(
-                &hooks.poststop().clone().unwrap_or_default(),
-                Some(oci_state.clone()),
-            )?;
-        }
-
-        match oci_state.status {
-            ContainerState::Stopped => {
-                self.destroy()?;
-            }
-            ContainerState::Created => {
-                // Kill an init process
-                self.kill(SIGKILL, false)?;
-                self.destroy()?;
-            }
-            _ => {
-                if force {
-                    self.kill(SIGKILL, true)?;
-                    self.destroy()?;
-                } else {
-                    return Err(anyhow!(
-                        "cannot delete container {} that is not stopped",
-                        &status.id
-                    ));
-                }
-            }
-        }
-
-        Ok(())
-    }
-
-    pub fn pause(&self) -> Result<()> {
-        if self.state != ContainerState::Running && self.state != ContainerState::Created {
-            return Err(anyhow!(
-                "failed to pause container: current status is: {:?}",
-                self.state
-            ));
-        }
-        freeze(&self.cgroup, FreezerState::Frozen)?;
-        Ok(())
-    }
-
-    pub fn resume(&self) -> Result<()> {
-        if self.state != ContainerState::Paused {
-            return Err(anyhow!(
-                "failed to resume container: current status is: {:?}",
-                self.state
-            ));
-        }
-        freeze(&self.cgroup, FreezerState::Thawed)?;
-        Ok(())
-    }
-
-    pub fn destroy(&self) -> Result<()> {
-        remove_cgroup_dir(&self.cgroup)?;
-        self.status.remove_dir()
-    }
-}
-
-/// Used to run a process. If init is set, it will create a container and run the process in it.
-/// If init is not set, it will run the process in an existing container.
-#[derive(Debug)]
-pub struct ContainerLauncher {
-    pub id: String,
-    pub bundle: PathBuf,
-    pub state_root: PathBuf,
-    pub init: bool,
-    pub runner: LinuxContainer,
-    pub pid_file: Option<PathBuf>,
-}
-
-impl ContainerLauncher {
-    pub fn new(
-        id: &str,
-        bundle: &Path,
-        state_root: &Path,
-        init: bool,
-        runner: LinuxContainer,
-        pid_file: Option<PathBuf>,
-    ) -> Self {
-        ContainerLauncher {
-            id: id.to_string(),
-            bundle: bundle.to_path_buf(),
-            state_root: state_root.to_path_buf(),
-            init,
-            runner,
-            pid_file,
-        }
-    }
-
-    /// Launch a process. For init containers, we will create a container. For non-init, it will join an existing container.
-    pub async fn launch(&mut self, action: ContainerAction, logger: &Logger) -> Result<()> {
-        if self.init {
-            self.spawn_container(action, logger).await?;
-        } else {
-            if action == ContainerAction::Create {
-                return Err(anyhow!(
-                    "ContainerAction::Create is used for init-container only"
-                ));
-            }
-            self.spawn_process(action, logger).await?;
-        }
-        if let Some(pid_file) = self.pid_file.as_ref() {
-            fs::write(
-                pid_file,
-                format!("{}", self.runner.get_process(self.id.as_str())?.pid()),
-            )?;
-        }
-        Ok(())
-    }
-
-    /// Create the container by invoking runner to spawn the first process and save status.
-    async fn spawn_container(&mut self, action: ContainerAction, logger: &Logger) -> Result<()> {
-        // State root path root/id has been created in LinuxContainer::new(),
-        // so we don't have to create it again.
-
-        // Spawn a new process in the container by using the agent's codes.
-        self.spawn_process(action, logger).await?;
-
-        let status = self.get_status()?;
-        status.save()?;
-        debug!(logger, "saved status is {:?}", status);
-
-        // Clean up the fifo file created by LinuxContainer, which is used for block the created process.
-        if action == ContainerAction::Run || action == ContainerAction::Start {
-            let fifo_path = get_fifo_path(&status);
-            if fifo_path.exists() {
-                unlink(&fifo_path)?;
-            }
-        }
-        Ok(())
-    }
-
-    /// Generate rustjail::Process from OCI::Process
-    fn get_process(&self, logger: &Logger) -> Result<Process> {
-        let spec = self.runner.config.spec.as_ref().unwrap();
-        if spec.process().is_some() {
-            Ok(Process::new(
-                logger,
-                spec.process().as_ref().unwrap(),
-                // rustjail::LinuxContainer use the exec_id to identify processes in a container,
-                // so we can get the spawned process by ctr.get_process(exec_id) later.
-                // Since LinuxContainer is temporarily created to spawn one process in each runk invocation,
-                // we can use arbitrary string as the exec_id. Here we choose the container id.
-                &self.id,
-                self.init,
-                0,
-                None,
-            )?)
-        } else {
-            Err(anyhow!("no process configuration"))
-        }
-    }
-
-    /// Spawn a new process in the container by invoking runner.
-    async fn spawn_process(&mut self, action: ContainerAction, logger: &Logger) -> Result<()> {
-        // Agent will chdir to bundle_path before creating LinuxContainer. Just do the same as agent.
-        let current_dir = current_dir()?;
-        chdir(&self.bundle)?;
-        defer! {
-            chdir(&current_dir).unwrap();
-        }
-
-        let process = self.get_process(logger)?;
-        match action {
-            ContainerAction::Create => {
-                self.runner.start(process).await?;
-            }
-            ContainerAction::Start => {
-                self.runner.exec().await?;
-            }
-            ContainerAction::Run => {
-                self.runner.run(process).await?;
-            }
-        }
-        Ok(())
-    }
-
-    /// Generate runk specified Status
-    fn get_status(&self) -> Result<Status> {
-        let oci_state = self.runner.oci_state()?;
-        // read start time from /proc/<pid>/stat
-        let proc = procfs::process::Process::new(self.runner.init_process_pid)?;
-        let process_start_time = proc.stat()?.starttime;
-        Status::new(
-            &self.state_root,
-            &self.bundle,
-            oci_state,
-            process_start_time,
-            self.runner.created,
-            self.runner
-                .cgroup_manager
-                .as_ref()
-                .as_any()?
-                .downcast_ref::<CgroupManager>()
-                .unwrap()
-                .clone(),
-            self.runner.config.clone(),
-        )
-    }
-}
-
-pub fn create_linux_container(
-    id: &str,
-    root: &Path,
-    config: CreateOpts,
-    console_socket: Option<PathBuf>,
-    logger: &Logger,
-) -> Result<LinuxContainer> {
-    let mut container = LinuxContainer::new(
-        id,
-        root.to_str()
-            .map(|s| s.to_string())
-            .ok_or_else(|| anyhow!("failed to convert bundle path"))?
-            .as_str(),
-        None,
-        config,
-        logger,
-    )?;
-    if let Some(socket_path) = console_socket.as_ref() {
-        container.set_console_socket(socket_path)?;
-    }
-    Ok(container)
-}
-
-// Load rustjail's Linux container.
-// "uid_map_path" and "gid_map_path" are always empty, so they are not set.
-pub fn load_linux_container(
-    status: &Status,
-    console_socket: Option<PathBuf>,
-    logger: &Logger,
-) -> Result<LinuxContainer> {
-    let mut container = LinuxContainer::new(
-        &status.id,
-        &status
-            .root
-            .to_str()
-            .map(|s| s.to_string())
-            .ok_or_else(|| anyhow!("failed to convert a root path"))?,
-        None,
-        status.config.clone(),
-        logger,
-    )?;
-    if let Some(socket_path) = console_socket.as_ref() {
-        container.set_console_socket(socket_path)?;
-    }
-
-    container.init_process_pid = status.pid;
-    container.init_process_start_time = status.process_start_time;
-    container.created = status.created.into();
-    Ok(container)
-}
-
-pub fn get_config_path<P: AsRef<Path>>(bundle: P) -> PathBuf {
-    bundle.as_ref().join(CONFIG_FILE_NAME)
-}
-
-pub fn get_fifo_path(status: &Status) -> PathBuf {
-    status.root.join(&status.id).join(EXEC_FIFO_FILENAME)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::utils::test_utils::*;
-    use rustjail::container::EXEC_FIFO_FILENAME;
-    use std::path::PathBuf;
-
-    #[test]
-    fn test_get_config_path() {
-        let test_data = PathBuf::from(TEST_BUNDLE_PATH).join(CONFIG_FILE_NAME);
-        assert_eq!(get_config_path(TEST_BUNDLE_PATH), test_data);
-    }
-
-    #[test]
-    fn test_get_fifo_path() {
-        let test_data = PathBuf::from(TEST_STATE_ROOT_PATH)
-            .join(TEST_CONTAINER_ID)
-            .join(EXEC_FIFO_FILENAME);
-        let status = create_dummy_status();
-
-        assert_eq!(get_fifo_path(&status), test_data);
-    }
-}