Merge pull request #9833 from wainersm/qemu-rs_tests

tests/k8s: run for qemu-runtime-rs on AKS

.github/actionlint.yaml

@@ -0,0 +1,24 @@
+# Copyright (c) 2024 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Configuration file with rules for the actionlint tool.
+#
+self-hosted-runner:
+  # Labels of self-hosted runner that linter should ignore
+  labels:
+    - arm64-builder
+    - garm-ubuntu-2004
+    - garm-ubuntu-2004-smaller
+    - garm-ubuntu-2204
+    - garm-ubuntu-2304
+    - garm-ubuntu-2304-smaller
+    - garm-ubuntu-2204-smaller
+    - k8s-ppc64le
+    - metrics
+    - ppc64le
+    - sev
+    - sev-snp
+    - s390x
+    - s390x-large
+    - tdx

.github/workflows/basic-ci-amd64.yaml

@@ -176,6 +176,8 @@ jobs:
         vmm:
           - clh # cloud-hypervisor
           - qemu
+    # TODO: enable me when https://github.com/kata-containers/kata-containers/issues/9763 is fixed
+    if: false
     runs-on: garm-ubuntu-2204-smaller
     env:
       KATA_HYPERVISOR: ${{ matrix.vmm }}
@@ -211,7 +213,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        vmm: ['clh', 'qemu']
+        vmm:
+          - clh
+          - qemu
+        exclude:
+          # TODO: enable with clh when https://github.com/kata-containers/kata-containers/issues/9764 is fixed
+          - vmm: clh
     runs-on: garm-ubuntu-2304
     env:
       GOPATH: ${{ github.workspace }}

.github/workflows/build-checks.yaml

@@ -111,3 +111,4 @@ jobs:
           ${{ matrix.command }}
         env:
           RUST_BACKTRACE: "1"
+          SKIP_GO_VERSION_CHECK: "1"

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -60,14 +60,8 @@ jobs:
         stage:
           - ${{ inputs.stage }}
         exclude:
-          - asset: agent
-            stage: release
           - asset: cloud-hypervisor-glibc
             stage: release
-          - asset: pause-image
-            stage: release
-          - asset: coco-guest-components
-            stage: release
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -93,7 +87,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -102,8 +96,10 @@ jobs:
           ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
+        if: ${{ matrix.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -39,8 +39,6 @@ jobs:
           - rootfs-initrd
           - shim-v2
           - virtiofsd
-        stage:
-          - ${{ inputs.stage }}
     steps:
       - name: Adjust a permission for repo
         run: |
@@ -79,8 +77,10 @@ jobs:
           ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
+        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}

.github/workflows/build-kata-static-tarball-ppc64le.yaml

@@ -80,8 +80,10 @@ jobs:
           ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
+        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -39,13 +39,6 @@ jobs:
           - rootfs-initrd-confidential
           - shim-v2
           - virtiofsd
-        stage:
-          - ${{ inputs.stage }}
-        exclude:
-          - asset: pause-image
-            stage: release
-          - asset: coco-guest-components
-            stage: release
     steps:
       - name: Take a pre-action for self-hosted runner
         run: ${HOME}/script/pre_action.sh ubuntu-2204
@@ -84,8 +77,10 @@ jobs:
           ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
+        if: ${{ inputs.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}

.github/workflows/release-amd64.yaml

@@ -10,7 +10,9 @@ jobs:
   build-kata-static-tarball-amd64:
     uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
     with:
+      push-to-registry: yes
       stage: release
+    secrets: inherit
 
   kata-deploy:
     needs: build-kata-static-tarball-amd64

.github/workflows/release-arm64.yaml

@@ -10,7 +10,9 @@ jobs:
   build-kata-static-tarball-arm64:
     uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
     with:
+      push-to-registry: yes
       stage: release
+    secrets: inherit
 
   kata-deploy:
     needs: build-kata-static-tarball-arm64

.github/workflows/release-ppc64le.yaml

@@ -10,7 +10,9 @@ jobs:
   build-kata-static-tarball-ppc64le:
     uses: ./.github/workflows/build-kata-static-tarball-ppc64le.yaml
     with:
+      push-to-registry: yes
       stage: release
+    secrets: inherit
 
   kata-deploy:
     needs: build-kata-static-tarball-ppc64le

.github/workflows/release-s390x.yaml

@@ -10,6 +10,7 @@ jobs:
   build-kata-static-tarball-s390x:
     uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
     with:
+      push-to-registry: yes
       stage: release
     secrets: inherit
 

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -36,6 +36,7 @@ jobs:
           - clh
           - dragonball
           - qemu
+          - qemu-runtime-rs
           - stratovirt
           - cloud-hypervisor
         instance-type:

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -27,8 +27,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        vmm:
-          - qemu
         snapshotter:
           - devmapper
           - nydus
@@ -39,10 +37,12 @@ jobs:
             pull-type: default
             using-nfd: true
             deploy-cmd: configure-snapshotter
+            vmm: qemu
           - snapshotter: nydus
             pull-type: guest-pull
             using-nfd: false
             deploy-cmd: deploy-snapshotter
+            vmm: qemu-coco-dev
     runs-on: s390x-large
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}

.github/workflows/run-kata-coco-tests.yaml

@@ -42,7 +42,9 @@ jobs:
       KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBERNETES: "k3s"
       USING_NFD: "true"
+      KBS: "true"
       K8S_TEST_HOST_TYPE: "baremetal"
+      KBS_INGRESS: "nodeport"
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       PULL_TYPE: ${{ matrix.pull-type }}
     steps:
@@ -65,6 +67,18 @@ jobs:
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
 
+      - name: Uninstall previous `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+
       - name: Run tests
         timeout-minutes: 30
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
@@ -77,6 +91,10 @@ jobs:
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
+      - name: Delete CoCo KBS
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+
   run-k8s-tests-on-sev:
     strategy:
       fail-fast: false

.github/workflows/run-kata-deploy-tests-on-aks.yaml

@@ -33,6 +33,7 @@ jobs:
           - clh
           - dragonball
           - qemu
+          - qemu-runtime-rs
         include:
           - host_os: cbl-mariner
             vmm: clh

.github/workflows/run-kata-monitor-tests.yaml

@@ -26,6 +26,10 @@ jobs:
         include:
           - container_engine: containerd
             containerd_version: lts
+        exclude:
+          # TODO: enable with containerd when https://github.com/kata-containers/kata-containers/issues/9761 is fixed
+          - container_engine: containerd
+            vmm: qemu
     runs-on: garm-ubuntu-2204-smaller
     env:
       CONTAINER_ENGINE: ${{ matrix.container_engine }}

README.md

@@ -140,6 +140,7 @@ The table below lists the remaining parts of the project:
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
 | [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](.github/workflows) | CI | Continuous Integration configuration files and scripts. |
+| [`ocp-ci`](ci/openshift-ci/README.md) | CI | Continuous Integration configuration for the OpenShift pipelines. |
 | [`katacontainers.io`](https://github.com/kata-containers/www.katacontainers.io) | Source for the [`katacontainers.io`](https://www.katacontainers.io) site. |
 | [`Webhook`](tools/testing/kata-webhook/README.md) | utility | Example of a simple admission controller webhook to annotate pods with the Kata runtime class |
 

VERSION

@@ -1 +1 @@
-3.5.0
+3.6.0

ci/install_libseccomp.sh

@@ -23,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 # Variables for libseccomp
 libseccomp_version="${LIBSECCOMP_VERSION:-""}"
 if [ -z "${libseccomp_version}" ]; then
-    libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version")
+    libseccomp_version=$(get_from_kata_deps ".externals.libseccomp.version")
 fi
 libseccomp_url="${LIBSECCOMP_URL:-""}"
 if [ -z "${libseccomp_url}" ]; then
-    libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url")
+    libseccomp_url=$(get_from_kata_deps ".externals.libseccomp.url")
 fi
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
@@ -36,11 +36,11 @@ cflags="-O2"
 # Variables for gperf
 gperf_version="${GPERF_VERSION:-""}"
 if [ -z "${gperf_version}" ]; then
-    gperf_version=$(get_from_kata_deps "externals.gperf.version")
+    gperf_version=$(get_from_kata_deps ".externals.gperf.version")
 fi
 gperf_url="${GPERF_URL:-""}"
 if [ -z "${gperf_url}" ]; then
-    gperf_url=$(get_from_kata_deps "externals.gperf.url")
+    gperf_url=$(get_from_kata_deps ".externals.gperf.url")
 fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"

ci/install_yq.sh

@@ -5,6 +5,8 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+[ -n "$DEBUG" ] && set -o xtrace
+
 # If we fail for any reason a message will be displayed
 die() {
 	msg="$*"
@@ -16,7 +18,7 @@ die() {
 # Install via binary download, as we may not have golang installed at this point
 function install_yq() {
 	local yq_pkg="github.com/mikefarah/yq"
-	local yq_version=3.4.1
+	local yq_version=v4.40.7
 	local precmd=""
 	INSTALL_IN_GOPATH=${INSTALL_IN_GOPATH:-true}
 
@@ -36,7 +38,7 @@ function install_yq() {
 			fi
 		fi
 	fi
-	[ -x  "${yq_path}" ] && [ "`${yq_path} --version`"X == "yq version ${yq_version}"X ] && return
+	[ -x  "${yq_path}" ] && [ "`${yq_path} --version`"X == "yq (https://github.com/mikefarah/yq/) version ${yq_version}"X ] && return
 
 	read -r -a sysInfo <<< "$(uname -sm)"
 

ci/lib.sh

@@ -21,13 +21,7 @@ clone_tests_repo()
 {
 	if [ -d "$tests_repo_dir" ]; then
 		[ -n "${CI:-}" ] && return
-		# git config --global --add safe.directory will always append
-		# the target to .gitconfig without checking the existence of
-		# the target, so it's better to check it before adding the target repo.
-		local sd="$(git config --global --get safe.directory ${tests_repo_dir} || true)"
-		if [ -z "${sd}" ]; then
-			git config --global --add safe.directory ${tests_repo_dir}
-		fi
+
 		pushd "${tests_repo_dir}"
 		git checkout "${branch}"
 		git pull

ci/openshift-ci/README.md

@@ -0,0 +1,149 @@
+OpenShift CI
+============
+
+This directory contains scripts used by
+[the OpenShift CI](https://github.com/openshift/release/tree/master/ci-operator/config/kata-containers/kata-containers)
+pipelines to monitor selected functional tests on OpenShift.
+There are 2 pipelines, history and logs can be accessed here:
+
+* [main - currently supported OCP](https://prow.ci.openshift.org/job-history/gs/origin-ci-test/logs/periodic-ci-kata-containers-kata-containers-main-e2e-tests)
+* [next - currently under development OCP](https://prow.ci.openshift.org/job-history/gs/origin-ci-test/logs/periodic-ci-kata-containers-kata-containers-main-next-e2e-tests)
+
+
+Running openshift-tests on OCP with kata-containers manually
+============================================================
+
+To run openshift-tests (or other suites) with kata-containers one can use
+the kata-webhook. To deploy everything you can mimic the CI pipeline by:
+
+```bash
+#!/bin/bash -e
+# Setup your kubectl and check it's accessible by
+kubectl nodes
+# Deploy kata (set KATA_DEPLOY_IMAGE to override the default kata-deploy-ci:latest image)
+./test.sh
+# Deploy the webhook
+KATA_RUNTIME=kata-qemu cluster/deploy_webhook.sh
+```
+
+This should ensure kata-containers as well as kata-webhook are installed and
+working. Before running the openshift-tests it's (currently) recommended to
+ignore some security features by:
+
+```bash
+#!/bin/bash -e
+oc adm policy add-scc-to-group privileged system:authenticated system:serviceaccounts
+oc adm policy add-scc-to-group anyuid system:authenticated system:serviceaccounts
+oc label --overwrite ns default pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/warn=baseline pod-security.kubernetes.io/audit=baseline
+```
+
+Now you should be ready to run the openshift-tests. Our CI only uses a subset
+of tests, to get the current ``TEST_SKIPS`` see
+[the pipeline config](https://github.com/openshift/release/tree/master/ci-operator/config/kata-containers/kata-containers).
+Following steps require the [openshift tests](https://github.com/openshift/origin)
+being cloned and built in the current directory:
+
+```bash
+#!/bin/bash -e
+# Define tests to be skipped (see the pipeline config for the current version)
+TEST_SKIPS="\[sig-node\] Security Context should support seccomp runtime/default\|\[sig-node\] Variable Expansion should allow substituting values in a volume subpath\|\[k8s.io\] Probing container should be restarted with a docker exec liveness probe with timeout\|\[sig-node\] Pods Extended Pod Container lifecycle evicted pods should be terminal\|\[sig-node\] PodOSRejection \[NodeConformance\] Kubelet should reject pod when the node OS doesn't match pod's OS\|\[sig-network\].*for evicted pods\|\[sig-network\].*HAProxy router should override the route\|\[sig-network\].*HAProxy router should serve a route\|\[sig-network\].*HAProxy router should serve the correct\|\[sig-network\].*HAProxy router should run\|\[sig-network\].*when FIPS.*the HAProxy router\|\[sig-network\].*bond\|\[sig-network\].*all sysctl on whitelist\|\[sig-network\].*sysctls should not affect\|\[sig-network\] pods should successfully create sandboxes by adding pod to network"
+# Get the list of tests to be executed
+TESTS="$(./openshift-tests run --dry-run --provider "${TEST_PROVIDER}" "${TEST_SUITE}")"
+# Store the list of tests in /tmp/tsts file
+echo "${TESTS}" | grep -v "$TEST_SKIPS" > /tmp/tsts
+# Remove previously-existing temporarily files as well as previous results
+OUT=RESULTS/tmp
+rm -Rf /tmp/*test* /tmp/e2e-*
+rm -R $OUT
+mkdir -p $OUT
+# Run the tests ignoring the monitor health checks
+./openshift-tests run --provider azure -o "$OUT/job.log" --junit-dir "$OUT" --file /tmp/tsts --max-parallel-tests 5 --cluster-stability Disruptive --run '^\[sig-node\].*|^\[sig-network\]'
+```
+
+[!NOTE]
+Note we are ignoring the cluster stability checks because our public cloud is
+not that stable and running with VMs instead of containers results in minor
+stability issues. Some of the old monitor stability tests do not reflect
+the ``--cluster-stability`` setting, one should simply ignore these. If you
+get a message like ``invariant was violated`` or ``error: failed due to a
+MonitorTest failure``, it's usually an indication that only those kind of
+tests failed but the real tests passed. See
+[wrapped-openshift-tests.sh](https://github.com/openshift/release/blob/master/ci-operator/config/kata-containers/kata-containers/wrapped-openshift-tests.sh)
+for details how our pipeline deals with that.
+
+[!TIP]
+To compare multiple results locally one can use
+[junit2html](https://github.com/inorton/junit2html) tool.
+
+
+Best-effort kata-containers cleanup
+===================================
+
+If you need to cleanup the cluster after testing, you can use the
+``cleanup.sh`` script from the current directory. It tries to delete all
+resources created by ``test.sh`` as well as ``cluster/deploy_webhook.sh``
+ignoring all failures. The primary purpose of this script is to allow
+soft-cleanup after deployment to test different versions without
+re-provisioning everything.
+
+[!WARNING]
+Do not rely on this script in production, return codes are not checked!**
+
+
+Bisecting e2e tests failures
+============================
+
+Let's say the OCP pipeline passed running with
+``quay.io/kata-containers/kata-deploy-ci:kata-containers-d7afd31fd40e37a675b25c53618904ab57e74ccd-amd64``
+but failed running with
+``quay.io/kata-containers/kata-deploy-ci:kata-containers-9f512c016e75599a4a921bd84ea47559fe610057-amd64``
+and you'd like to know which PR caused the regression. You can either run with
+all the 60 tags between or you can utilize the [bisecter](https://github.com/ldoktor/bisecter)
+to optimize the number of steps in between.
+
+Before running the bisection you need a reproducer script. Sample one called
+``sample-test-reproducer.sh`` is provided in this directory but you might
+want to copy and modify it, especially:
+
+* ``OCP_DIR`` - directory where your openshift/release is located (can be exported)
+* ``E2E_TEST`` - openshift-test(s) to be executed (can be exported)
+* behaviour of SETUP (returning 125 skips the current image tag, returning
+  >=128 interrupts the execution, everything else reports the tag as failure
+* what should be executed (perhaps running the setup is enough for you or
+  you might want to be looking for specific failures...)
+* use ``timeout`` to interrupt execution in case you know things should be faster
+
+Executing that script with the GOOD commit should pass
+``./sample-test-reproducer.sh quay.io/kata-containers/kata-deploy-ci:kata-containers-d7afd31fd40e37a675b25c53618904ab57e74ccd-amd64``
+and fail when executed with the BAD commit
+``./sample-test-reproducer.sh quay.io/kata-containers/kata-deploy-ci:kata-containers-9f512c016e75599a4a921bd84ea47559fe610057-amd64``.
+
+To get the list of all tags in between those two PRs you can use the
+``bisect-range.sh`` script
+
+```bash
+./bisect-range.sh d7afd31fd40e37a675b25c53618904ab57e74ccd 9f512c016e75599a4a921bd84ea47559fe610057
+```
+
+[!NOTE]
+The tagged images are only built per PR, not for individual commits. See
+[kata-deploy-ci](https://quay.io/kata-containers/kata-deploy-ci) to see the
+available images.
+
+To find out which PR caused this regression, you can either manually try the
+individual commits or you can simply execute:
+
+```bash
+bisecter start "$(./bisect-range.sh d7afd31fd40 9f512c016)"
+OCP_DIR=/path/to/openshift/release bisecter run ./sample-test-reproducer.sh
+```
+
+[!NOTE]
+If you use ``KATA_WITH_SYSTEM_QEMU=yes`` you might want to deploy once with
+it and skip it for the cleanup. That way you might (in most cases) test
+all images with a single MCP update instead of per-image MCP update.
+
+[!TIP]
+You can check the bisection progress during/after execution by running
+``bisecter log`` from the current directory. Before starting a new
+bisection you need to execute ``bisecter reset``.

ci/openshift-ci/bisect-range.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright (c) 2024 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+if [ "$#" -gt 2 ] || [ "$#" -lt 1 ] ; then
+	echo "Usage: $0 GOOD [BAD]"
+	echo "Prints list of available kata-deploy-ci tags between GOOD and BAD commits (by default BAD is the latest available tag)"
+	exit 255
+fi
+GOOD="$1"
+[ -n "$2" ] && BAD="$2"
+ARCH=amd64
+REPO="quay.io/kata-containers/kata-deploy-ci"
+
+TAGS=$(skopeo list-tags "docker://$REPO")
+# Only amd64
+TAGS=$(echo "$TAGS" | jq '.Tags' | jq "map(select(endswith(\"$ARCH\")))" | jq -r '.[]')
+# Tags since $GOOD
+TAGS=$(echo "$TAGS" | sed -n -e "/$GOOD/,$$p")
+# Tags up to $BAD
+[ -n "$BAD" ] && TAGS=$(echo "$TAGS" | sed "/$BAD/q")
+# Comma separated tags with repo
+echo "$TAGS" | sed -e "s@^@$REPO:@" | paste -s -d, -

ci/openshift-ci/cleanup.sh

@@ -25,6 +25,10 @@ WORKAROUND_9206_CRIO=${WORKAROUND_9206_CRIO:-no}
 # Ignore errors as we want best-effort-approach here
 trap - ERR
 
+# Delete webhook resources
+oc delete -f "${scripts_dir}/../../tools/testing/kata-webhook/deploy"
+oc delete -f "${scripts_dir}/cluster/deployments/configmap_kata-webhook.yaml.in"
+
 # Delete potential smoke-test resources
 oc delete -f "${scripts_dir}/smoke/service.yaml"
 oc delete -f "${scripts_dir}/smoke/service_kubernetes.yaml"

ci/openshift-ci/images/Dockerfile.buildroot

@@ -4,7 +4,7 @@
 #
 # This is the build root image for Kata Containers on OpenShift CI.
 #
-FROM quay.io/centos/centos:stream8
+FROM quay.io/centos/centos:stream9
 
 RUN yum -y update && \
     yum -y install \

ci/openshift-ci/sample-test-reproducer.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Copyright (c) 2024 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# A sample script to deploy, configure, run E2E_TEST and soft-cleanup
+# afterwards OCP cluster using kata-containers primarily created for use
+# with https://github.com/ldoktor/bisecter
+
+[ "$#" -ne 1 ] && echo "Provide image as the first and only argument" && exit 255
+export KATA_DEPLOY_IMAGE="$1"
+OCP_DIR="${OCP_DIR:-/path/to/your/openshift/release/}"
+E2E_TEST="${E2E_TEST:-'"[sig-node] Container Runtime blackbox test on terminated container should report termination message as empty when pod succeeds and TerminationMessagePolicy FallbackToLogsOnError is set [NodeConformance] [Conformance] [Suite:openshift/conformance/parallel/minimal] [Suite:k8s]"'}"
+KATA_CI_DIR="${KATA_CI_DIR:-$(pwd)}"
+export KATA_RUNTIME="${KATA_RUNTIME:-kata-qemu}"
+
+## SETUP
+# Deploy kata
+SETUP=0
+pushd "$KATA_CI_DIR" || { echo "Failed to cd to '$KATA_CI_DIR'"; exit 255; }
+./test.sh || SETUP=125
+cluster/deploy_webhook.sh || SETUP=125
+if [ $SETUP != 0 ]; then
+    ./cleanup.sh
+    exit "$SETUP"
+fi
+popd || true
+# Disable security
+oc adm policy add-scc-to-group privileged system:authenticated system:serviceaccounts
+oc adm policy add-scc-to-group anyuid system:authenticated system:serviceaccounts
+oc label --overwrite ns default pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/warn=baseline pod-security.kubernetes.io/audit=baseline
+
+## TEST EXECUTION
+# Run the testing
+pushd "$OCP_DIR" || { echo "Failed to cd to '$OCP_DIR'"; exit 255; }
+echo "$E2E_TEST" > /tmp/tsts
+# Remove previously-existing temporarily files as well as previous results
+OUT=RESULTS/tmp
+rm -Rf /tmp/*test* /tmp/e2e-*
+rm -R $OUT
+mkdir -p $OUT
+# Run the tests ignoring the monitor health checks
+./openshift-tests run --provider azure -o "$OUT/job.log" --junit-dir "$OUT" --file /tmp/tsts --max-parallel-tests 5 --cluster-stability Disruptive
+RET=$?
+popd || true
+
+## CLEANUP
+./cleanup.sh
+exit "$RET"
+

docs/Blog-Post-Submission-Guide.md

@@ -0,0 +1,86 @@
+# Blog Post Contributor Guide
+
+This section describes the guidelines for contributing new blog posts to the
+Kata Containers website.
+
+## Share your stories on the Kata Containers website
+
+Are you experimenting with Kata Containers or have it deployed in production and
+would like to share your story as a case study? Do you have a use case that
+Kata Containers can make more secure, but the world doesn't know it yet? Do you
+have features in the runtime that you like and would like to highlight? Do you
+have a Kata Containers demo that you would like to draw attention to?
+
+Share your Kata Containers story on the [Kata Containers blog](https://www.katacontainers.io/blog/)!
+You are only a few steps away...
+
+### Kata Containers website source
+
+Like the rest of the Kata Containers artifacts, the project’s website code and
+content are stored in a [GitHub repository](https://github.com/kata-containers/www.katacontainers.io).
+
+The blog posts are written using markdown language that is mainly plain text
+with a few easy formatting conventions to create lists, add images or code blocks,
+or format the text.
+
+You can find many [cheat sheets](https://www.markdownguide.org/cheat-sheet/)
+floating on the web to get in terms of the basic syntax. You can also check the
+[source files of the already existing blog posts](https://github.com/kata-containers/www.katacontainers.io/tree/main/src/pages/blog),
+where you will find examples of all the basic items that you will need for your
+new entry.
+
+### Create a new blog post
+
+When you create a new blog post, you need to create a new file in the
+[`src/pages/blog/` folder](https://github.com/kata-containers/www.katacontainers.io/tree/main/src/pages/blog)
+with a `.md` extension.
+
+The markdown file has a few formatting conventions in its header to capture the
+title, author, publishing date and category of your blog post.
+
+The header looks like the following:
+
+```
+  ---
+  templateKey: blog-post
+  title: The Title of Your Amazing Blog Post
+  author: Your Name
+  date: 2021-01-28T16:23:52.741Z
+  category:
+    - value: category-6-wjkXzEM2
+      label: Features & Updates
+  ---
+```
+
+The categories give the possibility to filter on the web page and see only the
+blog posts that fall under one of the options. You can choose from the
+following options:
+
+* News & Announcements
+* Features & Updates
+
+The `Annual Report` category is reserved for the Kata Containers chapter in the
+Open Infrastructure Annual report that we are also re-posting on the Kata
+Containers website.
+
+Once you filled out the above fields in the header and got your one-liner all
+set, you can go ahead and type up the contents of your blog post using the
+conventional markdown formatting.
+
+If you have an image file to add, you need to place the file in the
+`static/img` folder.
+
+You can then insert the image into your blog post by using the following line:
+
+```
+  ![alt text](/img/the-file-name-of-your-image.jpg)
+```
+
+Once you are done with formatting your blog post and happy with the content, you
+need to upload it to GitHub and create a pull request. You can do that by using
+git commands on your laptop or you can also use the GitHub web interface to add
+files to the repository and create a pull request when you are ready.
+
+If you have an idea for a blog post and would like to get feedback from the
+community about it or have any questions about the process, please reach out
+on one of the community's [communication channels](https://katacontainers.io/community/).

docs/Developer-Guide.md

@@ -461,7 +461,7 @@ and repository utilized can be found by looking at the [versions file](../versio
 Find the correct version of QEMU from the versions file:
 ```bash
 $ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_version="$(get_from_kata_deps "assets.hypervisor.qemu.version")"
+$ qemu_version="$(get_from_kata_deps ".assets.hypervisor.qemu.version")"
 $ echo "${qemu_version}"
 ```
 Get source from the matching branch of QEMU:

docs/README.md

@@ -50,6 +50,7 @@ Documents that help to understand and contribute to Kata Containers.
 * [Developer Guide](Developer-Guide.md): Setup the Kata Containers developing environments
 * [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md)
 * [Code of Conduct](../CODE_OF_CONDUCT.md)
+* [How to submit a blog post](Blog-Post-Submission-Guide.md)
 
 ## Help Writing a Code PR
 

docs/how-to/how-to-pull-images-in-guest-with-kata.md

@@ -35,27 +35,23 @@ $ git clone -b "${nydus_snapshotter_version}" "${nydus_snapshotter_url}" "${nydu
 2. Configure DaemonSet file
 ```bash
 $ pushd "$nydus_snapshotter_install_dir"
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.FS_DRIVER' \
->	 "proxy" --style=double
+$ yq -i \
+>	 '.data.FS_DRIVER = "proxy"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 # Disable to read snapshotter config from configmap
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.ENABLE_CONFIG_FROM_VOLUME' \
->	 "false" --style=double
+$ yq -i \
+>	 'data.ENABLE_CONFIG_FROM_VOLUME = "false"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 # Enable to run snapshotter as a systemd service 
 # (skip if you want to run nydus snapshotter as a standalone process)
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.ENABLE_SYSTEMD_SERVICE' \
->	 "true" --style=double
+$ yq -i \
+>	 'data.ENABLE_SYSTEMD_SERVICE = "true"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 # Enable "runtime specific snapshotter" feature in containerd when configuring containerd for snapshotter
 # (skip if you want to configure nydus snapshotter as a global snapshotter in containerd)
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER' \
->	 "true" --style=double
+$ yq -i \
+>	 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER = "true"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 ```
 
 3. Install `nydus snapshotter` as a DaemonSet

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -44,8 +44,8 @@ $ popd
 - Build a custom QEMU
 ```bash
 $ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")"
-$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")"
+$ qemu_url="$(get_from_kata_deps ".assets.hypervisor.qemu-snp-experimental.url")"
+$ qemu_tag="$(get_from_kata_deps ".assets.hypervisor.qemu-snp-experimental.tag")"
 $ git clone "${qemu_url}"
 $ pushd qemu
 $ git checkout "${qemu_tag}"

docs/threat-model/threat-model.md

@@ -1,95 +1,157 @@
 # Kata Containers threat model
 
-This document discusses threat models associated with the Kata Containers project.
-Kata was designed to provide additional isolation of container workloads, protecting
-the host infrastructure from potentially malicious container users or workloads. Since
-Kata Containers adds a level of isolation on top of traditional containers, the focus
-is on the additional layer provided, not on traditional container security.
+This document discusses threat models associated with the Kata Containers
+project. Kata was designed to provide additional isolation of container
+workloads, protecting the host infrastructure from potentially malicious
+container users or workloads. Since Kata Containers adds a level of isolation on
+top of traditional containers, the focus is on the additional layer provided,
+not on traditional container security.
 
-This document provides a brief background on containers and layered security, describes
-the interface to Kata from CRI runtimes, a review of utilized virtual machine interfaces, and then
-a review of threats.
+This document provides a brief background on containers and layered security,
+describes the interface to Kata from CRI runtimes, a review of utilized virtual
+machine interfaces, and then a review of threats.
 
 ## Kata security objective
 
-Kata seeks to prevent an untrusted container workload or user of that container workload to gain
-control of, obtain information from, or tamper with the host infrastructure.
+Kata seeks to prevent an untrusted container workload or user of that container
+workload to gain control of, obtain information from, or tamper with the host
+infrastructure.
 
-In our scenario, an asset is anything on the host system, or elsewhere in the cluster
-infrastructure. The attacker is assumed to be either a malicious user or the workload itself
-running within the container. The goal of Kata is to prevent attacks which would allow
-any access to the defined assets.
+In our scenario, an asset is anything on the host system, or elsewhere in the
+cluster infrastructure. The attacker is assumed to be either a malicious user or
+the workload itself running within the container. The goal of Kata is to prevent
+attacks which would allow any access to the defined assets.
 
 ## Background on containers, layered security
 
-Traditional containers leverage several key Linux kernel features to provide isolation and
-a view that the container workload is the only entity running on the host. Key features include
-`Namespaces`, `cgroups`, `capablities`, `SELinux` and `seccomp`. The canonical runtime for creating such
-a container is `runc`. In the remainder of the document, the term `traditional-container` will be used
-to describe a container workload created by runc.
+Traditional containers leverage several key Linux kernel features to provide
+isolation and a view that the container workload is the only entity running on
+the host. Key features include `Namespaces`, `cgroups`, `capablities`, `SELinux`
+and `seccomp`. The canonical runtime for creating such a container is `runc`. In
+the remainder of the document, the term `traditional-container` will be used to
+describe a container workload created by runc.
 
-Kata Containers provides a second layer of isolation on top of those provided by traditional-containers.
-The hardware virtualization interface is the basis of this additional layer. Kata launches a lightweight
-virtual machine, and uses the guest’s Linux kernel to create a container workload, or workloads in the case
-of multi-container pods. In Kubernetes and in the Kata implementation, the sandbox is carried out at the
-pod level. In Kata, this sandbox is created using a virtual machine.
+Kata Containers provides a second layer of isolation on top of those provided by
+traditional-containers. The hardware virtualization interface is the basis of
+this additional layer. Kata launches a lightweight virtual machine, and uses the
+guest’s Linux kernel to create a container workload, or workloads in the case of
+multi-container pods. In Kubernetes and in the Kata implementation, the sandbox
+is carried out at the pod level. In Kata, this sandbox is created using a
+virtual machine.
 
 ## Interface to Kata Containers: CRI, v2-shim, OCI
 
 A typical Kata Containers deployment uses Kubernetes with a CRI implementation.
-On every node, Kubelet will interact with a CRI implementor, which will in turn interface with
-an OCI based runtime, such as Kata Containers. Typical CRI implementors are `cri-o` and `containerd`.
+On every node, Kubelet will interact with a CRI implementor, which will in turn
+interface with an OCI based runtime, such as Kata Containers. Typical CRI
+implementors are `cri-o` and `containerd`.
 
-The CRI API, as defined at the Kubernetes [CRI-API repo](https://github.com/kubernetes/cri-api/),
-results in a few constructs being supported by the CRI implementation, and ultimately in the OCI
-runtime creating the workloads.
+The CRI API, as defined at the Kubernetes [CRI-API
+repo](https://github.com/kubernetes/cri-api/), results in a few constructs being
+supported by the CRI implementation, and ultimately in the OCI runtime creating
+the workloads.
 
-In order to run a container inside of the Kata sandbox, several virtual machine devices and interfaces
-are required. Kata translates sandbox and container definitions to underlying virtualization technologies provided
-by a set of virtual machine monitors (VMMs) and hypervisors. These devices and their underlying
-implementations are discussed in detail in the following section.
+In order to run a container inside of the Kata sandbox, several virtual machine
+devices and interfaces are required. Kata translates sandbox and container
+definitions to underlying virtualization technologies provided by a set of
+virtual machine monitors (VMMs) and hypervisors. These devices and their
+underlying implementations are discussed in detail in the following section.
 
 ## Interface to the Kata sandbox/virtual machine
 
 In case of Kata, today the devices which we need in the guest are:
- - Storage: In the current design of Kata Containers, we are reliant on the CRI implementor to
- assist in image handling and volume management on the host. As a result, we need to support a way of passing to the sandbox the container rootfs, volumes requested
- by the workload, and any other volumes created to facilitate sharing of secrets and `configmaps` with the containers. Depending on how these are managed, a block based device or file-system
- sharing is required. Kata Containers does this by way of `virtio-blk` and/or `virtio-fs`.
- - Networking: A method for enabling network connectivity with the workload is required. Typically this will be done providing a `TAP` device
- to the VMM, and this will be exposed to the guest as a `virtio-net` device. It is feasible to pass in a NIC device directly, in which case `VFIO` is leveraged
- and the device itself will be exposed to the guest.
- - Control: In order to interact with the guest agent and retrieve `STDIO` from containers, a medium of communication is required.
- This is available via `virtio-vsock`.
- - Devices: `VFIO` is utilized when devices are passed directly to the virtual machine and exposed to the container.
-- Dynamic Resource Management: `ACPI` is utilized to allow for dynamic VM resource management (for example: CPU, memory, device hotplug). This is required when containers are resized,
- or more generally when containers are added to a pod. 
+ - Storage: In the current design of Kata Containers, we are reliant on the CRI
+ implementor to assist in image handling and volume management on the host. As a
+ result, we need to support a way of passing to the sandbox the container
+ rootfs, volumes requested by the workload, and any other volumes created to
+ facilitate sharing of secrets and `configmaps` with the containers. Depending
+ on how these are managed, a block based device or file-system sharing is
+ required. Kata Containers does this by way of `virtio-blk` and/or `virtio-fs`.
+ - Networking: A method for enabling network connectivity with the workload is
+ required. Typically this will be done providing a `TAP` device to the VMM, and
+ this will be exposed to the guest as a `virtio-net` device. It is feasible to
+ pass in a NIC device directly, in which case `VFIO` is leveraged and the device
+ itself will be exposed to the guest.
+ - Control: In order to interact with the guest agent and retrieve `STDIO` from
+ containers, a medium of communication is required. This is available via
+ `virtio-vsock`.
+ - Devices: `VFIO` is utilized when devices are passed directly to the virtual
+   machine and exposed to the container.
+- Dynamic Resource Management: `ACPI` is utilized to allow for dynamic VM
+ resource management (for example: CPU, memory, device hotplug). This is
+ required when containers are resized, or more generally when containers are
+ added to a pod. 
  
-How these devices are utilized varies depending on the VMM utilized. We clarify the default settings provided when integrating Kata
-with the QEMU, Firecracker and Cloud Hypervisor VMMs in the following sections.
+How these devices are utilized varies depending on the VMM utilized. We clarify
+the default settings provided when integrating Kata with the QEMU, Dragonball,
+Firecracker and Cloud Hypervisor VMMs in the following sections.
+
+### Virtual Machine Monitor(s)
+
+In a KVM/QEMU (any other VMM utilizing KVM) virtualization setup, all virtual
+machines (VMs) share the same host kernel. This shared environment can lead to
+scenarios where one VM could potentially impact the performance or stability of
+other VMs, including the possibility of a Denial of Service attack.
+
+- Kernel Vulnerabilities: Since all VMs rely on the host's kernel, a
+vulnerability in the kernel could be exploited by a process running within one
+VM to affect the entire system. This could lead to scenarios where the
+compromised VM impacts other VMs or even takes down the host.
+
+- Improper Isolation and Containment: If the virtualization environment is not
+correctly configured, processes in one VM might impact other VMs. This could
+occur through improper isolation of network traffic, shared file systems, or
+other inter-VM communication channels.
+
+- Hypervisor Vulnerabilities: Flaws in the KVM hypervisor or QEMU could be
+exploited to cause information disclosure, data tampering, elevation of
+privileges, denial of service, and others. Since KVM/QEMU leverages the host
+kernel for its operation, any exploit at this level can have widespread impacts.
+
+- Malicious or Flawed Guest Operating Systems: A guest operating system that is
+maliciously designed or has serious flaws could engage in activities that
+disrupt the normal operation of the host or other guests. This might include
+aggressive network activity or interactions with the virtualization stack that
+lead to instability.
+
+- Resource Exhaustion: A VM could consume excessive shared resources such as
+CPU, memory, or I/O bandwidth, leading to resource starvation for other VMs.
+This could be due to misconfiguration, a runaway process, or a deliberate
+denial of service attack from a compromised VM.
 
 ### Devices
 
-Each virtio device is implemented by a backend, which may execute within userspace on the host (vhost-user), the VMM itself, or within the host kernel (vhost). While it may provide enhanced performance,
-vhost devices are often seen as higher risk since an exploit would be already running within the kernel space. While VMM and vhost-user are both in userspace on the host, `vhost-user` generally allows for the back-end process to require less system calls and capabilities compared to a full VMM.
+Each virtio device is implemented by a backend, which may execute within
+userspace on the host (vhost-user), the VMM itself, or within the host kernel
+(vhost). While it may provide enhanced performance, vhost devices are often seen
+as higher risk since an exploit would be already running within the kernel
+space. While VMM and vhost-user are both in userspace on the host, `vhost-user`
+generally allows for the back-end process to require less system calls and
+capabilities compared to a full VMM.
 
 #### `virtio-blk` and `virtio-scsi`
 
-The backend for `virtio-blk` and `virtio-scsi` are based in the VMM itself (ring3 in the context of x86) by default for Cloud Hypervisor, Firecracker and QEMU.
-While `vhost` based back-ends are available for QEMU, it is not recommended. `vhost-user` back-ends are being added for Cloud Hypervisor, they are not utilized in Kata today.
+The backend for `virtio-blk` and `virtio-scsi` are based in the VMM itself
+(ring3 in the context of x86) by default for Cloud Hypervisor, Firecracker and
+QEMU. While `vhost` based back-ends are available for QEMU, it is not
+recommended. `vhost-user` back-ends are being added for Cloud Hypervisor, they
+are not utilized in Kata today.
 
 #### `virtio-fs`
 
-`virtio-fs` is supported in Cloud Hypervisor and QEMU. `virtio-fs`'s interaction with the host filesystem is done through a vhost-user daemon, `virtiofsd`.
-The `virtio-fs` client, running in the guest, will generate requests to access files. `virtiofsd` will receive requests, open the file, and request the VMM
-to `mmap` it into the guest. When DAX is utilized, the guest will access the host's page cache, avoiding the need for copy and duplication. DAX is still an experimental feature,
-and is not enabled by default.
+`virtio-fs` is supported in Cloud Hypervisor and QEMU. `virtio-fs`'s interaction
+with the host filesystem is done through a vhost-user daemon, `virtiofsd`. The
+`virtio-fs` client, running in the guest, will generate requests to access
+files. `virtiofsd` will receive requests, open the file, and request the VMM to
+`mmap` it into the guest. When DAX is utilized, the guest will access the host's
+page cache, avoiding the need for copy and duplication. DAX is still an
+experimental feature, and is not enabled by default.
 
 From the `virtiofsd` [documentation](https://gitlab.com/virtio-fs/virtiofsd/-/blob/main/README.md):
 ```This program must be run as the root user. Upon startup the program will switch into a new file system namespace with the shared directory tree as its root. This prevents “file system escapes” due to symlinks and other file system objects that might lead to files outside the shared directory. The program also sandboxes itself using seccomp(2) to prevent ptrace(2) and other vectors that could allow an attacker to compromise the system after gaining control of the virtiofsd process.```
 
-DAX-less support for `virtio-fs` is available as of the 5.4 Linux kernel. QEMU VMM supports virtio-fs as of v4.2. Cloud Hypervisor
-supports `virtio-fs`.
+DAX-less support for `virtio-fs` is available as of the 5.4 Linux kernel. QEMU
+VMM supports virtio-fs as of v4.2. Cloud Hypervisor supports `virtio-fs`.
 
 #### `virtio-net`
 
@@ -97,9 +159,9 @@ supports `virtio-fs`.
 
 ##### QEMU networking
 
-While QEMU has options for `vhost`, `virtio-net` and `vhost-user`, the `virtio-net` backend
-for Kata defaults to `vhost-net` for performance reasons. The default configuration is being
-reevaluated.
+While QEMU has options for `vhost`, `virtio-net` and `vhost-user`, the
+`virtio-net` backend for Kata defaults to `vhost-net` for performance reasons.
+The default configuration is being reevaluated.
 
 ##### Firecracker networking
 
@@ -107,8 +169,14 @@ For Firecracker, the `virtio-net` backend is within Firecracker's VMM.
 
 ##### Cloud Hypervisor networking
 
-For Cloud Hypervisor, the current backend default is within the VMM. `vhost-user-net` support
-is being added (written in rust, Cloud Hypervisor specific).
+For Cloud Hypervisor, the current backend default is within the VMM.
+`vhost-user-net` support is being added (written in rust, Cloud Hypervisor
+specific).
+
+##### Dragonball networking
+
+For Dragonball, the `virtio-net` backend default is within Dragonbasll's VMM.
+
 
 #### virtio-vsock
 
@@ -116,22 +184,88 @@ is being added (written in rust, Cloud Hypervisor specific).
 
 In QEMU, vsock is backed by `vhost_vsock`, which runs within the kernel itself.
 
-##### Firecracker and Cloud Hypervisor
+##### Dragonball, Firecracker and Cloud Hypervisor
 
-In Firecracker and Cloud Hypervisor, vsock is backed by a unix-domain-socket in the hosts userspace.
+In Dragonball, Firecracker and Cloud Hypervisor, vsock is backed by a unix-domain-socket in
+the hosts userspace.
 
 #### VFIO
 
-Utilizing VFIO, devices can be passed through to the virtual machine. We will assess this separately. Exposure to
-host is limited to gaps in device pass-through handling. This is supported in QEMU and Cloud Hypervisor, but not
-Firecracker.
+Utilizing VFIO, devices can be passed through to the virtual machine. Exposure
+to the host is limited to gaps in device pass-through handling. This is
+supported in QEMU and Cloud Hypervisor, but not Firecracker.
+
+- Device Isolation Failure: One of the primary risks associated with VFIO is the
+failure to isolate the physical device. If a VM can affect the operation of the
+physical device in a way that impacts other VMs or the host system, it could
+lead to security breaches or system instability.
+
+- DMA Attacks: Direct Memory Access (DMA) attacks are a significant concern with
+VFIO. Since the device has direct access to the system's memory, there's a risk
+that a compromised VM could use its assigned device to read or write memory
+outside of its allocated space, potentially accessing sensitive information or
+affecting the host or other VMs.
+
+- Firmware Vulnerabilities: Devices attached via VFIO rely on their firmware,
+which can have vulnerabilities. A compromised device firmware could be exploited
+to gain unauthorized access or to disrupt the system. Resource Starvation:
+Improperly managed, a VM with direct access to hardware resources could
+monopolize those resources, leading to performance degradation or denial of
+service for other VMs or the host system.
+
+- Escalation of Privileges: If a VM with VFIO access is compromised, it could
+potentially be used to gain higher privileges than intended, especially if the
+I/O devices have capabilities that are not adequately controlled or monitored.
+
+- Improper Configuration and Management: Human errors in configuring VFIO, such
+as incorrect group or user permissions, can expose the system to risks.
+Additionally, inadequate monitoring and management of the VMs and their devices
+can lead to security lapses.
+
+- Software Vulnerabilities: Like any software, the components of VFIO (like the
+kernel modules, device drivers, and management tools) can have vulnerabilities
+that might be exploited by an attacker to compromise the security of the system.
+Inter-VM Interference and Side-Channel Attacks: Even with device assignment,
+there could be side-channel attacks where an attacker VM infers sensitive
+information from the physical device's behavior or through shared resources like
+cache.
+
+#### ACPI (Dragonball uses Upcall)
+
+ACPI is necessary for hotplugging of CPU, memory and devices. ACPI is available
+in QEMU and Cloud Hypervisor. Device, CPU and memory hotplug are not available
+in Firecracker.
+
+- Hypervisor Vulnerabilities: In virtualized environments, the hypervisor
+manages ACPI calls for virtual machines (VMs). If the hypervisor has
+vulnerabilities in handling ACPI requests, it could lead to escalated privileges
+or other security breaches.
+
+- VM Escape: A sophisticated attack could exploit ACPI functionality to achieve
+a VM escape, where malicious code in a VM breaks out to the host system or other
+VMs. Firmware Attacks in a Virtualized Context: Similar to physical
+environments, firmware-based attacks (including those targeting ACPI) in
+virtualized systems can be persistent and difficult to detect. In a virtualized
+environment, such attacks might not only compromise the host system but also all
+the VMs running on it.
+
+- Resource Starvation Attacks: ACPI functionality could be exploited to
+manipulate power management features, causing denial of service through
+resource starvation. For example, an attacker could force a VM into a low-power
+state, degrading its performance or availability.
+
+- Compromised VMs Affecting Host ACPI Settings: If a VM is compromised, it might
+be used to alter ACPI settings on the host, affecting all VMs on that host. This
+could lead to various impacts, from performance degradation to system
+instability.
+
+- Supply Chain Risks: As with non-virtualized environments, the firmware,
+including ACPI firmware used in virtualized environments, could be compromised
+during the supply chain process, leading to vulnerabilities that affect all VMs
+running on the hardware.
 
-#### ACPI
 
-ACPI is necessary for hotplug of CPU, memory and devices. ACPI is available in QEMU and Cloud Hypervisor. Device, CPU and memory hotplug
-are not available in Firecracker.
 
 ## Devices and threat model
 
 ![Threat model](threat-model-boundaries.svg "threat-model")
-

src/agent/README.md

@@ -126,7 +126,8 @@ The kata agent has the ability to configure agent options in guest kernel comman
 | `agent.debug_console_vport` | Debug console port | Allow to specify the `vsock` port to connect the debugging console | integer | `0` |
 | `agent.devmode` | Developer mode | Allow the agent process to coredump | boolean | `false` |
 | `agent.hotplug_timeout` | Hotplug timeout | Allow to configure hotplug timeout(seconds) of block devices | integer | `3` |
-| `agent.guest_components_rest_api` | `api-server-rest` configuration | Select the features that the API Server Rest attestation component will run with. Valid values are `all`, `attestation`, `resource`, or `none` to not launch the `api-server-rest` component | string | `resource` |
+| `agent.guest_components_rest_api` | `api-server-rest` configuration | Select the features that the API Server Rest attestation component will run with. Valid values are `all`, `attestation`, `resource` | string | `resource` |
+| `agent.guest_components_procs` | guest-components processes | Attestation-related processes that should be spawned as children of the guest. Valid values are `none`, `attestation-agent`, `confidential-data-hub` (implies `attestation-agent`), `api-server-rest` (implies `attestation-agent` and `confidential-data-hub`) | string | `api-server-rest` |
 | `agent.https_proxy` | HTTPS proxy | Allow to configure `https_proxy` in the guest | string | `""` |
 | `agent.log` | Log level | Allow the agent log level to be changed (produces more or less output) | string | `"info"` |
 | `agent.log_vport` | Log port | Allow to specify the `vsock` port to read logs | integer | `0` |

src/agent/rustjail/src/container.rs

@@ -601,6 +601,22 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::close(mount_fd)?;
     }
 
+    if init {
+        // CreateContainer Hooks:
+        // before pivot_root after prestart, createruntime
+        state.pid = std::process::id() as i32;
+        state.status = oci::ContainerState::Created;
+        if let Some(hooks) = spec.hooks.as_ref() {
+            log_child!(
+                cfd_log,
+                "create_container hooks {:?}",
+                hooks.create_container
+            );
+            let mut create_container_states = HookStates::new();
+            create_container_states.execute_hooks(&hooks.create_container, Some(state.clone()))?;
+        }
+    }
+
     if to_new.contains(CloneFlags::CLONE_NEWNS) {
         // unistd::chroot(rootfs)?;
         if no_pivot {

src/agent/src/config.rs

@@ -28,6 +28,7 @@ const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "systemd.unified_cgroup_hierarchy";
 const CONFIG_FILE: &str = "agent.config_file";
 const GUEST_COMPONENTS_REST_API_OPTION: &str = "agent.guest_components_rest_api";
+const GUEST_COMPONENTS_PROCS_OPTION: &str = "agent.guest_components_procs";
 
 // Configure the proxy settings for HTTPS requests in the guest,
 // to solve the problem of not being able to access the specified image in some cases.
@@ -59,19 +60,34 @@ const ERR_INVALID_CONTAINER_PIPE_SIZE_PARAM: &str = "unable to parse container p
 const ERR_INVALID_CONTAINER_PIPE_SIZE_KEY: &str = "invalid container pipe size key name";
 const ERR_INVALID_CONTAINER_PIPE_NEGATIVE: &str = "container pipe size should not be negative";
 
-const ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE: &str = "invalid guest components rest api feature given. Valid values are `all`, `attestation`, `resource`, or `none`";
+const ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE: &str = "invalid guest components rest api feature given. Valid values are `all`, `attestation`, `resource`";
+const ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE: &str = "invalid guest components process param given. Valid values are `attestation-agent`, `confidential-data-hub`, `api-server-rest`, or `none`";
 
 #[derive(Clone, Copy, Debug, Default, Display, Deserialize, EnumString, PartialEq)]
 // Features seem to typically be in kebab-case format, but we only have single words at the moment
 #[strum(serialize_all = "kebab-case")]
+#[serde(rename_all = "kebab-case")]
 pub enum GuestComponentsFeatures {
     All,
     Attestation,
-    None,
     #[default]
     Resource,
 }
 
+#[derive(Clone, Copy, Debug, Default, Display, Deserialize, EnumString, PartialEq)]
+/// Attestation-related processes that we want to spawn as children of the agent
+#[strum(serialize_all = "kebab-case")]
+#[serde(rename_all = "kebab-case")]
+pub enum GuestComponentsProcs {
+    None,
+    /// ApiServerRest implies ConfidentialDataHub and AttestationAgent
+    #[default]
+    ApiServerRest,
+    AttestationAgent,
+    /// ConfidentialDataHub implies AttestationAgent
+    ConfidentialDataHub,
+}
+
 #[derive(Debug)]
 pub struct AgentConfig {
     pub debug_console: bool,
@@ -89,6 +105,7 @@ pub struct AgentConfig {
     pub https_proxy: String,
     pub no_proxy: String,
     pub guest_components_rest_api: GuestComponentsFeatures,
+    pub guest_components_procs: GuestComponentsProcs,
 }
 
 #[derive(Debug, Deserialize)]
@@ -107,6 +124,7 @@ pub struct AgentConfigBuilder {
     pub https_proxy: Option<String>,
     pub no_proxy: Option<String>,
     pub guest_components_rest_api: Option<GuestComponentsFeatures>,
+    pub guest_components_procs: Option<GuestComponentsProcs>,
 }
 
 macro_rules! config_override {
@@ -171,6 +189,7 @@ impl Default for AgentConfig {
             https_proxy: String::from(""),
             no_proxy: String::from(""),
             guest_components_rest_api: GuestComponentsFeatures::default(),
+            guest_components_procs: GuestComponentsProcs::default(),
         }
     }
 }
@@ -207,6 +226,7 @@ impl FromStr for AgentConfig {
             agent_config,
             guest_components_rest_api
         );
+        config_override!(agent_config_builder, agent_config, guest_components_procs);
 
         Ok(agent_config)
     }
@@ -220,7 +240,10 @@ impl AgentConfig {
         let config_position = args.iter().position(|a| a == "--config" || a == "-c");
         if let Some(config_position) = config_position {
             if let Some(config_file) = args.get(config_position + 1) {
-                return AgentConfig::from_config_file(config_file).context("AgentConfig from args");
+                let mut config =
+                    AgentConfig::from_config_file(config_file).context("AgentConfig from args")?;
+                config.override_config_from_envs();
+                return Ok(config);
             } else {
                 panic!("The config argument wasn't formed properly: {:?}", args);
             }
@@ -314,23 +337,15 @@ impl AgentConfig {
                 config.guest_components_rest_api,
                 get_guest_components_features_value
             );
+            parse_cmdline_param!(
+                param,
+                GUEST_COMPONENTS_PROCS_OPTION,
+                config.guest_components_procs,
+                get_guest_components_procs_value
+            );
         }
 
-        if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
-            config.server_addr = addr;
-        }
-
-        if let Ok(addr) = env::var(LOG_LEVEL_ENV_VAR) {
-            if let Ok(level) = logrus_to_slog_level(&addr) {
-                config.log_level = level;
-            }
-        }
-
-        if let Ok(value) = env::var(TRACING_ENV_VAR) {
-            let name_value = format!("{}={}", TRACING_ENV_VAR, value);
-
-            config.tracing = get_bool_value(&name_value)?;
-        }
+        config.override_config_from_envs();
 
         Ok(config)
     }
@@ -341,6 +356,25 @@ impl AgentConfig {
             .with_context(|| format!("Failed to read config file {}", file))?;
         AgentConfig::from_str(&config)
     }
+
+    #[instrument]
+    fn override_config_from_envs(&mut self) {
+        if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
+            self.server_addr = addr;
+        }
+
+        if let Ok(addr) = env::var(LOG_LEVEL_ENV_VAR) {
+            if let Ok(level) = logrus_to_slog_level(&addr) {
+                self.log_level = level;
+            }
+        }
+
+        if let Ok(value) = env::var(TRACING_ENV_VAR) {
+            let name_value = format!("{}={}", TRACING_ENV_VAR, value);
+
+            self.tracing = get_bool_value(&name_value).unwrap_or(false);
+        }
+    }
 }
 
 #[instrument]
@@ -480,12 +514,26 @@ fn get_guest_components_features_value(param: &str) -> Result<GuestComponentsFea
         .map_err(|_| anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE))
 }
 
+#[instrument]
+fn get_guest_components_procs_value(param: &str) -> Result<GuestComponentsProcs> {
+    let fields: Vec<&str> = param.split('=').collect();
+    ensure!(fields.len() >= 2, ERR_INVALID_GET_VALUE_PARAM);
+
+    // We need name (but the value can be blank)
+    ensure!(!fields[0].is_empty(), ERR_INVALID_GET_VALUE_NO_NAME);
+
+    let value = fields[1..].join("=");
+    GuestComponentsProcs::from_str(&value)
+        .map_err(|_| anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE))
+}
+
 #[cfg(test)]
 mod tests {
     use test_utils::assert_result;
 
     use super::*;
     use anyhow::anyhow;
+    use serial_test::serial;
     use std::fs::File;
     use std::io::Write;
     use std::time;
@@ -501,6 +549,8 @@ mod tests {
     }
 
     #[test]
+    // Run in serial to stop the env set interfering with test_from_cmdline_with_args_overwrites
+    #[serial]
     fn test_from_cmdline() {
         const TEST_SERVER_ADDR: &str = "vsock://-1:1024";
 
@@ -519,6 +569,7 @@ mod tests {
             https_proxy: &'a str,
             no_proxy: &'a str,
             guest_components_rest_api: GuestComponentsFeatures,
+            guest_components_procs: GuestComponentsProcs,
         }
 
         impl Default for TestData<'_> {
@@ -537,6 +588,7 @@ mod tests {
                     https_proxy: "",
                     no_proxy: "",
                     guest_components_rest_api: GuestComponentsFeatures::default(),
+                    guest_components_procs: GuestComponentsProcs::default(),
                 }
             }
         }
@@ -745,6 +797,13 @@ mod tests {
                 server_addr: "unix://@/tmp/foo.socket",
                 ..Default::default()
             },
+            // Test that env_var has precedence over the command line (which is the current behaviour)
+            TestData {
+                contents: "agent.server_addr=unix:///tmp/ignored.socket",
+                env_vars: vec!["KATA_AGENT_SERVER_ADDR=unix:///tmp/foo.socket"],
+                server_addr: "unix:///tmp/foo.socket",
+                ..Default::default()
+            },
             TestData {
                 contents: "",
                 env_vars: vec!["KATA_AGENT_LOG_LEVEL="],
@@ -942,8 +1001,23 @@ mod tests {
                 ..Default::default()
             },
             TestData {
-                contents: "agent.guest_components_rest_api=none",
-                guest_components_rest_api: GuestComponentsFeatures::None,
+               contents: "agent.guest_components_procs=api-server-rest",
+               guest_components_procs: GuestComponentsProcs::ApiServerRest,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.guest_components_procs=confidential-data-hub",
+                guest_components_procs: GuestComponentsProcs::ConfidentialDataHub,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.guest_components_procs=attestation-agent",
+                guest_components_procs: GuestComponentsProcs::AttestationAgent,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.guest_components_procs=none",
+                guest_components_procs: GuestComponentsProcs::None,
                 ..Default::default()
             },
         ];
@@ -1000,6 +1074,11 @@ mod tests {
                 "{}",
                 msg
             );
+            assert_eq!(
+                d.guest_components_procs, config.guest_components_procs,
+                "{}",
+                msg
+            );
 
             for v in vars_to_unset {
                 env::remove_var(v);
@@ -1008,15 +1087,17 @@ mod tests {
     }
 
     #[test]
+    // Run in serial to stop the env set interfering with test_from_cmdline
+    #[serial]
     fn test_from_cmdline_with_args_overwrites() {
         let expected = AgentConfig {
             dev_mode: true,
-            server_addr: "unix://@/tmp/foo.socket".to_string(),
+            server_addr: "unix:///tmp/overwrite.socket".to_string(),
             ..Default::default()
         };
 
         let example_config_file_contents =
-            "dev_mode = true\nserver_addr = 'unix://@/tmp/foo.socket'";
+            "dev_mode = true\nserver_addr = 'unix:///tmp/ignored.socket'";
         let dir = tempdir().expect("failed to create tmpdir");
         let file_path = dir.path().join("config.toml");
         let filename = file_path.to_str().expect("failed to create filename");
@@ -1024,10 +1105,15 @@ mod tests {
         file.write_all(example_config_file_contents.as_bytes())
             .unwrap_or_else(|_| panic!("failed to write file contents"));
 
+        // Ensure that the env has precedence over agent config file
+        env::set_var("KATA_AGENT_SERVER_ADDR", "unix:///tmp/overwrite.socket");
+
         let config =
             AgentConfig::from_cmdline("", vec!["--config".to_string(), filename.to_string()])
                 .expect("Failed to parse command line");
 
+        env::remove_var("KATA_AGENT_SERVER_ADDR");
+
         assert_eq!(expected.debug_console, config.debug_console);
         assert_eq!(expected.dev_mode, config.dev_mode);
         assert_eq!(
@@ -1500,10 +1586,6 @@ Caused by:
                 param: "x=attestation",
                 result: Ok(GuestComponentsFeatures::Attestation),
             },
-            TestData {
-                param: "x=none",
-                result: Ok(GuestComponentsFeatures::None),
-            },
             TestData {
                 param: "x=resource",
                 result: Ok(GuestComponentsFeatures::Resource),
@@ -1533,12 +1615,76 @@ Caused by:
         }
     }
 
+    #[test]
+    fn test_get_guest_components_procs_value() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            param: &'a str,
+            result: Result<GuestComponentsProcs>,
+        }
+
+        let tests = &[
+            TestData {
+                param: "",
+                result: Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)),
+            },
+            TestData {
+                param: "=",
+                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
+            },
+            TestData {
+                param: "==",
+                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
+            },
+            TestData {
+                param: "x=attestation-agent",
+                result: Ok(GuestComponentsProcs::AttestationAgent),
+            },
+            TestData {
+                param: "x=confidential-data-hub",
+                result: Ok(GuestComponentsProcs::ConfidentialDataHub),
+            },
+            TestData {
+                param: "x=none",
+                result: Ok(GuestComponentsProcs::None),
+            },
+            TestData {
+                param: "x=api-server-rest",
+                result: Ok(GuestComponentsProcs::ApiServerRest),
+            },
+            TestData {
+                param: "x===",
+                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
+            },
+            TestData {
+                param: "x==x",
+                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
+            },
+            TestData {
+                param: "x=x",
+                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let result = get_guest_components_procs_value(d.param);
+
+            let msg = format!("{}: result: {:?}", msg, result);
+
+            assert_result!(d.result, result, msg);
+        }
+    }
+
     #[test]
     fn test_config_builder_from_string() {
         let config = AgentConfig::from_str(
             r#"
                dev_mode = true
                server_addr = 'vsock://8:2048'
+               guest_components_procs = "api-server-rest"
+               guest_components_rest_api = "all"
               "#,
         )
         .unwrap();
@@ -1546,6 +1692,14 @@ Caused by:
         // Verify that the override worked
         assert!(config.dev_mode);
         assert_eq!(config.server_addr, "vsock://8:2048");
+        assert_eq!(
+            config.guest_components_procs,
+            GuestComponentsProcs::ApiServerRest
+        );
+        assert_eq!(
+            config.guest_components_rest_api,
+            GuestComponentsFeatures::All
+        );
 
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);

src/agent/src/device.rs

@@ -710,7 +710,14 @@ pub fn update_env_pci(
     env: &mut [String],
     pcimap: &HashMap<pci::Address, pci::Address>,
 ) -> Result<()> {
-    for envvar in env {
+    // SR-IOV device plugin may add two environment variables for one resource:
+    // - PCIDEVICE_<prefix>_<resource-name>: a list of PCI device ids separated by comma
+    // - PCIDEVICE_<prefix>_<resource-name>_INFO: detailed info in JSON for above PCI devices
+    // Both environment variables hold information about the same set of PCI devices.
+    // Below code updates both of them in two passes:
+    // - 1st pass updates PCIDEVICE_<prefix>_<resource-name> and collects host to guest PCI address mapping
+    let mut pci_dev_map: HashMap<String, HashMap<String, String>> = HashMap::new();
+    for envvar in env.iter_mut() {
         let eqpos = envvar
             .find('=')
             .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
@@ -718,24 +725,46 @@ pub fn update_env_pci(
         let (name, eqval) = envvar.split_at(eqpos);
         let val = &eqval[1..];
 
-        if !name.starts_with("PCIDEVICE_") {
+        if !name.starts_with("PCIDEVICE_") || name.ends_with("_INFO") {
             continue;
         }
 
+        let mut addr_map: HashMap<String, String> = HashMap::new();
         let mut guest_addrs = Vec::<String>::new();
-
-        for host_addr in val.split(',') {
-            let host_addr = pci::Address::from_str(host_addr)
+        for host_addr_str in val.split(',') {
+            let host_addr = pci::Address::from_str(host_addr_str)
                 .with_context(|| format!("Can't parse {} environment variable", name))?;
             let guest_addr = pcimap
                 .get(&host_addr)
                 .ok_or_else(|| anyhow!("Unable to translate host PCI address {}", host_addr))?;
+
             guest_addrs.push(format!("{}", guest_addr));
+            addr_map.insert(host_addr_str.to_string(), format!("{}", guest_addr));
         }
 
+        pci_dev_map.insert(format!("{}_INFO", name), addr_map);
+
         envvar.replace_range(eqpos + 1.., guest_addrs.join(",").as_str());
     }
 
+    // - 2nd pass update PCIDEVICE_<prefix>_<resource-name>_INFO if it exists
+    for envvar in env.iter_mut() {
+        let eqpos = envvar
+            .find('=')
+            .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
+
+        let (name, _) = envvar.split_at(eqpos);
+        if !(name.starts_with("PCIDEVICE_") && name.ends_with("_INFO")) {
+            continue;
+        }
+
+        if let Some(addr_map) = pci_dev_map.get(name) {
+            for (host_addr, guest_addr) in addr_map {
+                *envvar = envvar.replace(host_addr, guest_addr);
+            }
+        }
+    }
+
     Ok(())
 }
 
@@ -867,9 +896,10 @@ async fn vfio_pci_device_handler(
             }
 
             group = Some(devgroup);
-
-            pci_fixups.push((host, guestdev));
         }
+
+        // collect PCI address mapping for both vfio-pci-gk and vfio-pci device
+        pci_fixups.push((host, guestdev));
     }
 
     let dev_update = if vfio_in_guest {
@@ -932,22 +962,22 @@ pub async fn add_devices(
                 ));
             }
 
-            let mut sb = sandbox.lock().await;
-            for (host, guest) in update.pci {
-                if let Some(other_guest) = sb.pcimap.insert(host, guest) {
-                    return Err(anyhow!(
-                        "Conflicting guest address for host device {} ({} versus {})",
-                        host,
-                        guest,
-                        other_guest
-                    ));
-                }
-            }
-
             // Update cgroup to allow all devices added to guest.
             insert_devices_cgroup_rule(spec, &dev_update.info, true, "rwm")
                 .context("Update device cgroup")?;
         }
+
+        let mut sb = sandbox.lock().await;
+        for (host, guest) in update.pci {
+            if let Some(other_guest) = sb.pcimap.insert(host, guest) {
+                return Err(anyhow!(
+                    "Conflicting guest address for host device {} ({} versus {})",
+                    host,
+                    guest,
+                    other_guest
+                ));
+            }
+        }
     }
 
     if let Some(process) = spec.process.as_mut() {
@@ -1382,8 +1412,11 @@ mod tests {
             ("0000:01:01.0", "ffff:02:1f.7"),
         ];
 
+        let pci_dev_info_original = r#"PCIDEVICE_x_INFO={"0000:1a:01.0":{"generic":{"deviceID":"0000:1a:01.0"}},"0000:1b:02.0":{"generic":{"deviceID":"0000:1b:02.0"}}}"#;
+        let pci_dev_info_expected = r#"PCIDEVICE_x_INFO={"0000:01:01.0":{"generic":{"deviceID":"0000:01:01.0"}},"0000:01:02.0":{"generic":{"deviceID":"0000:01:02.0"}}}"#;
         let mut env = vec![
             "PCIDEVICE_x=0000:1a:01.0,0000:1b:02.0".to_string(),
+            pci_dev_info_original.to_string(),
             "PCIDEVICE_y=0000:01:01.0".to_string(),
             "NOTAPCIDEVICE_blah=abcd:ef:01.0".to_string(),
         ];
@@ -1399,11 +1432,12 @@ mod tests {
             .collect();
 
         let res = update_env_pci(&mut env, &pci_fixups);
-        assert!(res.is_ok());
+        assert!(res.is_ok(), "error: {}", res.err().unwrap());
 
         assert_eq!(env[0], "PCIDEVICE_x=0000:01:01.0,0000:01:02.0");
-        assert_eq!(env[1], "PCIDEVICE_y=ffff:02:1f.7");
-        assert_eq!(env[2], "NOTAPCIDEVICE_blah=abcd:ef:01.0");
+        assert_eq!(env[1], pci_dev_info_expected);
+        assert_eq!(env[2], "PCIDEVICE_y=ffff:02:1f.7");
+        assert_eq!(env[3], "NOTAPCIDEVICE_blah=abcd:ef:01.0");
     }
 
     #[test]

src/agent/src/main.rs

@@ -59,11 +59,11 @@ mod util;
 mod version;
 mod watcher;
 
-use config::GuestComponentsFeatures;
+use config::GuestComponentsProcs;
 use mount::{cgroups_mount, general_mount};
 use sandbox::Sandbox;
 use signal::setup_signal_handler;
-use slog::{error, info, o, warn, Logger};
+use slog::{debug, error, info, o, warn, Logger};
 use uevent::watch_uevents;
 
 use futures::future::join_all;
@@ -403,8 +403,16 @@ async fn start_sandbox(
     let (tx, rx) = tokio::sync::oneshot::channel();
     sandbox.lock().await.sender = Some(tx);
 
-    if Path::new(CDH_PATH).exists() && Path::new(AA_PATH).exists() {
-        init_attestation_components(logger, config)?;
+    let gc_procs = config.guest_components_procs;
+    if gc_procs != GuestComponentsProcs::None {
+        if !attestation_binaries_available(logger, &gc_procs) {
+            warn!(
+                logger,
+                "attestation binaries requested for launch not available"
+            );
+        } else {
+            init_attestation_components(logger, config)?;
+        }
     }
 
     // vsock:///dev/vsock, port
@@ -417,9 +425,33 @@ async fn start_sandbox(
     Ok(())
 }
 
+// Check if required attestation binaries are available on the rootfs.
+fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs) -> bool {
+    let binaries = match procs {
+        GuestComponentsProcs::AttestationAgent => vec![AA_PATH],
+        GuestComponentsProcs::ConfidentialDataHub => vec![AA_PATH, CDH_PATH],
+        GuestComponentsProcs::ApiServerRest => vec![AA_PATH, CDH_PATH, API_SERVER_PATH],
+        _ => vec![],
+    };
+    for binary in binaries.iter() {
+        if !Path::new(binary).exists() {
+            warn!(logger, "{} not found", binary);
+            return false;
+        }
+    }
+    true
+}
+
 // Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
-fn init_attestation_components(logger: &Logger, _config: &AgentConfig) -> Result<()> {
-    // The Attestation Agent will run for the duration of the guest.
+// and the corresponding procs are enabled in the agent configuration. the process will be
+// launched in the background and the function will return immediately.
+fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> {
+    // skip launch of any guest-component
+    if config.guest_components_procs == GuestComponentsProcs::None {
+        return Ok(());
+    }
+
+    debug!(logger, "spawning attestation-agent process {}", AA_PATH);
     launch_process(
         logger,
         AA_PATH,
@@ -429,32 +461,43 @@ fn init_attestation_components(logger: &Logger, _config: &AgentConfig) -> Result
     )
     .map_err(|e| anyhow!("launch_process {} failed: {:?}", AA_PATH, e))?;
 
-    if let Err(e) = launch_process(
+    // skip launch of confidential-data-hub and api-server-rest
+    if config.guest_components_procs == GuestComponentsProcs::AttestationAgent {
+        return Ok(());
+    }
+
+    debug!(
+        logger,
+        "spawning confidential-data-hub process {}", CDH_PATH
+    );
+    launch_process(
         logger,
         CDH_PATH,
         &vec![],
         CDH_SOCKET,
         DEFAULT_LAUNCH_PROCESS_TIMEOUT,
-    ) {
-        error!(logger, "launch_process {} failed: {:?}", CDH_PATH, e);
-    } else {
-        let features = _config.guest_components_rest_api;
-        match features {
-            GuestComponentsFeatures::None => {}
-            _ => {
-                if let Err(e) = launch_process(
-                    logger,
-                    API_SERVER_PATH,
-                    &vec!["--features", &features.to_string()],
-                    "",
-                    0,
-                ) {
-                    error!(logger, "launch_process {} failed: {:?}", API_SERVER_PATH, e);
-                }
-            }
-        }
+    )
+    .map_err(|e| anyhow!("launch_process {} failed: {:?}", CDH_PATH, e))?;
+
+    // skip launch of api-server-rest
+    if config.guest_components_procs == GuestComponentsProcs::ConfidentialDataHub {
+        return Ok(());
     }
 
+    let features = config.guest_components_rest_api;
+    debug!(
+        logger,
+        "spawning api-server-rest process {} --features {}", API_SERVER_PATH, features
+    );
+    launch_process(
+        logger,
+        API_SERVER_PATH,
+        &vec!["--features", &features.to_string()],
+        "",
+        0,
+    )
+    .map_err(|e| anyhow!("launch_process {} failed: {:?}", API_SERVER_PATH, e))?;
+
     Ok(())
 }
 

src/dragonball/src/address_space_manager.rs

@@ -644,10 +644,7 @@ impl AddressSpaceMgr {
         guest_numa_node_id: u32,
         vcpu_ids: &[u32],
     ) {
-        let node = self
-            .numa_nodes
-            .entry(guest_numa_node_id)
-            .or_insert_with(NumaNode::new);
+        let node = self.numa_nodes.entry(guest_numa_node_id).or_default();
         node.add_info(&NumaNodeInfo {
             base: region.start_addr(),
             size: region.len(),

src/dragonball/src/device_manager/mod.rs

@@ -424,7 +424,7 @@ impl DeviceOpContext {
     fn generate_virtio_device_info(&self) -> Result<HashMap<(DeviceType, String), MMIODeviceInfo>> {
         let mut dev_info = HashMap::new();
         #[cfg(feature = "dbs-virtio-devices")]
-        for (_index, device) in self.virtio_devices.iter().enumerate() {
+        for device in self.virtio_devices.iter() {
             let (mmio_base, mmio_size, irq) = DeviceManager::get_virtio_mmio_device_info(device)?;
             let dev_type;
             let device_id;

src/libs/Cargo.lock

@@ -748,9 +748,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.151"
+version = "0.2.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
 
 [[package]]
 name = "lock_api"
@@ -944,9 +944,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.17.1"
+version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
 [[package]]
 name = "parking_lot"
@@ -1588,9 +1588,9 @@ dependencies = [
 
 [[package]]
 name = "sysinfo"
-version = "0.29.11"
+version = "0.30.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666"
+checksum = "732ffa00f53e6b2af46208fba5718d9662a421049204e156328b66791ffa15ae"
 dependencies = [
  "cfg-if",
  "core-foundation-sys",
@@ -1598,7 +1598,7 @@ dependencies = [
  "ntapi 0.4.1",
  "once_cell",
  "rayon",
- "winapi",
+ "windows",
 ]
 
 [[package]]
@@ -2007,13 +2007,32 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
+dependencies = [
+ "windows-core",
+ "windows-targets 0.52.5",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
+dependencies = [
+ "windows-targets 0.52.5",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]
@@ -2022,13 +2041,29 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
 dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.5",
+ "windows_aarch64_msvc 0.52.5",
+ "windows_i686_gnu 0.52.5",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc 0.52.5",
+ "windows_x86_64_gnu 0.52.5",
+ "windows_x86_64_gnullvm 0.52.5",
+ "windows_x86_64_msvc 0.52.5",
 ]
 
 [[package]]
@@ -2037,42 +2072,90 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
+
 [[package]]
 name = "wyz"
 version = "0.5.1"

src/libs/kata-types/src/capabilities.rs

@@ -127,7 +127,7 @@ mod tests {
         assert!(cap.is_fs_sharing_supported());
 
         // test set hybrid-vsock support
-        cap.set(CapabilityBits::HybridVsockSupport);
+        cap.add(CapabilityBits::HybridVsockSupport);
         assert!(cap.is_hybrid_vsock_supported());
         // test append capabilities
         cap.add(CapabilityBits::GuestMemoryProbe);

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -411,6 +411,20 @@ pub struct DebugInfo {
     /// much disk space.
     #[serde(default)]
     pub guest_memory_dump_path: String,
+
+    /// This option allows to add a debug monitor socket when `enable_debug = true`
+    /// WARNING: Anyone with access to the monitor socket can take full control of
+    /// Qemu. This is for debugging purpose only and must *NEVER* be used in
+    /// production.
+    /// Valid values are :
+    /// - "hmp"
+    /// - "qmp"
+    /// - "qmp-pretty" (same as "qmp" with pretty json formatting)
+    /// If set to the empty string "", no debug monitor socket is added. This is
+    /// the default.
+    /// dbg_monitor_socket = "hmp"
+    #[serde(default)]
+    pub dbg_monitor_socket: String,
 }
 
 impl DebugInfo {

src/libs/kata-types/tests/texture/configuration-anno-0.toml

@@ -84,6 +84,7 @@ sandbox_bind_mounts=["/proc/self"]
 vfio_mode="vfio"
 experimental=["a", "b"]
 enable_pprof = true
+name="virt-container"
 hypervisor_name = "qemu"
 agent_name = "agent0"
 

src/libs/kata-types/tests/texture/configuration-anno-1.toml

@@ -83,6 +83,7 @@ sandbox_bind_mounts=["/proc/self"]
 vfio_mode="vfio"
 experimental=["a", "b"]
 enable_pprof = true
+name="virt-container"
 hypervisor_name = "qemu"
 agent_name = "agent0"
 

src/runtime-rs/Makefile

@@ -208,18 +208,18 @@ ifneq (,$(DBCMD))
     SYSCONFIG_PATHS += $(SYSCONFIG_DB)
     CONFIGS += $(CONFIG_DB)
     # dragonball-specific options (all should be suffixed by "_DB")
-	VMROOTFSDRIVER_DB := virtio-blk-pci
+    VMROOTFSDRIVER_DB := virtio-blk-pci
     DEFMAXVCPUS_DB := 1
     DEFBLOCKSTORAGEDRIVER_DB := virtio-blk-mmio
     DEFNETWORKMODEL_DB := tcfilter
     KERNELPARAMS_DB = console=ttyS1 agent.log_vport=1025
-	KERNELTYPE_DB = uncompressed
+    KERNELTYPE_DB = uncompressed
     KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME_DB,$(KERNELTYPE_DB))
     KERNELPATH_DB = $(KERNELDIR)/$(KERNEL_NAME_DB)
     DEFSANDBOXCGROUPONLY = true
     RUNTIMENAME := virt_container
     PIPESIZE := 1
-	DBSHAREDFS := inline-virtio-fs
+    DBSHAREDFS := inline-virtio-fs
 endif
 
 ifneq (,$(CLHCMD))

src/runtime-rs/config/configuration-cloud-hypervisor.toml.in

@@ -226,9 +226,18 @@ container_pipe_size=@PIPESIZE@
 
 #debug_console_enabled = true
 
-# Agent connection dialing timeout value in seconds
-# (default: 45)
-dial_timeout = 45
+# Agent dial timeout in millisecond.
+# (default: 10)
+#dial_timeout_ms = 10
+
+# Agent reconnect timeout in millisecond.
+# Retry times = reconnect_timeout_ms / dial_timeout_ms (default: 300)
+# If you find pod cannot connect to the agent when starting, please
+# consider increasing this value to increase the retry times.
+# You'd better not change the value of dial_timeout_ms, unless you have an
+# idea of what you are doing.
+# (default: 3000)
+#reconnect_timeout_ms = 3000
 
 [runtime]
 # If enabled, the runtime will log additional debug messages to the

src/runtime-rs/config/configuration-dragonball.toml.in

@@ -239,9 +239,18 @@ container_pipe_size=@PIPESIZE@
 
 #debug_console_enabled = true
 
-# Agent connection dialing timeout value in seconds
-# (default: 45)
-dial_timeout = 45
+# Agent dial timeout in millisecond.
+# (default: 10)
+#dial_timeout_ms = 10
+
+# Agent reconnect timeout in millisecond.
+# Retry times = reconnect_timeout_ms / dial_timeout_ms (default: 300)
+# If you find pod cannot connect to the agent when starting, please
+# consider increasing this value to increase the retry times.
+# You'd better not change the value of dial_timeout_ms, unless you have an
+# idea of what you are doing.
+# (default: 3000)
+#reconnect_timeout_ms = 3000
 
 [runtime]
 # If enabled, the runtime will log additional debug messages to the

src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in

@@ -346,7 +346,7 @@ pflashes = []
 #
 # If set to the empty string "", no extra monitor socket is added. This is
 # the default.
-#extra_monitor_socket = hmp
+#extra_monitor_socket = "hmp"
 
 # Disable the customizations done in the runtime when it detects
 # that it is running on top a VMM. This will result in the runtime
@@ -567,9 +567,18 @@ kernel_modules=[]
 
 #debug_console_enabled = true
 
-# Agent connection dialing timeout value in seconds
-# (default: 45)
-dial_timeout = 45
+# Agent dial timeout in millisecond.
+# (default: 10)
+#dial_timeout_ms = 10
+
+# Agent reconnect timeout in millisecond.
+# Retry times = reconnect_timeout_ms / dial_timeout_ms (default: 300)
+# If you find pod cannot connect to the agent when starting, please
+# consider increasing this value to increase the retry times.
+# You'd better not change the value of dial_timeout_ms, unless you have an
+# idea of what you are doing.
+# (default: 3000)
+#reconnect_timeout_ms = 3000
 
 [runtime]
 # If enabled, the runtime will log additional debug messages to the

src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs

@@ -11,7 +11,9 @@ use async_trait::async_trait;
 use std::collections::HashMap;
 use std::fmt::Display;
 use std::fs::{read_to_string, File};
-use std::os::fd::AsRawFd;
+use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd};
+use std::os::unix::net::UnixListener;
+use std::path::PathBuf;
 use tokio;
 
 // These should have been called MiB and GiB for better readability but the
@@ -19,6 +21,9 @@ use tokio;
 const MI_B: u64 = 1024 * 1024;
 const GI_B: u64 = 1024 * MI_B;
 
+const QMP_SOCKET_FILE: &str = "qmp.sock";
+const DEBUG_MONITOR_SOCKET: &str = "debug-monitor.sock";
+
 // The approach taken here is inspired by govmm.  We build structs, each
 // corresponding to a qemu command line parameter, like Kernel, or a device,
 // for instance MemoryBackendFile.  Members of these structs mostly directly
@@ -65,6 +70,81 @@ impl Display for VirtioBusType {
     }
 }
 
+// Conventions used in qemu command line generation
+// ================================================
+//
+// While acknowledging govmm inspiration, this implementation differs in
+// several important aspects.
+//
+// 1:1 correspondence between qemu switches and cmdline generator structs
+// ----------------------------------------------------------------------
+//
+// There should mostly be a 1:1 mapping between qemu command line switches and
+// objects (structs) that represent them.  That is, each struct's
+// ToQemuParams::qemu_params() is expected to generate a single
+// `-whatever <params...>`.  Notably, a lot of devices are assembled from a
+// frontend (`-device`) and a backend (`-object`), each of which should be
+// represented and generated by a separate object (struct).  If setting up
+// a qemu construct (device or other) requires assembly from multiple such
+// parts, this is expected to be done in a `QemuCmdLine::add_*()` function.
+//
+// As an example, a network device might consist of `-netdev tap` and
+// `-device virtio-net-pci`.  The former is represented by struct Netdev,
+// the latter by struct DeviceVirtioNet, both of which are instantiated &
+// set up in QemuCmdLine::add_network_device().
+//
+// There is a couple of exceptions to this convention.  struct Kernel might
+// be required to generate multiple qemu cmdline switches to perform full
+// kernel set-up, struct Knobs (directly lifted from govmm) does that too
+// due to its purpose of being a catch-all grabbag.  Both are singletons
+// dictated mostly by nature of qemu's cmdline syntax.  Going forward, most
+// if not all future additions to qemu cmdline generation are expected to
+// follow this convention.
+//
+// This convention is a departure from govmm style which tends to lump all
+// information necessary to set up a piece of VM equipment into a single
+// object.  This might make sense at the first sight, after all a network
+// device needs both its frontend and backend so why not represent it with
+// a single struct.  However, as can be observed in govmm, the resulting
+// structs tend to be big and hard to read since the frontend and backend
+// params are distinct so the big struct is actually two separate structs
+// really.  If a need ever arises to have explicit representation of a whole
+// device it would still seem preferable to assemble it from the explicit
+// individual representations of frontend and backend.
+//
+// Naming conventions
+// ------------------
+//
+// The idea is to name entities in this qemu cmdline generator as close
+// as possible to the qemu cmdline switches and params that they represent.
+// This is to reduce the intellectual overhead of having to map between names
+// on qemu command line and in this source code.
+//
+// Most of the cmdline switch generating structs in this file are expected
+// to generate a single qemu cmdline switch as detailed above.  It follows
+// that most member fields of these objects will represent additional params
+// of the qemu switches and their names should ideally be as close to the
+// corresponding qemu switch params as possible, within reason and Rust
+// syntax limitations (e.g. dashes in qemu param names obviously need to be
+// converted to underscores for code to even compile).  As an example,
+// struct MemoryBackendFile members' names are identical to qemu's
+// `-object memory-backend-file,...` params, with the exception of `mem-path`
+// which is called `mem_path` to comply with Rust syntax.
+//
+// The struct names should follow the same logic, their names should ideally
+// be as close as possible to the qemu switch they represent (e.g. `-smp` is
+// generated by struct Smp), or to common parlance if that's not descriptive
+// enough.  This applies above all to devices which are commonly set up by
+// `-device` & `-object`.  In those cases, e.g. `-chardev socket` might be
+// referred to as "chardev socket" in common speech so the corresponding struct
+// is called ChardevSocket, or `-device vhost-user-fs-pci` will likely be
+// pronounced as "device vhost user fs" so its struct is called
+// DeviceVhostUserFs, too.  Admittedly, this approach can be rather squishy and
+// a matter of taste ultimately, but it should still be useful to keep in mind
+// that the idea is to keep the names in this source code as close as possible
+// to what folks working with qemu in kata are likely to use and be familiar
+// with already.
+
 #[derive(Debug)]
 struct Kernel {
     // PathBuf would seem more appropriae but since we get the kernel path
@@ -363,10 +443,6 @@ struct Knobs {
     nodefaults: bool,
     nographic: bool,
     no_reboot: bool,
-    no_shutdown: bool,
-    daemonize: bool,
-    stopped: bool,
-
     vga: String,
 }
 
@@ -377,9 +453,6 @@ impl Knobs {
             nodefaults: true,
             nographic: true,
             no_reboot: true,
-            no_shutdown: false,
-            daemonize: false,
-            stopped: false,
             vga: "none".to_owned(),
         }
     }
@@ -403,15 +476,6 @@ impl ToQemuParams for Knobs {
         if self.no_reboot {
             result.push("-no-reboot".to_owned());
         }
-        if self.no_shutdown {
-            result.push("-no-shutdown".to_owned());
-        }
-        if self.daemonize {
-            result.push("-daemonize".to_owned());
-        }
-        if self.stopped {
-            result.push("-S".to_owned());
-        }
         Ok(result)
     }
 }
@@ -1148,6 +1212,63 @@ impl ToQemuParams for Rtc {
     }
 }
 
+#[derive(Debug)]
+struct ObjectRngRandom {
+    // id is the device ID
+    id: String,
+
+    // filename is the entropy source on the host
+    filename: String,
+}
+
+impl ObjectRngRandom {
+    fn new() -> ObjectRngRandom {
+        ObjectRngRandom {
+            id: "rng0".to_owned(),
+            filename: "/dev/urandom".to_owned(),
+        }
+    }
+}
+
+#[async_trait]
+impl ToQemuParams for ObjectRngRandom {
+    async fn qemu_params(&self) -> Result<Vec<String>> {
+        let mut object_params = Vec::new();
+
+        object_params.push("rng-random".to_owned());
+        object_params.push(format!("id={}", self.id));
+        object_params.push(format!("filename={}", self.filename));
+
+        Ok(vec!["-object".to_owned(), object_params.join(",")])
+    }
+}
+
+#[derive(Debug)]
+struct DeviceRng {
+    // transport is the virtio transport for this device.
+    transport: String,
+}
+
+impl DeviceRng {
+    fn new() -> DeviceRng {
+        DeviceRng {
+            transport: "virtio-rng-pci".to_owned(),
+        }
+    }
+}
+
+#[async_trait]
+impl ToQemuParams for DeviceRng {
+    async fn qemu_params(&self) -> Result<Vec<String>> {
+        let mut device_params = Vec::new();
+
+        device_params.push(self.transport.clone());
+        device_params.push(format!("rng={}", "rng0".to_owned()));
+
+        Ok(vec!["-device".to_owned(), device_params.join(",")])
+    }
+}
+
 #[derive(Debug)]
 struct DeviceIntelIommu {
     intremap: bool,
@@ -1178,6 +1299,121 @@ impl ToQemuParams for DeviceIntelIommu {
     }
 }
 
+// Qemu provides methods and types for managing QEMU instances.
+// To manage a qemu instance after it has been launched you need
+// to pass the -qmp option during launch requesting the qemu instance
+// to create a QMP unix domain manageent socket, e.g.,
+// -qmp unix:fd=SOCK_FD,server=on,wait=off.
+// -monitor unix:path=SOCK_PATH,server=on,wait=off.
+#[derive(Debug, Default, PartialEq)]
+pub enum MonitorProtocol {
+    // Socket using a human-friendly text-based protocol.
+    Hmp,
+
+    // Socket using a richer json-based protocol.
+    #[default]
+    Qmp,
+
+    // Same as Qmp with pretty json formatting.
+    QmpPretty,
+}
+
+impl MonitorProtocol {
+    pub fn new(proto: &str) -> Self {
+        match proto {
+            "hmp" => MonitorProtocol::Hmp,
+            "qmp-pretty" => MonitorProtocol::QmpPretty,
+            _ => MonitorProtocol::Qmp,
+        }
+    }
+}
+
+impl ToString for MonitorProtocol {
+    fn to_string(&self) -> String {
+        match *self {
+            MonitorProtocol::Hmp => "monitor".to_string(),
+            MonitorProtocol::QmpPretty => "qmp-pretty".to_string(),
+            _ => "qmp".to_string(),
+        }
+    }
+}
+
+#[derive(Debug)]
+enum QmpSockType {
+    Fd(File),
+    Path(PathBuf),
+}
+
+#[derive(Debug)]
+pub struct QmpSocket {
+    // protocol to be used on the socket.
+    protocol: MonitorProtocol,
+    // QMP unix socket to be passed to qemu
+    address: QmpSockType,
+    // server tells if this is a server socket.
+    server: bool,
+    // nowait tells if qemu should block waiting for a client to connect.
+    nowait: bool,
+}
+
+impl QmpSocket {
+    fn new(proto: MonitorProtocol) -> Result<Self> {
+        let qmp_socket = match proto {
+            MonitorProtocol::Qmp | MonitorProtocol::QmpPretty => {
+                // let sock_path = root_path.join(QMP_SOCKET_FILE);
+                let listener =
+                    UnixListener::bind(QMP_SOCKET_FILE).context("unix listener bind failed.")?;
+                let raw_fd = listener.into_raw_fd();
+                clear_cloexec(raw_fd).context("clearing unix listenser O_CLOEXEC failed")?;
+                let sock_file = unsafe { File::from_raw_fd(raw_fd) };
+                // The default QMP socket or called base socket is qmp.sock.
+                QmpSocket {
+                    protocol: MonitorProtocol::new("qmp"),
+                    address: QmpSockType::Fd(sock_file),
+                    server: true,
+                    nowait: true,
+                }
+            }
+            MonitorProtocol::Hmp => {
+                // If extra monitor needed, HMP socket with qmp-extra.sock will be added.
+                QmpSocket {
+                    protocol: MonitorProtocol::new("hmp"),
+                    address: QmpSockType::Path(PathBuf::from(DEBUG_MONITOR_SOCKET)),
+                    server: true,
+                    nowait: true,
+                }
+            }
+        };
+
+        Ok(qmp_socket)
+    }
+}
+
+#[async_trait]
+impl ToQemuParams for QmpSocket {
+    async fn qemu_params(&self) -> Result<Vec<String>> {
+        let param_qmp = format!("-{}", self.protocol.to_string());
+
+        let mut params: Vec<String> = Vec::new();
+
+        match &self.address {
+            // -qmp unix:fd=SOCK_FD,server=on,wait=off
+            QmpSockType::Fd(f) => params.push(format!("unix:fd={}", f.as_raw_fd())),
+            // -monitor unix:path=SOCK_PATH,server=on,wait=off
+            QmpSockType::Path(p) => params.push(format!("unix:path={}", p.display())),
+        }
+
+        if self.server {
+            params.push("server=on".to_owned());
+            if self.nowait {
+                params.push("wait=off".to_owned());
+            }
+        }
+
+        Ok(vec![param_qmp, params.join(",")])
+    }
+}
+
 fn is_running_in_vm() -> Result<bool> {
     let res = read_to_string("/proc/cpuinfo")?
         .lines()
@@ -1207,11 +1443,21 @@ pub struct QemuCmdLine<'a> {
     id: String,
     config: &'a HypervisorConfig,
 
+    // In principle, all objects implementing ToQemuParams could be just stored
+    // in the `devices` container.  However, there are several special cases
+    // that might need to be set up in several steps after having been
+    // initially constructed (from HypervisorConfig, mostly).  For instance,
+    // adding an NVDIMM needs to modify Machine and query Memory, adding a
+    // virtiofs share modifies both Memory and Machine, adding a serial console
+    // modifies Kernel etc.  For convenience accessing them, we store these
+    // singletons in named QemuCmdLine member.  The rest should go to the
+    // anonymous `devices` container.
     kernel: Kernel,
     memory: Memory,
     smp: Smp,
     machine: Machine,
     cpu: Cpu,
+    qmp_socket: QmpSocket,
 
     knobs: Knobs,
 
@@ -1228,6 +1474,7 @@ impl<'a> QemuCmdLine<'a> {
             smp: Smp::new(config),
             machine: Machine::new(config),
             cpu: Cpu::new(config),
+            qmp_socket: QmpSocket::new(MonitorProtocol::Qmp)?,
             knobs: Knobs::new(config),
             devices: Vec::new(),
         };
@@ -1236,16 +1483,39 @@ impl<'a> QemuCmdLine<'a> {
             qemu_cmd_line.add_iommu();
         }
 
+        if config.debug_info.enable_debug && !config.debug_info.dbg_monitor_socket.is_empty() {
+            qemu_cmd_line.add_monitor(&config.debug_info.dbg_monitor_socket)?;
+        }
+
         qemu_cmd_line.add_rtc();
 
+        if qemu_cmd_line.bus_type() != VirtioBusType::Ccw {
+            qemu_cmd_line.add_rng();
+        }
+
         Ok(qemu_cmd_line)
     }
 
+    fn add_monitor(&mut self, proto: &str) -> Result<()> {
+        let monitor = QmpSocket::new(MonitorProtocol::new(proto))?;
+        self.devices.push(Box::new(monitor));
+
+        Ok(())
+    }
+
     fn add_rtc(&mut self) {
         let rtc = Rtc::new();
         self.devices.push(Box::new(rtc));
     }
 
+    fn add_rng(&mut self) {
+        let rng_object = ObjectRngRandom::new();
+        let rng_device = DeviceRng::new();
+
+        self.devices.push(Box::new(rng_object));
+        self.devices.push(Box::new(rng_device));
+    }
+
     fn bus_type(&self) -> VirtioBusType {
         if self.config.machine_info.machine_type.contains("-ccw-") {
             VirtioBusType::Ccw
@@ -1437,6 +1707,7 @@ impl<'a> QemuCmdLine<'a> {
         result.append(&mut self.machine.qemu_params().await?);
         result.append(&mut self.cpu.qemu_params().await?);
         result.append(&mut self.memory.qemu_params().await?);
+        result.append(&mut self.qmp_socket.qemu_params().await?);
 
         for device in &self.devices {
             result.append(&mut device.qemu_params().await?);

src/runtime-rs/crates/runtimes/src/manager.rs

@@ -528,20 +528,6 @@ fn load_config(spec: &oci::Spec, option: &Option<Vec<u8>>) -> Result<TomlConfig>
     // validate configuration and return the error
     toml_config.validate()?;
 
-    // Sandbox sizing information *may* be provided in two scenarios:
-    //   1. The upper layer runtime (ie, containerd or crio) provide sandbox sizing information as an annotation
-    //	in the 'sandbox container's' spec. This would typically be a scenario where as part of a create sandbox
-    //	request the upper layer runtime receives this information as part of a pod, and makes it available to us
-    //	for sizing purposes.
-    //   2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."),
-    //	then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is
-    //	a single container.
-    let mut initial_size_manager =
-        InitialSizeManager::new(spec).context("failed to construct static resource manager")?;
-    initial_size_manager
-        .setup_config(&mut toml_config)
-        .context("failed to setup static resource mgmt config")?;
-
     info!(logger, "get config content {:?}", &toml_config);
     Ok(toml_config)
 }

src/runtime/Makefile

@@ -4,8 +4,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-
-SKIP_GO_VERSION_CHECK=
 include golang.mk
 
 #Get ARCH.
@@ -151,7 +149,7 @@ FIRMWARETDVFPATH := PLACEHOLDER_FOR_DISTRO_OVMF_WITH_TDX_SUPPORT
 FIRMWARETDVFVOLUMEPATH :=
 
 FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
-FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
+FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd
 
 ROOTMEASURECONFIG ?= ""
 KERNELPARAMS += $(ROOTMEASURECONFIG)
@@ -179,6 +177,7 @@ QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"]
 #QEMUTDXPATH := $(QEMUBINDIR)/$(QEMUTDXCMD)
 QEMUTDXPATH := PLACEHOLDER_FOR_DISTRO_QEMU_WITH_TDX_SUPPORT
 QEMUTDXVALIDHYPERVISORPATHS := [\"$(QEMUTDXPATH)\"]
+QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT := 4050
 
 QEMUSNPPATH := $(QEMUBINDIR)/$(QEMUSNPCMD)
 QEMUSNPVALIDHYPERVISORPATHS := [\"$(QEMUSNPPATH)\"]
@@ -219,6 +218,7 @@ DEFMAXMEMSZ := 0
 #Default number of bridges
 DEFBRIDGES := 1
 DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\"]
+DEFENABLEANNOTATIONSTEE := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"default_vcpus\", \"default_memory\"]
 DEFDISABLEGUESTSECCOMP := true
 DEFDISABLEGUESTEMPTYDIR := false
 #Default experimental features enabled
@@ -246,9 +246,9 @@ DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
 # Please keep DEFSHAREDFS_QEMU_COCO_DEV_VIRTIOFS in sync with TDX/SEV/SNP
 DEFSHAREDFS_QEMU_COCO_DEV_VIRTIOFS := virtio-9p
 DEFSHAREDFS_STRATOVIRT_VIRTIOFS := virtio-fs
-DEFSHAREDFS_QEMU_TDX_VIRTIOFS := virtio-9p
-DEFSHAREDFS_QEMU_SEV_VIRTIOFS := virtio-9p
-DEFSHAREDFS_QEMU_SNP_VIRTIOFS := virtio-9p
+DEFSHAREDFS_QEMU_TDX_VIRTIOFS := none
+DEFSHAREDFS_QEMU_SEV_VIRTIOFS := none
+DEFSHAREDFS_QEMU_SNP_VIRTIOFS := none
 DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd
 DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"]
 # Default DAX mapping cache size in MiB
@@ -705,6 +705,7 @@ USER_VARS += QEMUTDXCMD
 USER_VARS += QEMUSNPCMD
 USER_VARS += QEMUPATH
 USER_VARS += QEMUTDXPATH
+USER_VARS += QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT
 USER_VARS += QEMUSNPPATH
 USER_VARS += QEMUVALIDHYPERVISORPATHS
 USER_VARS += QEMUTDXVALIDHYPERVISORPATHS
@@ -752,6 +753,7 @@ USER_VARS += DEFVIRTIOFSCACHE
 USER_VARS += DEFVIRTIOFSQUEUESIZE
 USER_VARS += DEFVIRTIOFSEXTRAARGS
 USER_VARS += DEFENABLEANNOTATIONS
+USER_VARS += DEFENABLEANNOTATIONSTEE
 USER_VARS += DEFENABLEIOTHREADS
 USER_VARS += DEFSECCOMPSANDBOXPARAM
 USER_VARS += DEFENABLEVHOSTUSERSTORE

src/runtime/cmd/kata-runtime/kata-check.go

@@ -471,16 +471,15 @@ func genericArchKernelParamHandler(onVMM bool, fields logrus.Fields, msg string)
 // genericKvmIsUsable determines if it will be possible to create a full virtual machine
 // by creating a minimal VM and then deleting it.
 func genericKvmIsUsable() error {
-	flags := syscall.O_RDWR | syscall.O_CLOEXEC
+	fieldLogger := kataLog.WithField("check-type", "full")
 
-	f, err := syscall.Open(kvmDevice, flags, 0)
+	f, err := syscall.Open(kvmDevice, syscall.O_RDWR|syscall.O_CLOEXEC, 0)
 	if err != nil {
+		fieldLogger.WithField("device", kvmDevice).Errorf("cannot open kvm device: %v", err)
 		return err
 	}
 	defer syscall.Close(f)
 
-	fieldLogger := kataLog.WithField("check-type", "full")
-
 	fieldLogger.WithField("device", kvmDevice).Info("device available")
 
 	vm, _, errno := syscall.Syscall(syscall.SYS_IOCTL,

src/runtime/cmd/kata-runtime/kata-env.go

@@ -123,6 +123,8 @@ type HypervisorInfo struct {
 	MemorySlots       uint32
 	HotPlugVFIO       config.PCIePort
 	ColdPlugVFIO      config.PCIePort
+	PCIeRootPort      uint32
+	PCIeSwitchPort    uint32
 	Debug             bool
 	SecurityInfo      SecurityInfo
 }
@@ -339,6 +341,8 @@ func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) {
 		VirtioFSDaemon:    config.HypervisorConfig.VirtioFSDaemon,
 		HotPlugVFIO:       config.HypervisorConfig.HotPlugVFIO,
 		ColdPlugVFIO:      config.HypervisorConfig.ColdPlugVFIO,
+		PCIeRootPort:      config.HypervisorConfig.PCIeRootPort,
+		PCIeSwitchPort:    config.HypervisorConfig.PCIeSwitchPort,
 		SocketPath:        socketPath,
 		SecurityInfo:      securityInfo,
 	}, nil

src/runtime/cmd/kata-runtime/kata-env_test.go

@@ -89,6 +89,8 @@ func makeRuntimeConfig(prefixDir string) (configFile string, ociConfig oci.Runti
 	enableIOThreads := true
 	hotPlugVFIO = config.BridgePort
 	coldPlugVFIO = config.NoPort
+	pcieRootPort := uint32(0)
+	pcieSwitchPort := uint32(0)
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := filepath.Join(prefixDir, "virtiofsd")
@@ -133,6 +135,8 @@ func makeRuntimeConfig(prefixDir string) (configFile string, ociConfig oci.Runti
 		EnableIOThreads:      enableIOThreads,
 		HotPlugVFIO:          hotPlugVFIO,
 		ColdPlugVFIO:         coldPlugVFIO,
+		PCIeRootPort:         pcieRootPort,
+		PCIeSwitchPort:       pcieSwitchPort,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     hypConfig.NumVCPUs(),
 		DefaultMaxVCPUCount:  hypConfig.DefaultMaxVCPUs,
@@ -276,6 +280,8 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
 		VirtioFSDaemon:    config.HypervisorConfig.VirtioFSDaemon,
 		HotPlugVFIO:       config.HypervisorConfig.HotPlugVFIO,
 		ColdPlugVFIO:      config.HypervisorConfig.ColdPlugVFIO,
+		PCIeRootPort:      config.HypervisorConfig.PCIeRootPort,
+		PCIeSwitchPort:    config.HypervisorConfig.PCIeSwitchPort,
 	}
 
 	if os.Geteuid() == 0 {

src/runtime/config/configuration-qemu-tdx.toml.in

@@ -17,6 +17,7 @@ kernel = "@KERNELCONFIDENTIALPATH@"
 image = "@IMAGECONFIDENTIALPATH@"
 # initrd = "@INITRDPATH@"
 machine_type = "@MACHINETYPE@"
+tdx_quote_generation_service_socket_port = @QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT@
 
 # rootfs filesystem type:
 #   - ext4 (default)
@@ -48,7 +49,7 @@ confidential_guest = true
 # List of valid annotation names for the hypervisor
 # Each member of the list is a regular expression, which is the base name
 # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
-enable_annotations = @DEFENABLEANNOTATIONS@
+enable_annotations = @DEFENABLEANNOTATIONSTEE@
 
 # List of valid annotations values for the hypervisor
 # Each member of the list is a path pattern as described by glob(3).

src/runtime/go.mod

@@ -1,6 +1,6 @@
 module github.com/kata-containers/kata-containers/src/runtime
 
-go 1.19
+go 1.21
 
 require (
 	code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5
@@ -10,10 +10,10 @@ require (
 	github.com/container-orchestrated-devices/container-device-interface v0.6.0
 	github.com/containerd/cgroups v1.1.0
 	github.com/containerd/console v1.0.3
-	github.com/containerd/containerd v1.7.11
+	github.com/containerd/containerd v1.7.16
 	github.com/containerd/cri-containerd v1.19.0
 	github.com/containerd/fifo v1.1.0
-	github.com/containerd/ttrpc v1.2.2
+	github.com/containerd/ttrpc v1.2.3
 	github.com/containerd/typeurl/v2 v2.1.1
 	github.com/containernetworking/plugins v1.3.0
 	github.com/containers/podman/v4 v4.9.4
@@ -51,7 +51,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.19.0
 	go.opentelemetry.io/otel/trace v1.19.0
 	golang.org/x/oauth2 v0.14.0
-	golang.org/x/sys v0.19.0
+	golang.org/x/sys v0.20.0
 	google.golang.org/grpc v1.58.3
 	google.golang.org/protobuf v1.33.0
 	k8s.io/apimachinery v0.26.2
@@ -86,7 +86,7 @@ require (
 	github.com/go-openapi/spec v0.20.9 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/uuid v1.4.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -101,6 +101,7 @@ require (
 	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/moby/sys/signal v0.7.0 // indirect
 	github.com/moby/sys/symlink v0.2.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
@@ -116,9 +117,9 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
 	go.opentelemetry.io/otel/metric v1.19.0 // indirect
 	golang.org/x/mod v0.13.0 // indirect
-	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
@@ -127,6 +128,8 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
+	tags.cncf.io/container-device-interface v0.7.2 // indirect
+	tags.cncf.io/container-device-interface/specs-go v0.7.0 // indirect
 )
 
 replace (

src/runtime/go.sum

@@ -86,8 +86,8 @@ github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxz
 github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
 github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw=
-github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE=
+github.com/containerd/containerd v1.7.16 h1:7Zsfe8Fkj4Wi2My6DXGQ87hiqIrmOXolm72ZEkFU5Mg=
+github.com/containerd/containerd v1.7.16/go.mod h1:NL49g7A/Fui7ccmxV6zkBWwqMgmMxFWzujYCc+JLt7k=
 github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM=
 github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
 github.com/containerd/cri-containerd v1.19.0 h1:PcTvvl+SHaekCMQZFQkYjn1RKlYrK6khYbuhOeF68k0=
@@ -98,8 +98,8 @@ github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nN
 github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
-github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs=
-github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak=
+github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz0=
+github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM=
 github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4=
 github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0=
 github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=
@@ -126,6 +126,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
+github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -186,6 +187,7 @@ github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUri
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
 github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
 github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
@@ -244,8 +246,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -266,6 +268,7 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -277,6 +280,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk=
+github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -285,6 +289,7 @@ github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
+github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
@@ -310,6 +315,7 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
@@ -325,6 +331,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -356,6 +363,8 @@ github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI
 github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
 github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc=
 github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs=
+github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
+github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -368,6 +377,7 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
+github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -376,11 +386,13 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/ginkgo/v2 v2.13.1 h1:LNGfMbR2OVGBfXjvRZIZ2YCTQdGKtPLvuI1rMCCj3OU=
+github.com/onsi/ginkgo/v2 v2.13.1/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
+github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
@@ -443,21 +455,23 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0=
 github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs=
 github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY=
+github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
+github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
 github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
 github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -480,8 +494,11 @@ github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23n
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -519,6 +536,7 @@ go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1
 go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
@@ -600,8 +618,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -683,10 +701,9 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
@@ -825,6 +842,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
@@ -857,3 +875,7 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
+tags.cncf.io/container-device-interface v0.7.2 h1:MLqGnWfOr1wB7m08ieI4YJ3IoLKKozEnnNYBtacDPQU=
+tags.cncf.io/container-device-interface v0.7.2/go.mod h1:Xb1PvXv2BhfNb3tla4r9JL129ck1Lxv9KuU6eVOfKto=
+tags.cncf.io/container-device-interface/specs-go v0.7.0 h1:w/maMGVeLP6TIQJVYT5pbqTi8SCw/iHZ+n4ignuGHqg=
+tags.cncf.io/container-device-interface/specs-go v0.7.0/go.mod h1:hMAwAbMZyBLdmYqWgYcKH0F/yctNpV3P35f+/088A80=

src/runtime/golang.mk

@@ -27,7 +27,17 @@ ifeq (,$(not_check_version))
     ifneq (,$(install_yq))
         $(error "ERROR: install yq failed")
     endif
-    golang_version_min=$(shell $(GOPATH)/bin/yq r ../../versions.yaml languages.golang.version)
+
+    YQ_VERSION=$(shell $(GOPATH)/bin/yq --version | grep -oE "version v?[0-9]+" | grep -oE "[0-9]+")
+    QUERY="languages.golang.version"
+
+    ifneq (,$(findstring 4,$(YQ_VERSION)))
+        YQ_CMD=$(GOPATH)/bin/yq eval .$(QUERY) ../../versions.yaml
+    else
+        YQ_CMD=$(GOPATH)/bin/yq r ../../versions.yaml $(QUERY)
+    endif
+
+    golang_version_min=$(shell $(YQ_CMD))
 
     ifeq (,$(golang_version_min))
         $(error "ERROR: cannot determine minimum golang version")

src/runtime/pkg/containerd-shim-v2/create.go

@@ -19,11 +19,11 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	taskAPI "github.com/containerd/containerd/api/runtime/task/v2"
 	containerd_types "github.com/containerd/containerd/api/types"
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/typeurl/v2"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
@@ -74,63 +74,6 @@ func copyLayersToMounts(rootFs *vc.RootFs, spec *specs.Spec) error {
 	return nil
 }
 
-// CDI (Container Device Interface), is a specification, for container- runtimes,
-// to support third-party devices.
-// It introduces an abstract notion of a device as a resource. Such devices are
-// uniquely specified by a fully-qualified name that is constructed from a
-// vendor ID, a device class, and a name that is unique per vendor ID-device
-// class pair.
-//
-// vendor.com/class=unique_name
-//
-// The combination of vendor ID and device class (vendor.com/class in the
-// above example) is referred to as the device kind.
-// CDI concerns itself only with enabling containers to be device aware.
-// Areas like resource management are explicitly left out of CDI (and are
-// expected to be handled by the orchestrator). Because of this focus, the CDI
-// specification is simple to implement and allows great flexibility for
-// runtimes and orchestrators.
-func withCDI(annotations map[string]string, cdiSpecDirs []string, spec *specs.Spec) (*specs.Spec, error) {
-	// Add devices from CDI annotations
-	_, devsFromAnnotations, err := cdi.ParseAnnotations(annotations)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse CDI device annotations: %w", err)
-	}
-
-	if len(devsFromAnnotations) == 0 {
-		// No devices found, skip device injection
-		return spec, nil
-	}
-
-	var registry cdi.Registry
-	if len(cdiSpecDirs) > 0 {
-		// We can override the directories where to search for CDI specs
-		// if needed, the default is /etc/cdi /var/run/cdi
-		registry = cdi.GetRegistry(cdi.WithSpecDirs(cdiSpecDirs...))
-	} else {
-		registry = cdi.GetRegistry()
-	}
-
-	if err = registry.Refresh(); err != nil {
-		// We don't consider registry refresh failure a fatal error.
-		// For instance, a dynamically generated invalid CDI Spec file for
-		// any particular vendor shouldn't prevent injection of devices of
-		// different vendors. CDI itself knows better and it will fail the
-		// injection if necessary.
-		return nil, fmt.Errorf("CDI registry refresh failed: %w", err)
-	}
-
-	if _, err := registry.InjectDevices(spec, devsFromAnnotations...); err != nil {
-		return nil, fmt.Errorf("CDI device injection failed: %w", err)
-	}
-
-	// One crucial thing to keep in mind is that CDI device injection
-	// might add OCI Spec environment variables, hooks, and mounts as
-	// well. Therefore it is important that none of the corresponding
-	// OCI Spec fields are reset up in the call stack once we return.
-	return spec, nil
-}
-
 func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*container, error) {
 	rootFs := vc.RootFs{}
 	if len(r.Rootfs) == 1 {
@@ -175,9 +118,13 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 		//
 		// _, err = withCDI(ociSpec.Annotations, []string{"/opt/cdi"}, ociSpec)
 		//
-		_, err = withCDI(ociSpec.Annotations, []string{}, ociSpec)
-		if err != nil {
-			return nil, fmt.Errorf("adding CDI devices failed")
+		// Only inject CDI devices if single_container we do not want
+		// CDI devices in the pod_sandbox
+		if containerType == vc.SingleContainer {
+			_, err = config.WithCDI(ociSpec.Annotations, []string{}, ociSpec)
+			if err != nil {
+				return nil, fmt.Errorf("adding CDI devices failed")
+			}
 		}
 
 		s.config = runtimeConfig

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -335,6 +335,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (runtimeConfig string,
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
 	hotPlugVFIO = config.BridgePort
 	coldPlugVFIO = config.NoPort
+	pcieRootPort := uint32(0)
+	pcieSwitchPort := uint32(0)
 
 	configFileOptions := ktu.RuntimeConfigOptions{
 		Hypervisor:        "qemu",
@@ -353,6 +355,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (runtimeConfig string,
 		VirtioFSDaemon:    virtioFSdaemon,
 		HotPlugVFIO:       hotPlugVFIO,
 		ColdPlugVFIO:      coldPlugVFIO,
+		PCIeRootPort:      pcieRootPort,
+		PCIeSwitchPort:    pcieSwitchPort,
 	}
 
 	runtimeConfigFileData := ktu.MakeRuntimeConfigFileData(configFileOptions)

src/runtime/pkg/device/config/config.go

@@ -13,8 +13,10 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/go-ini/ini"
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/sys/unix"
 )
 
@@ -227,13 +229,11 @@ func (p PCIePort) Valid() bool {
 	return false
 }
 
-type PCIePortMapping map[string]bool
-
 var (
 	// Each of this structures keeps track of the devices attached to the
 	// different types of PCI ports. We can deduces the Bus number from it
 	// and eliminate duplicates being assigned.
-	PCIeDevices = map[PCIePort]PCIePortMapping{}
+	PCIeDevicesPerPort = map[PCIePort][]VFIODev{}
 )
 
 // DeviceInfo is an embedded type that contains device data common to all types of devices.
@@ -420,11 +420,12 @@ type VFIODev struct {
 	// APDevices are the Adjunct Processor devices assigned to the mdev
 	APDevices []string
 
-	// Rank identifies a device in a IOMMU group
-	Rank int
-
 	// Port is the PCIe port type to which the device is attached
 	Port PCIePort
+
+	// HostPath is the path to the device on the host we need it as a reference
+	// to match a /dev/vfio/<num> device to a device in GK mode
+	HostPath string
 }
 
 // RNGDev represents a random number generator device
@@ -643,3 +644,60 @@ type DeviceState struct {
 	// or hot plugged (false).
 	ColdPlug bool
 }
+
+// CDI (Container Device Interface), is a specification, for container- runtimes,
+// to support third-party devices.
+// It introduces an abstract notion of a device as a resource. Such devices are
+// uniquely specified by a fully-qualified name that is constructed from a
+// vendor ID, a device class, and a name that is unique per vendor ID-device
+// class pair.
+//
+// vendor.com/class=unique_name
+//
+// The combination of vendor ID and device class (vendor.com/class in the
+// above example) is referred to as the device kind.
+// CDI concerns itself only with enabling containers to be device aware.
+// Areas like resource management are explicitly left out of CDI (and are
+// expected to be handled by the orchestrator). Because of this focus, the CDI
+// specification is simple to implement and allows great flexibility for
+// runtimes and orchestrators.
+func WithCDI(annotations map[string]string, cdiSpecDirs []string, spec *specs.Spec) (*specs.Spec, error) {
+	// Add devices from CDI annotations
+	_, devsFromAnnotations, err := cdi.ParseAnnotations(annotations)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI device annotations: %w", err)
+	}
+
+	if len(devsFromAnnotations) == 0 {
+		// No devices found, skip device injection
+		return spec, nil
+	}
+
+	var registry cdi.Registry
+	if len(cdiSpecDirs) > 0 {
+		// We can override the directories where to search for CDI specs
+		// if needed, the default is /etc/cdi /var/run/cdi
+		registry = cdi.GetRegistry(cdi.WithSpecDirs(cdiSpecDirs...))
+	} else {
+		registry = cdi.GetRegistry()
+	}
+
+	if err = registry.Refresh(); err != nil {
+		// We don't consider registry refresh failure a fatal error.
+		// For instance, a dynamically generated invalid CDI Spec file for
+		// any particular vendor shouldn't prevent injection of devices of
+		// different vendors. CDI itself knows better and it will fail the
+		// injection if necessary.
+		return nil, fmt.Errorf("CDI registry refresh failed: %w", err)
+	}
+
+	if _, err := registry.InjectDevices(spec, devsFromAnnotations...); err != nil {
+		return nil, fmt.Errorf("CDI device injection failed: %w", err)
+	}
+
+	// One crucial thing to keep in mind is that CDI device injection
+	// might add OCI Spec environment variables, hooks, and mounts as
+	// well. Therefore it is important that none of the corresponding
+	// OCI Spec fields are reset up in the call stack once we return.
+	return spec, nil
+}

src/runtime/pkg/device/drivers/utils.go

@@ -41,6 +41,8 @@ var (
 	PCISysFsDevicesClass     PCISysFsProperty = "class"         // /sys/bus/pci/devices/xxx/class
 	PCISysFsSlotsAddress     PCISysFsProperty = "address"       // /sys/bus/pci/slots/xxx/address
 	PCISysFsSlotsMaxBusSpeed PCISysFsProperty = "max_bus_speed" // /sys/bus/pci/slots/xxx/max_bus_speed
+	PCISysFsDevicesVendor    PCISysFsProperty = "vendor"        // /sys/bus/pci/devices/xxx/vendor
+	PCISysFsDevicesDevice    PCISysFsProperty = "device"        // /sys/bus/pci/devices/xxx/device
 )
 
 func deviceLogger() *logrus.Entry {
@@ -194,6 +196,10 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe
 			if ignorePCIDevice {
 				continue
 			}
+			// Fetch the PCI Vendor ID and Device ID
+			vendorID := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesVendor)
+			deviceID := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesDevice)
+
 			// Do not directly assign to `vfio` -- need to access field still
 			vfio = config.VFIODev{
 				ID:       id,
@@ -202,8 +208,10 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe
 				SysfsDev: deviceSysfsDev,
 				IsPCIe:   IsPCIeDevice(deviceBDF),
 				Class:    pciClass,
-				Rank:     -1,
+				VendorID: vendorID,
+				DeviceID: deviceID,
 				Port:     device.Port,
+				HostPath: device.HostPath,
 			}
 
 		case config.VFIOAPDeviceMediatedType:

src/runtime/pkg/device/drivers/vfio.go

@@ -78,9 +78,12 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 		}
 
 		if vfio.IsPCIe {
-			busIndex := len(config.PCIeDevices[vfio.Port])
+			busIndex := len(config.PCIeDevicesPerPort[vfio.Port])
 			vfio.Bus = fmt.Sprintf("%s%d", config.PCIePortPrefixMapping[vfio.Port], busIndex)
-			config.PCIeDevices[vfio.Port][vfio.BDF] = true
+			// We need to keep track the number of devices per port to deduce
+			// the corectu bus number, additionally we can use the VFIO device
+			// info to act upon different Vendor IDs and Device IDs.
+			config.PCIeDevicesPerPort[vfio.Port] = append(config.PCIeDevicesPerPort[vfio.Port], *vfio)
 		}
 	}
 
@@ -138,6 +141,16 @@ func (device *VFIODevice) Detach(ctx context.Context, devReceiver api.DeviceRece
 		deviceLogger().WithError(err).Error("Failed to remove device")
 		return err
 	}
+	for _, vfio := range device.VfioDevs {
+		if vfio.IsPCIe {
+			for ix, dev := range config.PCIeDevicesPerPort[vfio.Port] {
+				if dev.BDF == vfio.BDF {
+					config.PCIeDevicesPerPort[vfio.Port] = append(config.PCIeDevicesPerPort[vfio.Port][:ix], config.PCIeDevicesPerPort[vfio.Port][ix+1:]...)
+					break
+				}
+			}
+		}
+	}
 
 	deviceLogger().WithFields(logrus.Fields{
 		"device-group": device.DeviceInfo.HostPath,

src/runtime/pkg/device/manager/manager.go

@@ -71,11 +71,10 @@ func NewDeviceManager(blockDriver string, vhostUserStoreEnabled bool, vhostUserS
 		dm.blockDriver = config.VirtioSCSI
 	}
 
-	config.PCIeDevices = make(map[config.PCIePort]config.PCIePortMapping)
-
-	config.PCIeDevices[config.RootPort] = make(map[string]bool)
-	config.PCIeDevices[config.SwitchPort] = make(map[string]bool)
-	config.PCIeDevices[config.BridgePort] = make(map[string]bool)
+	config.PCIeDevicesPerPort = make(map[config.PCIePort][]config.VFIODev)
+	config.PCIeDevicesPerPort[config.RootPort] = make([]config.VFIODev, 0)
+	config.PCIeDevicesPerPort[config.SwitchPort] = make([]config.VFIODev, 0)
+	config.PCIeDevicesPerPort[config.BridgePort] = make([]config.VFIODev, 0)
 
 	for _, dev := range devices {
 		dm.devices[dev.DeviceID()] = dev

src/runtime/pkg/govmm/qemu/qemu.go

@@ -15,6 +15,7 @@ package qemu
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"log"
@@ -44,6 +45,12 @@ const (
 	MachineTypeMicrovm string = "microvm"
 )
 
+const (
+	// Well known vsock CID for host system.
+	// https://man7.org/linux/man-pages/man7/vsock.7.html
+	VsockHostCid uint64 = 2
+)
+
 // Device is the qemu device interface.
 type Device interface {
 	Valid() bool
@@ -306,6 +313,9 @@ type Object struct {
 
 	// Prealloc enables memory preallocation
 	Prealloc bool
+
+	// QgsPort defines Intel Quote Generation Service port exposed from the host
+	QgsPort uint32
 }
 
 // Valid returns true if the Object structure is valid and complete.
@@ -316,7 +326,7 @@ func (object Object) Valid() bool {
 	case MemoryBackendEPC:
 		return object.ID != "" && object.Size != 0
 	case TDXGuest:
-		return object.ID != "" && object.File != "" && object.DeviceID != ""
+		return object.ID != "" && object.File != "" && object.DeviceID != "" && object.QgsPort != 0
 	case SEVGuest:
 		fallthrough
 	case SNPGuest:
@@ -362,19 +372,22 @@ func (object Object) QemuParams(config *Config) []string {
 		}
 
 	case TDXGuest:
-		objectParams = append(objectParams, string(object.Type))
-		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
-		if object.Debug {
-			objectParams = append(objectParams, "debug=on")
-		}
+		objectParams = append(objectParams, prepareObjectWithTdxQgs(object))
 		config.Bios = object.File
 	case SEVGuest:
-		fallthrough
+		objectParams = append(objectParams, string(object.Type))
+		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
+		objectParams = append(objectParams, fmt.Sprintf("cbitpos=%d", object.CBitPos))
+		objectParams = append(objectParams, fmt.Sprintf("reduced-phys-bits=%d", object.ReducedPhysBits))
+
+		driveParams = append(driveParams, "if=pflash,format=raw,readonly=on")
+		driveParams = append(driveParams, fmt.Sprintf("file=%s", object.File))
 	case SNPGuest:
 		objectParams = append(objectParams, string(object.Type))
 		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
 		objectParams = append(objectParams, fmt.Sprintf("cbitpos=%d", object.CBitPos))
 		objectParams = append(objectParams, fmt.Sprintf("reduced-phys-bits=%d", object.ReducedPhysBits))
+		objectParams = append(objectParams, "kernel-hashes=on")
 
 		driveParams = append(driveParams, "if=pflash,format=raw,readonly=on")
 		driveParams = append(driveParams, fmt.Sprintf("file=%s", object.File))
@@ -408,6 +421,52 @@ func (object Object) QemuParams(config *Config) []string {
 	return qemuParams
 }
 
+type SocketAddress struct {
+	Type string `json:"type"`
+	Cid  string `json:"cid"`
+	Port string `json:"port"`
+}
+
+type TdxQomObject struct {
+	QomType               string        `json:"qom-type"`
+	Id                    string        `json:"id"`
+	QuoteGenerationSocket SocketAddress `json:"quote-generation-socket"`
+	Debug                 *bool         `json:"debug,omitempty"`
+}
+
+func (this *SocketAddress) String() string {
+	b, err := json.Marshal(*this)
+
+	if err != nil {
+		log.Fatalf("Unable to marshal SocketAddress object: %s", err.Error())
+		return ""
+	}
+
+	return string(b)
+}
+
+func (this *TdxQomObject) String() string {
+	b, err := json.Marshal(*this)
+
+	if err != nil {
+		log.Fatalf("Unable to marshal TDX QOM object: %s", err.Error())
+		return ""
+	}
+
+	return string(b)
+}
+
+func prepareObjectWithTdxQgs(object Object) string {
+	qgsSocket := SocketAddress{"vsock", fmt.Sprint(VsockHostCid), fmt.Sprint(object.QgsPort)}
+	tdxObject := TdxQomObject{string(object.Type), object.ID, qgsSocket, nil}
+
+	if object.Debug {
+		*tdxObject.Debug = true
+	}
+
+	return tdxObject.String()
+}
+
 // Virtio9PMultidev filesystem behaviour to deal
 // with multiple devices being shared with a 9p export.
 type Virtio9PMultidev string
@@ -2625,9 +2684,6 @@ type Knobs struct {
 	// NoGraphic completely disables graphic output.
 	NoGraphic bool
 
-	// Daemonize will turn the qemu process into a daemon
-	Daemonize bool
-
 	// Both HugePages and MemPrealloc require the Memory.Size of the VM
 	// to be set, as they need to reserve the memory upfront in order
 	// for the VM to boot without errors.
@@ -2658,9 +2714,6 @@ type Knobs struct {
 	// Prevents QEMU from rebooting in the event of a Triple Fault.
 	NoReboot bool
 
-	// Don’t exit QEMU on guest shutdown, but instead only stop the emulation.
-	NoShutdown bool
-
 	// IOMMUPlatform will enable IOMMU for supported devices
 	IOMMUPlatform bool
 }
@@ -3062,14 +3115,6 @@ func (config *Config) appendKnobs() {
 		config.qemuParams = append(config.qemuParams, "--no-reboot")
 	}
 
-	if config.Knobs.NoShutdown {
-		config.qemuParams = append(config.qemuParams, "--no-shutdown")
-	}
-
-	if config.Knobs.Daemonize {
-		config.qemuParams = append(config.qemuParams, "-daemonize")
-	}
-
 	config.appendMemoryKnobs()
 
 	if config.Knobs.Mlock {

src/runtime/pkg/govmm/qemu/qemu_test.go

@@ -445,13 +445,12 @@ func TestAppendEmptyDevice(t *testing.T) {
 }
 
 func TestAppendKnobsAllTrue(t *testing.T) {
-	var knobsString = "-no-user-config -nodefaults -nographic --no-reboot -daemonize -overcommit mem-lock=on -S"
+	var knobsString = "-no-user-config -nodefaults -nographic --no-reboot -overcommit mem-lock=on -S"
 	knobs := Knobs{
 		NoUserConfig:  true,
 		NoDefaults:    true,
 		NoGraphic:     true,
 		NoReboot:      true,
-		Daemonize:     true,
 		MemPrealloc:   true,
 		FileBackedMem: true,
 		MemShared:     true,

src/runtime/pkg/govmm/qemu/qmp.go

@@ -457,7 +457,7 @@ func failOutstandingCommands(cmdQueue *list.List) {
 		cmd := e.Value.(*qmpCommand)
 		select {
 		case cmd.res <- qmpResult{
-			err: errors.New("exitting QMP loop, command cancelled"),
+			err: errors.New("exiting QMP loop, command cancelled"),
 		}:
 		case <-cmd.ctx.Done():
 		}
@@ -626,7 +626,7 @@ func (q *QMP) executeCommandWithResponse(ctx context.Context, name string, args
 	resCh := make(chan qmpResult)
 	select {
 	case <-q.disconnectedCh:
-		err = errors.New("exitting QMP loop, command cancelled")
+		err = errors.New("exiting QMP loop, command cancelled")
 	case q.cmdCh <- qmpCommand{
 		ctx:    ctx,
 		res:    resCh,

src/runtime/pkg/hypervisors/hypervisor_state.go

@@ -47,4 +47,6 @@ type HypervisorState struct {
 	Pid               int
 	HotPlugVFIO       config.PCIePort
 	ColdPlugVFIO      config.PCIePort
+	PCIeRootPort      uint32
+	PCIeSwitchPort    uint32
 }

src/runtime/pkg/katatestutils/utils.go

@@ -226,6 +226,8 @@ type RuntimeConfigOptions struct {
 	PFlash               []string
 	HotPlugVFIO          config.PCIePort
 	ColdPlugVFIO         config.PCIePort
+	PCIeRootPort         uint32
+	PCIeSwitchPort       uint32
 	DefaultVCPUCount     uint32
 	DefaultMaxVCPUCount  uint32
 	DefaultMemSize       uint32
@@ -318,6 +320,8 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	enable_iothreads =  ` + strconv.FormatBool(config.EnableIOThreads) + `
 	cold_plug_vfio =  "` + config.ColdPlugVFIO.String() + `"
 	hot_plug_vfio =   "` + config.HotPlugVFIO.String() + `"
+	pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + `
+	pcie_switch_port = ` + strconv.FormatUint(uint64(config.PCIeSwitchPort), 10) + `
 	msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + `
 	enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + `
 	guest_hook_path = "` + config.DefaultGuestHookPath + `"

src/runtime/pkg/katautils/config-settings.go.in

@@ -58,6 +58,7 @@ var systemdUnitName = "kata-containers.target"
 
 const defaultKernelParams = ""
 const defaultMachineType = "q35"
+const defaultQgsPort = 4050
 
 const defaultVCPUCount uint32 = 1
 const defaultMaxVCPUCount uint32 = 0
@@ -111,5 +112,8 @@ var defaultRuntimeConfiguration = "@CONFIG_PATH@"
 const defaultHotPlugVFIO = config.NoPort
 const defaultColdPlugVFIO = config.NoPort
 
+const defaultPCIeRootPort = 0
+const defaultPCIeSwitchPort = 0
+
 const defaultRemoteHypervisorSocket = "/run/peerpod/hypervisor.sock"
 const defaultRemoteHypervisorTimeout = 600

src/runtime/pkg/katautils/config.go

@@ -57,6 +57,10 @@ const (
 
 	// the maximum amount of PCI bridges that can be cold plugged in a VM
 	maxPCIBridges uint32 = 5
+	// For more info on why these values were chosen, see:
+	// https://github.com/kata-containers/kata-containers/blob/main/docs/design/kata-vra.md#hypervisor-resource-limits
+	maxPCIeRootPorts   uint32 = 16
+	maxPCIeSwitchPorts uint32 = 16
 
 	errInvalidHypervisorPrefix = "configuration file contains invalid hypervisor section"
 )
@@ -89,6 +93,7 @@ type hypervisor struct {
 	CPUFeatures                    string                    `toml:"cpu_features"`
 	KernelParams                   string                    `toml:"kernel_params"`
 	MachineType                    string                    `toml:"machine_type"`
+	QgsPort                        uint32                    `toml:"tdx_quote_generation_service_socket_port"`
 	BlockDeviceDriver              string                    `toml:"block_device_driver"`
 	EntropySource                  string                    `toml:"entropy_source"`
 	SharedFS                       string                    `toml:"shared_fs"`
@@ -149,6 +154,8 @@ type hypervisor struct {
 	DisableImageNvdimm             bool                      `toml:"disable_image_nvdimm"`
 	HotPlugVFIO                    config.PCIePort           `toml:"hot_plug_vfio"`
 	ColdPlugVFIO                   config.PCIePort           `toml:"cold_plug_vfio"`
+	PCIeRootPort                   uint32                    `toml:"pcie_root_port"`
+	PCIeSwitchPort                 uint32                    `toml:"pcie_switch_port"`
 	DisableVhostNet                bool                      `toml:"disable_vhost_net"`
 	GuestMemoryDumpPaging          bool                      `toml:"guest_memory_dump_paging"`
 	ConfidentialGuest              bool                      `toml:"confidential_guest"`
@@ -301,6 +308,20 @@ func (h hypervisor) hotPlugVFIO() config.PCIePort {
 	return h.HotPlugVFIO
 }
 
+func (h hypervisor) pcieRootPort() uint32 {
+	if h.PCIeRootPort > maxPCIeRootPorts {
+		return maxPCIeRootPorts
+	}
+	return h.PCIeRootPort
+}
+
+func (h hypervisor) pcieSwitchPort() uint32 {
+	if h.PCIeSwitchPort > maxPCIeSwitchPorts {
+		return maxPCIeSwitchPorts
+	}
+	return h.PCIeSwitchPort
+}
+
 func (h hypervisor) firmwareVolume() (string, error) {
 	p := h.FirmwareVolume
 
@@ -373,6 +394,14 @@ func (h hypervisor) machineType() string {
 	return h.MachineType
 }
 
+func (h hypervisor) qgsPort() uint32 {
+	if h.QgsPort == 0 {
+		return defaultQgsPort
+	}
+
+	return h.QgsPort
+}
+
 func (h hypervisor) GetEntropySource() string {
 	if h.EntropySource == "" {
 		return defaultEntropySource
@@ -890,6 +919,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		CPUFeatures:             cpuFeatures,
 		KernelParams:            vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
 		HypervisorMachineType:   machineType,
+		QgsPort:                 h.qgsPort(),
 		NumVCPUsF:               h.defaultVCPUs(),
 		DefaultMaxVCPUs:         h.defaultMaxVCPUs(),
 		MemorySize:              h.defaultMemSz(),
@@ -926,6 +956,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		DisableImageNvdimm:      h.DisableImageNvdimm,
 		HotPlugVFIO:             h.hotPlugVFIO(),
 		ColdPlugVFIO:            h.coldPlugVFIO(),
+		PCIeRootPort:            h.pcieRootPort(),
+		PCIeSwitchPort:          h.pcieSwitchPort(),
 		DisableVhostNet:         h.DisableVhostNet,
 		EnableVhostUserStore:    h.EnableVhostUserStore,
 		VhostUserStorePath:      h.vhostUserStorePath(),
@@ -1121,6 +1153,8 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		Msize9p:                        h.msize9p(),
 		ColdPlugVFIO:                   h.coldPlugVFIO(),
 		HotPlugVFIO:                    h.hotPlugVFIO(),
+		PCIeRootPort:                   h.pcieRootPort(),
+		PCIeSwitchPort:                 h.pcieSwitchPort(),
 		DisableVhostNet:                true,
 		GuestHookPath:                  h.guestHookPath(),
 		VirtioFSExtraArgs:              h.VirtioFSExtraArgs,
@@ -1474,6 +1508,8 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		Msize9p:                  defaultMsize9p,
 		ColdPlugVFIO:             defaultColdPlugVFIO,
 		HotPlugVFIO:              defaultHotPlugVFIO,
+		PCIeRootPort:             defaultPCIeRootPort,
+		PCIeSwitchPort:           defaultPCIeSwitchPort,
 		GuestHookPath:            defaultGuestHookPath,
 		VhostUserStorePath:       defaultVhostUserStorePath,
 		VhostUserDeviceReconnect: defaultVhostUserDeviceReconnect,

src/runtime/pkg/katautils/config_test.go

@@ -87,6 +87,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 	enableIOThreads := true
 	hotPlugVFIO = config.NoPort
 	coldPlugVFIO = config.BridgePort
+	pcieRootPort := uint32(0)
+	pcieSwitchPort := uint32(0)
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
@@ -109,6 +111,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 		EnableIOThreads:      enableIOThreads,
 		HotPlugVFIO:          hotPlugVFIO,
 		ColdPlugVFIO:         coldPlugVFIO,
+		PCIeRootPort:         pcieRootPort,
+		PCIeSwitchPort:       pcieSwitchPort,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     defaultVCPUCount,
 		DefaultMaxVCPUCount:  defaultMaxVCPUCount,
@@ -172,6 +176,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 		EnableIOThreads:       enableIOThreads,
 		HotPlugVFIO:           hotPlugVFIO,
 		ColdPlugVFIO:          coldPlugVFIO,
+		PCIeRootPort:          pcieRootPort,
+		PCIeSwitchPort:        pcieSwitchPort,
 		Msize9p:               defaultMsize9p,
 		MemSlots:              defaultMemSlots,
 		EntropySource:         defaultEntropySource,
@@ -182,6 +188,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 		VirtioFSCache:         defaultVirtioFSCacheMode,
 		PFlash:                []string{},
 		SGXEPCSize:            epcSize,
+		QgsPort:               defaultQgsPort,
 	}
 
 	if goruntime.GOARCH == "arm64" && len(hypervisorConfig.PFlash) == 0 && hypervisorConfig.FirmwarePath == "" {
@@ -568,6 +575,8 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		DisableGuestSeLinux:   defaultDisableGuestSeLinux,
 		HotPlugVFIO:           defaultHotPlugVFIO,
 		ColdPlugVFIO:          defaultColdPlugVFIO,
+		PCIeRootPort:          defaultPCIeRootPort,
+		PCIeSwitchPort:        defaultPCIeSwitchPort,
 	}
 
 	expectedAgentConfig := vc.KataAgentConfig{
@@ -609,6 +618,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 	disableBlock := true
 	enableIOThreads := true
 	coldPlugVFIO = config.BridgePort
+	pcieRootPort := uint32(0)
+	pcieSwitchPort := uint32(0)
 	orgVHostVSockDevicePath := utils.VHostVSockDevicePath
 	blockDeviceAIO := "io_uring"
 	defer func() {
@@ -627,6 +638,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		DisableBlockDeviceUse: disableBlock,
 		EnableIOThreads:       enableIOThreads,
 		ColdPlugVFIO:          coldPlugVFIO,
+		PCIeRootPort:          pcieRootPort,
+		PCIeSwitchPort:        pcieSwitchPort,
 		RxRateLimiterMaxRate:  rxRateLimiterMaxRate,
 		TxRateLimiterMaxRate:  txRateLimiterMaxRate,
 		SharedFS:              "virtio-fs",

src/runtime/pkg/oci/utils.go

@@ -463,6 +463,14 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig,
 		return err
 	}
 
+	if err := addHypervisorPCIeRootPortOverrides(ocispec, config); err != nil {
+		return err
+	}
+
+	if err := addHypervisorPCIeSwitchPortOverrides(ocispec, config); err != nil {
+		return err
+	}
+
 	if value, ok := ocispec.Annotations[vcAnnotations.MachineType]; ok {
 		if value != "" {
 			config.HypervisorConfig.HypervisorMachineType = value
@@ -605,10 +613,33 @@ func addHypervisorHotColdPlugVfioOverrides(ocispec specs.Spec, sbConfig *vc.Sand
 	return nil
 }
 
+func addHypervisorPCIeRootPortOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
+
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.PCIeRootPort).setUint(func(pcieRootPort uint64) {
+		if pcieRootPort > 0 {
+			sbConfig.HypervisorConfig.PCIeRootPort = uint32(pcieRootPort)
+		}
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func addHypervisorPCIeSwitchPortOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.PCIeSwitchPort).setUint(func(pcieSwitchPort uint64) {
+		if pcieSwitchPort > 0 {
+			sbConfig.HypervisorConfig.PCIeSwitchPort = uint32(pcieSwitchPort)
+		}
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
 func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, runtime RuntimeConfig) error {
 
 	if err := newAnnotationConfiguration(ocispec, vcAnnotations.DefaultMemory).setUintWithCheck(func(memorySz uint64) error {
-		if memorySz < vc.MinHypervisorMemory {
+		if memorySz < vc.MinHypervisorMemory && sbConfig.HypervisorType != vc.RemoteHypervisor {
 			return fmt.Errorf("Memory specified in annotation %s is less than minimum required %d, please specify a larger value", vcAnnotations.DefaultMemory, vc.MinHypervisorMemory)
 		}
 		sbConfig.HypervisorConfig.MemorySize = uint32(memorySz)
@@ -689,7 +720,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
 	numCPUs := goruntime.NumCPU()
 
 	if err := newAnnotationConfiguration(ocispec, vcAnnotations.DefaultVCPUs).setFloat32WithCheck(func(vcpus float32) error {
-		if vcpus > float32(numCPUs) {
+		if vcpus > float32(numCPUs) && sbConfig.HypervisorType != vc.RemoteHypervisor {
 			return fmt.Errorf("Number of cpus %f specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
 		}
 		sbConfig.HypervisorConfig.NumVCPUsF = float32(vcpus)
@@ -701,11 +732,11 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
 	return newAnnotationConfiguration(ocispec, vcAnnotations.DefaultMaxVCPUs).setUintWithCheck(func(maxVCPUs uint64) error {
 		max := uint32(maxVCPUs)
 
-		if max > uint32(numCPUs) {
+		if max > uint32(numCPUs) && sbConfig.HypervisorType != vc.RemoteHypervisor {
 			return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than the number of CPUs %d on the system", max, numCPUs)
 		}
 
-		if sbConfig.HypervisorType == vc.QemuHypervisor && max > govmm.MaxVCPUs() {
+		if sbConfig.HypervisorType == vc.QemuHypervisor && max > govmm.MaxVCPUs() && sbConfig.HypervisorType != vc.RemoteHypervisor {
 			return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than max no of CPUs %d supported for qemu", max, govmm.MaxVCPUs())
 		}
 		sbConfig.HypervisorConfig.DefaultMaxVCPUs = max

src/runtime/pkg/oci/utils_test.go

@@ -661,6 +661,8 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	ocispec.Annotations[vcAnnotations.DisableImageNvdimm] = "true"
 	ocispec.Annotations[vcAnnotations.ColdPlugVFIO] = config.BridgePort
 	ocispec.Annotations[vcAnnotations.HotPlugVFIO] = config.NoPort
+	ocispec.Annotations[vcAnnotations.PCIeRootPort] = "1"
+	ocispec.Annotations[vcAnnotations.PCIeSwitchPort] = "1"
 	ocispec.Annotations[vcAnnotations.IOMMUPlatform] = "true"
 	ocispec.Annotations[vcAnnotations.SGXEPC] = "64Mi"
 	ocispec.Annotations[vcAnnotations.UseLegacySerial] = "true"
@@ -701,6 +703,8 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	assert.Equal(sbConfig.HypervisorConfig.DisableImageNvdimm, true)
 	assert.Equal(string(sbConfig.HypervisorConfig.ColdPlugVFIO), string(config.BridgePort))
 	assert.Equal(string(sbConfig.HypervisorConfig.HotPlugVFIO), string(config.NoPort))
+	assert.Equal(sbConfig.HypervisorConfig.PCIeRootPort, uint32(1))
+	assert.Equal(sbConfig.HypervisorConfig.PCIeSwitchPort, uint32(1))
 	assert.Equal(sbConfig.HypervisorConfig.IOMMUPlatform, true)
 	assert.Equal(sbConfig.HypervisorConfig.SGXEPCSize, int64(67108864))
 	assert.Equal(sbConfig.HypervisorConfig.LegacySerial, true)
@@ -726,6 +730,51 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	assert.Error(err)
 }
 
+func TestAddRemoteHypervisorAnnotations(t *testing.T) {
+	// Remote hypervisor uses DefaultVCPUs, DefaultMemory etc as annotations to pick the size of the separate VM to create,
+	// so doesn't need to be bound by the host's capacity limits.
+	assert := assert.New(t)
+
+	config := vc.SandboxConfig{
+		Annotations: make(map[string]string),
+	}
+
+	sbConfig := vc.SandboxConfig{
+		Annotations:    make(map[string]string),
+		HypervisorType: vc.RemoteHypervisor,
+	}
+
+	ocispec := specs.Spec{
+		Annotations: make(map[string]string),
+	}
+
+	runtimeConfig := RuntimeConfig{
+		HypervisorType: vc.RemoteHypervisor,
+	}
+
+	err := addAnnotations(ocispec, &config, runtimeConfig)
+	assert.NoError(err)
+	assert.Exactly(vc.HypervisorConfig{}, config.HypervisorConfig)
+
+	// Enable annotations
+	runtimeConfig.HypervisorConfig.EnableAnnotations = []string{".*"}
+
+	// When DefaultVCPUs is more than the number of cpus on the host, remote hypervisor annotations don't throw an error
+	ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "2000"
+	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
+	assert.NoError(err)
+
+	// When DefaultMaxVCPUs is more than the number of cpus on the host, remote hypervisor annotations don't throw an error
+	ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "2000"
+	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
+	assert.NoError(err)
+
+	// When memory is smaller than the minimum Hypervisor memory, remote hypervisor annotations don't throw an error
+	ocispec.Annotations[vcAnnotations.DefaultMemory] = "1"
+	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
+	assert.NoError(err)
+}
+
 func TestAddProtectedHypervisorAnnotations(t *testing.T) {
 	assert := assert.New(t)
 

src/runtime/pkg/utils/utils.go

@@ -13,7 +13,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/sirupsen/logrus"
 )
@@ -125,7 +124,6 @@ func CreateVmmUser() (string, error) {
 	// Add retries to mitigate temporary errors and race conditions. For example, the user already exists
 	// or another instance of the runtime is also creating a user.
 	maxAttempt := 5
-	rand.Seed(time.Now().UnixNano())
 	for i := 0; i < maxAttempt; i++ {
 		userName = fmt.Sprintf("kata-%v", rand.Intn(100000))
 		_, err = RunCommand([]string{useraddPath, "-M", "-s", nologinPath, userName, "-c", "\"Kata Containers temporary hypervisor user\""})

src/runtime/vendor/github.com/containerd/containerd/.golangci.yml

@@ -32,6 +32,24 @@ issues:
     - path: 'archive[\\/]tarheader[\\/]'
       # conversion is necessary on Linux, unnecessary on macOS
       text: "unnecessary conversion"
+    - linters:
+        - revive
+      text: "if-return"
+    - linters:
+        - revive
+      text: "empty-block"
+    - linters:
+        - revive
+      text: "superfluous-else"
+    - linters:
+        - revive
+      text: "unused-parameter"
+    - linters:
+        - revive
+      text: "unreachable-code"
+    - linters:
+        - revive
+      text: "redefines-builtin-id"
 
     # FIXME temporarily suppress deprecation warnings for the logs package. See https://github.com/containerd/containerd/pull/9086
     - text: "SA1019: log\\.(G|L|Fields|Entry|RFC3339NanoFixed|Level|TraceLevel|DebugLevel|InfoLevel|WarnLevel|ErrorLevel|FatalLevel|PanicLevel|SetLevel|GetLevel|OutputFormat|TextFormat|JSONFormat|SetFormat|WithLogger|GetLogger)"

src/runtime/vendor/github.com/containerd/containerd/.mailmap

@@ -2,6 +2,7 @@ Abhinandan Prativadi <abhi@docker.com>
 Abhinandan Prativadi <abhi@docker.com> <aprativadi@gmail.com>
 Ace-Tang <aceapril@126.com>
 Adam Korcz <adam@adalogics.com> <Adam@adalogics.com>
+Akhil Mohan <akhilerm@gmail.com> <akhil.mohan@broadcom.com>
 Aditi Sharma <adi.sky17@gmail.com> <sharmaad@vmware.com>
 Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp> <suda.akihiro@lab.ntt.co.jp>
 Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp> <suda.kyoto@gmail.com>

src/runtime/vendor/github.com/containerd/containerd/BUILDING.md

@@ -14,7 +14,8 @@ This doc includes:
 
 To build the `containerd` daemon, and the `ctr` simple test client, the following build system dependencies are required:
 
-* Go 1.19.x or above
+
+* Go 1.21.x or above
 * Protoc 3.x compiler and headers (download at the [Google protobuf releases page](https://github.com/protocolbuffers/protobuf/releases))
 * Btrfs headers and libraries for your distribution. Note that building the btrfs driver can be disabled via the build tag `no_btrfs`, removing this dependency.
 

src/runtime/vendor/github.com/containerd/containerd/RELEASES.md

@@ -394,6 +394,9 @@ The deprecated properties in [`config.toml`](./docs/cri/config.md) are shown in
 |`[plugins."io.containerd.grpc.v1.cri".registry]`                      | `auths`                      | containerd v1.3     | containerd v2.0            | Use [`ImagePullSecrets`](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). See also [#8228](https://github.com/containerd/containerd/issues/8228). |
 |`[plugins."io.containerd.grpc.v1.cri".registry]`                      | `configs`                    | containerd v1.5     | containerd v2.0            | Use [`config_path`](./docs/hosts.md)            |
 |`[plugins."io.containerd.grpc.v1.cri".registry]`                      | `mirrors`                    | containerd v1.5     | containerd v2.0            | Use [`config_path`](./docs/hosts.md)            |
+|`[plugins."io.containerd.tracing.processor.v1.otlp"]`                 | `endpoint`, `protocol`, `insecure` | containerd v1.6.29 | containerd v2.0       | Use [OTLP environment variables](https://opentelemetry.io/docs/specs/otel/protocol/exporter/), e.g. OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_SDK_DISABLED    |
+|`[plugins."io.containerd.internal.v1.tracing"]`                       | `service_name`, `sampling_ratio`   | containerd v1.6.29 | containerd v2.0       | Instead use [OTel environment variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/), e.g. OTEL_SERVICE_NAME, OTEL_TRACES_SAMPLER*  |
+
 
 > **Note**
 >

src/runtime/vendor/github.com/containerd/containerd/Vagrantfile

@@ -102,7 +102,7 @@ EOF
   config.vm.provision "install-golang", type: "shell", run: "once" do |sh|
     sh.upload_path = "/tmp/vagrant-install-golang"
     sh.env = {
-        'GO_VERSION': ENV['GO_VERSION'] || "1.20.12",
+        'GO_VERSION': ENV['GO_VERSION'] || "1.21.9",
     }
     sh.inline = <<~SHELL
         #!/usr/bin/env bash

src/runtime/vendor/github.com/containerd/containerd/api/services/diff/v1/diff.pb.go

@@ -47,6 +47,8 @@ type ApplyRequest struct {
 	Diff     *types.Descriptor     `protobuf:"bytes,1,opt,name=diff,proto3" json:"diff,omitempty"`
 	Mounts   []*types.Mount        `protobuf:"bytes,2,rep,name=mounts,proto3" json:"mounts,omitempty"`
 	Payloads map[string]*anypb.Any `protobuf:"bytes,3,rep,name=payloads,proto3" json:"payloads,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// SyncFs is to synchronize the underlying filesystem containing files.
+	SyncFs bool `protobuf:"varint,4,opt,name=sync_fs,json=syncFs,proto3" json:"sync_fs,omitempty"`
 }
 
 func (x *ApplyRequest) Reset() {
@@ -102,6 +104,13 @@ func (x *ApplyRequest) GetPayloads() map[string]*anypb.Any {
 	return nil
 }
 
+func (x *ApplyRequest) GetSyncFs() bool {
+	if x != nil {
+		return x.SyncFs
+	}
+	return false
+}
+
 type ApplyResponse struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -319,7 +328,7 @@ var file_github_com_containerd_containerd_api_services_diff_v1_diff_proto_rawDes
 	0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61,
 	0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f,
 	0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x22, 0x99, 0x02, 0x0a, 0x0c, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x22, 0xb2, 0x02, 0x0a, 0x0c, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
 	0x74, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x69, 0x66, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
 	0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70,
 	0x65, 0x73, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x04, 0x64,
@@ -331,61 +340,62 @@ var file_github_com_containerd_containerd_api_services_diff_v1_diff_proto_rawDes
 	0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x64, 0x69, 0x66,
 	0x66, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
 	0x74, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
-	0x08, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x1a, 0x51, 0x0a, 0x0d, 0x50, 0x61, 0x79,
-	0x6c, 0x6f, 0x61, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65,
-	0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05,
-	0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f,
-	0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e,
-	0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x47, 0x0a, 0x0d,
-	0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a,
-	0x07, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c,
-	0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65,
-	0x73, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x61, 0x70,
-	0x70, 0x6c, 0x69, 0x65, 0x64, 0x22, 0xeb, 0x02, 0x0a, 0x0b, 0x44, 0x69, 0x66, 0x66, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x6c, 0x65, 0x66, 0x74, 0x18, 0x01, 0x20,
-	0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64,
-	0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x04, 0x6c, 0x65,
-	0x66, 0x74, 0x12, 0x2d, 0x0a, 0x05, 0x72, 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28,
-	0x0b, 0x32, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74,
-	0x79, 0x70, 0x65, 0x73, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x05, 0x72, 0x69, 0x67, 0x68,
-	0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65,
-	0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x72,
-	0x65, 0x66, 0x12, 0x4c, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x03,
-	0x28, 0x0b, 0x32, 0x34, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e,
-	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x2e, 0x76, 0x31,
-	0x2e, 0x44, 0x69, 0x66, 0x66, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4c, 0x61, 0x62,
-	0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73,
-	0x12, 0x46, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x5f,
-	0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f,
-	0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69,
-	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44,
-	0x61, 0x74, 0x65, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65,
-	0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,
-	0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
-	0x02, 0x38, 0x01, 0x22, 0x40, 0x0a, 0x0c, 0x44, 0x69, 0x66, 0x66, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x69, 0x66, 0x66, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74,
-	0x79, 0x70, 0x65, 0x73, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52,
-	0x04, 0x64, 0x69, 0x66, 0x66, 0x32, 0xc3, 0x01, 0x0a, 0x04, 0x44, 0x69, 0x66, 0x66, 0x12, 0x5e,
-	0x0a, 0x05, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
-	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x64, 0x69,
-	0x66, 0x66, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e,
-	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x2e, 0x76, 0x31,
-	0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b,
-	0x0a, 0x04, 0x44, 0x69, 0x66, 0x66, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e,
-	0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x64, 0x69, 0x66,
-	0x66, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x66, 0x66, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65,
-	0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x2e, 0x76, 0x31, 0x2e, 0x44,
-	0x69, 0x66, 0x66, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3c, 0x5a, 0x3a, 0x67,
-	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
-	0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f,
-	0x61, 0x70, 0x69, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x64, 0x69, 0x66,
-	0x66, 0x2f, 0x76, 0x31, 0x3b, 0x64, 0x69, 0x66, 0x66, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x33,
+	0x08, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x79, 0x6e,
+	0x63, 0x5f, 0x66, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x79, 0x6e, 0x63,
+	0x46, 0x73, 0x1a, 0x51, 0x0a, 0x0d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x45, 0x6e,
+	0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75,
+	0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x47, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x65,
+	0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
+	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72,
+	0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x22, 0xeb,
+	0x02, 0x0a, 0x0b, 0x44, 0x69, 0x66, 0x66, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b,
+	0x0a, 0x04, 0x6c, 0x65, 0x66, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x63,
+	0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e,
+	0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x04, 0x6c, 0x65, 0x66, 0x74, 0x12, 0x2d, 0x0a, 0x05, 0x72,
+	0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x63, 0x6f, 0x6e,
+	0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x4d, 0x6f,
+	0x75, 0x6e, 0x74, 0x52, 0x05, 0x72, 0x69, 0x67, 0x68, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65,
+	0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,
+	0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x4c, 0x0a, 0x06, 0x6c,
+	0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x63, 0x6f,
+	0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x73, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x66, 0x66, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x46, 0x0a, 0x11, 0x73, 0x6f, 0x75,
+	0x72, 0x63, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x06,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
+	0x52, 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x61, 0x74, 0x65, 0x45, 0x70, 0x6f, 0x63,
+	0x68, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79,
+	0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b,
+	0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x40, 0x0a, 0x0c,
+	0x44, 0x69, 0x66, 0x66, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x04,
+	0x64, 0x69, 0x66, 0x66, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x6e,
+	0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x44, 0x65,
+	0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x04, 0x64, 0x69, 0x66, 0x66, 0x32, 0xc3,
+	0x01, 0x0a, 0x04, 0x44, 0x69, 0x66, 0x66, 0x12, 0x5e, 0x0a, 0x05, 0x41, 0x70, 0x70, 0x6c, 0x79,
+	0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x2e, 0x76, 0x31, 0x2e, 0x41,
+	0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f,
+	0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x73, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b, 0x0a, 0x04, 0x44, 0x69, 0x66, 0x66, 0x12,
+	0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69,
+	0x66, 0x66, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74,
+	0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e,
+	0x64, 0x69, 0x66, 0x66, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x66, 0x66, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
+	0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f,
+	0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x64, 0x69, 0x66, 0x66, 0x2f, 0x76, 0x31, 0x3b, 0x64, 0x69,
+	0x66, 0x66, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

src/runtime/vendor/github.com/containerd/containerd/api/services/diff/v1/diff.proto

@@ -44,6 +44,8 @@ message ApplyRequest {
 	repeated containerd.types.Mount mounts = 2;
 
 	map<string, google.protobuf.Any> payloads = 3;
+	// SyncFs is to synchronize the underlying filesystem containing files.
+	bool sync_fs = 4;
 }
 
 message ApplyResponse {

src/runtime/vendor/github.com/containerd/containerd/archive/compression/compression.go

@@ -25,12 +25,12 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
 	"strconv"
 	"sync"
 
 	"github.com/containerd/containerd/log"
 	"github.com/klauspost/compress/zstd"
-	exec "golang.org/x/sys/execabs"
 )
 
 type (

src/runtime/vendor/github.com/containerd/containerd/content/content.go

@@ -108,6 +108,12 @@ type Status struct {
 // WalkFunc defines the callback for a blob walk.
 type WalkFunc func(Info) error
 
+// InfoReaderProvider provides both info and reader for the specific content.
+type InfoReaderProvider interface {
+	InfoProvider
+	Provider
+}
+
 // InfoProvider provides info for content inspection.
 type InfoProvider interface {
 	// Info will return metadata about content available in the content store.

src/runtime/vendor/github.com/containerd/containerd/content/helpers.go

@@ -31,9 +31,6 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-// maxResets is the no.of times the Copy() method can tolerate a reset of the body
-const maxResets = 5
-
 var ErrReset = errors.New("writer has been reset")
 
 var bufPool = sync.Pool{
@@ -160,7 +157,7 @@ func Copy(ctx context.Context, cw Writer, or io.Reader, size int64, expected dig
 		}
 	}
 
-	for i := 0; i < maxResets; i++ {
+	for i := 0; ; i++ {
 		if i >= 1 {
 			log.G(ctx).WithField("digest", expected).Debugf("retrying copy due to reset")
 		}
@@ -201,9 +198,6 @@ func Copy(ctx context.Context, cw Writer, or io.Reader, size int64, expected dig
 		}
 		return nil
 	}
-
-	log.G(ctx).WithField("digest", expected).Errorf("failed to copy after %d retries", maxResets)
-	return fmt.Errorf("failed to copy after %d retries", maxResets)
 }
 
 // CopyReaderAt copies to a writer from a given reader at for the given
@@ -332,3 +326,14 @@ func copyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) {
 	}
 	return
 }
+
+// Exists returns whether an attempt to access the content would not error out
+// with an ErrNotFound error. It will return an encountered error if it was
+// different than ErrNotFound.
+func Exists(ctx context.Context, provider InfoProvider, desc ocispec.Descriptor) (bool, error) {
+	_, err := provider.Info(ctx, desc.Digest)
+	if errdefs.IsNotFound(err) {
+		return false, nil
+	}
+	return err == nil, err
+}

src/runtime/vendor/github.com/containerd/containerd/diff/diff.go

@@ -67,6 +67,8 @@ type Comparer interface {
 type ApplyConfig struct {
 	// ProcessorPayloads specifies the payload sent to various processors
 	ProcessorPayloads map[string]typeurl.Any
+	// SyncFs is to synchronize the underlying filesystem containing files
+	SyncFs bool
 }
 
 // ApplyOpt is used to configure an Apply operation
@@ -133,3 +135,11 @@ func WithSourceDateEpoch(tm *time.Time) Opt {
 		return nil
 	}
 }
+
+// WithSyncFs sets sync flag to the config.
+func WithSyncFs(sync bool) ApplyOpt {
+	return func(_ context.Context, _ ocispec.Descriptor, c *ApplyConfig) error {
+		c.SyncFs = sync
+		return nil
+	}
+}

src/runtime/vendor/github.com/containerd/containerd/diff/proxy/differ.go

@@ -62,6 +62,7 @@ func (r *diffRemote) Apply(ctx context.Context, desc ocispec.Descriptor, mounts
 		Diff:     fromDescriptor(desc),
 		Mounts:   fromMounts(mounts),
 		Payloads: payloads,
+		SyncFs:   config.SyncFs,
 	}
 	resp, err := r.client.Apply(ctx, req)
 	if err != nil {

src/runtime/vendor/github.com/containerd/containerd/diff/stream_unix.go

@@ -25,12 +25,12 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
 	"sync"
 
 	"github.com/containerd/containerd/protobuf"
 	"github.com/containerd/containerd/protobuf/proto"
 	"github.com/containerd/typeurl/v2"
-	exec "golang.org/x/sys/execabs"
 )
 
 // NewBinaryProcessor returns a binary processor for use with processing content streams

src/runtime/vendor/github.com/containerd/containerd/diff/stream_windows.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"sync"
 
@@ -31,7 +32,6 @@ import (
 	"github.com/containerd/containerd/protobuf/proto"
 	"github.com/containerd/typeurl/v2"
 	"github.com/sirupsen/logrus"
-	exec "golang.org/x/sys/execabs"
 )
 
 const processorPipe = "STREAM_PROCESSOR_PIPE"

src/runtime/vendor/github.com/containerd/containerd/events/exchange/exchange.go

@@ -67,7 +67,7 @@ func (e *Exchange) Forward(ctx context.Context, envelope *events.Envelope) (err
 		if err != nil {
 			logger.WithError(err).Error("error forwarding event")
 		} else {
-			logger.Debug("event forwarded")
+			logger.Trace("event forwarded")
 		}
 	}()
 
@@ -111,7 +111,7 @@ func (e *Exchange) Publish(ctx context.Context, topic string, event events.Event
 		if err != nil {
 			logger.WithError(err).Error("error publishing event")
 		} else {
-			logger.Debug("event published")
+			logger.Trace("event published")
 		}
 	}()
 

src/runtime/vendor/github.com/containerd/containerd/image.go

@@ -336,6 +336,14 @@ func WithUnpackDuplicationSuppressor(suppressor kmutex.KeyedLocker) UnpackOpt {
 	}
 }
 
+// WithUnpackApplyOpts appends new apply options on the UnpackConfig.
+func WithUnpackApplyOpts(opts ...diff.ApplyOpt) UnpackOpt {
+	return func(ctx context.Context, uc *UnpackConfig) error {
+		uc.ApplyOpts = append(uc.ApplyOpts, opts...)
+		return nil
+	}
+}
+
 func (i *image) Unpack(ctx context.Context, snapshotterName string, opts ...UnpackOpt) error {
 	ctx, done, err := i.client.WithLease(ctx)
 	if err != nil {

src/runtime/vendor/github.com/containerd/containerd/images/archive/exporter.go

@@ -24,11 +24,14 @@ import (
 	"io"
 	"path"
 	"sort"
+	"strings"
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/labels"
 	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/log"
 	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -140,6 +143,45 @@ func WithSkipNonDistributableBlobs() ExportOpt {
 	return WithBlobFilter(f)
 }
 
+// WithSkipMissing excludes blobs referenced by manifests if not all blobs
+// would be included in the archive.
+// The manifest itself is excluded only if it's not present locally.
+// This allows to export multi-platform images if not all platforms are present
+// while still persisting the multi-platform index.
+func WithSkipMissing(store content.InfoReaderProvider) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		o.blobRecordOptions.childrenHandler = images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) {
+			children, err := images.Children(ctx, store, desc)
+			if !images.IsManifestType(desc.MediaType) {
+				return children, err
+			}
+
+			if err != nil {
+				// If manifest itself is missing, skip it from export.
+				if errdefs.IsNotFound(err) {
+					return nil, images.ErrSkipDesc
+				}
+				return nil, err
+			}
+
+			// Don't export manifest descendants if any of them doesn't exist.
+			for _, child := range children {
+				exists, err := content.Exists(ctx, store, child)
+				if err != nil {
+					return nil, err
+				}
+
+				// If any child is missing, only export the manifest, but don't export its descendants.
+				if !exists {
+					return nil, nil
+				}
+			}
+			return children, nil
+		})
+		return nil
+	}
+}
+
 func addNameAnnotation(name string, base map[string]string) map[string]string {
 	annotations := map[string]string{}
 	for k, v := range base {
@@ -152,6 +194,23 @@ func addNameAnnotation(name string, base map[string]string) map[string]string {
 	return annotations
 }
 
+func copySourceLabels(ctx context.Context, infoProvider content.InfoProvider, desc ocispec.Descriptor) (ocispec.Descriptor, error) {
+	info, err := infoProvider.Info(ctx, desc.Digest)
+	if err != nil {
+		return desc, err
+	}
+	for k, v := range info.Labels {
+		if strings.HasPrefix(k, labels.LabelDistributionSource) {
+			if desc.Annotations == nil {
+				desc.Annotations = map[string]string{k: v}
+			} else {
+				desc.Annotations[k] = v
+			}
+		}
+	}
+	return desc, nil
+}
+
 // Export implements Exporter.
 func Export(ctx context.Context, store content.Provider, writer io.Writer, opts ...ExportOpt) error {
 	var eo exportOptions
@@ -163,15 +222,27 @@ func Export(ctx context.Context, store content.Provider, writer io.Writer, opts
 
 	records := []tarRecord{
 		ociLayoutFile(""),
-		ociIndexRecord(eo.manifests),
+	}
+
+	manifests := make([]ocispec.Descriptor, 0, len(eo.manifests))
+	if infoProvider, ok := store.(content.InfoProvider); ok {
+		for _, desc := range eo.manifests {
+			d, err := copySourceLabels(ctx, infoProvider, desc)
+			if err != nil {
+				log.G(ctx).WithError(err).WithField("desc", desc).Warn("failed to copy distribution.source labels")
+				continue
+			}
+			manifests = append(manifests, d)
+		}
+	} else {
+		manifests = append(manifests, eo.manifests...)
 	}
 
 	algorithms := map[string]struct{}{}
 	dManifests := map[digest.Digest]*exportManifest{}
 	resolvedIndex := map[digest.Digest]digest.Digest{}
-	for _, desc := range eo.manifests {
-		switch desc.MediaType {
-		case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
+	for _, desc := range manifests {
+		if images.IsManifestType(desc.MediaType) {
 			mt, ok := dManifests[desc.Digest]
 			if !ok {
 				// TODO(containerd): Skip if already added
@@ -191,7 +262,7 @@ func Export(ctx context.Context, store content.Provider, writer io.Writer, opts
 			if name != "" {
 				mt.names = append(mt.names, name)
 			}
-		case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
+		} else if images.IsIndexType(desc.MediaType) {
 			d, ok := resolvedIndex[desc.Digest]
 			if !ok {
 				if err := desc.Digest.Validate(); err != nil {
@@ -255,11 +326,13 @@ func Export(ctx context.Context, store content.Provider, writer io.Writer, opts
 				}
 
 			}
-		default:
+		} else {
 			return fmt.Errorf("only manifests may be exported: %w", errdefs.ErrInvalidArgument)
 		}
 	}
 
+	records = append(records, ociIndexRecord(manifests))
+
 	if !eo.skipDockerManifest && len(dManifests) > 0 {
 		tr, err := manifestsRecord(ctx, store, dManifests)
 		if err != nil {
@@ -292,7 +365,10 @@ func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descri
 		return nil, nil
 	}
 
-	childrenHandler := images.ChildrenHandler(store)
+	childrenHandler := brOpts.childrenHandler
+	if childrenHandler == nil {
+		childrenHandler = images.ChildrenHandler(store)
+	}
 
 	handlers := images.Handlers(
 		childrenHandler,
@@ -314,7 +390,8 @@ type tarRecord struct {
 }
 
 type blobRecordOptions struct {
-	blobFilter BlobFilter
+	blobFilter      BlobFilter
+	childrenHandler images.HandlerFunc
 }
 
 func blobRecord(cs content.Provider, desc ocispec.Descriptor, opts *blobRecordOptions) tarRecord {
@@ -394,6 +471,7 @@ func ociIndexRecord(manifests []ocispec.Descriptor) tarRecord {
 		Versioned: ocispecs.Versioned{
 			SchemaVersion: 2,
 		},
+		MediaType: ocispec.MediaTypeImageIndex,
 		Manifests: manifests,
 	}
 

src/runtime/vendor/github.com/containerd/containerd/import.go

@@ -39,6 +39,7 @@ type importOpts struct {
 	platformMatcher platforms.MatchComparer
 	compress        bool
 	discardLayers   bool
+	skipMissing     bool
 }
 
 // ImportOpt allows the caller to specify import specific options
@@ -115,6 +116,15 @@ func WithDiscardUnpackedLayers() ImportOpt {
 	}
 }
 
+// WithSkipMissing allows to import an archive which doesn't contain all the
+// referenced blobs.
+func WithSkipMissing() ImportOpt {
+	return func(c *importOpts) error {
+		c.skipMissing = true
+		return nil
+	}
+}
+
 // Import imports an image from a Tar stream using reader.
 // Caller needs to specify importer. Future version may use oci.v1 as the default.
 // Note that unreferenced blobs may be imported to the content store as well.
@@ -164,7 +174,12 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt
 	var handler images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
 		// Only save images at top level
 		if desc.Digest != index.Digest {
-			return images.Children(ctx, cs, desc)
+			// Don't set labels on missing content.
+			children, err := images.Children(ctx, cs, desc)
+			if iopts.skipMissing && errdefs.IsNotFound(err) {
+				return nil, images.ErrSkipDesc
+			}
+			return children, err
 		}
 
 		p, err := content.ReadBlob(ctx, cs, desc)

src/runtime/vendor/github.com/containerd/containerd/mount/mount_freebsd.go

@@ -20,9 +20,9 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"time"
 
-	exec "golang.org/x/sys/execabs"
 	"golang.org/x/sys/unix"
 )
 

src/runtime/vendor/github.com/containerd/containerd/mount/mount_linux.go

@@ -20,12 +20,12 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"path"
 	"runtime"
 	"strings"
 	"time"
 
-	exec "golang.org/x/sys/execabs"
 	"golang.org/x/sys/unix"
 )
 

src/runtime/vendor/github.com/containerd/containerd/oci/spec_opts.go

@@ -35,8 +35,8 @@ import (
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/continuity/fs"
+	"github.com/moby/sys/user"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/opencontainers/runc/libcontainer/user"
 	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -893,9 +893,9 @@ func WithAppendAdditionalGroups(groups ...string) SpecOpts {
 			if err != nil {
 				return err
 			}
-			ugroups, err := user.ParseGroupFile(gpath)
-			if err != nil {
-				return err
+			ugroups, groupErr := user.ParseGroupFile(gpath)
+			if groupErr != nil && !os.IsNotExist(groupErr) {
+				return groupErr
 			}
 			groupMap := make(map[string]user.Group)
 			for _, group := range ugroups {
@@ -909,6 +909,9 @@ func WithAppendAdditionalGroups(groups ...string) SpecOpts {
 				} else {
 					g, ok := groupMap[group]
 					if !ok {
+						if groupErr != nil {
+							return fmt.Errorf("unable to find group %s: %w", group, groupErr)
+						}
 						return fmt.Errorf("unable to find group %s", group)
 					}
 					gids = append(gids, uint32(g.Gid))

src/runtime/vendor/github.com/containerd/containerd/pkg/cri/opts/spec_darwin_opts.go

@@ -58,7 +58,7 @@ func WithDarwinMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra
 
 		// Sort mounts in number of parts. This ensures that high level mounts don't
 		// shadow other mounts.
-		sort.Sort(orderedMounts(mounts))
+		sort.Stable(orderedMounts(mounts))
 
 		// Copy all mounts from default mounts, except for
 		// - mounts overridden by supplied mount;

src/runtime/vendor/github.com/containerd/containerd/pkg/cri/opts/spec_linux.go

@@ -26,11 +26,11 @@ import (
 	"sync"
 	"syscall"
 
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/containerd/cgroups/v3"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
 
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/log"
@@ -176,8 +176,7 @@ func WithCDI(annotations map[string]string, CDIDevices []*runtime.CDIDevice) oci
 			return nil
 		}
 
-		registry := cdi.GetRegistry()
-		if err = registry.Refresh(); err != nil {
+		if err = cdi.Refresh(); err != nil {
 			// We don't consider registry refresh failure a fatal error.
 			// For instance, a dynamically generated invalid CDI Spec file for
 			// any particular vendor shouldn't prevent injection of devices of
@@ -186,7 +185,7 @@ func WithCDI(annotations map[string]string, CDIDevices []*runtime.CDIDevice) oci
 			log.G(ctx).Warnf("CDI registry refresh failed: %v", err)
 		}
 
-		if _, err := registry.InjectDevices(s, devices...); err != nil {
+		if _, err := cdi.InjectDevices(s, devices...); err != nil {
 			return fmt.Errorf("CDI device injection failed: %w", err)
 		}
 

src/runtime/vendor/github.com/containerd/containerd/pkg/cri/opts/spec_linux_opts.go

@@ -65,7 +65,7 @@ func WithMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*ru
 
 		// Sort mounts in number of parts. This ensures that high level mounts don't
 		// shadow other mounts.
-		sort.Sort(orderedMounts(mounts))
+		sort.Stable(orderedMounts(mounts))
 
 		// Mount cgroup into the container as readonly, which inherits docker's behavior.
 		s.Mounts = append(s.Mounts, runtimespec.Mount{

src/runtime/vendor/github.com/containerd/containerd/pkg/cri/opts/spec_windows_opts.go

@@ -127,7 +127,7 @@ func WithWindowsMounts(osi osinterface.OS, config *runtime.ContainerConfig, extr
 
 		// Sort mounts in number of parts. This ensures that high level mounts don't
 		// shadow other mounts.
-		sort.Sort(orderedMounts(mounts))
+		sort.Stable(orderedMounts(mounts))
 
 		// Copy all mounts from default mounts, except for
 		// mounts overridden by supplied mount;