Merge pull request #10021 from GabyCT/topic/fixarchdoc

docs: Update devmapper docs

.github/actionlint.yaml

@@ -0,0 +1,24 @@
+# Copyright (c) 2024 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Configuration file with rules for the actionlint tool.
+#
+self-hosted-runner:
+  # Labels of self-hosted runner that linter should ignore
+  labels:
+    - arm64-builder
+    - garm-ubuntu-2004
+    - garm-ubuntu-2004-smaller
+    - garm-ubuntu-2204
+    - garm-ubuntu-2304
+    - garm-ubuntu-2304-smaller
+    - garm-ubuntu-2204-smaller
+    - k8s-ppc64le
+    - metrics
+    - ppc64le
+    - sev
+    - sev-snp
+    - s390x
+    - s390x-large
+    - tdx

.github/cargo-deny-composite-action/cargo-deny-generator.sh

@@ -8,7 +8,7 @@
 script_dir=$(dirname "$(readlink -f "$0")")
 parent_dir=$(realpath "${script_dir}/../..")
 cidir="${parent_dir}/ci"
-source "${cidir}/lib.sh"
+source "${cidir}/../tests/common.bash"
 
 cargo_deny_file="${script_dir}/action.yaml"
 

.github/workflows/basic-ci-amd64.yaml

@@ -23,7 +23,7 @@ jobs:
       matrix:
         containerd_version: ['lts', 'active']
         vmm: ['clh', 'dragonball', 'qemu', 'stratovirt', 'cloud-hypervisor', 'qemu-runtime-rs']
-    runs-on: garm-ubuntu-2204-smaller
+    runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: ${{ matrix.containerd_version }}
       GOPATH: ${{ github.workspace }}
@@ -62,7 +62,7 @@ jobs:
       matrix:
         containerd_version: ['lts', 'active']
         vmm: ['clh', 'cloud-hypervisor', 'dragonball', 'qemu', 'stratovirt']
-    runs-on: garm-ubuntu-2204-smaller
+    runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: ${{ matrix.containerd_version }}
       GOPATH: ${{ github.workspace }}
@@ -104,7 +104,7 @@ jobs:
       matrix:
         containerd_version: ['lts', 'active']
         vmm: ['clh', 'qemu', 'dragonball', 'stratovirt']
-    runs-on: garm-ubuntu-2204-smaller
+    runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: ${{ matrix.containerd_version }}
       GOPATH: ${{ github.workspace }}
@@ -138,7 +138,7 @@ jobs:
         run: bash tests/integration/nydus/gha-run.sh run
 
   run-runk:
-    runs-on: garm-ubuntu-2204-smaller
+    runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: lts
     steps:
@@ -168,6 +168,37 @@ jobs:
       - name: Run runk tests
         timeout-minutes: 10
         run: bash tests/integration/runk/gha-run.sh run
+  run-stdio:
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/stdio/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/stdio/gha-run.sh install-kata kata-artifacts
+
+      - name: Run stdio tests
+        timeout-minutes: 10
+        run: bash tests/integration/stdio/gha-run.sh
 
   run-tracing:
     strategy:
@@ -176,6 +207,9 @@ jobs:
         vmm:
           - clh # cloud-hypervisor
           - qemu
+    # TODO: enable me when https://github.com/kata-containers/kata-containers/issues/9763 is fixed
+    # TODO: Transition to free runner (see #9940).
+    if: false
     runs-on: garm-ubuntu-2204-smaller
     env:
       KATA_HYPERVISOR: ${{ matrix.vmm }}
@@ -211,7 +245,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        vmm: ['clh', 'qemu']
+        vmm:
+          - clh
+          - qemu
+    # TODO: enable with clh when https://github.com/kata-containers/kata-containers/issues/9764 is fixed
+    # TODO: enable with qemu when https://github.com/kata-containers/kata-containers/issues/9851 is fixed
+    # TODO: Transition to free runner (see #9940).
+    if: false
     runs-on: garm-ubuntu-2304
     env:
       GOPATH: ${{ github.workspace }}
@@ -251,7 +291,7 @@ jobs:
         vmm:
           - clh
           - qemu
-    runs-on: garm-ubuntu-2304-smaller
+    runs-on: ubuntu-22.04
     env:
       KATA_HYPERVISOR: ${{ matrix.vmm }}
     steps:
@@ -294,7 +334,10 @@ jobs:
           - dragonball
           - qemu
           - cloud-hypervisor
-    runs-on: garm-ubuntu-2304-smaller
+        # TODO: enable with clh when https://github.com/kata-containers/kata-containers/issues/9852 is fixed
+        exclude:
+          - vmm: clh
+    runs-on: ubuntu-22.04
     env:
       KATA_HYPERVISOR: ${{ matrix.vmm }}
     steps:
@@ -326,7 +369,9 @@ jobs:
         run: bash tests/integration/nerdctl/gha-run.sh run
 
       - name: Collect artifacts ${{ matrix.vmm }}
+        if: always()
         run: bash tests/integration/nerdctl/gha-run.sh collect-artifacts
+        continue-on-error: true
 
       - name: Archive artifacts ${{ matrix.vmm }}
         uses: actions/upload-artifact@v4

.github/workflows/build-checks.yaml

@@ -111,3 +111,4 @@ jobs:
           ${{ matrix.command }}
         env:
           RUST_BACKTRACE: "1"
+          SKIP_GO_VERSION_CHECK: "1"

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -60,14 +60,8 @@ jobs:
         stage:
           - ${{ inputs.stage }}
         exclude:
-          - asset: agent
-            stage: release
           - asset: cloud-hypervisor-glibc
             stage: release
-          - asset: pause-image
-            stage: release
-          - asset: coco-guest-components
-            stage: release
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -93,7 +87,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -102,8 +96,10 @@ jobs:
           ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
+        if: ${{ matrix.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -39,13 +39,7 @@ jobs:
           - rootfs-initrd
           - shim-v2
           - virtiofsd
-        stage:
-          - ${{ inputs.stage }}
     steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
         uses: docker/login-action@v3
@@ -70,7 +64,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -79,8 +73,10 @@ jobs:
           ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
+        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}

.github/workflows/build-kata-static-tarball-ppc64le.yaml

@@ -36,16 +36,11 @@ jobs:
         stage:
           - ${{ inputs.stage }}
     steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
       - name: Prepare the self-hosted runner
         run: |
             ${HOME}/scripts/prepare_runner.sh
             sudo rm -rf $GITHUB_WORKSPACE/*
 
-
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
         uses: docker/login-action@v3
@@ -70,8 +65,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-          sudo chown -R $(id -u):$(id -g) "kata-build"
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -80,8 +74,10 @@ jobs:
           ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
+        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -39,13 +39,6 @@ jobs:
           - rootfs-initrd-confidential
           - shim-v2
           - virtiofsd
-        stage:
-          - ${{ inputs.stage }}
-        exclude:
-          - asset: pause-image
-            stage: release
-          - asset: coco-guest-components
-            stage: release
     steps:
       - name: Take a pre-action for self-hosted runner
         run: ${HOME}/script/pre_action.sh ubuntu-2204
@@ -74,8 +67,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-          sudo chown -R $(id -u):$(id -g) "kata-build"
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -84,8 +76,10 @@ jobs:
           ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
+        if: ${{ inputs.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}

.github/workflows/ci.yaml

@@ -43,7 +43,7 @@ jobs:
       commit-hash: ${{ inputs.commit-hash }}
       target-branch: ${{ inputs.target-branch }}
     secrets: inherit
-  
+
   build-kata-static-tarball-ppc64le:
     uses: ./.github/workflows/build-kata-static-tarball-ppc64le.yaml
     with:
@@ -62,7 +62,7 @@ jobs:
       commit-hash: ${{ inputs.commit-hash }}
       target-branch: ${{ inputs.target-branch }}
     secrets: inherit
-  
+
   publish-kata-deploy-payload-ppc64le:
     needs: build-kata-static-tarball-ppc64le
     uses: ./.github/workflows/publish-kata-deploy-payload-ppc64le.yaml
@@ -113,6 +113,8 @@ jobs:
           file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
 
   run-kata-deploy-tests-on-aks:
+    # TODO: Reenable when Azure CI budget is secured (see #9939).
+    if: false
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-kata-deploy-tests-on-aks.yaml
     with:
@@ -125,6 +127,8 @@ jobs:
     secrets: inherit
 
   run-kata-deploy-tests-on-garm:
+    # TODO: Transition to free runner (see #9940).
+    if: false
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-kata-deploy-tests-on-garm.yaml
     with:
@@ -203,7 +207,8 @@ jobs:
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
-      
+    secrets: inherit
+
   run-k8s-tests-on-ppc64le:
     needs: publish-kata-deploy-payload-ppc64le
     uses: ./.github/workflows/run-k8s-tests-on-ppc64le.yaml

.github/workflows/cleanup-resources.yaml

@@ -0,0 +1,31 @@
+name: Cleanup dangling Azure resources
+on:
+  schedule:
+    - cron: "0 */6 * * *"
+  workflow_dispatch:
+
+jobs:
+  cleanup-resources:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log into Azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+        run: bash tests/integration/kubernetes/gha-run.sh login-azure
+
+      - name: Install Python dependencies
+        run: |
+          pip3 install --user --upgrade \
+            azure-identity==1.16.0 \
+            azure-mgmt-resource==23.0.1
+
+      - name: Cleanup resources
+        env:
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+          CLEANUP_AFTER_HOURS: 6 # Clean up resources created more than this many hours ago.
+        run: python3 tests/cleanup_resources.py

.github/workflows/docs-url-alive-check.yaml

@@ -26,11 +26,6 @@ jobs:
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}
-    - name: Setup
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
     # docs url alive check
     - name: Docs URL Alive Check
       run: |

.github/workflows/release-amd64.yaml

@@ -10,7 +10,9 @@ jobs:
   build-kata-static-tarball-amd64:
     uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
     with:
+      push-to-registry: yes
       stage: release
+    secrets: inherit
 
   kata-deploy:
     needs: build-kata-static-tarball-amd64

.github/workflows/release-arm64.yaml

@@ -10,7 +10,9 @@ jobs:
   build-kata-static-tarball-arm64:
     uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
     with:
+      push-to-registry: yes
       stage: release
+    secrets: inherit
 
   kata-deploy:
     needs: build-kata-static-tarball-arm64

.github/workflows/release-ppc64le.yaml

@@ -10,7 +10,9 @@ jobs:
   build-kata-static-tarball-ppc64le:
     uses: ./.github/workflows/build-kata-static-tarball-ppc64le.yaml
     with:
+      push-to-registry: yes
       stage: release
+    secrets: inherit
 
   kata-deploy:
     needs: build-kata-static-tarball-ppc64le

.github/workflows/release-s390x.yaml

@@ -10,6 +10,7 @@ jobs:
   build-kata-static-tarball-s390x:
     uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
     with:
+      push-to-registry: yes
       stage: release
     secrets: inherit
 

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -36,6 +36,7 @@ jobs:
           - clh
           - dragonball
           - qemu
+          - qemu-runtime-rs
           - stratovirt
           - cloud-hypervisor
         instance-type:

.github/workflows/run-k8s-tests-on-garm.yaml

@@ -86,7 +86,9 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
   
       - name: Collect artifacts ${{ matrix.vmm }}
+        if: always()
         run: bash tests/integration/kubernetes/gha-run.sh collect-artifacts
+        continue-on-error: true
 
       - name: Archive artifacts ${{ matrix.vmm }}
         uses: actions/upload-artifact@v4

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -27,8 +27,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        vmm:
-          - qemu
         snapshotter:
           - devmapper
           - nydus
@@ -39,10 +37,12 @@ jobs:
             pull-type: default
             using-nfd: true
             deploy-cmd: configure-snapshotter
+            vmm: qemu
           - snapshotter: nydus
             pull-type: guest-pull
             using-nfd: false
             deploy-cmd: deploy-snapshotter
+            vmm: qemu-coco-dev
     runs-on: s390x-large
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
@@ -57,9 +57,12 @@ jobs:
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: ${{ matrix.using-nfd }}
       TARGET_ARCH: "s390x"
+      AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
     steps:
       - name: Take a pre-action for self-hosted runner
-        run: ${HOME}/script/pre_action.sh ubuntu-2204
+        run: |
+          "${HOME}/script/pre_action.sh" ubuntu-2204
 
       - uses: actions/checkout@v4
         with:
@@ -90,4 +93,4 @@ jobs:
         if: always()
         run: |
           bash tests/integration/kubernetes/gha-run.sh cleanup-zvsi || true
-          ${HOME}/script/post_action.sh ubuntu-2204
+          "${HOME}/script/post_action.sh" ubuntu-2204

.github/workflows/run-kata-coco-tests.yaml

@@ -40,11 +40,15 @@ jobs:
       DOCKER_TAG: ${{ inputs.tag }}
       PR_NUMBER: ${{ inputs.pr-number }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
-      KUBERNETES: "k3s"
+      KUBERNETES: "vanilla"
       USING_NFD: "true"
+      KBS: "true"
       K8S_TEST_HOST_TYPE: "baremetal"
+      KBS_INGRESS: "nodeport"
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       PULL_TYPE: ${{ matrix.pull-type }}
+      AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -65,8 +69,20 @@ jobs:
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
 
+      - name: Uninstall previous `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+
       - name: Run tests
-        timeout-minutes: 30
+        timeout-minutes: 50
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete kata-deploy
@@ -77,6 +93,10 @@ jobs:
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
+      - name: Delete CoCo KBS
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+
   run-k8s-tests-on-sev:
     strategy:
       fail-fast: false
@@ -100,6 +120,8 @@ jobs:
       K8S_TEST_HOST_TYPE: "baremetal"
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       PULL_TYPE: ${{ matrix.pull-type }}
+      AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -152,9 +174,13 @@ jobs:
       KUBECONFIG: /home/kata/.kube/config
       KUBERNETES: "vanilla"
       USING_NFD: "false"
+      KBS: "true"
+      KBS_INGRESS: "nodeport"
       K8S_TEST_HOST_TYPE: "baremetal"
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       PULL_TYPE: ${{ matrix.pull-type }}
+      AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -175,6 +201,18 @@ jobs:
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
 
+      - name: Uninstall previous `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+
       - name: Run tests
         timeout-minutes: 30
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
@@ -187,6 +225,10 @@ jobs:
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
+      - name: Delete CoCo KBS
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+
   # Generate jobs for testing CoCo on non-TEE environments
   run-k8s-tests-coco-nontee:
     strategy:
@@ -212,6 +254,8 @@ jobs:
       KBS_INGRESS: "aks"
       KUBERNETES: "vanilla"
       PULL_TYPE: ${{ matrix.pull-type }}
+      AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: "false"
     steps:

.github/workflows/run-kata-deploy-tests-on-aks.yaml

@@ -33,6 +33,7 @@ jobs:
           - clh
           - dragonball
           - qemu
+          - qemu-runtime-rs
         include:
           - host_os: cbl-mariner
             vmm: clh

.github/workflows/run-kata-deploy-tests-on-garm.yaml

@@ -34,6 +34,10 @@ jobs:
           - k0s
           - k3s
           - rke2
+    # TODO: There are a couple of vmm/k8s combination failing (https://github.com/kata-containers/kata-containers/issues/9854)
+    # and we will put the entire kata-deploy-tests on GARM on maintenance.
+    # TODO: Transition to free runner (see #9940).
+    if: false
     runs-on: garm-ubuntu-2004-smaller
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}

.github/workflows/run-kata-monitor-tests.yaml

@@ -15,6 +15,8 @@ on:
 
 jobs:
   run-monitor:
+    # TODO: Transition to free runner (see #9940).
+    if: false
     strategy:
       fail-fast: false
       matrix:
@@ -23,13 +25,18 @@ jobs:
         container_engine:
           - crio
           - containerd
-        include:
+        # TODO: enable when https://github.com/kata-containers/kata-containers/issues/9853 is fixed
+        #include:
+        #  - container_engine: containerd
+        #    containerd_version: lts
+        exclude:
+          # TODO: enable with containerd when https://github.com/kata-containers/kata-containers/issues/9761 is fixed
           - container_engine: containerd
-            containerd_version: lts
+            vmm: qemu
     runs-on: garm-ubuntu-2204-smaller
     env:
       CONTAINER_ENGINE: ${{ matrix.container_engine }}
-      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      #CONTAINERD_VERSION: ${{ matrix.containerd_version }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
     steps:
       - uses: actions/checkout@v4

.github/workflows/run-runk-tests.yaml

@@ -15,6 +15,8 @@ on:
 
 jobs:
   run-runk:
+    # TODO: Transition to free runner (see #9940).
+    if: false
     runs-on: garm-ubuntu-2204-smaller
     env:
       CONTAINERD_VERSION: lts

.github/workflows/static-checks.yaml

@@ -40,6 +40,8 @@ jobs:
       instance: ubuntu-20.04
 
   build-checks-depending-on-kvm:
+    # TODO: Transition to free runner (see #9940).
+    if: false
     runs-on: garm-ubuntu-2004-smaller
     strategy:
       fail-fast: false

README.md

@@ -140,6 +140,7 @@ The table below lists the remaining parts of the project:
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
 | [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](.github/workflows) | CI | Continuous Integration configuration files and scripts. |
+| [`ocp-ci`](ci/openshift-ci/README.md) | CI | Continuous Integration configuration for the OpenShift pipelines. |
 | [`katacontainers.io`](https://github.com/kata-containers/www.katacontainers.io) | Source for the [`katacontainers.io`](https://www.katacontainers.io) site. |
 | [`Webhook`](tools/testing/kata-webhook/README.md) | utility | Example of a simple admission controller webhook to annotate pods with the Kata runtime class |
 

VERSION

@@ -1 +1 @@
-3.5.0
+3.7.0

ci/docs-url-alive-check.sh

@@ -7,6 +7,6 @@
 set -e
 
 cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
+source "${cidir}/../tests/common.bash"
 
 run_docs_url_alive_check

ci/install_go.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-
-clone_tests_repo
-
-new_goroot=/usr/local/go
-
-pushd "${tests_repo_dir}"
-# Force overwrite the current version of golang
-[ -z "${GOROOT}" ] || rm -rf "${GOROOT}"
-.ci/install_go.sh -p -f -d "$(dirname ${new_goroot})"
-[ -z "${GOROOT}" ] || sudo ln -sf "${new_goroot}" "${GOROOT}"
-go version
-popd

ci/install_libseccomp.sh

@@ -23,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 # Variables for libseccomp
 libseccomp_version="${LIBSECCOMP_VERSION:-""}"
 if [ -z "${libseccomp_version}" ]; then
-    libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version")
+    libseccomp_version=$(get_from_kata_deps ".externals.libseccomp.version")
 fi
 libseccomp_url="${LIBSECCOMP_URL:-""}"
 if [ -z "${libseccomp_url}" ]; then
-    libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url")
+    libseccomp_url=$(get_from_kata_deps ".externals.libseccomp.url")
 fi
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
@@ -36,11 +36,11 @@ cflags="-O2"
 # Variables for gperf
 gperf_version="${GPERF_VERSION:-""}"
 if [ -z "${gperf_version}" ]; then
-    gperf_version=$(get_from_kata_deps "externals.gperf.version")
+    gperf_version=$(get_from_kata_deps ".externals.gperf.version")
 fi
 gperf_url="${GPERF_URL:-""}"
 if [ -z "${gperf_url}" ]; then
-    gperf_url=$(get_from_kata_deps "externals.gperf.url")
+    gperf_url=$(get_from_kata_deps ".externals.gperf.url")
 fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"

ci/install_rust.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) 2019 Ant Financial
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -e
-
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-
-clone_tests_repo
-
-pushd ${tests_repo_dir}
-.ci/install_rust.sh ${1:-}
-popd

ci/install_vc.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-cidir=$(dirname "$0")
-vcdir="${cidir}/../src/runtime/virtcontainers/"
-source "${cidir}/lib.sh"
-export CI_JOB="${CI_JOB:-default}"
-
-clone_tests_repo
-
-if [ "${CI_JOB}" != "PODMAN" ]; then
-	echo "Install virtcontainers"
-	make -C "${vcdir}" && chronic sudo make -C "${vcdir}" install
-fi

ci/install_yq.sh

@@ -5,6 +5,8 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+[ -n "$DEBUG" ] && set -o xtrace
+
 # If we fail for any reason a message will be displayed
 die() {
 	msg="$*"
@@ -16,7 +18,7 @@ die() {
 # Install via binary download, as we may not have golang installed at this point
 function install_yq() {
 	local yq_pkg="github.com/mikefarah/yq"
-	local yq_version=3.4.1
+	local yq_version=v4.40.7
 	local precmd=""
 	INSTALL_IN_GOPATH=${INSTALL_IN_GOPATH:-true}
 
@@ -36,7 +38,7 @@ function install_yq() {
 			fi
 		fi
 	fi
-	[ -x  "${yq_path}" ] && [ "`${yq_path} --version`"X == "yq version ${yq_version}"X ] && return
+	[ -x  "${yq_path}" ] && [ "`${yq_path} --version`"X == "yq (https://github.com/mikefarah/yq/) version ${yq_version}"X ] && return
 
 	read -r -a sysInfo <<< "$(uname -sm)"
 

ci/lib.sh

@@ -1,87 +0,0 @@
-#
-# Copyright (c) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -o nounset
-
-GOPATH=${GOPATH:-${HOME}/go}
-export kata_repo="github.com/kata-containers/kata-containers"
-export kata_repo_dir="$GOPATH/src/$kata_repo"
-export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
-export tests_repo_dir="$GOPATH/src/$tests_repo"
-export branch="${target_branch:-main}"
-
-# Clones the tests repository and checkout to the branch pointed out by
-# the global $branch variable.
-# If the clone exists and `CI` is exported then it does nothing. Otherwise
-# it will clone the repository or `git pull` the latest code.
-#
-clone_tests_repo()
-{
-	if [ -d "$tests_repo_dir" ]; then
-		[ -n "${CI:-}" ] && return
-		# git config --global --add safe.directory will always append
-		# the target to .gitconfig without checking the existence of
-		# the target, so it's better to check it before adding the target repo.
-		local sd="$(git config --global --get safe.directory ${tests_repo_dir} || true)"
-		if [ -z "${sd}" ]; then
-			git config --global --add safe.directory ${tests_repo_dir}
-		fi
-		pushd "${tests_repo_dir}"
-		git checkout "${branch}"
-		git pull
-		popd
-	else
-		git clone -q "https://${tests_repo}" "$tests_repo_dir"
-		pushd "${tests_repo_dir}"
-		git checkout "${branch}"
-		popd
-	fi
-}
-
-run_static_checks()
-{
-	# Make sure we have the targeting branch
-	git remote set-branches --add origin "${branch}"
-	git fetch -a
-	bash "$kata_repo_dir/tests/static-checks.sh" "$@"
-}
-
-run_docs_url_alive_check()
-{
-	# Make sure we have the targeting branch
-	git remote set-branches --add origin "${branch}"
-	git fetch -a
-	bash "$kata_repo_dir/tests/static-checks.sh" --docs --all "$kata_repo"
-}
-
-run_get_pr_changed_file_details()
-{
-	# Make sure we have the targeting branch
-	git remote set-branches --add origin "${branch}"
-	git fetch -a
-	source "$kata_repo_dir/tests/common.bash"
-	get_pr_changed_file_details
-}
-
-# Check if the 1st argument version is greater than and equal to 2nd one
-# Version format: [0-9]+ separated by period (e.g. 2.4.6, 1.11.3 and etc.)
-#
-# Parameters:
-#	$1	- a version to be tested
-#	$2	- a target version
-#
-# Return:
-# 	0 if $1 is greater than and equal to $2
-#	1 otherwise
-version_greater_than_equal() {
-	local current_version=$1
-	local target_version=$2
-	smaller_version=$(echo -e "$current_version\n$target_version" | sort -V | head -1)
-	if [ "${smaller_version}" = "${target_version}" ]; then
-		return 0
-	else
-		return 1
-	fi
-}

ci/openshift-ci/README.md

@@ -0,0 +1,149 @@
+OpenShift CI
+============
+
+This directory contains scripts used by
+[the OpenShift CI](https://github.com/openshift/release/tree/master/ci-operator/config/kata-containers/kata-containers)
+pipelines to monitor selected functional tests on OpenShift.
+There are 2 pipelines, history and logs can be accessed here:
+
+* [main - currently supported OCP](https://prow.ci.openshift.org/job-history/gs/origin-ci-test/logs/periodic-ci-kata-containers-kata-containers-main-e2e-tests)
+* [next - currently under development OCP](https://prow.ci.openshift.org/job-history/gs/origin-ci-test/logs/periodic-ci-kata-containers-kata-containers-main-next-e2e-tests)
+
+
+Running openshift-tests on OCP with kata-containers manually
+============================================================
+
+To run openshift-tests (or other suites) with kata-containers one can use
+the kata-webhook. To deploy everything you can mimic the CI pipeline by:
+
+```bash
+#!/bin/bash -e
+# Setup your kubectl and check it's accessible by
+kubectl nodes
+# Deploy kata (set KATA_DEPLOY_IMAGE to override the default kata-deploy-ci:latest image)
+./test.sh
+# Deploy the webhook
+KATA_RUNTIME=kata-qemu cluster/deploy_webhook.sh
+```
+
+This should ensure kata-containers as well as kata-webhook are installed and
+working. Before running the openshift-tests it's (currently) recommended to
+ignore some security features by:
+
+```bash
+#!/bin/bash -e
+oc adm policy add-scc-to-group privileged system:authenticated system:serviceaccounts
+oc adm policy add-scc-to-group anyuid system:authenticated system:serviceaccounts
+oc label --overwrite ns default pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/warn=baseline pod-security.kubernetes.io/audit=baseline
+```
+
+Now you should be ready to run the openshift-tests. Our CI only uses a subset
+of tests, to get the current ``TEST_SKIPS`` see
+[the pipeline config](https://github.com/openshift/release/tree/master/ci-operator/config/kata-containers/kata-containers).
+Following steps require the [openshift tests](https://github.com/openshift/origin)
+being cloned and built in the current directory:
+
+```bash
+#!/bin/bash -e
+# Define tests to be skipped (see the pipeline config for the current version)
+TEST_SKIPS="\[sig-node\] Security Context should support seccomp runtime/default\|\[sig-node\] Variable Expansion should allow substituting values in a volume subpath\|\[k8s.io\] Probing container should be restarted with a docker exec liveness probe with timeout\|\[sig-node\] Pods Extended Pod Container lifecycle evicted pods should be terminal\|\[sig-node\] PodOSRejection \[NodeConformance\] Kubelet should reject pod when the node OS doesn't match pod's OS\|\[sig-network\].*for evicted pods\|\[sig-network\].*HAProxy router should override the route\|\[sig-network\].*HAProxy router should serve a route\|\[sig-network\].*HAProxy router should serve the correct\|\[sig-network\].*HAProxy router should run\|\[sig-network\].*when FIPS.*the HAProxy router\|\[sig-network\].*bond\|\[sig-network\].*all sysctl on whitelist\|\[sig-network\].*sysctls should not affect\|\[sig-network\] pods should successfully create sandboxes by adding pod to network"
+# Get the list of tests to be executed
+TESTS="$(./openshift-tests run --dry-run --provider "${TEST_PROVIDER}" "${TEST_SUITE}")"
+# Store the list of tests in /tmp/tsts file
+echo "${TESTS}" | grep -v "$TEST_SKIPS" > /tmp/tsts
+# Remove previously-existing temporarily files as well as previous results
+OUT=RESULTS/tmp
+rm -Rf /tmp/*test* /tmp/e2e-*
+rm -R $OUT
+mkdir -p $OUT
+# Run the tests ignoring the monitor health checks
+./openshift-tests run --provider azure -o "$OUT/job.log" --junit-dir "$OUT" --file /tmp/tsts --max-parallel-tests 5 --cluster-stability Disruptive --run '^\[sig-node\].*|^\[sig-network\]'
+```
+
+[!NOTE]
+Note we are ignoring the cluster stability checks because our public cloud is
+not that stable and running with VMs instead of containers results in minor
+stability issues. Some of the old monitor stability tests do not reflect
+the ``--cluster-stability`` setting, one should simply ignore these. If you
+get a message like ``invariant was violated`` or ``error: failed due to a
+MonitorTest failure``, it's usually an indication that only those kind of
+tests failed but the real tests passed. See
+[wrapped-openshift-tests.sh](https://github.com/openshift/release/blob/master/ci-operator/config/kata-containers/kata-containers/wrapped-openshift-tests.sh)
+for details how our pipeline deals with that.
+
+[!TIP]
+To compare multiple results locally one can use
+[junit2html](https://github.com/inorton/junit2html) tool.
+
+
+Best-effort kata-containers cleanup
+===================================
+
+If you need to cleanup the cluster after testing, you can use the
+``cleanup.sh`` script from the current directory. It tries to delete all
+resources created by ``test.sh`` as well as ``cluster/deploy_webhook.sh``
+ignoring all failures. The primary purpose of this script is to allow
+soft-cleanup after deployment to test different versions without
+re-provisioning everything.
+
+[!WARNING]
+Do not rely on this script in production, return codes are not checked!**
+
+
+Bisecting e2e tests failures
+============================
+
+Let's say the OCP pipeline passed running with
+``quay.io/kata-containers/kata-deploy-ci:kata-containers-d7afd31fd40e37a675b25c53618904ab57e74ccd-amd64``
+but failed running with
+``quay.io/kata-containers/kata-deploy-ci:kata-containers-9f512c016e75599a4a921bd84ea47559fe610057-amd64``
+and you'd like to know which PR caused the regression. You can either run with
+all the 60 tags between or you can utilize the [bisecter](https://github.com/ldoktor/bisecter)
+to optimize the number of steps in between.
+
+Before running the bisection you need a reproducer script. Sample one called
+``sample-test-reproducer.sh`` is provided in this directory but you might
+want to copy and modify it, especially:
+
+* ``OCP_DIR`` - directory where your openshift/release is located (can be exported)
+* ``E2E_TEST`` - openshift-test(s) to be executed (can be exported)
+* behaviour of SETUP (returning 125 skips the current image tag, returning
+  >=128 interrupts the execution, everything else reports the tag as failure
+* what should be executed (perhaps running the setup is enough for you or
+  you might want to be looking for specific failures...)
+* use ``timeout`` to interrupt execution in case you know things should be faster
+
+Executing that script with the GOOD commit should pass
+``./sample-test-reproducer.sh quay.io/kata-containers/kata-deploy-ci:kata-containers-d7afd31fd40e37a675b25c53618904ab57e74ccd-amd64``
+and fail when executed with the BAD commit
+``./sample-test-reproducer.sh quay.io/kata-containers/kata-deploy-ci:kata-containers-9f512c016e75599a4a921bd84ea47559fe610057-amd64``.
+
+To get the list of all tags in between those two PRs you can use the
+``bisect-range.sh`` script
+
+```bash
+./bisect-range.sh d7afd31fd40e37a675b25c53618904ab57e74ccd 9f512c016e75599a4a921bd84ea47559fe610057
+```
+
+[!NOTE]
+The tagged images are only built per PR, not for individual commits. See
+[kata-deploy-ci](https://quay.io/kata-containers/kata-deploy-ci) to see the
+available images.
+
+To find out which PR caused this regression, you can either manually try the
+individual commits or you can simply execute:
+
+```bash
+bisecter start "$(./bisect-range.sh d7afd31fd40 9f512c016)"
+OCP_DIR=/path/to/openshift/release bisecter run ./sample-test-reproducer.sh
+```
+
+[!NOTE]
+If you use ``KATA_WITH_SYSTEM_QEMU=yes`` you might want to deploy once with
+it and skip it for the cleanup. That way you might (in most cases) test
+all images with a single MCP update instead of per-image MCP update.
+
+[!TIP]
+You can check the bisection progress during/after execution by running
+``bisecter log`` from the current directory. Before starting a new
+bisection you need to execute ``bisecter reset``.

ci/openshift-ci/bisect-range.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright (c) 2024 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+if [ "$#" -gt 2 ] || [ "$#" -lt 1 ] ; then
+	echo "Usage: $0 GOOD [BAD]"
+	echo "Prints list of available kata-deploy-ci tags between GOOD and BAD commits (by default BAD is the latest available tag)"
+	exit 255
+fi
+GOOD="$1"
+[ -n "$2" ] && BAD="$2"
+ARCH=amd64
+REPO="quay.io/kata-containers/kata-deploy-ci"
+
+TAGS=$(skopeo list-tags "docker://$REPO")
+# Only amd64
+TAGS=$(echo "$TAGS" | jq '.Tags' | jq "map(select(endswith(\"$ARCH\")))" | jq -r '.[]')
+# Tags since $GOOD
+TAGS=$(echo "$TAGS" | sed -n -e "/$GOOD/,$$p")
+# Tags up to $BAD
+[ -n "$BAD" ] && TAGS=$(echo "$TAGS" | sed "/$BAD/q")
+# Comma separated tags with repo
+echo "$TAGS" | sed -e "s@^@$REPO:@" | paste -s -d, -

ci/openshift-ci/cleanup.sh

@@ -25,6 +25,10 @@ WORKAROUND_9206_CRIO=${WORKAROUND_9206_CRIO:-no}
 # Ignore errors as we want best-effort-approach here
 trap - ERR
 
+# Delete webhook resources
+oc delete -f "${scripts_dir}/../../tools/testing/kata-webhook/deploy"
+oc delete -f "${scripts_dir}/cluster/deployments/configmap_kata-webhook.yaml.in"
+
 # Delete potential smoke-test resources
 oc delete -f "${scripts_dir}/smoke/service.yaml"
 oc delete -f "${scripts_dir}/smoke/service_kubernetes.yaml"

ci/openshift-ci/images/Dockerfile.buildroot

@@ -4,7 +4,7 @@
 #
 # This is the build root image for Kata Containers on OpenShift CI.
 #
-FROM quay.io/centos/centos:stream8
+FROM quay.io/centos/centos:stream9
 
 RUN yum -y update && \
     yum -y install \

ci/openshift-ci/run_smoke_test.sh

@@ -15,7 +15,9 @@ pod='http-server'
 # Create a pod.
 #
 info "Creating the ${pod} pod"
-oc apply -f ${script_dir}/smoke/${pod}.yaml || \
+[ -z "$KATA_RUNTIME" ] && die "Please set the KATA_RUNTIME first"
+envsubst < "${script_dir}/smoke/${pod}.yaml.in" | \
+	oc apply -f - || \
 	die "failed to create ${pod} pod"
 
 # Check it eventually goes to 'running'

ci/openshift-ci/sample-test-reproducer.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Copyright (c) 2024 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# A sample script to deploy, configure, run E2E_TEST and soft-cleanup
+# afterwards OCP cluster using kata-containers primarily created for use
+# with https://github.com/ldoktor/bisecter
+
+[ "$#" -ne 1 ] && echo "Provide image as the first and only argument" && exit 255
+export KATA_DEPLOY_IMAGE="$1"
+OCP_DIR="${OCP_DIR:-/path/to/your/openshift/release/}"
+E2E_TEST="${E2E_TEST:-'"[sig-node] Container Runtime blackbox test on terminated container should report termination message as empty when pod succeeds and TerminationMessagePolicy FallbackToLogsOnError is set [NodeConformance] [Conformance] [Suite:openshift/conformance/parallel/minimal] [Suite:k8s]"'}"
+KATA_CI_DIR="${KATA_CI_DIR:-$(pwd)}"
+export KATA_RUNTIME="${KATA_RUNTIME:-kata-qemu}"
+
+## SETUP
+# Deploy kata
+SETUP=0
+pushd "$KATA_CI_DIR" || { echo "Failed to cd to '$KATA_CI_DIR'"; exit 255; }
+./test.sh || SETUP=125
+cluster/deploy_webhook.sh || SETUP=125
+if [ $SETUP != 0 ]; then
+    ./cleanup.sh
+    exit "$SETUP"
+fi
+popd || true
+# Disable security
+oc adm policy add-scc-to-group privileged system:authenticated system:serviceaccounts
+oc adm policy add-scc-to-group anyuid system:authenticated system:serviceaccounts
+oc label --overwrite ns default pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/warn=baseline pod-security.kubernetes.io/audit=baseline
+
+## TEST EXECUTION
+# Run the testing
+pushd "$OCP_DIR" || { echo "Failed to cd to '$OCP_DIR'"; exit 255; }
+echo "$E2E_TEST" > /tmp/tsts
+# Remove previously-existing temporarily files as well as previous results
+OUT=RESULTS/tmp
+rm -Rf /tmp/*test* /tmp/e2e-*
+rm -R $OUT
+mkdir -p $OUT
+# Run the tests ignoring the monitor health checks
+./openshift-tests run --provider azure -o "$OUT/job.log" --junit-dir "$OUT" --file /tmp/tsts --max-parallel-tests 5 --cluster-stability Disruptive
+RET=$?
+popd || true
+
+## CLEANUP
+./cleanup.sh
+exit "$RET"
+

ci/openshift-ci/smoke/http-server.yaml.in

@@ -27,4 +27,4 @@ spec:
         runAsUser: 1000
         seccompProfile:
           type: RuntimeDefault
-  runtimeClassName: kata-qemu
+  runtimeClassName: ${KATA_RUNTIME}

ci/openshift-ci/test.sh

@@ -5,6 +5,9 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+# The kata shim to be used
+export KATA_RUNTIME=${KATA_RUNTIME:-kata-qemu}
+
 script_dir=$(dirname $0)
 source ${script_dir}/lib.sh
 

ci/run.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 Ant Financial
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -e
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-export CI_JOB="${CI_JOB:-}"
-
-clone_tests_repo
-
-pushd ${tests_repo_dir}
-.ci/run.sh
-# temporary fix, see https://github.com/kata-containers/tests/issues/3878
-if [ "$(uname -m)" != "s390x" ] && [ "$CI_JOB" == "CRI_CONTAINERD_K8S_MINIMAL" ]; then
-	tracing/test-agent-shutdown.sh
-fi
-popd

ci/setup.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-
-clone_tests_repo
-
-pushd "${tests_repo_dir}"
-.ci/setup.sh
-popd

ci/static-checks.sh

@@ -7,6 +7,6 @@
 set -e
 
 cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
+source "${cidir}/../tests/common.bash"
 
 run_static_checks "${@:-github.com/kata-containers/kata-containers}"

docs/Blog-Post-Submission-Guide.md

@@ -0,0 +1,86 @@
+# Blog Post Contributor Guide
+
+This section describes the guidelines for contributing new blog posts to the
+Kata Containers website.
+
+## Share your stories on the Kata Containers website
+
+Are you experimenting with Kata Containers or have it deployed in production and
+would like to share your story as a case study? Do you have a use case that
+Kata Containers can make more secure, but the world doesn't know it yet? Do you
+have features in the runtime that you like and would like to highlight? Do you
+have a Kata Containers demo that you would like to draw attention to?
+
+Share your Kata Containers story on the [Kata Containers blog](https://www.katacontainers.io/blog/)!
+You are only a few steps away...
+
+### Kata Containers website source
+
+Like the rest of the Kata Containers artifacts, the project’s website code and
+content are stored in a [GitHub repository](https://github.com/kata-containers/www.katacontainers.io).
+
+The blog posts are written using markdown language that is mainly plain text
+with a few easy formatting conventions to create lists, add images or code blocks,
+or format the text.
+
+You can find many [cheat sheets](https://www.markdownguide.org/cheat-sheet/)
+floating on the web to get in terms of the basic syntax. You can also check the
+[source files of the already existing blog posts](https://github.com/kata-containers/www.katacontainers.io/tree/main/src/pages/blog),
+where you will find examples of all the basic items that you will need for your
+new entry.
+
+### Create a new blog post
+
+When you create a new blog post, you need to create a new file in the
+[`src/pages/blog/` folder](https://github.com/kata-containers/www.katacontainers.io/tree/main/src/pages/blog)
+with a `.md` extension.
+
+The markdown file has a few formatting conventions in its header to capture the
+title, author, publishing date and category of your blog post.
+
+The header looks like the following:
+
+```
+  ---
+  templateKey: blog-post
+  title: The Title of Your Amazing Blog Post
+  author: Your Name
+  date: 2021-01-28T16:23:52.741Z
+  category:
+    - value: category-6-wjkXzEM2
+      label: Features & Updates
+  ---
+```
+
+The categories give the possibility to filter on the web page and see only the
+blog posts that fall under one of the options. You can choose from the
+following options:
+
+* News & Announcements
+* Features & Updates
+
+The `Annual Report` category is reserved for the Kata Containers chapter in the
+Open Infrastructure Annual report that we are also re-posting on the Kata
+Containers website.
+
+Once you filled out the above fields in the header and got your one-liner all
+set, you can go ahead and type up the contents of your blog post using the
+conventional markdown formatting.
+
+If you have an image file to add, you need to place the file in the
+`static/img` folder.
+
+You can then insert the image into your blog post by using the following line:
+
+```
+  ![alt text](/img/the-file-name-of-your-image.jpg)
+```
+
+Once you are done with formatting your blog post and happy with the content, you
+need to upload it to GitHub and create a pull request. You can do that by using
+git commands on your laptop or you can also use the GitHub web interface to add
+files to the repository and create a pull request when you are ready.
+
+If you have an idea for a blog post and would like to get feedback from the
+community about it or have any questions about the process, please reach out
+on one of the community's [communication channels](https://katacontainers.io/community/).

docs/Developer-Guide.md

@@ -461,7 +461,7 @@ and repository utilized can be found by looking at the [versions file](../versio
 Find the correct version of QEMU from the versions file:
 ```bash
 $ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_version="$(get_from_kata_deps "assets.hypervisor.qemu.version")"
+$ qemu_version="$(get_from_kata_deps ".assets.hypervisor.qemu.version")"
 $ echo "${qemu_version}"
 ```
 Get source from the matching branch of QEMU:

docs/README.md

@@ -50,6 +50,7 @@ Documents that help to understand and contribute to Kata Containers.
 * [Developer Guide](Developer-Guide.md): Setup the Kata Containers developing environments
 * [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md)
 * [Code of Conduct](../CODE_OF_CONDUCT.md)
+* [How to submit a blog post](Blog-Post-Submission-Guide.md)
 
 ## Help Writing a Code PR
 

docs/design/architecture/storage.md

@@ -32,7 +32,7 @@ For virtio-fs, the [runtime](README.md#runtime) starts one `virtiofsd` daemon
 ## Devicemapper
 
 The
-[devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/main/snapshots/devmapper)
+[devicemapper `snapshotter`](https://github.com/containerd/containerd/blob/main/docs/snapshotters/devmapper.md)
 is a special case. The `snapshotter` uses dedicated block devices
 rather than formatted filesystems, and operates at the block level
 rather than the file level. This knowledge is used to directly use the

docs/how-to/containerd-kata.md

@@ -40,7 +40,7 @@ use `RuntimeClass` instead of the deprecated annotations.
 ### Containerd Runtime V2 API: Shim V2 API
 
 The [`containerd-shim-kata-v2` (short as `shimv2` in this documentation)](../../src/runtime/cmd/containerd-shim-kata-v2/)
-implements the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/main/runtime/v2) for Kata.
+implements the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/main/core/runtime/v2) for Kata.
 With `shimv2`, Kubernetes can launch Pod and OCI-compatible containers with one shim per Pod. Prior to `shimv2`, `2N+1` 
 shims (i.e. a `containerd-shim` and a `kata-shim` for each container and the Pod sandbox itself) and no standalone `kata-proxy` 
 process were used, even with VSOCK not available.
@@ -62,7 +62,7 @@ Follow the instructions to [install Kata Containers](../install/README.md).
 > You do not need to install `cri` if you have containerd 1.1 or above. Just remove the `cri` plugin from the list of
 > `disabled_plugins` in the containerd configuration file (`/etc/containerd/config.toml`).
 
-Follow the instructions from the [CRI installation guide](https://github.com/containerd/containerd/blob/main/docs/cri/installation.md).
+Follow the instructions from the [CRI installation guide](https://github.com/containerd/containerd/blob/main/docs/cri/crictl.md#install-crictl).
 
 Then, check if `containerd` is now available:
 
@@ -132,9 +132,9 @@ The `RuntimeClass` is suggested.
 
 The following configuration includes two runtime classes:
 - `plugins.cri.containerd.runtimes.runc`: the runc, and it is the default runtime.
-- `plugins.cri.containerd.runtimes.kata`: The function in containerd (reference [the document here](https://github.com/containerd/containerd/tree/main/runtime/v2#binary-naming)) 
+- `plugins.cri.containerd.runtimes.kata`: The function in containerd (reference [the document here](https://github.com/containerd/containerd/tree/main/core/runtime/v2)) 
   where the dot-connected string `io.containerd.kata.v2` is translated to `containerd-shim-kata-v2` (i.e. the 
-  binary name of the Kata implementation of [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/main/runtime/v2)).
+  binary name of the Kata implementation of [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/main/core/runtime/v2)).
 
 ```toml
     [plugins.cri.containerd]

docs/how-to/how-to-pull-images-in-guest-with-kata.md

@@ -35,27 +35,23 @@ $ git clone -b "${nydus_snapshotter_version}" "${nydus_snapshotter_url}" "${nydu
 2. Configure DaemonSet file
 ```bash
 $ pushd "$nydus_snapshotter_install_dir"
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.FS_DRIVER' \
->	 "proxy" --style=double
+$ yq -i \
+>	 '.data.FS_DRIVER = "proxy"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 # Disable to read snapshotter config from configmap
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.ENABLE_CONFIG_FROM_VOLUME' \
->	 "false" --style=double
+$ yq -i \
+>	 'data.ENABLE_CONFIG_FROM_VOLUME = "false"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 # Enable to run snapshotter as a systemd service 
 # (skip if you want to run nydus snapshotter as a standalone process)
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.ENABLE_SYSTEMD_SERVICE' \
->	 "true" --style=double
+$ yq -i \
+>	 'data.ENABLE_SYSTEMD_SERVICE = "true"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 # Enable "runtime specific snapshotter" feature in containerd when configuring containerd for snapshotter
 # (skip if you want to configure nydus snapshotter as a global snapshotter in containerd)
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER' \
->	 "true" --style=double
+$ yq -i \
+>	 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER = "true"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 ```
 
 3. Install `nydus snapshotter` as a DaemonSet

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -44,8 +44,8 @@ $ popd
 - Build a custom QEMU
 ```bash
 $ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")"
-$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")"
+$ qemu_url="$(get_from_kata_deps ".assets.hypervisor.qemu-snp-experimental.url")"
+$ qemu_tag="$(get_from_kata_deps ".assets.hypervisor.qemu-snp-experimental.tag")"
 $ git clone "${qemu_url}"
 $ pushd qemu
 $ git checkout "${qemu_tag}"
@@ -53,7 +53,14 @@ $ ./configure --enable-virtfs --target-list=x86_64-softmmu --enable-debug
 $ make -j "$(nproc)"
 $ popd
 ```
-
+- Create cert-chain for SNP attestation ( using [snphost](https://github.com/virtee/snphost/blob/main/docs/snphost.1.adoc) )
+```bash
+$ git clone https://github.com/virtee/snphost.git && cd snphost/
+$ cargo build
+$ mkdir /tmp/certs
+$ ./target/debug/snphost fetch vcek der /tmp/certs
+$ ./target/debug/snphost import /tmp/certs /opt/snp/cert_chain.cert
+```
 ### Kata Containers Configuration for SNP
 
 The configuration file located at `/etc/kata-containers/configuration.toml` must be adapted as follows to support SNP-VMs:
@@ -100,6 +107,10 @@ sev_snp_guest = true
   - Configure an OVMF (add path)
 ```toml
 firmware = "/path/to/kata-containers/tools/packaging/static-build/ovmf/opt/kata/share/ovmf/OVMF.fd"
+```
+  - SNP attestation (add cert-chain to default path or add the path with cert-chain)
+```toml
+snp_certs_path = "/path/to/cert-chain"
 ```
 
 ## Test Kata Containers with Containerd

docs/presentations/unit-testing/kata-containers-unit-testing.md

@@ -202,11 +202,6 @@ attributes of each environment (local and CI):
 - The hardware architecture.
 - Number (and spec) of the CPUs.
 
-## Gotchas (part 3)
-
-If in doubt, look at the
-["test artifacts" attached to the failing CI test](http://jenkins.katacontainers.io).
-
 ## Before raising a PR
 
 - Remember to check that the test runs locally:

docs/threat-model/threat-model.md

@@ -1,95 +1,157 @@
 # Kata Containers threat model
 
-This document discusses threat models associated with the Kata Containers project.
-Kata was designed to provide additional isolation of container workloads, protecting
-the host infrastructure from potentially malicious container users or workloads. Since
-Kata Containers adds a level of isolation on top of traditional containers, the focus
-is on the additional layer provided, not on traditional container security.
+This document discusses threat models associated with the Kata Containers
+project. Kata was designed to provide additional isolation of container
+workloads, protecting the host infrastructure from potentially malicious
+container users or workloads. Since Kata Containers adds a level of isolation on
+top of traditional containers, the focus is on the additional layer provided,
+not on traditional container security.
 
-This document provides a brief background on containers and layered security, describes
-the interface to Kata from CRI runtimes, a review of utilized virtual machine interfaces, and then
-a review of threats.
+This document provides a brief background on containers and layered security,
+describes the interface to Kata from CRI runtimes, a review of utilized virtual
+machine interfaces, and then a review of threats.
 
 ## Kata security objective
 
-Kata seeks to prevent an untrusted container workload or user of that container workload to gain
-control of, obtain information from, or tamper with the host infrastructure.
+Kata seeks to prevent an untrusted container workload or user of that container
+workload to gain control of, obtain information from, or tamper with the host
+infrastructure.
 
-In our scenario, an asset is anything on the host system, or elsewhere in the cluster
-infrastructure. The attacker is assumed to be either a malicious user or the workload itself
-running within the container. The goal of Kata is to prevent attacks which would allow
-any access to the defined assets.
+In our scenario, an asset is anything on the host system, or elsewhere in the
+cluster infrastructure. The attacker is assumed to be either a malicious user or
+the workload itself running within the container. The goal of Kata is to prevent
+attacks which would allow any access to the defined assets.
 
 ## Background on containers, layered security
 
-Traditional containers leverage several key Linux kernel features to provide isolation and
-a view that the container workload is the only entity running on the host. Key features include
-`Namespaces`, `cgroups`, `capablities`, `SELinux` and `seccomp`. The canonical runtime for creating such
-a container is `runc`. In the remainder of the document, the term `traditional-container` will be used
-to describe a container workload created by runc.
+Traditional containers leverage several key Linux kernel features to provide
+isolation and a view that the container workload is the only entity running on
+the host. Key features include `Namespaces`, `cgroups`, `capablities`, `SELinux`
+and `seccomp`. The canonical runtime for creating such a container is `runc`. In
+the remainder of the document, the term `traditional-container` will be used to
+describe a container workload created by runc.
 
-Kata Containers provides a second layer of isolation on top of those provided by traditional-containers.
-The hardware virtualization interface is the basis of this additional layer. Kata launches a lightweight
-virtual machine, and uses the guest’s Linux kernel to create a container workload, or workloads in the case
-of multi-container pods. In Kubernetes and in the Kata implementation, the sandbox is carried out at the
-pod level. In Kata, this sandbox is created using a virtual machine.
+Kata Containers provides a second layer of isolation on top of those provided by
+traditional-containers. The hardware virtualization interface is the basis of
+this additional layer. Kata launches a lightweight virtual machine, and uses the
+guest’s Linux kernel to create a container workload, or workloads in the case of
+multi-container pods. In Kubernetes and in the Kata implementation, the sandbox
+is carried out at the pod level. In Kata, this sandbox is created using a
+virtual machine.
 
 ## Interface to Kata Containers: CRI, v2-shim, OCI
 
 A typical Kata Containers deployment uses Kubernetes with a CRI implementation.
-On every node, Kubelet will interact with a CRI implementor, which will in turn interface with
-an OCI based runtime, such as Kata Containers. Typical CRI implementors are `cri-o` and `containerd`.
+On every node, Kubelet will interact with a CRI implementor, which will in turn
+interface with an OCI based runtime, such as Kata Containers. Typical CRI
+implementors are `cri-o` and `containerd`.
 
-The CRI API, as defined at the Kubernetes [CRI-API repo](https://github.com/kubernetes/cri-api/),
-results in a few constructs being supported by the CRI implementation, and ultimately in the OCI
-runtime creating the workloads.
+The CRI API, as defined at the Kubernetes [CRI-API
+repo](https://github.com/kubernetes/cri-api/), results in a few constructs being
+supported by the CRI implementation, and ultimately in the OCI runtime creating
+the workloads.
 
-In order to run a container inside of the Kata sandbox, several virtual machine devices and interfaces
-are required. Kata translates sandbox and container definitions to underlying virtualization technologies provided
-by a set of virtual machine monitors (VMMs) and hypervisors. These devices and their underlying
-implementations are discussed in detail in the following section.
+In order to run a container inside of the Kata sandbox, several virtual machine
+devices and interfaces are required. Kata translates sandbox and container
+definitions to underlying virtualization technologies provided by a set of
+virtual machine monitors (VMMs) and hypervisors. These devices and their
+underlying implementations are discussed in detail in the following section.
 
 ## Interface to the Kata sandbox/virtual machine
 
 In case of Kata, today the devices which we need in the guest are:
- - Storage: In the current design of Kata Containers, we are reliant on the CRI implementor to
- assist in image handling and volume management on the host. As a result, we need to support a way of passing to the sandbox the container rootfs, volumes requested
- by the workload, and any other volumes created to facilitate sharing of secrets and `configmaps` with the containers. Depending on how these are managed, a block based device or file-system
- sharing is required. Kata Containers does this by way of `virtio-blk` and/or `virtio-fs`.
- - Networking: A method for enabling network connectivity with the workload is required. Typically this will be done providing a `TAP` device
- to the VMM, and this will be exposed to the guest as a `virtio-net` device. It is feasible to pass in a NIC device directly, in which case `VFIO` is leveraged
- and the device itself will be exposed to the guest.
- - Control: In order to interact with the guest agent and retrieve `STDIO` from containers, a medium of communication is required.
- This is available via `virtio-vsock`.
- - Devices: `VFIO` is utilized when devices are passed directly to the virtual machine and exposed to the container.
-- Dynamic Resource Management: `ACPI` is utilized to allow for dynamic VM resource management (for example: CPU, memory, device hotplug). This is required when containers are resized,
- or more generally when containers are added to a pod. 
+ - Storage: In the current design of Kata Containers, we are reliant on the CRI
+ implementor to assist in image handling and volume management on the host. As a
+ result, we need to support a way of passing to the sandbox the container
+ rootfs, volumes requested by the workload, and any other volumes created to
+ facilitate sharing of secrets and `configmaps` with the containers. Depending
+ on how these are managed, a block based device or file-system sharing is
+ required. Kata Containers does this by way of `virtio-blk` and/or `virtio-fs`.
+ - Networking: A method for enabling network connectivity with the workload is
+ required. Typically this will be done providing a `TAP` device to the VMM, and
+ this will be exposed to the guest as a `virtio-net` device. It is feasible to
+ pass in a NIC device directly, in which case `VFIO` is leveraged and the device
+ itself will be exposed to the guest.
+ - Control: In order to interact with the guest agent and retrieve `STDIO` from
+ containers, a medium of communication is required. This is available via
+ `virtio-vsock`.
+ - Devices: `VFIO` is utilized when devices are passed directly to the virtual
+   machine and exposed to the container.
+- Dynamic Resource Management: `ACPI` is utilized to allow for dynamic VM
+ resource management (for example: CPU, memory, device hotplug). This is
+ required when containers are resized, or more generally when containers are
+ added to a pod. 
  
-How these devices are utilized varies depending on the VMM utilized. We clarify the default settings provided when integrating Kata
-with the QEMU, Firecracker and Cloud Hypervisor VMMs in the following sections.
+How these devices are utilized varies depending on the VMM utilized. We clarify
+the default settings provided when integrating Kata with the QEMU, Dragonball,
+Firecracker and Cloud Hypervisor VMMs in the following sections.
+
+### Virtual Machine Monitor(s)
+
+In a KVM/QEMU (any other VMM utilizing KVM) virtualization setup, all virtual
+machines (VMs) share the same host kernel. This shared environment can lead to
+scenarios where one VM could potentially impact the performance or stability of
+other VMs, including the possibility of a Denial of Service attack.
+
+- Kernel Vulnerabilities: Since all VMs rely on the host's kernel, a
+vulnerability in the kernel could be exploited by a process running within one
+VM to affect the entire system. This could lead to scenarios where the
+compromised VM impacts other VMs or even takes down the host.
+
+- Improper Isolation and Containment: If the virtualization environment is not
+correctly configured, processes in one VM might impact other VMs. This could
+occur through improper isolation of network traffic, shared file systems, or
+other inter-VM communication channels.
+
+- Hypervisor Vulnerabilities: Flaws in the KVM hypervisor or QEMU could be
+exploited to cause information disclosure, data tampering, elevation of
+privileges, denial of service, and others. Since KVM/QEMU leverages the host
+kernel for its operation, any exploit at this level can have widespread impacts.
+
+- Malicious or Flawed Guest Operating Systems: A guest operating system that is
+maliciously designed or has serious flaws could engage in activities that
+disrupt the normal operation of the host or other guests. This might include
+aggressive network activity or interactions with the virtualization stack that
+lead to instability.
+
+- Resource Exhaustion: A VM could consume excessive shared resources such as
+CPU, memory, or I/O bandwidth, leading to resource starvation for other VMs.
+This could be due to misconfiguration, a runaway process, or a deliberate
+denial of service attack from a compromised VM.
 
 ### Devices
 
-Each virtio device is implemented by a backend, which may execute within userspace on the host (vhost-user), the VMM itself, or within the host kernel (vhost). While it may provide enhanced performance,
-vhost devices are often seen as higher risk since an exploit would be already running within the kernel space. While VMM and vhost-user are both in userspace on the host, `vhost-user` generally allows for the back-end process to require less system calls and capabilities compared to a full VMM.
+Each virtio device is implemented by a backend, which may execute within
+userspace on the host (vhost-user), the VMM itself, or within the host kernel
+(vhost). While it may provide enhanced performance, vhost devices are often seen
+as higher risk since an exploit would be already running within the kernel
+space. While VMM and vhost-user are both in userspace on the host, `vhost-user`
+generally allows for the back-end process to require less system calls and
+capabilities compared to a full VMM.
 
 #### `virtio-blk` and `virtio-scsi`
 
-The backend for `virtio-blk` and `virtio-scsi` are based in the VMM itself (ring3 in the context of x86) by default for Cloud Hypervisor, Firecracker and QEMU.
-While `vhost` based back-ends are available for QEMU, it is not recommended. `vhost-user` back-ends are being added for Cloud Hypervisor, they are not utilized in Kata today.
+The backend for `virtio-blk` and `virtio-scsi` are based in the VMM itself
+(ring3 in the context of x86) by default for Cloud Hypervisor, Firecracker and
+QEMU. While `vhost` based back-ends are available for QEMU, it is not
+recommended. `vhost-user` back-ends are being added for Cloud Hypervisor, they
+are not utilized in Kata today.
 
 #### `virtio-fs`
 
-`virtio-fs` is supported in Cloud Hypervisor and QEMU. `virtio-fs`'s interaction with the host filesystem is done through a vhost-user daemon, `virtiofsd`.
-The `virtio-fs` client, running in the guest, will generate requests to access files. `virtiofsd` will receive requests, open the file, and request the VMM
-to `mmap` it into the guest. When DAX is utilized, the guest will access the host's page cache, avoiding the need for copy and duplication. DAX is still an experimental feature,
-and is not enabled by default.
+`virtio-fs` is supported in Cloud Hypervisor and QEMU. `virtio-fs`'s interaction
+with the host filesystem is done through a vhost-user daemon, `virtiofsd`. The
+`virtio-fs` client, running in the guest, will generate requests to access
+files. `virtiofsd` will receive requests, open the file, and request the VMM to
+`mmap` it into the guest. When DAX is utilized, the guest will access the host's
+page cache, avoiding the need for copy and duplication. DAX is still an
+experimental feature, and is not enabled by default.
 
 From the `virtiofsd` [documentation](https://gitlab.com/virtio-fs/virtiofsd/-/blob/main/README.md):
 ```This program must be run as the root user. Upon startup the program will switch into a new file system namespace with the shared directory tree as its root. This prevents “file system escapes” due to symlinks and other file system objects that might lead to files outside the shared directory. The program also sandboxes itself using seccomp(2) to prevent ptrace(2) and other vectors that could allow an attacker to compromise the system after gaining control of the virtiofsd process.```
 
-DAX-less support for `virtio-fs` is available as of the 5.4 Linux kernel. QEMU VMM supports virtio-fs as of v4.2. Cloud Hypervisor
-supports `virtio-fs`.
+DAX-less support for `virtio-fs` is available as of the 5.4 Linux kernel. QEMU
+VMM supports virtio-fs as of v4.2. Cloud Hypervisor supports `virtio-fs`.
 
 #### `virtio-net`
 
@@ -97,9 +159,9 @@ supports `virtio-fs`.
 
 ##### QEMU networking
 
-While QEMU has options for `vhost`, `virtio-net` and `vhost-user`, the `virtio-net` backend
-for Kata defaults to `vhost-net` for performance reasons. The default configuration is being
-reevaluated.
+While QEMU has options for `vhost`, `virtio-net` and `vhost-user`, the
+`virtio-net` backend for Kata defaults to `vhost-net` for performance reasons.
+The default configuration is being reevaluated.
 
 ##### Firecracker networking
 
@@ -107,8 +169,14 @@ For Firecracker, the `virtio-net` backend is within Firecracker's VMM.
 
 ##### Cloud Hypervisor networking
 
-For Cloud Hypervisor, the current backend default is within the VMM. `vhost-user-net` support
-is being added (written in rust, Cloud Hypervisor specific).
+For Cloud Hypervisor, the current backend default is within the VMM.
+`vhost-user-net` support is being added (written in rust, Cloud Hypervisor
+specific).
+
+##### Dragonball networking
+
+For Dragonball, the `virtio-net` backend default is within Dragonbasll's VMM.
+
 
 #### virtio-vsock
 
@@ -116,22 +184,88 @@ is being added (written in rust, Cloud Hypervisor specific).
 
 In QEMU, vsock is backed by `vhost_vsock`, which runs within the kernel itself.
 
-##### Firecracker and Cloud Hypervisor
+##### Dragonball, Firecracker and Cloud Hypervisor
 
-In Firecracker and Cloud Hypervisor, vsock is backed by a unix-domain-socket in the hosts userspace.
+In Dragonball, Firecracker and Cloud Hypervisor, vsock is backed by a unix-domain-socket in
+the hosts userspace.
 
 #### VFIO
 
-Utilizing VFIO, devices can be passed through to the virtual machine. We will assess this separately. Exposure to
-host is limited to gaps in device pass-through handling. This is supported in QEMU and Cloud Hypervisor, but not
-Firecracker.
+Utilizing VFIO, devices can be passed through to the virtual machine. Exposure
+to the host is limited to gaps in device pass-through handling. This is
+supported in QEMU and Cloud Hypervisor, but not Firecracker.
+
+- Device Isolation Failure: One of the primary risks associated with VFIO is the
+failure to isolate the physical device. If a VM can affect the operation of the
+physical device in a way that impacts other VMs or the host system, it could
+lead to security breaches or system instability.
+
+- DMA Attacks: Direct Memory Access (DMA) attacks are a significant concern with
+VFIO. Since the device has direct access to the system's memory, there's a risk
+that a compromised VM could use its assigned device to read or write memory
+outside of its allocated space, potentially accessing sensitive information or
+affecting the host or other VMs.
+
+- Firmware Vulnerabilities: Devices attached via VFIO rely on their firmware,
+which can have vulnerabilities. A compromised device firmware could be exploited
+to gain unauthorized access or to disrupt the system. Resource Starvation:
+Improperly managed, a VM with direct access to hardware resources could
+monopolize those resources, leading to performance degradation or denial of
+service for other VMs or the host system.
+
+- Escalation of Privileges: If a VM with VFIO access is compromised, it could
+potentially be used to gain higher privileges than intended, especially if the
+I/O devices have capabilities that are not adequately controlled or monitored.
+
+- Improper Configuration and Management: Human errors in configuring VFIO, such
+as incorrect group or user permissions, can expose the system to risks.
+Additionally, inadequate monitoring and management of the VMs and their devices
+can lead to security lapses.
+
+- Software Vulnerabilities: Like any software, the components of VFIO (like the
+kernel modules, device drivers, and management tools) can have vulnerabilities
+that might be exploited by an attacker to compromise the security of the system.
+Inter-VM Interference and Side-Channel Attacks: Even with device assignment,
+there could be side-channel attacks where an attacker VM infers sensitive
+information from the physical device's behavior or through shared resources like
+cache.
+
+#### ACPI (Dragonball uses Upcall)
+
+ACPI is necessary for hotplugging of CPU, memory and devices. ACPI is available
+in QEMU and Cloud Hypervisor. Device, CPU and memory hotplug are not available
+in Firecracker.
+
+- Hypervisor Vulnerabilities: In virtualized environments, the hypervisor
+manages ACPI calls for virtual machines (VMs). If the hypervisor has
+vulnerabilities in handling ACPI requests, it could lead to escalated privileges
+or other security breaches.
+
+- VM Escape: A sophisticated attack could exploit ACPI functionality to achieve
+a VM escape, where malicious code in a VM breaks out to the host system or other
+VMs. Firmware Attacks in a Virtualized Context: Similar to physical
+environments, firmware-based attacks (including those targeting ACPI) in
+virtualized systems can be persistent and difficult to detect. In a virtualized
+environment, such attacks might not only compromise the host system but also all
+the VMs running on it.
+
+- Resource Starvation Attacks: ACPI functionality could be exploited to
+manipulate power management features, causing denial of service through
+resource starvation. For example, an attacker could force a VM into a low-power
+state, degrading its performance or availability.
+
+- Compromised VMs Affecting Host ACPI Settings: If a VM is compromised, it might
+be used to alter ACPI settings on the host, affecting all VMs on that host. This
+could lead to various impacts, from performance degradation to system
+instability.
+
+- Supply Chain Risks: As with non-virtualized environments, the firmware,
+including ACPI firmware used in virtualized environments, could be compromised
+during the supply chain process, leading to vulnerabilities that affect all VMs
+running on the hardware.
 
-#### ACPI
 
-ACPI is necessary for hotplug of CPU, memory and devices. ACPI is available in QEMU and Cloud Hypervisor. Device, CPU and memory hotplug
-are not available in Firecracker.
 
 ## Devices and threat model
 
 ![Threat model](threat-model-boundaries.svg "threat-model")
-

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -279,8 +279,8 @@ $ export KERNEL_EXTRAVERSION=$(awk '/^EXTRAVERSION =/{print $NF}' $GOPATH/$LINUX
 $ export KERNEL_ROOTFS_DIR=${KERNEL_MAJOR_VERSION}.${KERNEL_PATHLEVEL}.${KERNEL_SUBLEVEL}${KERNEL_EXTRAVERSION}
 $ cd $QAT_SRC
 $ KERNEL_SOURCE_ROOT=$GOPATH/$LINUX_VER ./configure --enable-icp-sriov=guest
-$ sudo -E make all -j $($(nproc ${CI:+--ignore 1}))
-$ sudo -E make INSTALL_MOD_PATH=$ROOTFS_DIR qat-driver-install -j $($(nproc ${CI:+--ignore 1}))
+$ sudo -E make all -j $(nproc)
+$ sudo -E make INSTALL_MOD_PATH=$ROOTFS_DIR qat-driver-install -j $(nproc)
 ```
 
 The `usdm_drv` module also needs to be copied into the rootfs modules path and

src/agent/Cargo.toml

@@ -21,8 +21,8 @@ scopeguard = "1.0.0"
 thiserror = "1.0.26"
 regex = "1.10.4"
 serial_test = "0.5.1"
-oci-distribution = "0.10.0"
 url = "2.5.0"
+derivative = "2.2.0"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
 safe-path = { path = "../libs/safe-path" }
@@ -34,8 +34,8 @@ async-recursion = "0.3.2"
 futures = "0.3.30"
 
 # Async runtime
-tokio = { version = "1.28.1", features = ["full"] }
-tokio-vsock = "0.3.1"
+tokio = { version = "1.38.0", features = ["full"] }
+tokio-vsock = "0.3.4"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket"] }
 rtnetlink = "0.8.0"
@@ -57,12 +57,7 @@ cfg-if = "1.0.0"
 prometheus = { version = "0.13.0", features = ["process"] }
 procfs = "0.12.0"
 
-# anyhow is currently locked at 1.0.58 because:
-# - Versions between 1.0.59 - 1.0.76 have not been tested yet using Kata CI.
-#   However, those versions are passing "make test" for the Kata Agent.
-# - Versions 1.0.77 or newer fail during "make test" - see
-#   https://github.com/kata-containers/kata-containers/issues/9538
-anyhow = "=1.0.58"
+anyhow = "1"
 
 cgroups = { package = "cgroups-rs", version = "0.3.3" }
 
@@ -81,11 +76,13 @@ strum = "0.26.2"
 strum_macros = "0.26.2"
 
 # Image pull/decrypt
-image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "ca6b438", default-features = true, optional = true }
-openssl = { version = "0.10.54", features = ["vendored"], optional = true }
+image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "2c5ac6b01aafcb0be3875f5743c77d654a548146", default-features = false, optional = true }
 
 # Agent Policy
-regorus = { version = "0.1.4", default-features = false, features = ["arc", "regex"], optional = true }
+regorus = { version = "0.1.4", default-features = false, features = [
+    "arc",
+    "regex",
+], optional = true }
 
 [dev-dependencies]
 tempfile = "3.1.0"
@@ -106,7 +103,7 @@ default-pull = ["guest-pull"]
 seccomp = ["rustjail/seccomp"]
 standard-oci-runtime = ["rustjail/standard-oci-runtime"]
 agent-policy = ["regorus"]
-guest-pull = ["image-rs", "openssl"]
+guest-pull = ["image-rs/kata-cc-rustls-tls"]
 
 [[bin]]
 name = "kata-agent"

src/agent/Makefile

@@ -15,7 +15,7 @@ PROJECT_COMPONENT = kata-agent
 TARGET = $(PROJECT_COMPONENT)
 
 VERSION_FILE := ./VERSION
-VERSION := $(shell grep -v ^\# $(VERSION_FILE))
+VERSION := $(shell grep -v ^\# $(VERSION_FILE) 2>/dev/null || echo "unknown")
 COMMIT_NO := $(shell git rev-parse HEAD 2>/dev/null || true)
 COMMIT := $(if $(shell git status --porcelain --untracked-files=no 2>/dev/null || true),${COMMIT_NO}-dirty,${COMMIT_NO})
 COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown)
@@ -159,7 +159,7 @@ vendor:
 
 #TARGET test: run cargo tests
 test: $(GENERATED_FILES)
-	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
+	@RUST_LIB_BACKTRACE=0 cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test
 check: $(GENERATED_FILES) standard_rust_check

src/agent/README.md

@@ -125,9 +125,11 @@ The kata agent has the ability to configure agent options in guest kernel comman
 | `agent.debug_console` | Debug console flag | Allow to connect guest OS running inside hypervisor Connect using `kata-runtime exec <sandbox-id>` | boolean | `false` |
 | `agent.debug_console_vport` | Debug console port | Allow to specify the `vsock` port to connect the debugging console | integer | `0` |
 | `agent.devmode` | Developer mode | Allow the agent process to coredump | boolean | `false` |
+| `agent.guest_components_rest_api` | `api-server-rest` configuration | Select the features that the API Server Rest attestation component will run with. Valid values are `all`, `attestation`, `resource` | string | `resource` |
+| `agent.guest_components_procs` | guest-components processes | Attestation-related processes that should be spawned as children of the guest. Valid values are `none`, `attestation-agent`, `confidential-data-hub` (implies `attestation-agent`), `api-server-rest` (implies `attestation-agent` and `confidential-data-hub`) | string | `api-server-rest` |
 | `agent.hotplug_timeout` | Hotplug timeout | Allow to configure hotplug timeout(seconds) of block devices | integer | `3` |
-| `agent.guest_components_rest_api` | `api-server-rest` configuration | Select the features that the API Server Rest attestation component will run with. Valid values are `all`, `attestation`, `resource`, or `none` to not launch the `api-server-rest` component | string | `resource` |
 | `agent.https_proxy` | HTTPS proxy | Allow to configure `https_proxy` in the guest | string | `""` |
+| `agent.image_registry_auth` | Image registry credential URI | The URI to where image-rs can find the credentials for pulling images from private registries e.g. `file:///root/.docker/config.json` to read from a file in the guest image, or `kbs:///default/credentials/test` to get the file from the KBS| string | `""` |
 | `agent.log` | Log level | Allow the agent log level to be changed (produces more or less output) | string | `"info"` |
 | `agent.log_vport` | Log port | Allow to specify the `vsock` port to read logs | integer | `0` |
 | `agent.no_proxy` | NO proxy | Allow to configure `no_proxy` in the guest | string | `""` |

src/agent/rustjail/Cargo.toml

@@ -30,8 +30,8 @@ cgroups = { package = "cgroups-rs", version = "0.3.3" }
 rlimit = "0.5.3"
 cfg-if = "0.1.0"
 
-tokio = { version = "1.28.1", features = ["sync", "io-util", "process", "time", "macros", "rt", "fs"] }
-tokio-vsock = "0.3.1"
+tokio = { version = "1.38.0", features = ["sync", "io-util", "process", "time", "macros", "rt", "fs"] }
+tokio-vsock = "0.3.4"
 futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"

src/agent/rustjail/src/container.rs

@@ -601,6 +601,22 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::close(mount_fd)?;
     }
 
+    if init {
+        // CreateContainer Hooks:
+        // before pivot_root after prestart, createruntime
+        state.pid = std::process::id() as i32;
+        state.status = oci::ContainerState::Created;
+        if let Some(hooks) = spec.hooks.as_ref() {
+            log_child!(
+                cfd_log,
+                "create_container hooks {:?}",
+                hooks.create_container
+            );
+            let mut create_container_states = HookStates::new();
+            create_container_states.execute_hooks(&hooks.create_container, Some(state.clone()))?;
+        }
+    }
+
     if to_new.contains(CloneFlags::CLONE_NEWNS) {
         // unistd::chroot(rootfs)?;
         if no_pivot {

src/agent/rustjail/src/process.rs

@@ -200,15 +200,8 @@ impl Process {
     }
 
     pub async fn close_stdin(&mut self) {
-        // stdin will be closed automatically in passfd-io senario
-        if self.proc_io.is_some() {
-            return;
-        }
-
         close_process_stream!(self, term_master, TermMaster);
         close_process_stream!(self, parent_stdin, ParentStdin);
-
-        self.notify_term_close();
     }
 
     pub fn cleanup_process_stream(&mut self) {

src/agent/src/cdh.rs

@@ -0,0 +1,150 @@
+// Copyright (c) 2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// Confidential Data Hub client wrapper.
+// Confidential Data Hub is a service running inside guest to provide resource related APIs.
+// https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub
+
+use anyhow::Result;
+use derivative::Derivative;
+use protocols::{
+    sealed_secret, sealed_secret_ttrpc_async, sealed_secret_ttrpc_async::SealedSecretServiceClient,
+};
+
+use crate::CDH_SOCKET_URI;
+
+// Nanoseconds
+const CDH_UNSEAL_TIMEOUT: i64 = 50 * 1000 * 1000 * 1000;
+const SEALED_SECRET_PREFIX: &str = "sealed.";
+
+#[derive(Derivative)]
+#[derivative(Clone, Debug)]
+pub struct CDHClient {
+    #[derivative(Debug = "ignore")]
+    sealed_secret_client: SealedSecretServiceClient,
+}
+
+impl CDHClient {
+    pub fn new() -> Result<Self> {
+        let client = ttrpc::asynchronous::Client::connect(CDH_SOCKET_URI)?;
+        let sealed_secret_client =
+            sealed_secret_ttrpc_async::SealedSecretServiceClient::new(client);
+
+        Ok(CDHClient {
+            sealed_secret_client,
+        })
+    }
+
+    pub async fn unseal_secret_async(&self, sealed_secret: &str) -> Result<Vec<u8>> {
+        let mut input = sealed_secret::UnsealSecretInput::new();
+        input.set_secret(sealed_secret.into());
+
+        let unsealed_secret = self
+            .sealed_secret_client
+            .unseal_secret(ttrpc::context::with_timeout(CDH_UNSEAL_TIMEOUT), &input)
+            .await?;
+        Ok(unsealed_secret.plaintext)
+    }
+
+    pub async fn unseal_env(&self, env: &str) -> Result<String> {
+        if let Some((key, value)) = env.split_once('=') {
+            if value.starts_with(SEALED_SECRET_PREFIX) {
+                let unsealed_value = self.unseal_secret_async(value).await?;
+                let unsealed_env = format!("{}={}", key, std::str::from_utf8(&unsealed_value)?);
+
+                return Ok(unsealed_env);
+            }
+        }
+
+        Ok((*env.to_owned()).to_string())
+    }
+}
+
+#[cfg(test)]
+#[cfg(feature = "sealed-secret")]
+mod tests {
+    use crate::cdh::CDHClient;
+    use crate::cdh::CDH_ADDR;
+    use anyhow::anyhow;
+    use async_trait::async_trait;
+    use protocols::{sealed_secret, sealed_secret_ttrpc_async};
+    use std::sync::Arc;
+    use test_utils::skip_if_not_root;
+    use tokio::signal::unix::{signal, SignalKind};
+
+    struct TestService;
+
+    #[async_trait]
+    impl sealed_secret_ttrpc_async::SealedSecretService for TestService {
+        async fn unseal_secret(
+            &self,
+            _ctx: &::ttrpc::asynchronous::TtrpcContext,
+            _req: sealed_secret::UnsealSecretInput,
+        ) -> ttrpc::error::Result<sealed_secret::UnsealSecretOutput> {
+            let mut output = sealed_secret::UnsealSecretOutput::new();
+            output.set_plaintext("unsealed".into());
+            Ok(output)
+        }
+    }
+
+    fn remove_if_sock_exist(sock_addr: &str) -> std::io::Result<()> {
+        let path = sock_addr
+            .strip_prefix("unix://")
+            .expect("socket address does not have the expected format.");
+
+        if std::path::Path::new(path).exists() {
+            std::fs::remove_file(path)?;
+        }
+
+        Ok(())
+    }
+
+    fn start_ttrpc_server() {
+        tokio::spawn(async move {
+            let ss = Box::new(TestService {})
+                as Box<dyn sealed_secret_ttrpc_async::SealedSecretService + Send + Sync>;
+            let ss = Arc::new(ss);
+            let ss_service = sealed_secret_ttrpc_async::create_sealed_secret_service(ss);
+
+            remove_if_sock_exist(CDH_ADDR).unwrap();
+
+            let mut server = ttrpc::asynchronous::Server::new()
+                .bind(CDH_ADDR)
+                .unwrap()
+                .register_service(ss_service);
+
+            server.start().await.unwrap();
+
+            let mut interrupt = signal(SignalKind::interrupt()).unwrap();
+            tokio::select! {
+                _ = interrupt.recv() => {
+                    server.shutdown().await.unwrap();
+                }
+            };
+        });
+    }
+
+    #[tokio::test]
+    async fn test_unseal_env() {
+        skip_if_not_root!();
+
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let _guard = rt.enter();
+        start_ttrpc_server();
+        std::thread::sleep(std::time::Duration::from_secs(2));
+
+        let cc = Some(CDHClient::new().unwrap());
+        let cdh_client = cc.as_ref().ok_or(anyhow!("get cdh_client failed")).unwrap();
+        let sealed_env = String::from("key=sealed.testdata");
+        let unsealed_env = cdh_client.unseal_env(&sealed_env).await.unwrap();
+        assert_eq!(unsealed_env, String::from("key=unsealed"));
+        let normal_env = String::from("key=testdata");
+        let unchanged_env = cdh_client.unseal_env(&normal_env).await.unwrap();
+        assert_eq!(unchanged_env, String::from("key=testdata"));
+
+        rt.shutdown_background();
+        std::thread::sleep(std::time::Duration::from_secs(2));
+    }
+}

src/agent/src/config.rs

@@ -28,6 +28,9 @@ const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "systemd.unified_cgroup_hierarchy";
 const CONFIG_FILE: &str = "agent.config_file";
 const GUEST_COMPONENTS_REST_API_OPTION: &str = "agent.guest_components_rest_api";
+const GUEST_COMPONENTS_PROCS_OPTION: &str = "agent.guest_components_procs";
+#[cfg(feature = "guest-pull")]
+const IMAGE_REGISTRY_AUTH_OPTION: &str = "agent.image_registry_auth";
 
 // Configure the proxy settings for HTTPS requests in the guest,
 // to solve the problem of not being able to access the specified image in some cases.
@@ -59,19 +62,34 @@ const ERR_INVALID_CONTAINER_PIPE_SIZE_PARAM: &str = "unable to parse container p
 const ERR_INVALID_CONTAINER_PIPE_SIZE_KEY: &str = "invalid container pipe size key name";
 const ERR_INVALID_CONTAINER_PIPE_NEGATIVE: &str = "container pipe size should not be negative";
 
-const ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE: &str = "invalid guest components rest api feature given. Valid values are `all`, `attestation`, `resource`, or `none`";
+const ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE: &str = "invalid guest components rest api feature given. Valid values are `all`, `attestation`, `resource`";
+const ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE: &str = "invalid guest components process param given. Valid values are `attestation-agent`, `confidential-data-hub`, `api-server-rest`, or `none`";
 
 #[derive(Clone, Copy, Debug, Default, Display, Deserialize, EnumString, PartialEq)]
 // Features seem to typically be in kebab-case format, but we only have single words at the moment
 #[strum(serialize_all = "kebab-case")]
+#[serde(rename_all = "kebab-case")]
 pub enum GuestComponentsFeatures {
     All,
     Attestation,
-    None,
     #[default]
     Resource,
 }
 
+#[derive(Clone, Copy, Debug, Default, Display, Deserialize, EnumString, PartialEq)]
+/// Attestation-related processes that we want to spawn as children of the agent
+#[strum(serialize_all = "kebab-case")]
+#[serde(rename_all = "kebab-case")]
+pub enum GuestComponentsProcs {
+    None,
+    /// ApiServerRest implies ConfidentialDataHub and AttestationAgent
+    #[default]
+    ApiServerRest,
+    AttestationAgent,
+    /// ConfidentialDataHub implies AttestationAgent
+    ConfidentialDataHub,
+}
+
 #[derive(Debug)]
 pub struct AgentConfig {
     pub debug_console: bool,
@@ -89,6 +107,9 @@ pub struct AgentConfig {
     pub https_proxy: String,
     pub no_proxy: String,
     pub guest_components_rest_api: GuestComponentsFeatures,
+    pub guest_components_procs: GuestComponentsProcs,
+    #[cfg(feature = "guest-pull")]
+    pub image_registry_auth: String,
 }
 
 #[derive(Debug, Deserialize)]
@@ -107,6 +128,9 @@ pub struct AgentConfigBuilder {
     pub https_proxy: Option<String>,
     pub no_proxy: Option<String>,
     pub guest_components_rest_api: Option<GuestComponentsFeatures>,
+    pub guest_components_procs: Option<GuestComponentsProcs>,
+    #[cfg(feature = "guest-pull")]
+    pub image_registry_auth: Option<String>,
 }
 
 macro_rules! config_override {
@@ -171,6 +195,9 @@ impl Default for AgentConfig {
             https_proxy: String::from(""),
             no_proxy: String::from(""),
             guest_components_rest_api: GuestComponentsFeatures::default(),
+            guest_components_procs: GuestComponentsProcs::default(),
+            #[cfg(feature = "guest-pull")]
+            image_registry_auth: String::from(""),
         }
     }
 }
@@ -207,6 +234,9 @@ impl FromStr for AgentConfig {
             agent_config,
             guest_components_rest_api
         );
+        config_override!(agent_config_builder, agent_config, guest_components_procs);
+        #[cfg(feature = "guest-pull")]
+        config_override!(agent_config_builder, agent_config, image_registry_auth);
 
         Ok(agent_config)
     }
@@ -220,7 +250,10 @@ impl AgentConfig {
         let config_position = args.iter().position(|a| a == "--config" || a == "-c");
         if let Some(config_position) = config_position {
             if let Some(config_file) = args.get(config_position + 1) {
-                return AgentConfig::from_config_file(config_file).context("AgentConfig from args");
+                let mut config =
+                    AgentConfig::from_config_file(config_file).context("AgentConfig from args")?;
+                config.override_config_from_envs();
+                return Ok(config);
             } else {
                 panic!("The config argument wasn't formed properly: {:?}", args);
             }
@@ -293,7 +326,6 @@ impl AgentConfig {
                 get_vsock_port,
                 |port| port > 0
             );
-
             parse_cmdline_param!(
                 param,
                 CONTAINER_PIPE_SIZE_OPTION,
@@ -314,23 +346,22 @@ impl AgentConfig {
                 config.guest_components_rest_api,
                 get_guest_components_features_value
             );
+            parse_cmdline_param!(
+                param,
+                GUEST_COMPONENTS_PROCS_OPTION,
+                config.guest_components_procs,
+                get_guest_components_procs_value
+            );
+            #[cfg(feature = "guest-pull")]
+            parse_cmdline_param!(
+                param,
+                IMAGE_REGISTRY_AUTH_OPTION,
+                config.image_registry_auth,
+                get_string_value
+            );
         }
 
-        if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
-            config.server_addr = addr;
-        }
-
-        if let Ok(addr) = env::var(LOG_LEVEL_ENV_VAR) {
-            if let Ok(level) = logrus_to_slog_level(&addr) {
-                config.log_level = level;
-            }
-        }
-
-        if let Ok(value) = env::var(TRACING_ENV_VAR) {
-            let name_value = format!("{}={}", TRACING_ENV_VAR, value);
-
-            config.tracing = get_bool_value(&name_value)?;
-        }
+        config.override_config_from_envs();
 
         Ok(config)
     }
@@ -341,6 +372,25 @@ impl AgentConfig {
             .with_context(|| format!("Failed to read config file {}", file))?;
         AgentConfig::from_str(&config)
     }
+
+    #[instrument]
+    fn override_config_from_envs(&mut self) {
+        if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
+            self.server_addr = addr;
+        }
+
+        if let Ok(addr) = env::var(LOG_LEVEL_ENV_VAR) {
+            if let Ok(level) = logrus_to_slog_level(&addr) {
+                self.log_level = level;
+            }
+        }
+
+        if let Ok(value) = env::var(TRACING_ENV_VAR) {
+            let name_value = format!("{}={}", TRACING_ENV_VAR, value);
+
+            self.tracing = get_bool_value(&name_value).unwrap_or(false);
+        }
+    }
 }
 
 #[instrument]
@@ -471,13 +521,24 @@ fn get_url_value(param: &str) -> Result<String> {
 fn get_guest_components_features_value(param: &str) -> Result<GuestComponentsFeatures> {
     let fields: Vec<&str> = param.split('=').collect();
     ensure!(fields.len() >= 2, ERR_INVALID_GET_VALUE_PARAM);
+    // We need name (but the value can be blank)
+    ensure!(!fields[0].is_empty(), ERR_INVALID_GET_VALUE_NO_NAME);
+    let value = fields[1..].join("=");
+    GuestComponentsFeatures::from_str(&value)
+        .map_err(|_| anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE))
+}
+
+#[instrument]
+fn get_guest_components_procs_value(param: &str) -> Result<GuestComponentsProcs> {
+    let fields: Vec<&str> = param.split('=').collect();
+    ensure!(fields.len() >= 2, ERR_INVALID_GET_VALUE_PARAM);
 
     // We need name (but the value can be blank)
     ensure!(!fields[0].is_empty(), ERR_INVALID_GET_VALUE_NO_NAME);
 
     let value = fields[1..].join("=");
-    GuestComponentsFeatures::from_str(&value)
-        .map_err(|_| anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE))
+    GuestComponentsProcs::from_str(&value)
+        .map_err(|_| anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE))
 }
 
 #[cfg(test)]
@@ -486,6 +547,7 @@ mod tests {
 
     use super::*;
     use anyhow::anyhow;
+    use serial_test::serial;
     use std::fs::File;
     use std::io::Write;
     use std::time;
@@ -501,6 +563,8 @@ mod tests {
     }
 
     #[test]
+    // Run in serial to stop the env set interfering with test_from_cmdline_with_args_overwrites
+    #[serial]
     fn test_from_cmdline() {
         const TEST_SERVER_ADDR: &str = "vsock://-1:1024";
 
@@ -519,6 +583,9 @@ mod tests {
             https_proxy: &'a str,
             no_proxy: &'a str,
             guest_components_rest_api: GuestComponentsFeatures,
+            guest_components_procs: GuestComponentsProcs,
+            #[cfg(feature = "guest-pull")]
+            image_registry_auth: &'a str,
         }
 
         impl Default for TestData<'_> {
@@ -537,6 +604,9 @@ mod tests {
                     https_proxy: "",
                     no_proxy: "",
                     guest_components_rest_api: GuestComponentsFeatures::default(),
+                    guest_components_procs: GuestComponentsProcs::default(),
+                    #[cfg(feature = "guest-pull")]
+                    image_registry_auth: "",
                 }
             }
         }
@@ -745,6 +815,13 @@ mod tests {
                 server_addr: "unix://@/tmp/foo.socket",
                 ..Default::default()
             },
+            // Test that env_var has precedence over the command line (which is the current behaviour)
+            TestData {
+                contents: "agent.server_addr=unix:///tmp/ignored.socket",
+                env_vars: vec!["KATA_AGENT_SERVER_ADDR=unix:///tmp/foo.socket"],
+                server_addr: "unix:///tmp/foo.socket",
+                ..Default::default()
+            },
             TestData {
                 contents: "",
                 env_vars: vec!["KATA_AGENT_LOG_LEVEL="],
@@ -942,8 +1019,35 @@ mod tests {
                 ..Default::default()
             },
             TestData {
-                contents: "agent.guest_components_rest_api=none",
-                guest_components_rest_api: GuestComponentsFeatures::None,
+               contents: "agent.guest_components_procs=api-server-rest",
+               guest_components_procs: GuestComponentsProcs::ApiServerRest,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.guest_components_procs=confidential-data-hub",
+                guest_components_procs: GuestComponentsProcs::ConfidentialDataHub,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.guest_components_procs=attestation-agent",
+                guest_components_procs: GuestComponentsProcs::AttestationAgent,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.guest_components_procs=none",
+                guest_components_procs: GuestComponentsProcs::None,
+                ..Default::default()
+            },
+            #[cfg(feature = "guest-pull")]
+            TestData {
+                contents: "agent.image_registry_auth=file:///root/.docker/config.json",
+                image_registry_auth: "file:///root/.docker/config.json",
+                ..Default::default()
+            },
+            #[cfg(feature = "guest-pull")]
+            TestData {
+                contents: "agent.image_registry_auth=kbs:///default/credentials/test",
+                image_registry_auth: "kbs:///default/credentials/test",
                 ..Default::default()
             },
         ];
@@ -1000,6 +1104,13 @@ mod tests {
                 "{}",
                 msg
             );
+            assert_eq!(
+                d.guest_components_procs, config.guest_components_procs,
+                "{}",
+                msg
+            );
+            #[cfg(feature = "guest-pull")]
+            assert_eq!(d.image_registry_auth, config.image_registry_auth, "{}", msg);
 
             for v in vars_to_unset {
                 env::remove_var(v);
@@ -1008,15 +1119,17 @@ mod tests {
     }
 
     #[test]
+    // Run in serial to stop the env set interfering with test_from_cmdline
+    #[serial]
     fn test_from_cmdline_with_args_overwrites() {
         let expected = AgentConfig {
             dev_mode: true,
-            server_addr: "unix://@/tmp/foo.socket".to_string(),
+            server_addr: "unix:///tmp/overwrite.socket".to_string(),
             ..Default::default()
         };
 
         let example_config_file_contents =
-            "dev_mode = true\nserver_addr = 'unix://@/tmp/foo.socket'";
+            "dev_mode = true\nserver_addr = 'unix:///tmp/ignored.socket'";
         let dir = tempdir().expect("failed to create tmpdir");
         let file_path = dir.path().join("config.toml");
         let filename = file_path.to_str().expect("failed to create filename");
@@ -1024,10 +1137,15 @@ mod tests {
         file.write_all(example_config_file_contents.as_bytes())
             .unwrap_or_else(|_| panic!("failed to write file contents"));
 
+        // Ensure that the env has precedence over agent config file
+        env::set_var("KATA_AGENT_SERVER_ADDR", "unix:///tmp/overwrite.socket");
+
         let config =
             AgentConfig::from_cmdline("", vec!["--config".to_string(), filename.to_string()])
                 .expect("Failed to parse command line");
 
+        env::remove_var("KATA_AGENT_SERVER_ADDR");
+
         assert_eq!(expected.debug_console, config.debug_console);
         assert_eq!(expected.dev_mode, config.dev_mode);
         assert_eq!(
@@ -1500,10 +1618,6 @@ Caused by:
                 param: "x=attestation",
                 result: Ok(GuestComponentsFeatures::Attestation),
             },
-            TestData {
-                param: "x=none",
-                result: Ok(GuestComponentsFeatures::None),
-            },
             TestData {
                 param: "x=resource",
                 result: Ok(GuestComponentsFeatures::Resource),
@@ -1533,12 +1647,76 @@ Caused by:
         }
     }
 
+    #[test]
+    fn test_get_guest_components_procs_value() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            param: &'a str,
+            result: Result<GuestComponentsProcs>,
+        }
+
+        let tests = &[
+            TestData {
+                param: "",
+                result: Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)),
+            },
+            TestData {
+                param: "=",
+                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
+            },
+            TestData {
+                param: "==",
+                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
+            },
+            TestData {
+                param: "x=attestation-agent",
+                result: Ok(GuestComponentsProcs::AttestationAgent),
+            },
+            TestData {
+                param: "x=confidential-data-hub",
+                result: Ok(GuestComponentsProcs::ConfidentialDataHub),
+            },
+            TestData {
+                param: "x=none",
+                result: Ok(GuestComponentsProcs::None),
+            },
+            TestData {
+                param: "x=api-server-rest",
+                result: Ok(GuestComponentsProcs::ApiServerRest),
+            },
+            TestData {
+                param: "x===",
+                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
+            },
+            TestData {
+                param: "x==x",
+                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
+            },
+            TestData {
+                param: "x=x",
+                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let result = get_guest_components_procs_value(d.param);
+
+            let msg = format!("{}: result: {:?}", msg, result);
+
+            assert_result!(d.result, result, msg);
+        }
+    }
+
     #[test]
     fn test_config_builder_from_string() {
         let config = AgentConfig::from_str(
             r#"
                dev_mode = true
                server_addr = 'vsock://8:2048'
+               guest_components_procs = "api-server-rest"
+               guest_components_rest_api = "all"
               "#,
         )
         .unwrap();
@@ -1546,6 +1724,14 @@ Caused by:
         // Verify that the override worked
         assert!(config.dev_mode);
         assert_eq!(config.server_addr, "vsock://8:2048");
+        assert_eq!(
+            config.guest_components_procs,
+            GuestComponentsProcs::ApiServerRest
+        );
+        assert_eq!(
+            config.guest_components_rest_api,
+            GuestComponentsFeatures::All
+        );
 
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);

src/agent/src/device.rs

@@ -710,7 +710,14 @@ pub fn update_env_pci(
     env: &mut [String],
     pcimap: &HashMap<pci::Address, pci::Address>,
 ) -> Result<()> {
-    for envvar in env {
+    // SR-IOV device plugin may add two environment variables for one resource:
+    // - PCIDEVICE_<prefix>_<resource-name>: a list of PCI device ids separated by comma
+    // - PCIDEVICE_<prefix>_<resource-name>_INFO: detailed info in JSON for above PCI devices
+    // Both environment variables hold information about the same set of PCI devices.
+    // Below code updates both of them in two passes:
+    // - 1st pass updates PCIDEVICE_<prefix>_<resource-name> and collects host to guest PCI address mapping
+    let mut pci_dev_map: HashMap<String, HashMap<String, String>> = HashMap::new();
+    for envvar in env.iter_mut() {
         let eqpos = envvar
             .find('=')
             .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
@@ -718,24 +725,46 @@ pub fn update_env_pci(
         let (name, eqval) = envvar.split_at(eqpos);
         let val = &eqval[1..];
 
-        if !name.starts_with("PCIDEVICE_") {
+        if !name.starts_with("PCIDEVICE_") || name.ends_with("_INFO") {
             continue;
         }
 
+        let mut addr_map: HashMap<String, String> = HashMap::new();
         let mut guest_addrs = Vec::<String>::new();
-
-        for host_addr in val.split(',') {
-            let host_addr = pci::Address::from_str(host_addr)
+        for host_addr_str in val.split(',') {
+            let host_addr = pci::Address::from_str(host_addr_str)
                 .with_context(|| format!("Can't parse {} environment variable", name))?;
             let guest_addr = pcimap
                 .get(&host_addr)
                 .ok_or_else(|| anyhow!("Unable to translate host PCI address {}", host_addr))?;
+
             guest_addrs.push(format!("{}", guest_addr));
+            addr_map.insert(host_addr_str.to_string(), format!("{}", guest_addr));
         }
 
+        pci_dev_map.insert(format!("{}_INFO", name), addr_map);
+
         envvar.replace_range(eqpos + 1.., guest_addrs.join(",").as_str());
     }
 
+    // - 2nd pass update PCIDEVICE_<prefix>_<resource-name>_INFO if it exists
+    for envvar in env.iter_mut() {
+        let eqpos = envvar
+            .find('=')
+            .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
+
+        let (name, _) = envvar.split_at(eqpos);
+        if !(name.starts_with("PCIDEVICE_") && name.ends_with("_INFO")) {
+            continue;
+        }
+
+        if let Some(addr_map) = pci_dev_map.get(name) {
+            for (host_addr, guest_addr) in addr_map {
+                *envvar = envvar.replace(host_addr, guest_addr);
+            }
+        }
+    }
+
     Ok(())
 }
 
@@ -867,9 +896,10 @@ async fn vfio_pci_device_handler(
             }
 
             group = Some(devgroup);
-
-            pci_fixups.push((host, guestdev));
         }
+
+        // collect PCI address mapping for both vfio-pci-gk and vfio-pci device
+        pci_fixups.push((host, guestdev));
     }
 
     let dev_update = if vfio_in_guest {
@@ -903,7 +933,11 @@ async fn vfio_ap_device_handler(
     for apqn in device.options.iter() {
         wait_for_ap_device(sandbox, ap::Address::from_str(apqn)?).await?;
     }
-    Ok(Default::default())
+    let dev_update = Some(DevUpdate::new(Z9_CRYPT_DEV_PATH, Z9_CRYPT_DEV_PATH)?);
+    Ok(SpecUpdate {
+        dev: dev_update,
+        pci: Vec::new(),
+    })
 }
 
 #[cfg(not(target_arch = "s390x"))]
@@ -932,22 +966,22 @@ pub async fn add_devices(
                 ));
             }
 
-            let mut sb = sandbox.lock().await;
-            for (host, guest) in update.pci {
-                if let Some(other_guest) = sb.pcimap.insert(host, guest) {
-                    return Err(anyhow!(
-                        "Conflicting guest address for host device {} ({} versus {})",
-                        host,
-                        guest,
-                        other_guest
-                    ));
-                }
-            }
-
             // Update cgroup to allow all devices added to guest.
             insert_devices_cgroup_rule(spec, &dev_update.info, true, "rwm")
                 .context("Update device cgroup")?;
         }
+
+        let mut sb = sandbox.lock().await;
+        for (host, guest) in update.pci {
+            if let Some(other_guest) = sb.pcimap.insert(host, guest) {
+                return Err(anyhow!(
+                    "Conflicting guest address for host device {} ({} versus {})",
+                    host,
+                    guest,
+                    other_guest
+                ));
+            }
+        }
     }
 
     if let Some(process) = spec.process.as_mut() {
@@ -1382,8 +1416,11 @@ mod tests {
             ("0000:01:01.0", "ffff:02:1f.7"),
         ];
 
+        let pci_dev_info_original = r#"PCIDEVICE_x_INFO={"0000:1a:01.0":{"generic":{"deviceID":"0000:1a:01.0"}},"0000:1b:02.0":{"generic":{"deviceID":"0000:1b:02.0"}}}"#;
+        let pci_dev_info_expected = r#"PCIDEVICE_x_INFO={"0000:01:01.0":{"generic":{"deviceID":"0000:01:01.0"}},"0000:01:02.0":{"generic":{"deviceID":"0000:01:02.0"}}}"#;
         let mut env = vec![
             "PCIDEVICE_x=0000:1a:01.0,0000:1b:02.0".to_string(),
+            pci_dev_info_original.to_string(),
             "PCIDEVICE_y=0000:01:01.0".to_string(),
             "NOTAPCIDEVICE_blah=abcd:ef:01.0".to_string(),
         ];
@@ -1399,11 +1436,12 @@ mod tests {
             .collect();
 
         let res = update_env_pci(&mut env, &pci_fixups);
-        assert!(res.is_ok());
+        assert!(res.is_ok(), "error: {}", res.err().unwrap());
 
         assert_eq!(env[0], "PCIDEVICE_x=0000:01:01.0,0000:01:02.0");
-        assert_eq!(env[1], "PCIDEVICE_y=ffff:02:1f.7");
-        assert_eq!(env[2], "NOTAPCIDEVICE_blah=abcd:ef:01.0");
+        assert_eq!(env[1], pci_dev_info_expected);
+        assert_eq!(env[2], "PCIDEVICE_y=ffff:02:1f.7");
+        assert_eq!(env[3], "NOTAPCIDEVICE_blah=abcd:ef:01.0");
     }
 
     #[test]

src/agent/src/image.rs

@@ -20,8 +20,6 @@ use tokio::sync::Mutex;
 use crate::rpc::CONTAINER_BASE;
 use crate::AGENT_CONFIG;
 
-// A marker to merge container spec for images pulled inside guest.
-const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name";
 const KATA_IMAGE_WORK_DIR: &str = "/run/kata-containers/image/";
 const CONFIG_JSON: &str = "config.json";
 const KATA_PAUSE_BUNDLE: &str = "/pause_bundle";
@@ -52,23 +50,24 @@ fn copy_if_not_exists(src: &Path, dst: &Path) -> Result<()> {
 
 pub struct ImageService {
     image_client: ImageClient,
-    images: HashMap<String, String>,
 }
 
 impl ImageService {
     pub fn new() -> Self {
-        Self {
-            image_client: ImageClient::new(PathBuf::from(KATA_IMAGE_WORK_DIR)),
-            images: HashMap::new(),
+        let mut image_client = ImageClient::new(PathBuf::from(KATA_IMAGE_WORK_DIR));
+        #[cfg(feature = "guest-pull")]
+        if !AGENT_CONFIG.image_registry_auth.is_empty() {
+            let registry_auth = &AGENT_CONFIG.image_registry_auth;
+            debug!(sl(), "Set registry auth file {:?}", registry_auth);
+            image_client.config.file_paths.auth_file = registry_auth.clone();
+            image_client.config.auth = true;
         }
-    }
 
-    async fn add_image(&mut self, image: String, cid: String) {
-        self.images.insert(image, cid);
+        Self { image_client }
     }
 
     /// pause image is packaged in rootfs
-    fn unpack_pause_image(cid: &str, target_subpath: &str) -> Result<String> {
+    fn unpack_pause_image(cid: &str) -> Result<String> {
         verify_id(cid).context("The guest pause image cid contains invalid characters.")?;
 
         let guest_pause_bundle = Path::new(KATA_PAUSE_BUNDLE);
@@ -102,9 +101,7 @@ impl ImageService {
             bail!("The number of args should be greater than or equal to one! Please check the pause image.");
         }
 
-        let container_bundle = scoped_join(CONTAINER_BASE, cid)?;
-        fs::create_dir_all(&container_bundle)?;
-        let pause_bundle = scoped_join(&container_bundle, target_subpath)?;
+        let pause_bundle = scoped_join(CONTAINER_BASE, cid)?;
         fs::create_dir_all(&pause_bundle)?;
         let pause_rootfs = scoped_join(&pause_bundle, "rootfs")?;
         fs::create_dir_all(&pause_rootfs)?;
@@ -125,7 +122,7 @@ impl ImageService {
     /// - `cid`: Container id
     /// - `image_metadata`: Annotations about the image (exp: "containerd.io/snapshot/cri.layer-digest": "sha256:24fb2886d6f6c5d16481dd7608b47e78a8e92a13d6e64d87d57cb16d5f766d63")
     /// # Returns
-    /// - The image rootfs bundle path. (exp. /run/kata-containers/cb0b47276ea66ee9f44cc53afa94d7980b57a52c3f306f68cb034e58d9fbd3c6/images/rootfs)
+    /// - The image rootfs bundle path. (exp. /run/kata-containers/cb0b47276ea66ee9f44cc53afa94d7980b57a52c3f306f68cb034e58d9fbd3c6/rootfs)
     pub async fn pull_image(
         &mut self,
         image: &str,
@@ -146,16 +143,13 @@ impl ImageService {
         }
 
         if is_sandbox {
-            let mount_path = Self::unpack_pause_image(cid, "pause")?;
-            self.add_image(String::from(image), String::from(cid)).await;
+            let mount_path = Self::unpack_pause_image(cid)?;
             return Ok(mount_path);
         }
 
         // Image layers will store at KATA_IMAGE_WORK_DIR, generated bundles
         // with rootfs and config.json will store under CONTAINER_BASE/cid/images.
-        let bundle_base_dir = scoped_join(CONTAINER_BASE, cid)?;
-        fs::create_dir_all(&bundle_base_dir)?;
-        let bundle_path = scoped_join(&bundle_base_dir, "images")?;
+        let bundle_path = scoped_join(CONTAINER_BASE, cid)?;
         fs::create_dir_all(&bundle_path)?;
         info!(sl(), "pull image {image:?}, bundle path {bundle_path:?}");
 
@@ -179,35 +173,9 @@ impl ImageService {
                 return Err(e);
             }
         };
-        self.add_image(String::from(image), String::from(cid)).await;
         let image_bundle_path = scoped_join(&bundle_path, "rootfs")?;
         Ok(image_bundle_path.as_path().display().to_string())
     }
-
-    /// Partially merge an OCI process specification into another one.
-    fn merge_oci_process(&self, target: &mut oci::Process, source: &oci::Process) {
-        // Override the target args only when the target args is empty and source.args is not empty
-        if target.args.is_empty() && !source.args.is_empty() {
-            target.args.append(&mut source.args.clone());
-        }
-
-        // Override the target cwd only when the target cwd is blank and source.cwd is not blank
-        if target.cwd == "/" && source.cwd != "/" {
-            target.cwd = String::from(&source.cwd);
-        }
-
-        for source_env in &source.env {
-            if let Some((variable_name, variable_value)) = source_env.split_once('=') {
-                debug!(
-                    sl(),
-                    "source spec environment variable: {variable_name:?} : {variable_value:?}"
-                );
-                if !target.env.iter().any(|i| i.contains(variable_name)) {
-                    target.env.push(source_env.to_string());
-                }
-            }
-        }
-    }
 }
 
 /// Set proxy environment from AGENT_CONFIG
@@ -237,55 +205,6 @@ pub async fn set_proxy_env_vars() {
     };
 }
 
-/// When being passed an image name through a container annotation, merge its
-/// corresponding bundle OCI specification into the passed container creation one.
-pub async fn merge_bundle_oci(container_oci: &mut oci::Spec) -> Result<()> {
-    let image_service = IMAGE_SERVICE.clone();
-    let mut image_service = image_service.lock().await;
-    let image_service = image_service
-        .as_mut()
-        .expect("Image Service not initialized");
-    if let Some(image_name) = container_oci.annotations.get(ANNO_K8S_IMAGE_NAME) {
-        if let Some(container_id) = image_service.images.get(image_name) {
-            let image_oci_config_path = Path::new(CONTAINER_BASE)
-                .join(container_id)
-                .join(CONFIG_JSON);
-            debug!(
-                sl(),
-                "Image bundle config path: {:?}", image_oci_config_path
-            );
-
-            let image_oci = oci::Spec::load(image_oci_config_path.to_str().ok_or_else(|| {
-                anyhow!(
-                    "Invalid container image OCI config path {:?}",
-                    image_oci_config_path
-                )
-            })?)
-            .context("load image bundle")?;
-
-            if let (Some(container_root), Some(image_root)) =
-                (container_oci.root.as_mut(), image_oci.root.as_ref())
-            {
-                let root_path = Path::new(CONTAINER_BASE)
-                    .join(container_id)
-                    .join(image_root.path.clone());
-                container_root.path =
-                    String::from(root_path.to_str().ok_or_else(|| {
-                        anyhow!("Invalid container image root path {:?}", root_path)
-                    })?);
-            }
-
-            if let (Some(container_process), Some(image_process)) =
-                (container_oci.process.as_mut(), image_oci.process.as_ref())
-            {
-                image_service.merge_oci_process(container_process, image_process);
-            }
-        }
-    }
-
-    Ok(())
-}
-
 /// Init the image service
 pub async fn init_image_service() {
     let image_service = ImageService::new();
@@ -305,71 +224,3 @@ pub async fn pull_image(
 
     image_service.pull_image(image, cid, image_metadata).await
 }
-
-#[cfg(test)]
-mod tests {
-    use super::ImageService;
-    use rstest::rstest;
-
-    #[rstest]
-    // TODO - how can we tell the user didn't specifically set it to `/` vs not setting at all? Is that scenario valid?
-    #[case::image_cwd_should_override_blank_container_cwd("/", "/imageDir", "/imageDir")]
-    #[case::container_cwd_should_override_image_cwd("/containerDir", "/imageDir", "/containerDir")]
-    #[case::container_cwd_should_override_blank_image_cwd("/containerDir", "/", "/containerDir")]
-    async fn test_merge_cwd(
-        #[case] container_process_cwd: &str,
-        #[case] image_process_cwd: &str,
-        #[case] expected: &str,
-    ) {
-        let image_service = ImageService::new();
-        let mut container_process = oci::Process {
-            cwd: container_process_cwd.to_string(),
-            ..Default::default()
-        };
-        let image_process = oci::Process {
-            cwd: image_process_cwd.to_string(),
-            ..Default::default()
-        };
-        image_service.merge_oci_process(&mut container_process, &image_process);
-        assert_eq!(expected, container_process.cwd);
-    }
-
-    #[rstest]
-    #[case::pods_environment_overrides_images(
-        vec!["ISPRODUCTION=true".to_string()],
-        vec!["ISPRODUCTION=false".to_string()],
-        vec!["ISPRODUCTION=true".to_string()]
-    )]
-    #[case::multiple_environment_variables_can_be_overrided(
-        vec!["ISPRODUCTION=true".to_string(), "ISDEVELOPMENT=false".to_string()],
-        vec!["ISPRODUCTION=false".to_string(), "ISDEVELOPMENT=true".to_string()],
-        vec!["ISPRODUCTION=true".to_string(), "ISDEVELOPMENT=false".to_string()]
-    )]
-    #[case::not_override_them_when_none_of_variables_match(
-        vec!["ANOTHERENV=TEST".to_string()],
-        vec!["ISPRODUCTION=false".to_string(), "ISDEVELOPMENT=true".to_string()],
-        vec!["ANOTHERENV=TEST".to_string(), "ISPRODUCTION=false".to_string(), "ISDEVELOPMENT=true".to_string()]
-    )]
-    #[case::a_mix_of_both_overriding_and_not(
-        vec!["ANOTHERENV=TEST".to_string(), "ISPRODUCTION=true".to_string()],
-        vec!["ISPRODUCTION=false".to_string(), "ISDEVELOPMENT=true".to_string()],
-        vec!["ANOTHERENV=TEST".to_string(), "ISPRODUCTION=true".to_string(), "ISDEVELOPMENT=true".to_string()]
-    )]
-    async fn test_merge_env(
-        #[case] container_process_env: Vec<String>,
-        #[case] image_process_env: Vec<String>,
-        #[case] expected: Vec<String>,
-    ) {
-        let image_service = ImageService::new();
-        let mut container_process = oci::Process {
-            env: container_process_env,
-            ..Default::default()
-        };
-        let image_process = oci::Process {
-            env: image_process_env,
-            ..Default::default()
-        };
-        image_service.merge_oci_process(&mut container_process, &image_process);
-        assert_eq!(expected, container_process.env);
-    }
-}

src/agent/src/linux_abi.rs

@@ -71,6 +71,7 @@ cfg_if! {
         pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0";
         pub const AP_ROOT_BUS_PATH: &str = "/devices/ap";
         pub const AP_SCANS_PATH: &str = "/sys/bus/ap/scans";
+        pub const Z9_CRYPT_DEV_PATH: &str = "/dev/z90crypt";
     }
 }
 

src/agent/src/main.rs

@@ -38,6 +38,7 @@ use std::process::Command;
 use std::sync::Arc;
 use tracing::{instrument, span};
 
+mod cdh;
 mod config;
 mod console;
 mod device;
@@ -59,11 +60,12 @@ mod util;
 mod version;
 mod watcher;
 
-use config::GuestComponentsFeatures;
+use cdh::CDHClient;
+use config::GuestComponentsProcs;
 use mount::{cgroups_mount, general_mount};
 use sandbox::Sandbox;
 use signal::setup_signal_handler;
-use slog::{error, info, o, warn, Logger};
+use slog::{debug, error, info, o, warn, Logger};
 use uevent::watch_uevents;
 
 use futures::future::join_all;
@@ -104,9 +106,13 @@ const AA_ATTESTATION_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, AA_ATTESTATION_SO
 
 const CDH_PATH: &str = "/usr/local/bin/confidential-data-hub";
 const CDH_SOCKET: &str = "/run/confidential-containers/cdh.sock";
+const CDH_SOCKET_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, CDH_SOCKET);
 
 const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest";
 
+/// Path of ocicrypt config file. This is used by image-rs when decrypting image.
+const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+
 const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6;
 
 lazy_static! {
@@ -403,12 +409,28 @@ async fn start_sandbox(
     let (tx, rx) = tokio::sync::oneshot::channel();
     sandbox.lock().await.sender = Some(tx);
 
-    if Path::new(CDH_PATH).exists() && Path::new(AA_PATH).exists() {
-        init_attestation_components(logger, config)?;
+    let mut cdh_client = None;
+    let gc_procs = config.guest_components_procs;
+    if gc_procs != GuestComponentsProcs::None {
+        if !attestation_binaries_available(logger, &gc_procs) {
+            warn!(
+                logger,
+                "attestation binaries requested for launch not available"
+            );
+        } else {
+            cdh_client = init_attestation_components(logger, config)?;
+        }
     }
 
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode).await?;
+    let mut server = rpc::start(
+        sandbox.clone(),
+        config.server_addr.as_str(),
+        init_mode,
+        cdh_client,
+    )
+    .await?;
+
     server.start().await?;
 
     rx.await?;
@@ -417,9 +439,34 @@ async fn start_sandbox(
     Ok(())
 }
 
+// Check if required attestation binaries are available on the rootfs.
+fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs) -> bool {
+    let binaries = match procs {
+        GuestComponentsProcs::AttestationAgent => vec![AA_PATH],
+        GuestComponentsProcs::ConfidentialDataHub => vec![AA_PATH, CDH_PATH],
+        GuestComponentsProcs::ApiServerRest => vec![AA_PATH, CDH_PATH, API_SERVER_PATH],
+        _ => vec![],
+    };
+    for binary in binaries.iter() {
+        if !Path::new(binary).exists() {
+            warn!(logger, "{} not found", binary);
+            return false;
+        }
+    }
+    true
+}
+
 // Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
-fn init_attestation_components(logger: &Logger, _config: &AgentConfig) -> Result<()> {
-    // The Attestation Agent will run for the duration of the guest.
+// and the corresponding procs are enabled in the agent configuration. the process will be
+// launched in the background and the function will return immediately.
+// If the CDH is started, a CDH client will be instantiated and returned.
+fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<Option<CDHClient>> {
+    // skip launch of any guest-component
+    if config.guest_components_procs == GuestComponentsProcs::None {
+        return Ok(None);
+    }
+
+    debug!(logger, "spawning attestation-agent process {}", AA_PATH);
     launch_process(
         logger,
         AA_PATH,
@@ -429,33 +476,58 @@ fn init_attestation_components(logger: &Logger, _config: &AgentConfig) -> Result
     )
     .map_err(|e| anyhow!("launch_process {} failed: {:?}", AA_PATH, e))?;
 
-    if let Err(e) = launch_process(
+    // skip launch of confidential-data-hub and api-server-rest
+    if config.guest_components_procs == GuestComponentsProcs::AttestationAgent {
+        return Ok(None);
+    }
+
+    let ocicrypt_config = serde_json::json!({
+        "key-providers": {
+            "attestation-agent":{
+                "ttrpc":CDH_SOCKET_URI
+            }
+        }
+    });
+
+    fs::write(OCICRYPT_CONFIG_PATH, ocicrypt_config.to_string().as_bytes())?;
+    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
+
+    debug!(
+        logger,
+        "spawning confidential-data-hub process {}", CDH_PATH
+    );
+
+    launch_process(
         logger,
         CDH_PATH,
         &vec![],
         CDH_SOCKET,
         DEFAULT_LAUNCH_PROCESS_TIMEOUT,
-    ) {
-        error!(logger, "launch_process {} failed: {:?}", CDH_PATH, e);
-    } else {
-        let features = _config.guest_components_rest_api;
-        match features {
-            GuestComponentsFeatures::None => {}
-            _ => {
-                if let Err(e) = launch_process(
-                    logger,
-                    API_SERVER_PATH,
-                    &vec!["--features", &features.to_string()],
-                    "",
-                    0,
-                ) {
-                    error!(logger, "launch_process {} failed: {:?}", API_SERVER_PATH, e);
-                }
-            }
-        }
+    )
+    .map_err(|e| anyhow!("launch_process {} failed: {:?}", CDH_PATH, e))?;
+
+    let cdh_client = CDHClient::new().context("Failed to create CDH Client")?;
+
+    // skip launch of api-server-rest
+    if config.guest_components_procs == GuestComponentsProcs::ConfidentialDataHub {
+        return Ok(Some(cdh_client));
     }
 
-    Ok(())
+    let features = config.guest_components_rest_api;
+    debug!(
+        logger,
+        "spawning api-server-rest process {} --features {}", API_SERVER_PATH, features
+    );
+    launch_process(
+        logger,
+        API_SERVER_PATH,
+        &vec!["--features", &features.to_string()],
+        "",
+        0,
+    )
+    .map_err(|e| anyhow!("launch_process {} failed: {:?}", API_SERVER_PATH, e))?;
+
+    Ok(Some(cdh_client))
 }
 
 fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> {

src/agent/src/rpc.rs

@@ -76,6 +76,8 @@ use crate::policy::{do_set_policy, is_allowed};
 #[cfg(feature = "guest-pull")]
 use crate::image;
 
+use crate::cdh::CDHClient;
+
 use opentelemetry::global;
 use tracing::span;
 use tracing_opentelemetry::OpenTelemetrySpanExt;
@@ -171,6 +173,7 @@ impl<T> OptionToTtrpcResult<T> for Option<T> {
 pub struct AgentService {
     sandbox: Arc<Mutex<Sandbox>>,
     init_mode: bool,
+    cdh_client: Option<CDHClient>,
 }
 
 impl AgentService {
@@ -210,11 +213,6 @@ impl AgentService {
             "receive createcontainer, storages: {:?}", &req.storages
         );
 
-        // In case of pulling image inside guest, we need to merge the image bundle OCI spec
-        // into the container creation request OCI spec.
-        #[cfg(feature = "guest-pull")]
-        image::merge_bundle_oci(&mut oci).await?;
-
         // Some devices need some extra processing (the ones invoked with
         // --device for instance), and that's what this call is doing. It
         // updates the devices listed in the OCI spec, so that they actually
@@ -222,6 +220,22 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&req.devices, &mut oci, &self.sandbox).await?;
 
+        if let Some(cdh) = self.cdh_client.as_ref() {
+            let process = oci
+                .process
+                .as_mut()
+                .ok_or_else(|| anyhow!("Spec didn't contain process field"))?;
+
+            for env in process.env.iter_mut() {
+                match cdh.unseal_env(env).await {
+                    Ok(unsealed_env) => *env = unsealed_env.to_string(),
+                    Err(e) => {
+                        warn!(sl(), "Failed to unseal secret: {}", e)
+                    }
+                }
+            }
+        }
+
         // Both rootfs and volumes (invoked with --volume for instance) will
         // be processed the same way. The idea is to always mount any provided
         // storage to the specified MountPoint, so that it will match what's
@@ -584,25 +598,32 @@ impl AgentService {
         let cid = req.container_id;
         let eid = req.exec_id;
 
-        let writer = {
-            let mut sandbox = self.sandbox.lock().await;
-            let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?;
-
-            // use ptmx io
-            if p.term_master.is_some() {
-                p.get_writer(StreamType::TermMaster)
-            } else {
-                // use piped io
-                p.get_writer(StreamType::ParentStdin)
-            }
-        };
-
-        let writer = writer.ok_or_else(|| anyhow!(ERR_CANNOT_GET_WRITER))?;
-        writer.lock().await.write_all(req.data.as_slice()).await?;
-
         let mut resp = WriteStreamResponse::new();
         resp.set_len(req.data.len() as u32);
 
+        // EOF of stdin
+        if req.data.is_empty() {
+            let mut sandbox = self.sandbox.lock().await;
+            let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?;
+            p.close_stdin().await;
+        } else {
+            let writer = {
+                let mut sandbox = self.sandbox.lock().await;
+                let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?;
+
+                // use ptmx io
+                if p.term_master.is_some() {
+                    p.get_writer(StreamType::TermMaster)
+                } else {
+                    // use piped io
+                    p.get_writer(StreamType::ParentStdin)
+                }
+            };
+
+            let writer = writer.ok_or_else(|| anyhow!(ERR_CANNOT_GET_WRITER))?;
+            writer.lock().await.write_all(req.data.as_slice()).await?;
+        }
+
         Ok(resp)
     }
 
@@ -645,6 +666,7 @@ impl AgentService {
             biased;
             v = read_stream(&reader, req.len as usize)  => {
                 let vector = v?;
+
                 let mut resp = ReadStreamResponse::new();
                 resp.set_data(vector);
 
@@ -845,6 +867,9 @@ impl agent_ttrpc::AgentService for AgentService {
         ctx: &TtrpcContext,
         req: protocols::agent::CloseStdinRequest,
     ) -> ttrpc::Result<Empty> {
+        // The stdin will be closed when EOF is got in rpc `write_stdin`[runtime-rs]
+        // so this rpc will not be called anymore by runtime-rs.
+
         trace_rpc_call!(ctx, "close_stdin", req);
         is_allowed(&req).await?;
 
@@ -1601,10 +1626,12 @@ pub async fn start(
     s: Arc<Mutex<Sandbox>>,
     server_address: &str,
     init_mode: bool,
+    cdh_client: Option<CDHClient>,
 ) -> Result<TtrpcServer> {
     let agent_service = Box::new(AgentService {
         sandbox: s,
         init_mode,
+        cdh_client,
     }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
     let aservice = agent_ttrpc::create_agent_service(Arc::new(agent_service));
 
@@ -1920,21 +1947,28 @@ pub fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
         return Err(anyhow!(nix::Error::EINVAL));
     };
 
-    let spec_root_path = Path::new(&spec_root.path);
-
     let bundle_path = Path::new(CONTAINER_BASE).join(cid);
     let config_path = bundle_path.join("config.json");
     let rootfs_path = bundle_path.join("rootfs");
+    let spec_root_path = Path::new(&spec_root.path);
 
-    fs::create_dir_all(&rootfs_path)?;
-    baremount(
-        spec_root_path,
-        &rootfs_path,
-        "bind",
-        MsFlags::MS_BIND,
-        "",
-        &sl(),
-    )?;
+    let rootfs_exists = Path::new(&rootfs_path).exists();
+    info!(
+        sl(),
+        "The rootfs_path is {:?} and exists: {}", rootfs_path, rootfs_exists
+    );
+
+    if !rootfs_exists {
+        fs::create_dir_all(&rootfs_path)?;
+        baremount(
+            spec_root_path,
+            &rootfs_path,
+            "bind",
+            MsFlags::MS_BIND,
+            "",
+            &sl(),
+        )?;
+    }
 
     let rootfs_path_name = rootfs_path
         .to_str()
@@ -2148,6 +2182,7 @@ mod tests {
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            cdh_client: None,
         });
 
         let req = protocols::agent::UpdateInterfaceRequest::default();
@@ -2162,10 +2197,10 @@ mod tests {
     async fn test_update_routes() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let sandbox = Sandbox::new(&logger).unwrap();
-
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            cdh_client: None,
         });
 
         let req = protocols::agent::UpdateRoutesRequest::default();
@@ -2180,10 +2215,10 @@ mod tests {
     async fn test_add_arp_neighbors() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let sandbox = Sandbox::new(&logger).unwrap();
-
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            cdh_client: None,
         });
 
         let req = protocols::agent::AddARPNeighborsRequest::default();
@@ -2322,6 +2357,7 @@ mod tests {
             let agent_service = Box::new(AgentService {
                 sandbox: Arc::new(Mutex::new(sandbox)),
                 init_mode: true,
+                cdh_client: None,
             });
 
             let result = agent_service
@@ -2811,6 +2847,7 @@ OtherField:other
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            cdh_client: None,
         });
 
         let ctx = mk_ttrpc_context();

src/agent/src/storage/image_pull_handler.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use super::new_device;
 use crate::image;
 use crate::storage::{StorageContext, StorageHandler};
 use anyhow::{anyhow, Result};
@@ -12,8 +13,6 @@ use protocols::agent::Storage;
 use std::sync::Arc;
 use tracing::instrument;
 
-use super::{common_storage_handler, new_device};
-
 #[derive(Debug)]
 pub struct ImagePullHandler {}
 
@@ -36,7 +35,7 @@ impl StorageHandler for ImagePullHandler {
     #[instrument]
     async fn create_device(
         &self,
-        mut storage: Storage,
+        storage: Storage,
         ctx: &mut StorageContext,
     ) -> Result<Arc<dyn StorageDevice>> {
         //Currently the image metadata is not used to pulling image in the guest.
@@ -51,12 +50,7 @@ impl StorageHandler for ImagePullHandler {
             .ok_or_else(|| anyhow!("failed to get container id"))?;
         let bundle_path = image::pull_image(image_name, &cid, &image_pull_volume.metadata).await?;
 
-        storage.source = bundle_path;
-        storage.options = vec!["bind".to_string(), "ro".to_string()];
-
-        common_storage_handler(ctx.logger, &storage)?;
-
-        new_device(storage.mount_point)
+        new_device(bundle_path)
     }
 }
 

src/dragonball/src/address_space_manager.rs

@@ -644,10 +644,7 @@ impl AddressSpaceMgr {
         guest_numa_node_id: u32,
         vcpu_ids: &[u32],
     ) {
-        let node = self
-            .numa_nodes
-            .entry(guest_numa_node_id)
-            .or_insert_with(NumaNode::new);
+        let node = self.numa_nodes.entry(guest_numa_node_id).or_default();
         node.add_info(&NumaNodeInfo {
             base: region.start_addr(),
             size: region.len(),

src/dragonball/src/api/v1/vmm_action.rs

@@ -281,6 +281,8 @@ pub enum VmmData {
     MachineConfiguration(Box<VmConfigInfo>),
     /// Prometheus Metrics represented by String.
     HypervisorMetrics(String),
+    /// Return vfio device's slot number in guest.
+    VfioDeviceData(Option<u8>),
     /// Sync Hotplug
     SyncHotplug((Sender<Option<i32>>, Receiver<Option<i32>>)),
 }
@@ -398,7 +400,9 @@ impl VmmService {
                 self.add_balloon_device(vmm, event_mgr, balloon_cfg)
             }
             #[cfg(feature = "host-device")]
-            VmmAction::InsertHostDevice(hostdev_cfg) => self.add_vfio_device(vmm, hostdev_cfg),
+            VmmAction::InsertHostDevice(mut hostdev_cfg) => {
+                self.add_vfio_device(vmm, &mut hostdev_cfg)
+            }
             #[cfg(feature = "host-device")]
             VmmAction::PrepareRemoveHostDevice(hostdev_id) => {
                 self.prepare_remove_vfio_device(vmm, &hostdev_id)
@@ -850,7 +854,7 @@ impl VmmService {
     }
 
     #[cfg(feature = "host-device")]
-    fn add_vfio_device(&self, vmm: &mut Vmm, config: HostDeviceConfig) -> VmmRequestResult {
+    fn add_vfio_device(&self, vmm: &mut Vmm, config: &mut HostDeviceConfig) -> VmmRequestResult {
         let vm = vmm.get_vm_mut().ok_or(VmmActionError::HostDeviceConfig(
             VfioDeviceError::InvalidVMID,
         ))?;
@@ -873,7 +877,8 @@ impl VmmService {
             .unwrap()
             .insert_device(&mut ctx, config)
             .map_err(VmmActionError::HostDeviceConfig)?;
-        Ok(VmmData::Empty)
+
+        Ok(VmmData::VfioDeviceData(config.dev_config.guest_dev_id))
     }
 
     // using upcall to unplug the pci device in the guest

src/dragonball/src/device_manager/mod.rs

@@ -424,7 +424,7 @@ impl DeviceOpContext {
     fn generate_virtio_device_info(&self) -> Result<HashMap<(DeviceType, String), MMIODeviceInfo>> {
         let mut dev_info = HashMap::new();
         #[cfg(feature = "dbs-virtio-devices")]
-        for (_index, device) in self.virtio_devices.iter().enumerate() {
+        for device in self.virtio_devices.iter() {
             let (mmio_base, mmio_size, irq) = DeviceManager::get_virtio_mmio_device_info(device)?;
             let dev_type;
             let device_id;
@@ -553,7 +553,7 @@ impl DeviceOpContext {
         &self,
         dev: &Arc<dyn DeviceIo>,
         callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>>,
-    ) -> Result<()> {
+    ) -> Result<u8> {
         if !self.is_hotplug || !self.pci_hotplug_enabled {
             return Err(DeviceMgrError::InvalidOperation);
         }
@@ -561,7 +561,12 @@ impl DeviceOpContext {
         let (busno, devfn) = DeviceManager::get_pci_device_info(dev)?;
         let req = DevMgrRequest::AddPciDev(PciDevRequest { busno, devfn });
 
-        self.call_hotplug_device(req, callback)
+        self.call_hotplug_device(req, callback)?;
+
+        // Extract the slot number from devfn
+        // Right shift by 3 to remove function bits (2:0) and
+        // align slot bits (7:3) to the least significant position
+        Ok(devfn >> 3)
     }
 
     #[cfg(feature = "host-device")]

src/dragonball/src/device_manager/vfio_dev_mgr/mod.rs

@@ -255,7 +255,7 @@ impl VfioDeviceMgr {
     pub fn insert_device(
         &mut self,
         ctx: &mut DeviceOpContext,
-        config: HostDeviceConfig,
+        config: &mut HostDeviceConfig,
     ) -> Result<()> {
         if !cfg!(feature = "hotplug") && ctx.is_hotplug {
             return Err(VfioDeviceError::UpdateNotAllowedPostBoot);
@@ -267,7 +267,7 @@ impl VfioDeviceMgr {
             "hostdev_id" => &config.hostdev_id,
             "bdf" => &config.dev_config.bus_slot_func,
         );
-        let device_index = self.info_list.insert_or_update(&config)?;
+        let device_index = self.info_list.insert_or_update(config)?;
         // Handle device hotplug case
         if ctx.is_hotplug {
             slog::info!(
@@ -277,7 +277,7 @@ impl VfioDeviceMgr {
                 "hostdev_id" => &config.hostdev_id,
                 "bdf" => &config.dev_config.bus_slot_func,
             );
-            self.add_device(ctx, &config, device_index)?;
+            self.add_device(ctx, config, device_index)?;
         }
 
         Ok(())
@@ -438,7 +438,7 @@ impl VfioDeviceMgr {
     fn add_device(
         &mut self,
         ctx: &mut DeviceOpContext,
-        cfg: &HostDeviceConfig,
+        cfg: &mut HostDeviceConfig,
         idx: usize,
     ) -> Result<()> {
         let dev = self.create_device(cfg, ctx, idx)?;
@@ -450,8 +450,13 @@ impl VfioDeviceMgr {
 
             self.register_memory(vm_memory.deref())?;
         }
-        ctx.insert_hotplug_pci_device(&dev, None)
-            .map_err(VfioDeviceError::VfioDeviceMgr)
+        let slot = ctx
+            .insert_hotplug_pci_device(&dev, None)
+            .map_err(VfioDeviceError::VfioDeviceMgr)?;
+
+        cfg.dev_config.guest_dev_id = Some(slot);
+
+        Ok(())
     }
 
     /// Gets the index of the device with the specified `hostdev_id` if it exists in the list.

src/libs/Cargo.lock

@@ -2,6 +2,21 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "addr2line"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
 [[package]]
 name = "ahash"
 version = "0.7.7"
@@ -48,7 +63,7 @@ checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -68,6 +83,21 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "backtrace"
+version = "0.3.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -87,7 +117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2"
 dependencies = [
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -122,7 +152,7 @@ dependencies = [
  "borsh-schema-derive-internal",
  "proc-macro-crate",
  "proc-macro2",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -133,7 +163,7 @@ checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -144,7 +174,7 @@ checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -183,7 +213,7 @@ checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -198,6 +228,12 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
 
+[[package]]
+name = "cc"
+version = "1.0.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -328,7 +364,7 @@ dependencies = [
  "ident_case",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -339,7 +375,7 @@ checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
 dependencies = [
  "darling_core",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -350,7 +386,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -474,7 +510,7 @@ checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -518,6 +554,12 @@ dependencies = [
  "wasi 0.10.2+wasi-snapshot-preview1",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
+
 [[package]]
 name = "glob"
 version = "0.3.0"
@@ -613,7 +655,7 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "socket2",
+ "socket2 0.4.7",
  "tokio",
  "tower-service",
  "tracing",
@@ -748,9 +790,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.151"
+version = "0.2.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
 
 [[package]]
 name = "lock_api"
@@ -811,26 +853,23 @@ dependencies = [
 ]
 
 [[package]]
-name = "mio"
-version = "0.8.2"
+name = "miniz_oxide"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9"
+checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae"
 dependencies = [
- "libc",
- "log",
- "miow",
- "ntapi 0.3.7",
- "wasi 0.11.0+wasi-snapshot-preview1",
- "winapi",
+ "adler",
 ]
 
 [[package]]
-name = "miow"
-version = "0.3.7"
+name = "mio"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
 dependencies = [
- "winapi",
+ "libc",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -876,15 +915,6 @@ dependencies = [
  "pin-utils",
 ]
 
-[[package]]
-name = "ntapi"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "ntapi"
 version = "0.4.1"
@@ -932,6 +962,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "object"
+version = "0.36.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "oci"
 version = "0.1.0"
@@ -944,9 +983,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.17.1"
+version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
 [[package]]
 name = "parking_lot"
@@ -1000,14 +1039,14 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.8"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c"
+checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
 
 [[package]]
 name = "pin-utils"
@@ -1032,11 +1071,11 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.37"
+version = "1.0.85"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
+checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
@@ -1077,7 +1116,7 @@ dependencies = [
  "itertools",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1186,14 +1225,14 @@ checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.18"
+version = "1.0.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
 dependencies = [
  "proc-macro2",
 ]
@@ -1334,7 +1373,7 @@ checksum = "b5c462a1328c8e67e4d6dbad1eb0355dd43e8ab432c6e227a43657f16ade5033"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1353,6 +1392,12 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
+
 [[package]]
 name = "rustversion"
 version = "1.0.12"
@@ -1402,7 +1447,7 @@ checksum = "6eb8ec7724e4e524b2492b510e66957fe1a2c76c26a6975ec80823f2439da685"
 dependencies = [
  "darling_core",
  "serde-rename-rule",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1415,7 +1460,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde-attributes",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1432,7 +1477,7 @@ checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1465,7 +1510,7 @@ checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1565,6 +1610,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "socket2"
+version = "0.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c"
+dependencies = [
+ "libc",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "subprocess"
 version = "0.2.9"
@@ -1587,18 +1642,29 @@ dependencies = [
 ]
 
 [[package]]
-name = "sysinfo"
-version = "0.29.11"
+name = "syn"
+version = "2.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666"
+checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "sysinfo"
+version = "0.30.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "732ffa00f53e6b2af46208fba5718d9662a421049204e156328b66791ffa15ae"
 dependencies = [
  "cfg-if",
  "core-foundation-sys",
  "libc",
- "ntapi 0.4.1",
+ "ntapi",
  "once_cell",
  "rayon",
- "winapi",
+ "windows",
 ]
 
 [[package]]
@@ -1662,7 +1728,7 @@ checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1730,30 +1796,30 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.17.0"
+version = "1.38.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"
+checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a"
 dependencies = [
+ "backtrace",
  "bytes",
  "libc",
- "memchr",
  "mio",
  "num_cpus",
  "pin-project-lite",
- "socket2",
+ "socket2 0.5.7",
  "tokio-macros",
- "winapi",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "1.7.0"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
+checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.66",
 ]
 
 [[package]]
@@ -1828,7 +1894,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tokio-vsock",
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -1858,6 +1924,12 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "unicode-ident"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+
 [[package]]
 name = "unicode-segmentation"
 version = "1.9.0"
@@ -1941,7 +2013,7 @@ dependencies = [
  "log",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
  "wasm-bindgen-shared",
 ]
 
@@ -1963,7 +2035,7 @@ checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -2007,13 +2079,41 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
+dependencies = [
+ "windows-core",
+ "windows-targets 0.52.5",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
+dependencies = [
+ "windows-targets 0.52.5",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.5",
 ]
 
 [[package]]
@@ -2022,13 +2122,29 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
 dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.5",
+ "windows_aarch64_msvc 0.52.5",
+ "windows_i686_gnu 0.52.5",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc 0.52.5",
+ "windows_x86_64_gnu 0.52.5",
+ "windows_x86_64_gnullvm 0.52.5",
+ "windows_x86_64_msvc 0.52.5",
 ]
 
 [[package]]
@@ -2037,42 +2153,90 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
+
 [[package]]
 name = "wyz"
 version = "0.5.1"

src/libs/kata-types/src/capabilities.rs

@@ -127,7 +127,7 @@ mod tests {
         assert!(cap.is_fs_sharing_supported());
 
         // test set hybrid-vsock support
-        cap.set(CapabilityBits::HybridVsockSupport);
+        cap.add(CapabilityBits::HybridVsockSupport);
         assert!(cap.is_hybrid_vsock_supported());
         // test append capabilities
         cap.add(CapabilityBits::GuestMemoryProbe);

src/libs/kata-types/src/config/default.rs

@@ -88,3 +88,13 @@ pub const DEFAULT_CH_PCI_BRIDGES: u32 = 2;
 pub const MAX_CH_PCI_BRIDGES: u32 = 5;
 pub const MAX_CH_VCPUS: u32 = 256;
 pub const MIN_CH_MEMORY_SIZE_MB: u32 = 64;
+
+//Default configuration for firecracker
+pub const DEFAULT_FIRECRACKER_ENTROPY_SOURCE: &str = "/dev/urandom";
+pub const DEFAULT_FIRECRACKER_MEMORY_SIZE_MB: u32 = 128;
+pub const DEFAULT_FIRECRACKER_MEMORY_SLOTS: u32 = 128;
+pub const DEFAULT_FIRECRACKER_VCPUS: u32 = 1;
+pub const DEFAULT_FIRECRACKER_GUEST_KERNEL_IMAGE: &str = "vmlinux";
+pub const DEFAULT_FIRECRACKER_GUEST_KERNEL_PARAMS: &str = "";
+pub const MAX_FIRECRACKER_VCPUS: u32 = 32;
+pub const MIN_FIRECRACKER_MEMORY_SIZE_MB: u32 = 128;

src/libs/kata-types/src/config/hypervisor/firecracker.rs

@@ -0,0 +1,116 @@
+// Copyright (c) 2019-2021 Alibaba Cloud
+// Copyright (c) 2022-2023 Nubificus LTD
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::io::Result;
+use std::path::Path;
+use std::sync::Arc;
+
+use super::{default, register_hypervisor_plugin};
+
+use crate::config::default::MAX_FIRECRACKER_VCPUS;
+use crate::config::default::MIN_FIRECRACKER_MEMORY_SIZE_MB;
+
+use crate::config::{ConfigPlugin, TomlConfig};
+use crate::{eother, validate_path};
+
+/// Hypervisor name for firecracker, used to index `TomlConfig::hypervisor`.
+pub const HYPERVISOR_NAME_FIRECRACKER: &str = "firecracker";
+
+/// Configuration information for firecracker.
+#[derive(Default, Debug)]
+pub struct FirecrackerConfig {}
+
+impl FirecrackerConfig {
+    /// Create a new instance of `FirecrackerConfig`.
+    pub fn new() -> Self {
+        FirecrackerConfig {}
+    }
+
+    /// Register the firecracker plugin.
+    pub fn register(self) {
+        let plugin = Arc::new(self);
+        register_hypervisor_plugin(HYPERVISOR_NAME_FIRECRACKER, plugin);
+    }
+}
+
+impl ConfigPlugin for FirecrackerConfig {
+    fn get_max_cpus(&self) -> u32 {
+        MAX_FIRECRACKER_VCPUS
+    }
+
+    fn get_min_memory(&self) -> u32 {
+        MIN_FIRECRACKER_MEMORY_SIZE_MB
+    }
+
+    fn name(&self) -> &str {
+        HYPERVISOR_NAME_FIRECRACKER
+    }
+
+    /// Adjust the configuration information after loading from configuration file.
+    fn adjust_config(&self, conf: &mut TomlConfig) -> Result<()> {
+        if let Some(firecracker) = conf.hypervisor.get_mut(HYPERVISOR_NAME_FIRECRACKER) {
+            if firecracker.boot_info.kernel.is_empty() {
+                firecracker.boot_info.kernel =
+                    default::DEFAULT_FIRECRACKER_GUEST_KERNEL_IMAGE.to_string();
+            }
+            if firecracker.boot_info.kernel_params.is_empty() {
+                firecracker.boot_info.kernel_params =
+                    default::DEFAULT_FIRECRACKER_GUEST_KERNEL_PARAMS.to_string();
+            }
+            if firecracker.machine_info.entropy_source.is_empty() {
+                firecracker.machine_info.entropy_source =
+                    default::DEFAULT_FIRECRACKER_ENTROPY_SOURCE.to_string();
+            }
+
+            if firecracker.memory_info.default_memory == 0 {
+                firecracker.memory_info.default_memory =
+                    default::DEFAULT_FIRECRACKER_MEMORY_SIZE_MB;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Validate the configuration information.
+    fn validate(&self, conf: &TomlConfig) -> Result<()> {
+        if let Some(firecracker) = conf.hypervisor.get(HYPERVISOR_NAME_FIRECRACKER) {
+            if firecracker.path.is_empty() {
+                return Err(eother!("Firecracker path is empty"));
+            }
+            validate_path!(
+                firecracker.path,
+                "FIRECRACKER binary path `{}` is invalid: {}"
+            )?;
+            if firecracker.boot_info.kernel.is_empty() {
+                return Err(eother!("Guest kernel image for firecracker is empty"));
+            }
+            if firecracker.boot_info.image.is_empty() {
+                return Err(eother!(
+                    "Both guest boot image and initrd for firecracker are empty"
+                ));
+            }
+
+            if (firecracker.cpu_info.default_vcpus > 0
+                && firecracker.cpu_info.default_vcpus as u32 > default::MAX_FIRECRACKER_VCPUS)
+                || firecracker.cpu_info.default_maxvcpus > default::MAX_FIRECRACKER_VCPUS
+            {
+                return Err(eother!(
+                    "Firecracker hypervisor can not support {} vCPUs",
+                    firecracker.cpu_info.default_maxvcpus
+                ));
+            }
+
+            if firecracker.memory_info.default_memory < MIN_FIRECRACKER_MEMORY_SIZE_MB {
+                return Err(eother!(
+                    "Firecracker hypervisor has minimal memory limitation {}",
+                    MIN_FIRECRACKER_MEMORY_SIZE_MB
+                ));
+            }
+        }
+
+        Ok(())
+    }
+}

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -59,6 +59,9 @@ pub const VIRTIO_SCSI: &str = "virtio-scsi";
 /// Virtual PMEM device driver.
 pub const VIRTIO_PMEM: &str = "virtio-pmem";
 
+mod firecracker;
+pub use self::firecracker::{FirecrackerConfig, HYPERVISOR_NAME_FIRECRACKER};
+
 const VIRTIO_9P: &str = "virtio-9p";
 const VIRTIO_FS: &str = "virtio-fs";
 const VIRTIO_FS_INLINE: &str = "inline-virtio-fs";
@@ -411,6 +414,20 @@ pub struct DebugInfo {
     /// much disk space.
     #[serde(default)]
     pub guest_memory_dump_path: String,
+
+    /// This option allows to add a debug monitor socket when `enable_debug = true`
+    /// WARNING: Anyone with access to the monitor socket can take full control of
+    /// Qemu. This is for debugging purpose only and must *NEVER* be used in
+    /// production.
+    /// Valid values are :
+    /// - "hmp"
+    /// - "qmp"
+    /// - "qmp-pretty" (same as "qmp" with pretty json formatting)
+    /// If set to the empty string "", no debug monitor socket is added. This is
+    /// the default.
+    /// dbg_monitor_socket = "hmp"
+    #[serde(default)]
+    pub dbg_monitor_socket: String,
 }
 
 impl DebugInfo {
@@ -516,6 +533,7 @@ impl TopologyConfigInfo {
             HYPERVISOR_NAME_QEMU,
             HYPERVISOR_NAME_CH,
             HYPERVISOR_NAME_DRAGONBALL,
+            HYPERVISOR_NAME_FIRECRACKER,
         ];
         let hypervisor_name = toml_config.runtime.hypervisor_name.as_str();
         if !hypervisor_names.contains(&hypervisor_name) {

src/libs/kata-types/src/config/mod.rs

@@ -25,8 +25,8 @@ pub mod hypervisor;
 pub use self::agent::Agent;
 use self::default::DEFAULT_AGENT_DBG_CONSOLE_PORT;
 pub use self::hypervisor::{
-    BootInfo, CloudHypervisorConfig, DragonballConfig, Hypervisor, QemuConfig,
-    HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_QEMU,
+    BootInfo, CloudHypervisorConfig, DragonballConfig, FirecrackerConfig, Hypervisor, QemuConfig,
+    HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_FIRECRACKER, HYPERVISOR_NAME_QEMU,
 };
 
 mod runtime;

src/libs/kata-types/src/k8s.rs

@@ -130,7 +130,11 @@ fn count_files<P: AsRef<Path>>(path: P, limit: i32) -> std::io::Result<i32> {
         let file = entry?;
         let p = file.path();
         if p.is_dir() {
-            num_files += count_files(&p, limit)?;
+            let inc = count_files(&p, limit - num_files)?;
+            if inc == -1 {
+                return Ok(-1);
+            }
+            num_files += inc;
         } else {
             num_files += 1;
         }
@@ -165,6 +169,40 @@ mod tests {
     use std::fs;
     use test_utils::skip_if_not_root;
 
+    #[test]
+    fn test_count_files() {
+        let limit = 8;
+        let test_tmp_dir = tempfile::tempdir().expect("failed to create tempdir");
+        let work_path = test_tmp_dir.path().join("work");
+
+        let result = fs::create_dir_all(&work_path);
+        assert!(result.is_ok());
+
+        let origin_dir = work_path.join("origin_dir");
+        let result = fs::create_dir_all(&origin_dir);
+        assert!(result.is_ok());
+        for n in 0..limit {
+            let tmp_file = origin_dir.join(format!("file{}", n));
+            let res = fs::File::create(tmp_file);
+            assert!(res.is_ok());
+        }
+
+        let symlink_origin_dir = work_path.join("symlink_origin_dir");
+        let result = std::os::unix::fs::symlink(&origin_dir, &symlink_origin_dir);
+        assert!(result.is_ok());
+        for n in 0..2 {
+            let tmp_file = work_path.join(format!("file{}", n));
+            let res = fs::File::create(tmp_file);
+            assert!(res.is_ok());
+        }
+
+        let count = count_files(&work_path, limit).unwrap_or(0);
+        assert_eq!(count, -1);
+
+        let count = count_files(&origin_dir, limit).unwrap_or(0);
+        assert_eq!(count, limit);
+    }
+
     #[test]
     fn test_is_watchable_mount() {
         skip_if_not_root!();

src/libs/kata-types/tests/texture/configuration-anno-0.toml

@@ -84,6 +84,7 @@ sandbox_bind_mounts=["/proc/self"]
 vfio_mode="vfio"
 experimental=["a", "b"]
 enable_pprof = true
+name="virt-container"
 hypervisor_name = "qemu"
 agent_name = "agent0"
 

src/libs/kata-types/tests/texture/configuration-anno-1.toml

@@ -83,6 +83,7 @@ sandbox_bind_mounts=["/proc/self"]
 vfio_mode="vfio"
 experimental=["a", "b"]
 enable_pprof = true
+name="virt-container"
 hypervisor_name = "qemu"
 agent_name = "agent0"
 

src/libs/protocols/build.rs

@@ -198,13 +198,34 @@ fn real_main() -> Result<(), std::io::Error> {
     // generate async
     #[cfg(feature = "async")]
     {
-        codegen("src", &["protos/agent.proto", "protos/health.proto"], true)?;
+
+        codegen(
+            "src",
+            &[
+                "protos/agent.proto",
+                "protos/health.proto",
+                "protos/sealed_secret.proto",
+            ],
+            true,
+        )?;
 
         fs::rename("src/agent_ttrpc.rs", "src/agent_ttrpc_async.rs")?;
         fs::rename("src/health_ttrpc.rs", "src/health_ttrpc_async.rs")?;
+        fs::rename(
+            "src/sealed_secret_ttrpc.rs",
+            "src/sealed_secret_ttrpc_async.rs",
+        )?;
     }
 
-    codegen("src", &["protos/agent.proto", "protos/health.proto"], false)?;
+    codegen(
+        "src",
+        &[
+            "protos/agent.proto",
+            "protos/health.proto",
+            "protos/sealed_secret.proto",
+        ],
+        false,
+    )?;
 
     // There is a message named 'Box' in oci.proto
     // so there is a struct named 'Box', we should replace Box<Self> to ::std::boxed::Box<Self>

src/libs/protocols/protos/sealed_secret.proto

@@ -0,0 +1,21 @@
+//
+// Copyright (c) 2024 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+syntax = "proto3";
+
+package api;
+
+message UnsealSecretInput {
+    bytes secret = 1;
+}
+
+message UnsealSecretOutput {
+    bytes plaintext = 1;
+}
+
+service SealedSecretService {
+    rpc UnsealSecret(UnsealSecretInput) returns (UnsealSecretOutput) {};
+}

src/libs/protocols/src/lib.rs

@@ -27,3 +27,9 @@ pub use serde_config::{
     deserialize_enum_or_unknown, deserialize_message_field, serialize_enum_or_unknown,
     serialize_message_field,
 };
+
+pub mod sealed_secret;
+pub mod sealed_secret_ttrpc;
+
+#[cfg(feature = "async")]
+pub mod sealed_secret_ttrpc_async;

src/libs/shim-interface/Cargo.toml

@@ -14,12 +14,12 @@ edition = "2018"
 [dependencies]
 anyhow = "^1.0"
 nix = "0.24.0"
-tokio = { version = "1.8.0", features = ["rt-multi-thread"] }
+tokio = { version = "1.38.0", features = ["rt-multi-thread"] }
 hyper = { version = "0.14.20", features = ["stream", "server", "http1"] }
 hyperlocal = "0.8"
 kata-types = { path = "../kata-types" }
-kata-sys-util = {path = "../kata-sys-util" }
+kata-sys-util = { path = "../kata-sys-util" }
 
 [dev-dependencies]
 tempfile = "3.2.0"
-test-utils = {path = "../test-utils"}
+test-utils = { path = "../test-utils" }

src/libs/shim-interface/src/lib.rs

@@ -37,6 +37,9 @@ fn get_uds_with_sid(short_id: &str, path: &str) -> Result<String> {
         return Ok(format!("unix://{}", p.display()));
     }
 
+    let _ = fs::create_dir_all(kata_run_path.join(short_id))
+            .context(format!("failed to create directory {:?}", kata_run_path.join(short_id)));
+
     let target_ids: Vec<String> = fs::read_dir(&kata_run_path)?
         .filter_map(|e| {
             let x = e.ok()?.file_name().to_string_lossy().into_owned();

src/runtime-rs/Cargo.lock

@@ -185,7 +185,7 @@ dependencies = [
  "polling",
  "rustix 0.37.23",
  "slab",
- "socket2",
+ "socket2 0.4.9",
  "waker-fn",
 ]
 
@@ -1589,7 +1589,7 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "socket2",
+ "socket2 0.4.9",
  "tokio",
  "tower-service",
  "tracing",
@@ -1635,6 +1635,8 @@ dependencies = [
  "dragonball",
  "futures 0.3.28",
  "go-flag",
+ "hyper",
+ "hyperlocal",
  "hypervisor",
  "kata-sys-util",
  "kata-types",
@@ -1644,6 +1646,9 @@ dependencies = [
  "nix 0.24.3",
  "path-clean",
  "persist",
+ "qapi",
+ "qapi-qmp",
+ "qapi-spec",
  "rand 0.8.5",
  "rust-ini",
  "safe-path 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2034,9 +2039,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.8.8"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
 dependencies = [
  "libc",
  "log",
@@ -2699,9 +2704,9 @@ dependencies = [
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.10"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57"
+checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
 
 [[package]]
 name = "pin-utils"
@@ -2959,6 +2964,65 @@ dependencies = [
  "ttrpc-codegen",
 ]
 
+[[package]]
+name = "qapi"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6412bdd014ebee03ddbbe79ac03a0b622cce4d80ba45254f6357c847f06fa38"
+dependencies = [
+ "bytes",
+ "futures 0.3.28",
+ "log",
+ "memchr",
+ "qapi-qmp",
+ "qapi-spec",
+ "serde",
+ "serde_json",
+ "tokio",
+ "tokio-util",
+]
+
+[[package]]
+name = "qapi-codegen"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ba4de731473de4c8bd508ddb38a9049e999b8a7429f3c052ba8735a178ff68c"
+dependencies = [
+ "qapi-parser",
+]
+
+[[package]]
+name = "qapi-parser"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80044db145aa2953ef5803d0376dcbca50f2763242547e856b7f37507adca677"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "qapi-qmp"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8b944db7e544d2fa97595e9a000a6ba5c62c426fa185e7e00aabe4b5640b538"
+dependencies = [
+ "qapi-codegen",
+ "qapi-spec",
+ "serde",
+]
+
+[[package]]
+name = "qapi-spec"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b360919a24ea5fc02fa762cb01bd8f43b643fee51c585f763257773b4dc5a9e8"
+dependencies = [
+ "base64 0.13.1",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.35"
@@ -3825,6 +3889,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "socket2"
+version = "0.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c"
+dependencies = [
+ "libc",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
@@ -4099,11 +4173,10 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.29.1"
+version = "1.38.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
+checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a"
 dependencies = [
- "autocfg",
  "backtrace",
  "bytes",
  "libc",
@@ -4112,16 +4185,16 @@ dependencies = [
  "parking_lot 0.12.1",
  "pin-project-lite",
  "signal-hook-registry",
- "socket2",
+ "socket2 0.5.7",
  "tokio-macros",
  "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "2.1.0"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a"
 dependencies = [
  "proc-macro2",
  "quote",

src/runtime-rs/Makefile

@@ -109,6 +109,12 @@ ROOTFSTYPE_XFS := \"xfs\"
 ROOTFSTYPE_EROFS := \"erofs\"
 DEFROOTFSTYPE := $(ROOTFSTYPE_EXT4)
 
+FCBINDIR := $(PREFIXDEPS)/bin
+FCPATH = $(FCBINDIR)/$(FCCMD)
+FCVALIDHYPERVISORPATHS := [\"$(FCPATH)\"]
+FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD)
+FCVALIDJAILERPATHS = [\"$(FCJAILERPATH)\"]
+
 PKGLIBEXECDIR := $(LIBEXECDIR)/$(PROJECT_DIR)
 FIRMWAREPATH :=
 FIRMWAREVOLUMEPATH :=
@@ -164,8 +170,11 @@ DEFMSIZE9P := 8192
 DEFVFIOMODE := guest-kernel
 ##VAR DEFSANDBOXCGROUPONLY=<bool> Default cgroup model
 DEFSANDBOXCGROUPONLY ?= false
+DEFSANDBOXCGROUPONLY_DB ?= true
+DEFSANDBOXCGROUPONLY_FC ?= true
 DEFSTATICRESOURCEMGMT ?= false
 DEFSTATICRESOURCEMGMT_DB ?= false
+DEFSTATICRESOURCEMGMT_FC ?= true
 DEFBINDMOUNTS := []
 DEFDANCONF := /run/kata-containers/dans
 SED = sed
@@ -208,18 +217,18 @@ ifneq (,$(DBCMD))
     SYSCONFIG_PATHS += $(SYSCONFIG_DB)
     CONFIGS += $(CONFIG_DB)
     # dragonball-specific options (all should be suffixed by "_DB")
-	VMROOTFSDRIVER_DB := virtio-blk-pci
+    VMROOTFSDRIVER_DB := virtio-blk-pci
     DEFMAXVCPUS_DB := 1
     DEFBLOCKSTORAGEDRIVER_DB := virtio-blk-mmio
     DEFNETWORKMODEL_DB := tcfilter
     KERNELPARAMS_DB = console=ttyS1 agent.log_vport=1025
-	KERNELTYPE_DB = uncompressed
+    KERNELTYPE_DB = uncompressed
     KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME_DB,$(KERNELTYPE_DB))
     KERNELPATH_DB = $(KERNELDIR)/$(KERNEL_NAME_DB)
-    DEFSANDBOXCGROUPONLY = true
+    DEFSANDBOXCGROUPONLY_DB = true
     RUNTIMENAME := virt_container
     PIPESIZE := 1
-	DBSHAREDFS := inline-virtio-fs
+    DBSHAREDFS := inline-virtio-fs
 endif
 
 ifneq (,$(CLHCMD))
@@ -244,6 +253,9 @@ ifneq (,$(CLHCMD))
     KERNEL_NAME_CLH = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_CLH))
     KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH)
     VMROOTFSDRIVER_CLH := virtio-pmem
+
+    DEFSTATICRESOURCEMGMT = true
+    DEFSANDBOXCGROUPONLY = true
 endif
 
 ifneq (,$(QEMUCMD))
@@ -288,6 +300,28 @@ endif
     DEFSECCOMPSANDBOXPARAM := on,obsolete=deny,spawn=deny,resourcecontrol=deny
     DEFGUESTSELINUXLABEL := system_u:system_r:container_t
 endif
+ifneq (,$(FCCMD))
+    KNOWN_HYPERVISORS += $(HYPERVISOR_FC)
+    CONFIG_FILE_FC = configuration-rs-fc.toml
+    CONFIG_FC = config/$(CONFIG_FILE_FC)
+    CONFIG_FC_IN = $(CONFIG_FC).in
+    CONFIG_PATH_FC = $(abspath $(CONFDIR)/$(CONFIG_FILE_FC))
+    CONFIG_PATHS += $(CONFIG_PATH_FC)
+    SYSCONFIG_FC = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_FC))
+    SYSCONFIG_PATHS += $(SYSCONFIG_FC)
+    CONFIGS += $(CONFIG_FC)
+    # firecracker-specific options (all should be suffixed by "_FC")
+    DEFBLOCKSTORAGEDRIVER_FC := virtio-blk-mmio
+    DEFMAXMEMSZ_FC := 2048
+    DEFNETWORKMODEL_FC := tcfilter
+    KERNELPARAMS = console=ttyS0 agent.log_vport=1025
+    KERNELTYPE_FC = uncompressed
+    KERNEL_NAME_FC = $(call MAKE_KERNEL_NAME_FC,$(KERNELTYPE_FC))
+    KERNELPATH_FC = $(KERNELDIR)/$(KERNEL_NAME_FC)
+    DEFSANDBOXCGROUPONLY_FC = true
+    RUNTIMENAME := virt_container
+    DEFSTATICRESOURCEMGMT_FC ?= true
+endif
 
 ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_DB))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_DB)
@@ -296,16 +330,21 @@ endif
 ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_QEMU))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_QEMU)
 endif
+ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_FC))
+    DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_FC)
+endif
 # list of variables the user may wish to override
 USER_VARS += ARCH
 USER_VARS += BINDIR
 USER_VARS += CONFIG_DB_IN
+USER_VARS += CONFIG_FC_IN
 USER_VARS += CONFIG_PATH
 USER_VARS += CONFIG_QEMU_IN
 USER_VARS += DESTDIR
 USER_VARS += DEFAULT_HYPERVISOR
 USER_VARS += DBCMD
 USER_VARS += DBCTLCMD
+USER_VARS += FCCTLCMD
 USER_VARS += DBPATH
 USER_VARS += DBVALIDHYPERVISORPATHS
 USER_VARS += DBCTLPATH
@@ -316,6 +355,13 @@ USER_VARS += QEMUPATH
 USER_VARS += QEMUVALIDHYPERVISORPATHS
 USER_VARS += FIRMWAREPATH_CLH
 USER_VARS += KERNELPATH_CLH
+USER_VARS += FCCMD
+USER_VARS += FCPATH
+USER_VARS += FCVALIDHYPERVISORPATHS
+USER_VARS += FCJAILERPATH
+USER_VARS += FCVALIDJAILERPATHS
+USER_VARS += FCVALIDJAILERPATHS
+USER_VARS += DEFMAXMEMSZ_FC
 USER_VARS += SYSCONFIG
 USER_VARS += IMAGENAME
 USER_VARS += IMAGEPATH
@@ -329,6 +375,8 @@ USER_VARS += KERNELDIR
 USER_VARS += KERNELTYPE
 USER_VARS += KERNELPATH_DB
 USER_VARS += KERNELPATH_QEMU
+USER_VARS += KERNELPATH_FC
+USER_VARS += KERNELPATH
 USER_VARS += KERNELVIRTIOFSPATH
 USER_VARS += FIRMWAREPATH
 USER_VARS += FIRMWAREVOLUMEPATH
@@ -365,6 +413,7 @@ USER_VARS += DEFBRIDGES
 USER_VARS += DEFNETWORKMODEL_DB
 USER_VARS += DEFNETWORKMODEL_CLH
 USER_VARS += DEFNETWORKMODEL_QEMU
+USER_VARS += DEFNETWORKMODEL_FC
 USER_VARS += DEFDISABLEGUESTEMPTYDIR
 USER_VARS += DEFDISABLEGUESTSECCOMP
 USER_VARS += DEFDISABLESELINUX
@@ -374,6 +423,7 @@ USER_VARS += DEFDISABLEBLOCK
 USER_VARS += DEFBLOCKSTORAGEDRIVER_DB
 USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU
 USER_VARS += DEFBLOCKDEVICEAIO_QEMU
+USER_VARS += DEFBLOCKSTORAGEDRIVER_FC
 USER_VARS += DEFSHAREDFS_CLH_VIRTIOFS
 USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS
 USER_VARS += DEFVIRTIOFSDAEMON
@@ -396,8 +446,11 @@ USER_VARS += DEFENTROPYSOURCE
 USER_VARS += DEFVALIDENTROPYSOURCES
 USER_VARS += DEFSANDBOXCGROUPONLY
 USER_VARS += DEFSANDBOXCGROUPONLY_QEMU
+USER_VARS += DEFSANDBOXCGROUPONLY_DB
+USER_VARS += DEFSANDBOXCGROUPONLY_FC
 USER_VARS += DEFSTATICRESOURCEMGMT
 USER_VARS += DEFSTATICRESOURCEMGMT_DB
+USER_VARS += DEFSTATICRESOURCEMGMT_FC
 USER_VARS += DEFBINDMOUNTS
 USER_VARS += DEFVFIOMODE
 USER_VARS += BUILDFLAGS
@@ -405,6 +458,7 @@ USER_VARS += RUNTIMENAME
 USER_VARS += HYPERVISOR_DB
 USER_VARS += HYPERVISOR_CLH
 USER_VARS += HYPERVISOR_QEMU
+USER_VARS += HYPERVISOR_FC
 USER_VARS += PIPESIZE
 USER_VARS += DBSHAREDFS
 USER_VARS += KATA_INSTALL_GROUP
@@ -417,7 +471,7 @@ SOURCES := \
   Cargo.toml
 
 VERSION_FILE := ./VERSION
-VERSION := $(shell grep -v ^\# $(VERSION_FILE))
+VERSION := $(shell grep -v ^\# $(VERSION_FILE) 2>/dev/null || echo "unknown")
 COMMIT_NO := $(shell git rev-parse HEAD 2>/dev/null || true)
 COMMIT := $(if $(shell git status --porcelain --untracked-files=no 2>/dev/null || true),${COMMIT_NO}-dirty,${COMMIT_NO})
 COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown)
@@ -442,6 +496,7 @@ RUNTIME_VERSION=$(VERSION)
 GENERATED_VARS = \
 		VERSION \
 		CONFIG_DB_IN \
+		CONFIG_FC_IN \
 		$(USER_VARS)
 
 
@@ -483,6 +538,9 @@ endef
 define MAKE_KERNEL_NAME_DB
 $(if $(findstring uncompressed,$1),vmlinux-dragonball-experimental.container,vmlinuz-dragonball-experimental.container)
 endef
+define MAKE_KERNEL_NAME_FC
+$(if $(findstring uncompressed,$1),vmlinux.container,vmlinuz.container)
+endef
 
 # Returns the name of the kernel file to use based on the provided KERNELTYPE.
 #   # $1 : KERNELTYPE (compressed or uncompressed)

src/runtime-rs/arch/aarch64-options.mk

@@ -13,3 +13,5 @@ QEMUCMD := qemu-system-aarch64
 
 # dragonball binary name
 DBCMD := dragonball
+FCCMD := firecracker
+FCJAILERCMD := jailer

src/runtime-rs/arch/x86_64-options.mk

@@ -16,3 +16,7 @@ DBCMD := dragonball
 
 # cloud-hypervisor binary name
 CLHCMD := cloud-hypervisor
+
+# firecracker binary (vmm and jailer)
+FCCMD := firecracker
+FCJAILERCMD := jailer

src/runtime-rs/config/configuration-cloud-hypervisor.toml.in

@@ -226,9 +226,18 @@ container_pipe_size=@PIPESIZE@
 
 #debug_console_enabled = true
 
-# Agent connection dialing timeout value in seconds
-# (default: 45)
-dial_timeout = 45
+# Agent dial timeout in millisecond.
+# (default: 10)
+#dial_timeout_ms = 10
+
+# Agent reconnect timeout in millisecond.
+# Retry times = reconnect_timeout_ms / dial_timeout_ms (default: 300)
+# If you find pod cannot connect to the agent when starting, please
+# consider increasing this value to increase the retry times.
+# You'd better not change the value of dial_timeout_ms, unless you have an
+# idea of what you are doing.
+# (default: 3000)
+#reconnect_timeout_ms = 3000
 
 [runtime]
 # If enabled, the runtime will log additional debug messages to the

src/runtime-rs/config/configuration-dragonball.toml.in

@@ -239,9 +239,18 @@ container_pipe_size=@PIPESIZE@
 
 #debug_console_enabled = true
 
-# Agent connection dialing timeout value in seconds
-# (default: 45)
-dial_timeout = 45
+# Agent dial timeout in millisecond.
+# (default: 10)
+#dial_timeout_ms = 10
+
+# Agent reconnect timeout in millisecond.
+# Retry times = reconnect_timeout_ms / dial_timeout_ms (default: 300)
+# If you find pod cannot connect to the agent when starting, please
+# consider increasing this value to increase the retry times.
+# You'd better not change the value of dial_timeout_ms, unless you have an
+# idea of what you are doing.
+# (default: 3000)
+#reconnect_timeout_ms = 3000
 
 [runtime]
 # If enabled, the runtime will log additional debug messages to the
@@ -332,7 +341,7 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 # The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
 # The sandbox cgroup is constrained if there is no container type annotation.
 # See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
-sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY_DB@
 
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,

src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in

@@ -346,7 +346,7 @@ pflashes = []
 #
 # If set to the empty string "", no extra monitor socket is added. This is
 # the default.
-#extra_monitor_socket = hmp
+#extra_monitor_socket = "hmp"
 
 # Disable the customizations done in the runtime when it detects
 # that it is running on top a VMM. This will result in the runtime
@@ -567,9 +567,18 @@ kernel_modules=[]
 
 #debug_console_enabled = true
 
-# Agent connection dialing timeout value in seconds
-# (default: 45)
-dial_timeout = 45
+# Agent dial timeout in millisecond.
+# (default: 10)
+#dial_timeout_ms = 10
+
+# Agent reconnect timeout in millisecond.
+# Retry times = reconnect_timeout_ms / dial_timeout_ms (default: 300)
+# If you find pod cannot connect to the agent when starting, please
+# consider increasing this value to increase the retry times.
+# You'd better not change the value of dial_timeout_ms, unless you have an
+# idea of what you are doing.
+# (default: 3000)
+#reconnect_timeout_ms = 3000
 
 [runtime]
 # If enabled, the runtime will log additional debug messages to the

src/runtime-rs/config/configuration-rs-fc.toml.in

@@ -0,0 +1,373 @@
+# Copyright (c) 2017-2023 Intel Corporation
+# Copyright (c) Adobe Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_FC_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.firecracker]
+path = "@FCPATH@"
+kernel = "@KERNELPATH_FC@"
+image = "@IMAGEPATH@"
+
+rootfs_type=@DEFROOTFSTYPE@
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+enable_annotations = @DEFENABLEANNOTATIONS@
+
+# List of valid annotations values for the hypervisor
+# Each member of the list is a path pattern as described by glob(3).
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @FCVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @FCVALIDHYPERVISORPATHS@
+
+# Path for the jailer specific to firecracker
+# If the jailer path is not set kata will launch firecracker
+# without a jail. If the jailer is set firecracker will be
+# launched in a jailed enviornment created by the jailer
+#jailer_path = "@FCJAILERPATH@"
+
+# List of valid jailer path values for the hypervisor
+# Each member of the list can be a regular expression
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @FCVALIDJAILERPATHS@
+valid_jailer_paths = @FCVALIDJAILERPATHS@
+
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+memory_slots = @DEFMEMSLOTS@
+
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM devie.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+#memory_offset = 0
+
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ_FC@
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_FC@"
+
+# Specifies cache-related options will be set to block devices or not.
+# Default false
+#block_device_cache_set = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
+# Default false
+#block_device_cache_direct = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether flush requests for the device are ignored.
+# Default false
+#block_device_cache_noflush = true
+
+# Enable pre allocation of VM RAM, default false
+# Enabling this will result in lower container density
+# as all of the memory will be allocated and locked
+# This is useful when you want to reserve all the memory
+# upfront or in the cases where you want memory latencies
+# to be very predictable
+# Default false
+#enable_mem_prealloc = true
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+# This is useful when you want to use vhost-user network
+# stacks within the container. This will automatically
+# result in memory pre allocation
+#enable_hugepages = true
+
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: intel_iommu=on,iommu=pt
+#enable_iommu = true
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available.
+#
+# Default false
+#enable_debug = true
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+#
+#disable_nesting_checks = true
+
+# This is the msize used for 9p shares. It is the number of bytes
+# used for 9p packet payload.
+#msize_9p = @DEFMSIZE9P@
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hotplugging on root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hotplugging on
+# a bridge.
+# Default false
+#hotplug_vfio_on_root_bus = true
+
+#
+# Default entropy source.
+# The path to a host source of entropy (including a real hardware RNG)
+# /dev/urandom and /dev/random are two main options.
+# Be aware that /dev/random is a blocking source of entropy.  If the host
+# runs out of entropy, the VMs boot time will increase leading to get startup
+# timeouts.
+# The source of entropy /dev/urandom is non-blocking and provides a
+# generally acceptable source of entropy. It should work well for pretty much
+# all practical purposes.
+#entropy_source= "@DEFENTROPYSOURCE@"
+
+# List of valid annotations values for entropy_source
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered will scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+#
+# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM).
+# In Firecracker, it provides a built-in rate limiter, which is based on TBF(Token Bucket Filter)
+# queueing discipline.
+# Default 0-sized value means unlimited rate.
+#rx_rate_limiter_max_rate = 0
+# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM).
+# In Firecracker, it provides a built-in rate limiter, which is based on TBF(Token Bucket Filter)
+# queueing discipline.
+# Default 0-sized value means unlimited rate.
+#tx_rate_limiter_max_rate = 0
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+[factory]
+# VM templating support. Once enabled, new VMs are created from template
+# using vm cloning. They will share the same initial kernel, initramfs and
+# agent memory by mapping it readonly. It helps speeding up new container
+# creation and saves a lot of memory if there are many kata containers running
+# on the same host.
+#
+# When disabled, new VMs are created from scratch.
+#
+# Note: Requires "initrd=" to be set ("image=" is not supported).
+#
+# Default false
+#enable_template = true
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+#enable_tracing = true
+
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 45)
+dial_timeout = 45 
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_FC@"
+
+name="@RUNTIMENAME@"
+hypervisor_name="@HYPERVISOR_FC@"
+agent_name="@PROJECT_TYPE@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY_FC@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_FC@
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true

src/runtime-rs/crates/agent/Cargo.toml

@@ -18,15 +18,15 @@ serde_json = ">=1.0.9"
 slog = "2.5.2"
 slog-scope = "4.4.0"
 ttrpc = "0.8"
-tokio = { version = "1.28.1", features = ["fs", "rt"] }
+tokio = { version = "1.38.0", features = ["fs", "rt"] }
 tracing = "0.1.36"
 url = "2.2.2"
 nix = "0.24.2"
 
-kata-types = { path = "../../../libs/kata-types"}
-logging = { path = "../../../libs/logging"}
+kata-types = { path = "../../../libs/kata-types" }
+logging = { path = "../../../libs/logging" }
 oci = { path = "../../../libs/oci" }
-protocols = { path = "../../../libs/protocols", features=["async"] }
+protocols = { path = "../../../libs/protocols", features = ["async"] }
 
 [features]
 default = []

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -22,7 +22,7 @@ serde_json = ">=1.0.9"
 slog = "2.5.2"
 slog-scope = "4.4.0"
 thiserror = "1.0"
-tokio = { version = "1.28.1", features = ["sync", "fs", "process", "io-util"] }
+tokio = { version = "1.38.0", features = ["sync", "fs", "process", "io-util"] }
 vmm-sys-util = "0.11.0"
 rand = "0.8.4"
 path-clean = "1.0.1"
@@ -43,8 +43,15 @@ safe-path = "0.1.0"
 crossbeam-channel = "0.5.6"
 tempdir = "0.3.7"
 
+qapi = { version = "0.14", features = [ "qmp", "async-tokio-all" ] }
+qapi-spec = "0.3.1"
+qapi-qmp = "0.14.0"
+
 [target.'cfg(not(target_arch = "s390x"))'.dependencies]
 dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall", "virtio-mem", "virtio-balloon", "vhost-user-net", "host-device"] }
+dbs-utils = { path = "../../../dragonball/src/dbs_utils" }
+hyperlocal = "0.8.0"
+hyper = {version = "0.14.18", features = ["client"]}
 
 [features]
 default = []

src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml

@@ -13,7 +13,7 @@ edition = "2021"
 anyhow = "1.0.68"
 serde = { version = "1.0.145", features = ["rc", "derive"] }
 serde_json = "1.0.91"
-tokio = { version = "1.28.1", features = ["sync", "rt"] }
+tokio = { version = "1.38.0", features = ["sync", "rt"] }
 
 # Cloud Hypervisor public HTTP API functions
 # Note that the version specified is not necessarily the version of CH

src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs

@@ -4,6 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use std::convert::TryFrom;
 use std::path::PathBuf;
 
 use anyhow::{anyhow, Context, Result};
@@ -19,6 +20,7 @@ use dragonball::device_manager::{
 };
 
 use super::{build_dragonball_network_config, DragonballInner};
+use crate::device::pci_path::PciPath;
 use crate::VhostUserConfig;
 use crate::{
     device::DeviceType, HybridVsockConfig, NetworkConfig, ShareFsConfig, ShareFsMountConfig,
@@ -37,46 +39,64 @@ pub(crate) fn drive_index_to_id(index: u64) -> String {
 }
 
 impl DragonballInner {
-    pub(crate) async fn add_device(&mut self, device: DeviceType) -> Result<()> {
+    pub(crate) async fn add_device(&mut self, device: DeviceType) -> Result<DeviceType> {
         if self.state == VmmState::NotReady {
             info!(sl!(), "VMM not ready, queueing device {}", device);
 
             // add the pending device by reverse order, thus the
             // start_vm would pop the devices in an right order
             // to add the devices.
-            self.pending_devices.insert(0, device);
-            return Ok(());
+            self.pending_devices.insert(0, device.clone());
+            return Ok(device);
         }
 
         info!(sl!(), "dragonball add device {:?}", &device);
         match device {
-            DeviceType::Network(network) => self
-                .add_net_device(&network.config)
-                .context("add net device"),
-            DeviceType::Vfio(hostdev) => self.add_vfio_device(&hostdev).context("add vfio device"),
-            DeviceType::Block(block) => self
-                .add_block_device(
+            DeviceType::Network(network) => {
+                self.add_net_device(&network.config)
+                    .context("add net device")?;
+                Ok(DeviceType::Network(network))
+            }
+            DeviceType::Vfio(mut hostdev) => {
+                self.add_vfio_device(&mut hostdev)
+                    .context("add vfio device")?;
+
+                Ok(DeviceType::Vfio(hostdev))
+            }
+            DeviceType::Block(block) => {
+                self.add_block_device(
                     block.config.path_on_host.as_str(),
                     block.device_id.as_str(),
                     block.config.is_readonly,
                     block.config.no_drop,
                 )
-                .context("add block device"),
-            DeviceType::VhostUserBlk(block) => self
-                .add_block_device(
+                .context("add block device")?;
+                Ok(DeviceType::Block(block))
+            }
+            DeviceType::VhostUserBlk(block) => {
+                self.add_block_device(
                     block.config.socket_path.as_str(),
                     block.device_id.as_str(),
                     block.is_readonly,
                     block.no_drop,
                 )
-                .context("add vhost user based block device"),
-            DeviceType::HybridVsock(hvsock) => self.add_hvsock(&hvsock.config).context("add vsock"),
-            DeviceType::ShareFs(sharefs) => self
-                .add_share_fs_device(&sharefs.config)
-                .context("add share fs device"),
-            DeviceType::VhostUserNetwork(dev) => self
-                .add_vhost_user_net_device(&dev.config)
-                .context("add vhost-user-net device"),
+                .context("add vhost user based block device")?;
+                Ok(DeviceType::VhostUserBlk(block))
+            }
+            DeviceType::HybridVsock(hvsock) => {
+                self.add_hvsock(&hvsock.config).context("add vsock")?;
+                Ok(DeviceType::HybridVsock(hvsock))
+            }
+            DeviceType::ShareFs(sharefs) => {
+                self.add_share_fs_device(&sharefs.config)
+                    .context("add share fs device")?;
+                Ok(DeviceType::ShareFs(sharefs))
+            }
+            DeviceType::VhostUserNetwork(dev) => {
+                self.add_vhost_user_net_device(&dev.config)
+                    .context("add vhost-user-net device")?;
+                Ok(DeviceType::VhostUserNetwork(dev))
+            }
             DeviceType::Vsock(_) => todo!(),
         }
     }
@@ -121,56 +141,49 @@ impl DragonballInner {
         }
     }
 
-    fn add_vfio_device(&mut self, device: &VfioDevice) -> Result<()> {
-        let vfio_device = device.clone();
-
+    fn add_vfio_device(&mut self, device: &mut VfioDevice) -> Result<()> {
         // FIXME:
         // A device with multi-funtions, or a IOMMU group with one more
         // devices, the Primary device is selected to be passed to VM.
         // And the the first one is Primary device.
         // safe here, devices is not empty.
-        let primary_device = vfio_device.devices.first().unwrap().clone();
-
-        let vendor_device_id = if let Some(vd) = primary_device.device_vendor {
+        let primary_device = device.devices.first_mut().unwrap();
+        let vendor_device_id = if let Some(vd) = primary_device.device_vendor.as_ref() {
             vd.get_device_vendor_id()?
         } else {
             0
         };
 
-        // It's safe to unwrap the guest_pci_path and get device slot,
-        // As it has been assigned in vfio device manager.
-        let pci_path = primary_device.guest_pci_path.unwrap();
-        let guest_dev_id = pci_path.get_device_slot().unwrap().0;
-
         info!(
             sl!(),
             "insert host device. 
             host device id: {:?}, 
             bus_slot_func: {:?}, 
-            guest device id: {:?}, 
             vendor/device id: {:?}",
             primary_device.hostdev_id,
             primary_device.bus_slot_func,
-            guest_dev_id,
             vendor_device_id,
         );
 
         let vfio_dev_config = VfioPciDeviceConfig {
-            bus_slot_func: primary_device.bus_slot_func,
+            bus_slot_func: primary_device.bus_slot_func.clone(),
             vendor_device_id,
-            guest_dev_id: Some(guest_dev_id),
             ..Default::default()
         };
         let host_dev_config = HostDeviceConfig {
-            hostdev_id: primary_device.hostdev_id,
+            hostdev_id: primary_device.hostdev_id.clone(),
             sysfs_path: primary_device.sysfs_path.clone(),
             dev_config: vfio_dev_config,
         };
 
-        self.vmm_instance
+        let guest_device_id = self
+            .vmm_instance
             .insert_host_device(host_dev_config)
             .context("insert host device failed")?;
 
+        // It's safe to unwrap guest_device_id as we can get a guest device id here.
+        primary_device.guest_pci_path = Some(PciPath::try_from(guest_device_id.unwrap() as u32)?);
+
         Ok(())
     }
 

src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs

@@ -104,10 +104,7 @@ impl Hypervisor for Dragonball {
 
     async fn add_device(&self, device: DeviceType) -> Result<DeviceType> {
         let mut inner = self.inner.write().await;
-        match inner.add_device(device.clone()).await {
-            Ok(_) => Ok(device),
-            Err(err) => Err(err),
-        }
+        inner.add_device(device.clone()).await
     }
 
     async fn remove_device(&self, device: DeviceType) -> Result<()> {