Merge pull request #10562 from stevenhorsman/remove-release-artifactor-skips

workflows: Remove skipping of artifact uploads

.github/workflows/basic-ci-amd64.yaml

@@ -138,6 +138,8 @@ jobs:
         run: bash tests/integration/nydus/gha-run.sh run
 
   run-runk:
+    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
+    if: false
     runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: lts
@@ -260,6 +262,8 @@ jobs:
         vmm:
           - clh
           - qemu
+          - dragonball
+          - cloud-hypervisor
     runs-on: ubuntu-22.04
     env:
       KATA_HYPERVISOR: ${{ matrix.vmm }}

.github/workflows/build-checks.yaml

@@ -19,7 +19,6 @@ jobs:
           - runtime-rs
           - agent-ctl
           - kata-ctl
-          - runk
           - trace-forwarder
           - genpolicy
         command:
@@ -40,15 +39,11 @@ jobs:
             component-path: src/tools/agent-ctl
           - component: kata-ctl
             component-path: src/tools/kata-ctl
-          - component: runk
-            component-path: src/tools/runk
           - component: trace-forwarder
             component-path: src/tools/trace-forwarder
           - install-libseccomp: no
           - component: agent
             install-libseccomp: yes
-          - component: runk
-            install-libseccomp: yes
           - component: genpolicy
             component-path: src/tools/genpolicy
     steps:
@@ -94,10 +89,10 @@ jobs:
           echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
           echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
       - name: Install protobuf-compiler
-        if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'runk' || matrix.component == 'genpolicy') }}
+        if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'genpolicy' || matrix.component == 'agent-ctl') }}
         run: sudo apt-get -y install protobuf-compiler
       - name: Install clang
-        if: ${{ matrix.command == 'make check' && matrix.component == 'agent' }}
+        if: ${{ matrix.command == 'make check' && (matrix.component == 'agent' || matrix.component == 'agent-ctl') }}
         run: sudo apt-get -y install clang
       - name: Setup XDG_RUNTIME_DIR for the `runtime` tests
         if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }}

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -24,6 +24,11 @@ on:
 jobs:
   build-asset:
     runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
     strategy:
       matrix:
         asset:
@@ -48,13 +53,6 @@ jobs:
           - qemu
           - qemu-snp-experimental
           - stratovirt
-          - rootfs-image
-          - rootfs-image-confidential
-          - rootfs-initrd
-          - rootfs-initrd-confidential
-          - rootfs-initrd-mariner
-          - runk
-          - shim-v2
           - trace-forwarder
           - virtiofsd
         stage:
@@ -62,6 +60,8 @@ jobs:
         exclude:
           - asset: cloud-hypervisor-glibc
             stage: release
+    env:
+      PERFORM_ATTESTATION: ${{ matrix.asset == 'agent' && inputs.push-to-registry == 'yes' && 'yes' || 'no' }}
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -83,6 +83,7 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Build ${{ matrix.asset }}
+        id: build
         run: |
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
@@ -98,8 +99,35 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
+      - name: Parse OCI image name and digest
+        id: parse-oci-segments
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        run: |
+          oci_image="$(<"build/${{ matrix.asset }}-oci-image")"
+          echo "oci-name=${oci_image%@*}" >> "$GITHUB_OUTPUT"
+          echo "oci-digest=${oci_image#*@}" >> "$GITHUB_OUTPUT"
+
+      - uses: oras-project/setup-oras@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          version: "1.2.0"
+
+      # for pushing attestations to the registry
+      - uses: docker/login-action@v3
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/attest-build-provenance@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          subject-name: ${{ steps.parse-oci-segments.outputs.oci-name }}
+          subject-digest: ${{ steps.parse-oci-segments.outputs.oci-digest }}
+          push-to-registry: true
+
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ matrix.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -107,9 +135,130 @@ jobs:
           retention-days: 15
           if-no-files-found: error
 
-  create-kata-tarball:
+  build-asset-rootfs:
     runs-on: ubuntu-22.04
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-image-confidential
+          - rootfs-image-mariner
+          - rootfs-initrd
+          - rootfs-initrd-confidential
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-shim-v2:
+    runs-on: ubuntu-22.04
+    needs: [build-asset, build-asset-rootfs]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-amd64-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-22.04
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -35,9 +35,6 @@ jobs:
           - nydus
           - qemu
           - stratovirt
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
           - virtiofsd
     steps:
       - name: Login to Kata Containers quay.io
@@ -76,7 +73,6 @@ jobs:
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -84,9 +80,124 @@ jobs:
           retention-days: 15
           if-no-files-found: error
 
-  create-kata-tarball:
+  build-asset-rootfs:
     runs-on: arm64-builder
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-initrd
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-shim-v2:
+    runs-on: arm64-builder
+    needs: [build-asset, build-asset-rootfs]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-arm64-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: arm64-builder
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - name: Adjust a permission for repo
         run: |

.github/workflows/build-kata-static-tarball-ppc64le.yaml

@@ -30,8 +30,6 @@ jobs:
           - agent
           - kernel
           - qemu
-          - rootfs-initrd
-          - shim-v2
           - virtiofsd
         stage:
           - ${{ inputs.stage }}
@@ -77,7 +75,6 @@ jobs:
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -85,9 +82,135 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
-  create-kata-tarball:
+  build-asset-rootfs:
     runs-on: ppc64le
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-initrd
+        stage:
+          - ${{ inputs.stage }}
+    steps:
+      - name: Prepare the self-hosted runner
+        run: |
+            ${HOME}/scripts/prepare_runner.sh
+            sudo rm -rf $GITHUB_WORKSPACE/*
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  build-asset-shim-v2:
+    runs-on: ppc64le
+    needs: [build-asset, build-asset-rootfs]
+    steps:
+      - name: Prepare the self-hosted runner
+        run: |
+            ${HOME}/scripts/prepare_runner.sh
+            sudo rm -rf $GITHUB_WORKSPACE/*
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-ppc64le-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ppc64le
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - name: Adjust a permission for repo
         run: |

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -24,6 +24,11 @@ on:
 jobs:
   build-asset:
     runs-on: s390x
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
     strategy:
       matrix:
         asset:
@@ -33,12 +38,9 @@ jobs:
           - kernel-confidential
           - pause-image
           - qemu
-          - rootfs-image
-          - rootfs-image-confidential
-          - rootfs-initrd
-          - rootfs-initrd-confidential
-          - shim-v2
           - virtiofsd
+    env:
+      PERFORM_ATTESTATION: ${{ matrix.asset == 'agent' && inputs.push-to-registry == 'yes' && 'yes' || 'no' }}
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -60,6 +62,7 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Build ${{ matrix.asset }}
+        id: build
         run: |
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
@@ -75,8 +78,93 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
+      - name: Parse OCI image name and digest
+        id: parse-oci-segments
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        run: |
+          oci_image="$(<"build/${{ matrix.asset }}-oci-image")"
+          echo "oci-name=${oci_image%@*}" >> "$GITHUB_OUTPUT"
+          echo "oci-digest=${oci_image#*@}" >> "$GITHUB_OUTPUT"
+
+      # for pushing attestations to the registry
+      - uses: docker/login-action@v3
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/attest-build-provenance@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          subject-name: ${{ steps.parse-oci-segments.outputs.oci-name }}
+          subject-digest: ${{ steps.parse-oci-segments.outputs.oci-digest }}
+          push-to-registry: true
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-rootfs:
+    runs-on: s390x
+    needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-image-confidential
+          - rootfs-initrd
+          - rootfs-initrd-confidential
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -86,7 +174,7 @@ jobs:
 
   build-asset-boot-image-se:
     runs-on: s390x
-    needs: build-asset
+    needs: [build-asset, build-asset-rootfs]
     steps:
       - uses: actions/checkout@v4
 
@@ -112,11 +200,7 @@ jobs:
 
       - name: Build boot-image-se
         run: |
-          base_dir=tools/packaging/kata-deploy/local-build/
-          cp -r kata-artifacts ${base_dir}/build
-          # Skip building dependant artifacts of boot-image-se-tarball
-          # because we already have them from the previous build
-          sed -i 's/\(^boot-image-se-tarball:\).*/\1/g' ${base_dir}/Makefile
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "boot-image-se"
           make boot-image-se-tarball
           build_dir=$(readlink -f build)
           sudo cp -r "${build_dir}" "kata-build"
@@ -132,9 +216,66 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
+  build-asset-shim-v2:
+    runs-on: s390x
+    needs: [build-asset, build-asset-rootfs]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-s390x-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
   create-kata-tarball:
     runs-on: s390x
-    needs: [build-asset, build-asset-boot-image-se]
+    needs: [build-asset, build-asset-rootfs, build-asset-boot-image-se, build-asset-shim-v2]
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/ci-devel.yaml

@@ -0,0 +1,13 @@
+name: Kata Containers CI (manually triggered)
+on:
+  workflow_dispatch:
+
+jobs:
+  kata-containers-ci-on-push:
+    uses: ./.github/workflows/ci.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      pr-number: "dev"
+      tag: ${{ github.sha }}-dev
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit

.github/workflows/ci-nightly.yaml

@@ -2,7 +2,6 @@ name: Kata Containers Nightly CI
 on:
   schedule:
     - cron: '0 0 * * *'
-  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/ci-on-push.yaml

@@ -19,12 +19,21 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  kata-containers-ci-on-push:
+  skipper:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
+    uses: ./.github/workflows/gatekeeper-skipper.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+
+  kata-containers-ci-on-push:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_build != 'yes' }}
     uses: ./.github/workflows/ci.yaml
     with:
       commit-hash: ${{ github.event.pull_request.head.sha }}
       pr-number: ${{ github.event.pull_request.number }}
       tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
       target-branch: ${{ github.event.pull_request.base.ref }}
+      skip-test: ${{ needs.skipper.outputs.skip_test }}
     secrets: inherit

.github/workflows/ci-weekly.yaml

@@ -37,7 +37,7 @@ jobs:
     secrets: inherit
 
   build-and-publish-tee-confidential-unencrypted-image:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/ci.yaml

@@ -15,6 +15,10 @@ on:
         required: false
         type: string
         default: ""
+      skip-test:
+        required: false
+        type: string
+        default: no
 
 jobs:
   build-kata-static-tarball-amd64:
@@ -132,6 +136,7 @@ jobs:
           file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
 
   run-kata-monitor-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-amd64
     uses: ./.github/workflows/run-kata-monitor-tests.yaml
     with:
@@ -140,6 +145,7 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
 
   run-k8s-tests-on-aks:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
     with:
@@ -153,6 +159,7 @@ jobs:
     secrets: inherit
 
   run-k8s-tests-on-amd64:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-k8s-tests-on-amd64.yaml
     with:
@@ -165,9 +172,11 @@ jobs:
     secrets: inherit
 
   run-kata-coco-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
     uses: ./.github/workflows/run-kata-coco-tests.yaml
     with:
+      tarball-suffix: -${{ inputs.tag }}
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
@@ -177,6 +186,7 @@ jobs:
     secrets: inherit
 
   run-k8s-tests-on-zvsi:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: [publish-kata-deploy-payload-s390x, build-and-publish-tee-confidential-unencrypted-image]
     uses: ./.github/workflows/run-k8s-tests-on-zvsi.yaml
     with:
@@ -189,6 +199,7 @@ jobs:
     secrets: inherit
 
   run-k8s-tests-on-ppc64le:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: publish-kata-deploy-payload-ppc64le
     uses: ./.github/workflows/run-k8s-tests-on-ppc64le.yaml
     with:
@@ -200,6 +211,7 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
 
   run-metrics-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-amd64
     uses: ./.github/workflows/run-metrics.yaml
     with:
@@ -208,6 +220,7 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
 
   run-basic-amd64-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-amd64
     uses: ./.github/workflows/basic-ci-amd64.yaml
     with:
@@ -216,6 +229,7 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
 
   run-cri-containerd-tests-s390x:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-s390x
     uses: ./.github/workflows/run-cri-containerd-tests-s390x.yaml
     with:
@@ -224,6 +238,7 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
 
   run-cri-containerd-tests-ppc64le:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-ppc64le
     uses: ./.github/workflows/run-cri-containerd-tests-ppc64le.yaml
     with:

.github/workflows/gatekeeper-skipper.yaml

@@ -0,0 +1,52 @@
+name: Skipper
+
+# This workflow sets various "skip_*" output values that can be used to
+# determine what workflows/jobs are expected to be executed. Sample usage:
+#
+#   skipper:
+#     uses: ./.github/workflows/gatekeeper-skipper.yaml
+#     with:
+#       commit-hash: ${{ github.event.pull_request.head.sha }}
+#       target-branch: ${{ github.event.pull_request.base.ref }}
+#
+#   your-workflow:
+#     needs: skipper
+#     if: ${{ needs.skipper.outputs.skip_build != 'yes' }}
+
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+    outputs:
+      skip_build:
+        value: ${{ jobs.skipper.outputs.skip_build }}
+      skip_test:
+        value: ${{ jobs.skipper.outputs.skip_test }}
+      skip_static:
+        value: ${{ jobs.skipper.outputs.skip_static }}
+
+
+jobs:
+  skipper:
+    runs-on: ubuntu-22.04
+    outputs:
+      skip_build: ${{ steps.skipper.outputs.skip_build }}
+      skip_test: ${{ steps.skipper.outputs.skip_test }}
+      skip_static: ${{ steps.skipper.outputs.skip_static }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - id: skipper
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+        run: |
+          python3 tools/testing/gatekeeper/skips.py | tee -a "$GITHUB_OUTPUT"
+        shell: /usr/bin/bash -x {0}

.github/workflows/gatekeeper.yaml

@@ -0,0 +1,44 @@
+name: Gatekeeper
+
+# Gatekeeper uses the "skips.py" to determine which job names/regexps are
+# required for given PR and waits for them to either complete or fail
+# reporting the status.
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  gatekeeper:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - id: gatekeeper
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMIT_HASH: ${{ github.event.pull_request.head.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          #!/usr/bin/env bash -x
+          mapfile -t lines < <(python3 tools/testing/gatekeeper/skips.py -t)
+          export REQUIRED_JOBS="${lines[0]}"
+          export REQUIRED_REGEXPS="${lines[1]}"
+          export REQUIRED_LABELS="${lines[2]}"
+          echo "REQUIRED_JOBS: $REQUIRED_JOBS"
+          echo "REQUIRED_REGEXPS: $REQUIRED_REGEXPS"
+          echo "REQUIRED_LABELS: $REQUIRED_LABELS"
+          python3 tools/testing/gatekeeper/jobs.py
+          exit $?
+        shell: /usr/bin/bash -x {0}

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -47,13 +47,16 @@ jobs:
             vmm: clh
             instance-type: small
             genpolicy-pull-method: oci-distribution
+            auto-generate-policy: yes
           - host_os: cbl-mariner
             vmm: clh
             instance-type: small
             genpolicy-pull-method: containerd
+            auto-generate-policy: yes
           - host_os: cbl-mariner
             vmm: clh
             instance-type: normal
+            auto-generate-policy: yes
     runs-on: ubuntu-22.04
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
@@ -66,6 +69,7 @@ jobs:
       USING_NFD: "false"
       K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
       GENPOLICY_PULL_METHOD: ${{ matrix.genpolicy-pull-method }}
+      AUTO_GENERATE_POLICY: ${{ matrix.auto-generate-policy }}
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/run-k8s-tests-on-amd64.yaml

@@ -56,6 +56,7 @@ jobs:
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: "false"
       K8S_TEST_HOST_TYPE: all
+      CONTAINER_RUNTIME: ${{ matrix.container_runtime }}
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -36,7 +36,7 @@ jobs:
           - qemu-runtime-rs
           - qemu-coco-dev
         k8s:
-          - k3s
+          - kubeadm
         include:
           - snapshotter: devmapper
             pull-type: default
@@ -97,9 +97,6 @@ jobs:
           echo "KBS_INGRESS=nodeport" >> $GITHUB_ENV
         if: ${{ matrix.vmm == 'qemu-coco-dev' }}
 
-      - name: Deploy ${{ matrix.k8s }}
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
-
       # qemu-runtime-rs only works with overlayfs
       # See: https://github.com/kata-containers/kata-containers/issues/10066
       - name: Configure the ${{ matrix.snapshotter }} snapshotter

.github/workflows/run-kata-coco-stability-tests.yaml

@@ -34,7 +34,7 @@ jobs:
           - nydus
         pull-type:
           - guest-pull
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
@@ -64,6 +64,15 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
 

.github/workflows/run-kata-coco-tests.yaml

@@ -2,6 +2,9 @@ name: CI | Run kata coco tests
 on:
   workflow_call:
     inputs:
+      tarball-suffix:
+        required: false
+        type: string
       registry:
         required: true
         type: string
@@ -33,7 +36,15 @@ jobs:
           - nydus
         pull-type:
           - guest-pull
-    runs-on: tdx
+        k8s-test-host-type:
+          - baremetal-attestation
+          - baremetal-no-attestation
+        include:
+          - k8s-test-host-type: baremetal-attestation
+            machine: tdx-attestation
+          - k8s-test-host-type: baremetal-no-attestation
+            machine: tdx-no-attestation
+    runs-on: ${{ matrix.machine }}
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
@@ -43,12 +54,14 @@ jobs:
       KUBERNETES: "vanilla"
       USING_NFD: "true"
       KBS: "true"
-      K8S_TEST_HOST_TYPE: "baremetal"
+      K8S_TEST_HOST_TYPE: ${{ matrix.k8s-test-host-type }}
       KBS_INGRESS: "nodeport"
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       PULL_TYPE: ${{ matrix.pull-type }}
       AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      ITA_KEY: ${{ secrets.ITA_KEY }}
+      AUTO_GENERATE_POLICY: "yes"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -70,19 +83,22 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
 
       - name: Uninstall previous `kbs-client`
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
 
       - name: Deploy CoCo KBS
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
 
       - name: Install `kbs-client`
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
       - name: Run tests
-        timeout-minutes: 50
+        timeout-minutes: 100
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete kata-deploy
@@ -94,7 +110,7 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
       - name: Delete CoCo KBS
-        if: always()
+        if: ${{ always() && matrix.machine != 'tdx-no-attestation' }}
         run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
 
   run-k8s-tests-on-sev:
@@ -122,6 +138,7 @@ jobs:
       PULL_TYPE: ${{ matrix.pull-type }}
       AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      AUTO_GENERATE_POLICY: "yes"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -155,6 +172,9 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
   run-k8s-tests-sev-snp:
+    # Skipping SNP tests to unblock the CI. 
+    # Will revert after issue is fixed.
+    if: false
     strategy:
       fail-fast: false
       matrix:
@@ -181,6 +201,7 @@ jobs:
       PULL_TYPE: ${{ matrix.pull-type }}
       AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      AUTO_GENERATE_POLICY: "yes"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -258,6 +279,7 @@ jobs:
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: "false"
+      AUTO_GENERATE_POLICY: "yes"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -270,6 +292,15 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
 
@@ -311,7 +342,7 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
       - name: Run tests
-        timeout-minutes: 60
+        timeout-minutes: 80
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete AKS cluster

.github/workflows/run-metrics.yaml

@@ -48,7 +48,7 @@ jobs:
       # all the tests due to a single flaky instance.
       fail-fast: false
       matrix:
-        vmm: ['clh', 'qemu', 'stratovirt']
+        vmm: ['clh', 'qemu']
       max-parallel: 1
     runs-on: metrics
     env:

.github/workflows/run-runk-tests.yaml

@@ -15,6 +15,8 @@ on:
 
 jobs:
   run-runk:
+    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
+    if: false
     runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: lts

.github/workflows/static-checks-self-hosted.yaml

@@ -12,8 +12,16 @@ concurrency:
 
 name: Static checks self-hosted
 jobs:
-  build-checks:
+  skipper:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
+    uses: ./.github/workflows/gatekeeper-skipper.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+
+  build-checks:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     strategy:
       fail-fast: false
       matrix:

.github/workflows/static-checks.yaml

@@ -12,7 +12,15 @@ concurrency:
 
 name: Static checks
 jobs:
+  skipper:
+    uses: ./.github/workflows/gatekeeper-skipper.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+
   check-kernel-config-version:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout the code
@@ -35,12 +43,16 @@ jobs:
           fi
 
   build-checks:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     uses: ./.github/workflows/build-checks.yaml
     with:
       instance: ubuntu-22.04
 
   build-checks-depending-on-kvm:
     runs-on: ubuntu-22.04
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     strategy:
       fail-fast: false
       matrix:
@@ -78,6 +90,8 @@ jobs:
 
   static-checks:
     runs-on: ubuntu-22.04
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     strategy:
       fail-fast: false
       matrix:

VERSION

@@ -1 +1 @@
-3.9.0
+3.11.0

ci/README.md

@@ -55,14 +55,14 @@ of a PR review), the following tests will be executed:
 - Run the following tests:
   - Tests depending on the generated tarball
     - Metrics (runs on bare-metal)
-    - `docker` (runs on Azure small instances)
-    - `nerdctl` (runs on Azure small instances)
-    - `kata-monitor` (runs on Azure small instances)
-    - `cri-containerd` (runs on Azure small instances)
-    - `nydus` (runs on Azure small instances)
-    - `vfio` (runs on Azure normal instances)
+    - `docker` (runs on cost free runners)
+    - `nerdctl` (runs on cost free runners)
+    - `kata-monitor` (runs on cost free runners)
+    - `cri-containerd` (runs on cost free runners)
+    - `nydus` (runs on cost free runners)
+    - `vfio` (runs on cost free runners)
   - Tests depending on the generated kata-deploy payload
-    - kata-deploy (runs on Azure small instances)
+    - kata-deploy (runs on cost free runners)
       - Tests are performed using different "Kubernetes flavors", such as k0s, k3s, rke2, and Azure Kubernetes Service (AKS).
     - Kubernetes (runs in Azure small and medium instances depending on what's required by each test, and on TEE bare-metal machines)
       - Tests are performed with different runtime engines, such as CRI-O and containerd.
@@ -77,8 +77,8 @@ them to merely debug issues.
 
 In the previous section we've mentioned using different runners, now in this section we'll go through each type of runner used.
 
-- Cost free runners:  Those are the runners provided by GIthub itself, and
-  those are fairly small machines with no virtualization capabilities enabled - 
+- Cost free runners:  Those are the runners provided by Github itself, and
+  those are fairly small machines with virtualization capabilities enabled. 
 - Azure small instances: Those are runners which have virtualization
   capabilities enabled, 2 CPUs, and 8GB of RAM.  These runners have a "-smaller"
   suffix to their name. 

ci/install_yq.sh

@@ -14,20 +14,38 @@ die() {
 	exit 1
 }
 
+function verify_yq_exists() {
+	local yq_path=$1
+	local yq_version=$2
+	local expected="yq (https://github.com/mikefarah/yq/) version $yq_version"
+	if [ -x  "${yq_path}" ] && [ "$($yq_path --version)"X == "$expected"X ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 # Install the yq yaml query package from the mikefarah github repo
 # Install via binary download, as we may not have golang installed at this point
 function install_yq() {
 	local yq_pkg="github.com/mikefarah/yq"
 	local yq_version=v4.40.7
 	local precmd=""
+	local yq_path=""
 	INSTALL_IN_GOPATH=${INSTALL_IN_GOPATH:-true}
 
-	if [ "${INSTALL_IN_GOPATH}"  == "true" ];then
+	if [ "${INSTALL_IN_GOPATH}" == "true" ]; then
 		GOPATH=${GOPATH:-${HOME}/go}
 		mkdir -p "${GOPATH}/bin"
-		local yq_path="${GOPATH}/bin/yq"
+		yq_path="${GOPATH}/bin/yq"
 	else
 		yq_path="/usr/local/bin/yq"
+	fi
+	if verify_yq_exists "$yq_path" "$yq_version"; then
+		echo "yq is already installed in correct version"
+		return
+	fi
+	if [ "${yq_path}" == "/usr/local/bin/yq" ]; then
 		# Check if we need sudo to install yq
 		if [ ! -w "/usr/local/bin" ]; then
 			# Check if we have sudo privileges
@@ -38,7 +56,6 @@ function install_yq() {
 			fi
 		fi
 	fi
-	[ -x  "${yq_path}" ] && [ "`${yq_path} --version`"X == "yq (https://github.com/mikefarah/yq/) version ${yq_version}"X ] && return
 
 	read -r -a sysInfo <<< "$(uname -sm)"
 

ci/openshift-ci/bisect-range.sh

@@ -16,9 +16,12 @@ REPO="quay.io/kata-containers/kata-deploy-ci"
 TAGS=$(skopeo list-tags "docker://$REPO")
 # Only amd64
 TAGS=$(echo "$TAGS" | jq '.Tags' | jq "map(select(endswith(\"$ARCH\")))" | jq -r '.[]')
-# Tags since $GOOD
-TAGS=$(echo "$TAGS" | sed -n -e "/$GOOD/,$$p")
-# Tags up to $BAD
-[ -n "$BAD" ] && TAGS=$(echo "$TAGS" | sed "/$BAD/q")
+# Sort by git
+SORTED=""
+[ -n "$BAD" ] && LOG_ARGS="$GOOD~1..$BAD" || LOG_ARGS="$GOOD~1.."
+for TAG in $(git log --merges --pretty=format:%H --reverse $LOG_ARGS); do
+	[[ "$TAGS" =~ "$TAG" ]] && SORTED+="
+kata-containers-$TAG-$ARCH"
+done
 # Comma separated tags with repo
-echo "$TAGS" | sed -e "s@^@$REPO:@" | paste -s -d, -
+echo "$SORTED" | tail -n +2 | sed -e "s@^@$REPO:@" | paste -s -d, -

ci/openshift-ci/cluster/deploy_webhook.sh

@@ -13,16 +13,11 @@ set -e
 set -o nounset
 set -o pipefail
 
-script_dir="$(dirname $0)"
+script_dir="$(realpath $(dirname $0))"
 webhook_dir="${script_dir}/../../../tools/testing/kata-webhook"
 source "${script_dir}/../lib.sh"
 KATA_RUNTIME=${KATA_RUNTIME:-kata-ci}
 
-info "Creates the kata-webhook ConfigMap"
-RUNTIME_CLASS="${KATA_RUNTIME}" \
-	envsubst < "${script_dir}/deployments/configmap_kata-webhook.yaml.in" \
-	| oc apply -f -
-
 pushd "${webhook_dir}" >/dev/null
 # Build and deploy the webhook
 #
@@ -30,6 +25,12 @@ info "Builds the kata-webhook"
 ./create-certs.sh
 info "Deploys the kata-webhook"
 oc apply -f deploy/
+
+info "Override our KATA_RUNTIME ConfigMap"
+RUNTIME_CLASS="${KATA_RUNTIME}" \
+	envsubst < "${script_dir}/deployments/configmap_kata-webhook.yaml.in" \
+	| oc apply -f -
+
 # Check the webhook was deployed and is working.
 RUNTIME_CLASS="${KATA_RUNTIME}" ./webhook-check.sh
 popd >/dev/null

ci/openshift-ci/smoke/http-server.yaml.in

@@ -13,7 +13,7 @@ metadata:
 spec:
   containers:
     - name: http-server
-      image: registry.fedoraproject.org/fedora
+      image: docker.io/library/python:3
       ports:
         - containerPort: 8080
       command: ["python3"]

docs/Developer-Guide.md

@@ -499,19 +499,6 @@ If you do not want to install the respective QEMU version, the configuration fil
 
 See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata.
 
-### Build a custom QEMU for aarch64/arm64 - REQUIRED
-> **Note:**
->
-> - You should only do this step if you are on aarch64/arm64.
-> - You should include [Eric Auger's latest PCDIMM/NVDIMM patches](https://patchwork.kernel.org/cover/10647305/) which are
->   under upstream review for supporting NVDIMM on aarch64.
->
-You could build the custom `qemu-system-aarch64` as required with the following command:
-```bash
-$ git clone https://github.com/kata-containers/tests.git
-$ script -fec 'sudo -E tests/.ci/install_qemu.sh'
-```
-
 ## Build `virtiofsd`
 
 When using the file system type virtio-fs (default), `virtiofsd` is required

docs/Release-Process.md

@@ -28,10 +28,22 @@ Bug fixes are released as part of `MINOR` or `MAJOR` releases only. `PATCH` is a
 
 ## Release Process
 
-### Bump the `VERSION` file
+### Bump the `VERSION` and `Chart.yaml` file
 
 When the `kata-containers/kata-containers` repository is ready for a new release,
-first create a PR to set the release in the `VERSION` file and have it merged.
+first create a PR to set the release in the [`VERSION`](./../VERSION) file and update the
+`version` and `appVersion` in the
+[`Chart.yaml`](./../tools/packaging/kata-deploy/helm-chart/kata-deploy/Chart.yaml) file and
+have it merged.
+
+### Lock the `main` branch
+
+In order to prevent any PRs getting merged during the release process, and slowing the release
+process down, by impacting the payload caches, we have recently trailed setting the `main`
+branch to read only whilst the release action runs.
+
+> [!NOTE]
+> Admin permission is needed to complete this task.
 
 ### Check GitHub Actions
 
@@ -40,6 +52,9 @@ We make use of [GitHub actions](https://github.com/features/actions) in the
 file from the `kata-containers/kata-containers` repository to build and upload
 release artifacts.
 
+> [!NOTE]
+> Write permissions to trigger the action.
+
 The action is manually triggered and is responsible for generating a new
 release (including a new tag), pushing those to the
 `kata-containers/kata-containers` repository. The new release is initially
@@ -59,6 +74,11 @@ If for some reason you need to cancel the workflow or re-run it entirely, go fir
 to the [Release page](https://github.com/kata-containers/kata-containers/releases) and
 delete the draft release from the previous run.
 
+### Unlock the `main` branch
+
+After the release process has concluded, either unlock the `main` branch, or ask
+an admin to do it.
+
 ### Improve the release notes
 
 Release notes are auto-generated by the GitHub CLI tool used as part of our

docs/design/architecture_3.0/README.md

@@ -50,7 +50,7 @@ We provide `Dragonball` Sandbox to enable built-in VMM by integrating VMM's func
 #### How To Support Async
 The kata-runtime is controlled by TOKIO_RUNTIME_WORKER_THREADS to run the OS thread, which is 2 threads by default. For TTRPC and container-related threads run in the `tokio` thread in a unified manner, and related dependencies need to be switched to Async, such as Timer, File, Netlink, etc. With the help of Async, we can easily support no-block I/O and timer. Currently, we only utilize Async for kata-runtime. The built-in VMM keeps the OS thread because it can ensure that the threads are controllable.
 
-**For N tokio worker threads and M containers**
+**For N `tokio` worker threads and M containers**
 
 - Sync runtime(both OS thread and `tokio` task are OS thread but without `tokio` worker thread)  OS thread number:  4 + 12*M
 - Async runtime(only OS thread is OS thread) OS thread number: 2 + N
@@ -103,7 +103,6 @@ In our case, there will be a variety of resources, and every resource has severa
 | `Cgroup V2`                |                     | Stage 2               |  🚧        |
 | Hypervisor                 | `Dragonball`        | Stage 1               |  🚧        |
 |                            | QEMU                | Stage 2               |  🚫        |
-|                            | ACRN                | Stage 3               |  🚫        |
 |                            | Cloud Hypervisor    | Stage 3               |  🚫        |
 |                            | Firecracker         | Stage 3               |  🚫        |
 
@@ -166,4 +165,4 @@ In our case, there will be a variety of resources, and every resource has severa
 
 - What is the security boundary for the monolithic / "Built-in VMM" case?
   
-  It has the security boundary of virtualization. More details will be provided in next stage.
+  It has the security boundary of virtualization. More details will be provided in next stage.

docs/design/virtualization.md

@@ -98,8 +98,7 @@ of Kata Containers, the Cloud Hypervisor configuration supports both CPU
 and memory resize, device hotplug (disk and VFIO), file-system sharing through virtio-fs,
 block-based volumes, booting from VM images backed by pmem device, and
 fine-grained seccomp filters for each VMM threads (e.g. all virtio
-device worker threads). Please check [this GitHub Project](https://github.com/orgs/kata-containers/projects/21)
-for details of ongoing integration efforts.
+device worker threads).
 
 Devices and features used:
 - virtio VSOCK or virtio serial

docs/how-to/README.md

@@ -20,12 +20,6 @@
    for the VM rootfs. Refer to the following guide for additional configuration
    steps:
    - [Setup Kata containers with `firecracker`](how-to-use-kata-containers-with-firecracker.md)
-- `ACRN`
-
-  While `qemu` , `cloud-hypervisor` and `firecracker` work out of the box with installation of Kata,
-  some additional configuration is needed in case of `ACRN`.
-  Refer to the following guides for additional configuration steps:
-- [Kata Containers with ACRN Hypervisor](how-to-use-kata-containers-with-acrn.md)
 
 ## Confidential Containers Policy
 
@@ -52,4 +46,4 @@
 - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
 - [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)
 - [How to use the Kata Agent Policy](how-to-use-the-kata-agent-policy.md)
-- [How to pull images in the guest](how-to-pull-images-in-guest-with-kata.md)
+- [How to pull images in the guest](how-to-pull-images-in-guest-with-kata.md)

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -46,7 +46,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.block_device_cache_set` | `boolean` | cache-related options will be set to block devices or not |
 | `io.katacontainers.config.hypervisor.block_device_driver` | string | the driver to be used for block device, valid values are `virtio-blk`, `virtio-scsi`, `nvdimm`|
 | `io.katacontainers.config.hypervisor.cpu_features` | `string` | Comma-separated list of CPU features to pass to the CPU (QEMU) |
-| `io.katacontainers.config.hypervisor.ctlpath` (R) | `string` | Path to the `acrnctl` binary for the ACRN hypervisor |
 | `io.katacontainers.config.hypervisor.default_max_vcpus` | uint32| the maximum number of vCPUs allocated for the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.default_memory` | uint32| the memory assigned for a VM by the hypervisor in `MiB` |
 | `io.katacontainers.config.hypervisor.default_vcpus` | float32| the default vCPUs assigned for a VM by the hypervisor |
@@ -95,6 +94,8 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
 | `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
+| `io.katacontainers.config.hypervisor.default_gpus` | uint32 | the minimum number of GPUs required for the VM. Only used by remote hypervisor to help with instance selection |
+| `io.katacontainers.config.hypervisor.default_gpu_model` | string | the GPU model required for the VM. Only used by remote hypervisor to help with instance selection |
 
 ## Container Options
 | Key | Value Type | Comments |
@@ -209,7 +210,6 @@ the configuration entry:
 
 | Key | Config file entry | Comments |
 |-------| ----- | ----- |
-| `ctlpath`  | `valid_ctlpaths` | Valid paths for `acrnctl` binary |
 | `entropy_source` | `valid_entropy_sources` | Valid entropy sources, e.g. `/dev/random` |
 | `file_mem_backend`  | `valid_file_mem_backends` | Valid locations for the file-based memory backend root directory |
 | `jailer_path`  | `valid_jailer_paths`| Valid paths for the jailer constraining the container VM (Firecracker) |

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -1,125 +0,0 @@
-# Kata Containers with ACRN
-
-This document provides an overview on how to run Kata containers with ACRN hypervisor and device model.
-
-## Introduction
-
-ACRN is a flexible, lightweight Type-1 reference hypervisor built with real-time and safety-criticality in mind. ACRN uses an open source platform making it optimized to streamline embedded development.
-
-Some of the key features being:
-
-- Small footprint - Approx. 25K lines of code (LOC).
-- Real Time - Low latency, faster boot time, improves overall responsiveness with hardware.
-- Adaptability - Multi-OS support for guest operating systems like Linux, Android, RTOSes.
-- Rich I/O mediators - Allows sharing of various I/O devices across VMs.
-- Optimized for a variety of IoT (Internet of Things) and embedded device solutions.
-
-Please refer to ACRN [documentation](https://projectacrn.github.io/latest/index.html) for more details on ACRN hypervisor and device model.
-
-## Pre-requisites
-
-This document requires the presence of the ACRN hypervisor and Kata Containers on your system. Install using the instructions available through the following links:
-
-- ACRN supported [Hardware](https://projectacrn.github.io/latest/hardware.html#supported-hardware).
-  > **Note:** Please make sure to have a minimum of 4 logical processors (HT) or cores.
-- ACRN [software](https://projectacrn.github.io/latest/tutorials/run_kata_containers.html) setup.
-- For networking, ACRN supports either MACVTAP or TAP. If MACVTAP is not enabled in the Service OS, please follow the below steps to update the kernel:
-
-  ```sh
-   $ git clone https://github.com/projectacrn/acrn-kernel.git
-   $ cd acrn-kernel
-   $ cp kernel_config_sos .config
-   $ sed -i "s/# CONFIG_MACVLAN is not set/CONFIG_MACVLAN=y/" .config
-   $ sed -i '$ i CONFIG_MACVTAP=y' .config
-   $ make clean && make olddefconfig && make && sudo make modules_install INSTALL_MOD_PATH=out/
-  ```
-  Login into Service OS and update the kernel with MACVTAP support:
-
-  ```sh
-  $ sudo mount /dev/sda1 /mnt
-  $ sudo scp -r <user name>@<host address>:<your workspace>/acrn-kernel/arch/x86/boot/bzImage /mnt/EFI/org.clearlinux/
-  $ sudo scp -r <user name>@<host address>:<your workspace>/acrn-kernel/out/lib/modules/* /lib/modules/
-  $ conf_file=$(sed -n '$ s/default //p' /mnt/loader/loader.conf).conf
-  $ kernel_img=$(sed -n 2p /mnt/loader/entries/$conf_file | cut -d'/' -f4)
-  $ sudo sed -i "s/$kernel_img/bzImage/g" /mnt/loader/entries/$conf_file
-  $ sync && sudo umount /mnt && sudo reboot
-  ```
-- Kata Containers installation: Automated installation does not seem to be supported for Clear Linux, so please use [manual installation](../Developer-Guide.md) steps.
-
-> **Note:** Create rootfs image and not initrd image.
-
-In order to run Kata with ACRN, your container stack must provide block-based storage, such as device-mapper.
-
-> **Note:** Currently, by design you can only launch one VM from Kata Containers using ACRN hypervisor (SDC scenario). Based on feedback from community we can increase number of VMs.
-
-## Configure Docker
-
-To configure Docker for device-mapper and Kata,
-
-1. Stop Docker daemon if it is already running.
-
-```bash
-$ sudo systemctl stop docker
-```
-
-2. Set `/etc/docker/daemon.json` with the following contents.
-
-```
-{
-  "storage-driver": "devicemapper"
-}
-```
-
-3. Restart docker.
-
-```bash
-$ sudo systemctl daemon-reload
-$ sudo systemctl restart docker
-```
-
-4. Configure [Docker](../Developer-Guide.md#update-the-docker-systemd-unit-file) to use `kata-runtime`.
-
-## Configure Kata Containers with ACRN
-
-To configure Kata Containers with ACRN, copy the generated `configuration-acrn.toml` file when building the `kata-runtime` to either `/etc/kata-containers/configuration.toml` or `/usr/share/defaults/kata-containers/configuration.toml`.
-
-The following command shows full paths to the `configuration.toml` files that the runtime loads. It will use the first path that exists. (Please make sure the kernel and image paths are set correctly in the `configuration.toml` file)
-
-```bash
-$ sudo kata-runtime --show-default-config-paths
-```
-
->**Warning:** Please offline CPUs using [this](offline_cpu.sh) script, else VM launches will fail.
-
-```bash
-$ sudo ./offline_cpu.sh
-```
-
-Start an ACRN based Kata Container,
-
-```bash
-$ sudo docker run -ti --runtime=kata-runtime busybox sh
-```
-
-You will see ACRN(`acrn-dm`) is now running on your system, as well as a `kata-shim`. You should obtain an interactive shell prompt. Verify that all the Kata processes terminate once you exit the container.
-
-```bash
-$ ps -ef | grep -E "kata|acrn"
-```
-
-Validate ACRN hypervisor by using `kata-runtime kata-env`,
-
-```sh
-$ kata-runtime kata-env | awk -v RS= '/\[Hypervisor\]/'
-[Hypervisor]
-  MachineType = ""
-  Version = "DM version is: 1.2-unstable-254577a6-dirty (daily tag:acrn-2019w27.4-140000p)
-  Path = "/usr/bin/acrn-dm"
-  BlockDeviceDriver = "virtio-blk"
-  EntropySource = "/dev/urandom"
-  Msize9p = 0
-  MemorySlots = 10
-  Debug = false
-  UseVSock = false
-  SharedFS = ""
-```

docs/how-to/offline_cpu.sh

@@ -18,7 +18,6 @@ for i in $(ls -d /sys/devices/system/cpu/cpu[1-9]*); do
                         echo 0 > $i/online
                         online=`cat $i/online`
                 done
-                echo $idx > /sys/class/vhm/acrn_vhm/offline_cpu
         fi
 done
 

docs/hypervisors.md

@@ -18,7 +18,6 @@ which hypervisors you may wish to investigate further.
 
 | Hypervisor | Written in | Architectures | Type |
 |-|-|-|-|
-|[ACRN] | C | `x86_64` | Type 1 (bare metal) |
 |[Cloud Hypervisor] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) |
 |[Firecracker] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) |
 |[QEMU] | C | all | Type 2 ([KVM]) | `configuration-qemu.toml` |
@@ -38,7 +37,6 @@ the hypervisors:
 
 | Hypervisor | Summary | Features | Limitations | Container Creation speed | Memory density | Use cases | Comment |
 |-|-|-|-|-|-|-|-|
-|[ACRN] | Safety critical and real-time workloads | | | excellent | excellent | Embedded and IOT systems | For advanced users |
 |[Cloud Hypervisor] | Low latency, small memory footprint, small attack surface | Minimal | | excellent | excellent | High performance modern cloud workloads | |
 |[Firecracker] | Very slimline | Extremely minimal | Doesn't support all device types | excellent | excellent | Serverless / FaaS | |
 |[QEMU] | Lots of features | Lots | | good | good | Good option for most users | |
@@ -57,7 +55,6 @@ are available, their default values and how each setting can be used.
 
 | Hypervisor | Golang runtime config file | golang runtime short name | golang runtime default | rust runtime config file | rust runtime short name | rust runtime default |
 |-|-|-|-|-|-|-|
-| [ACRN] | [`configuration-acrn.toml`](../src/runtime/config/configuration-acrn.toml.in) | `acrn` | | | | |
 | [Cloud Hypervisor] | [`configuration-clh.toml`](../src/runtime/config/configuration-clh.toml.in) | `clh` | | [`configuration-cloud-hypervisor.toml`](../src/runtime-rs/config/configuration-cloud-hypervisor.toml.in) | `cloud-hypervisor` | |
 | [Firecracker] | [`configuration-fc.toml`](../src/runtime/config/configuration-fc.toml.in) | `fc` | | | | |
 | [QEMU] | [`configuration-qemu.toml`](../src/runtime/config/configuration-qemu.toml.in) | `qemu` | yes | [`configuration-qemu.toml`](../src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in) | `qemu` | |
@@ -93,10 +90,9 @@ are available, their default values and how each setting can be used.
 To switch the configured hypervisor, you only need to run a single command.
 See [the `kata-manager` documentation](../utils/README.md#choose-a-hypervisor) for further details.
 
-[ACRN]: https://projectacrn.org
 [Cloud Hypervisor]: https://github.com/cloud-hypervisor/cloud-hypervisor
 [Firecracker]: https://github.com/firecracker-microvm/firecracker
 [KVM]: https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine
-[QEMU]: http://www.qemu-project.org
+[QEMU]: http://www.qemu.org
 [`Dragonball`]: https://github.com/kata-containers/kata-containers/blob/main/src/dragonball
 [StratoVirt]: https://gitee.com/openeuler/stratovirt

docs/install/kata-containers-3.0-rust-runtime-installation-guide.md

@@ -83,6 +83,23 @@ $ make && sudo make install
 ```
 After running the command above, the default config file `configuration.toml` will be installed under `/usr/share/defaults/kata-containers/`,  the binary file `containerd-shim-kata-v2` will be installed under `/usr/local/bin/` .
 
+### Install Shim Without Builtin Dragonball VMM
+
+By default, runtime-rs includes the `Dragonball` VMM. To build without the built-in `Dragonball` hypervisor, use `make USE_BUILDIN_DB=false`:
+```bash
+$ cd kata-containers/src/runtime-rs
+$ make USE_BUILDIN_DB=false
+```
+After building, specify the desired hypervisor during installation using `HYPERVISOR`. For example, to use `qemu` or `cloud-hypervisor`:
+
+```
+sudo make install HYPERVISOR=qemu
+```
+or
+```
+sudo make install HYPERVISOR=cloud-hypervisor
+```
+
 ### Build Kata Containers Kernel
 Follow the [Kernel installation guide](/tools/packaging/kernel/README.md).
 

src/agent/Cargo.lock

@@ -64,6 +64,20 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "ahash"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
+dependencies = [
+ "cfg-if 1.0.0",
+ "getrandom",
+ "once_cell",
+ "serde",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -97,6 +111,55 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "anstream"
+version = "0.6.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.86"
@@ -712,6 +775,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "bytecount"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -814,6 +883,30 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cdi"
+version = "0.1.0"
+source = "git+https://github.com/cncf-tags/container-device-interface-rs?rev=fba5677a8e7cc962fc6e495fcec98d7d765e332a#fba5677a8e7cc962fc6e495fcec98d7d765e332a"
+dependencies = [
+ "anyhow",
+ "clap 4.5.13",
+ "const_format",
+ "jsonschema",
+ "lazy_static",
+ "libc",
+ "nix 0.24.3",
+ "notify",
+ "oci-spec",
+ "once_cell",
+ "path-clean",
+ "regex",
+ "semver",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "serde_yaml",
+]
+
 [[package]]
 name = "cesu8"
 version = "1.1.0"
@@ -914,8 +1007,8 @@ checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
 dependencies = [
  "atty",
  "bitflags 1.3.2",
- "clap_derive",
- "clap_lex",
+ "clap_derive 3.2.25",
+ "clap_lex 0.2.4",
  "indexmap 1.9.3",
  "once_cell",
  "strsim 0.10.0",
@@ -923,6 +1016,28 @@ dependencies = [
  "textwrap",
 ]
 
+[[package]]
+name = "clap"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
+dependencies = [
+ "clap_builder",
+ "clap_derive 4.5.13",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex 0.7.2",
+ "strsim 0.11.1",
+]
+
 [[package]]
 name = "clap_derive"
 version = "3.2.25"
@@ -936,6 +1051,18 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "clap_derive"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.71",
+]
+
 [[package]]
 name = "clap_lex"
 version = "0.2.4"
@@ -945,6 +1072,12 @@ dependencies = [
  "os_str_bytes",
 ]
 
+[[package]]
+name = "clap_lex"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
+
 [[package]]
 name = "cmac"
 version = "0.7.2"
@@ -967,6 +1100,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "colorchoice"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -1741,6 +1880,17 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "fancy-regex"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
+dependencies = [
+ "bit-set",
+ "regex-automata 0.4.7",
+ "regex-syntax 0.8.4",
+]
+
 [[package]]
 name = "fastrand"
 version = "1.9.0"
@@ -1812,6 +1962,15 @@ dependencies = [
  "miniz_oxide",
 ]
 
+[[package]]
+name = "fluent-uri"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17c704e9dbe1ddd863da1e6ff3567795087b1eb201ce80d8fa81162e1516500d"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1827,6 +1986,25 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fraction"
+version = "0.15.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7"
+dependencies = [
+ "lazy_static",
+ "num",
+]
+
+[[package]]
+name = "fsevent-sys"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "funty"
 version = "2.0.0"
@@ -2052,7 +2230,7 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 dependencies = [
- "ahash",
+ "ahash 0.7.8",
 ]
 
 [[package]]
@@ -2605,6 +2783,21 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
+[[package]]
+name = "iso8601"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -2623,15 +2816,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -2690,6 +2874,18 @@ dependencies = [
  "smallvec",
 ]
 
+[[package]]
+name = "json-patch"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b1fb8864823fad91877e6caea0baca82e49e8db50f8e5c9f9a453e27d3330fc"
+dependencies = [
+ "jsonptr",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
 [[package]]
 name = "json-syntax"
 version = "0.12.5"
@@ -2709,6 +2905,47 @@ dependencies = [
  "utf8-decode",
 ]
 
+[[package]]
+name = "jsonptr"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627"
+dependencies = [
+ "fluent-uri",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "jsonschema"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0afd06142c9bcb03f4a8787c77897a87b6be9c4918f1946c33caa714c27578"
+dependencies = [
+ "ahash 0.8.11",
+ "anyhow",
+ "base64 0.22.1",
+ "bytecount",
+ "clap 4.5.13",
+ "fancy-regex",
+ "fraction",
+ "getrandom",
+ "iso8601",
+ "itoa",
+ "memchr",
+ "num-cmp",
+ "once_cell",
+ "parking_lot 0.12.3",
+ "percent-encoding",
+ "regex",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "time",
+ "url",
+ "uuid",
+]
+
 [[package]]
 name = "jwt"
 version = "0.16.0"
@@ -2773,14 +3010,16 @@ dependencies = [
  "async-std",
  "async-trait",
  "capctl",
+ "cdi",
  "cfg-if 1.0.0",
  "cgroups-rs",
- "clap",
+ "clap 3.2.25",
  "const_format",
  "derivative",
  "futures",
  "image-rs",
  "ipnetwork",
+ "json-patch",
  "kata-sys-util",
  "kata-types",
  "lazy_static",
@@ -2835,7 +3074,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -2934,6 +3172,26 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "kqueue"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c"
+dependencies = [
+ "kqueue-sys",
+ "libc",
+]
+
+[[package]]
+name = "kqueue-sys"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+]
+
 [[package]]
 name = "krata-tokio-tar"
 version = "0.4.2"
@@ -3285,6 +3543,18 @@ dependencies = [
  "adler",
 ]
 
+[[package]]
+name = "mio"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+dependencies = [
+ "libc",
+ "log",
+ "wasi",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "mio"
 version = "1.0.2"
@@ -3465,6 +3735,25 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "notify"
+version = "6.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
+dependencies = [
+ "bitflags 2.6.0",
+ "crossbeam-channel",
+ "filetime",
+ "fsevent-sys",
+ "inotify",
+ "kqueue",
+ "libc",
+ "log",
+ "mio 0.8.11",
+ "walkdir",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "ntapi"
 version = "0.4.1"
@@ -3515,6 +3804,12 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "num-cmp"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa"
+
 [[package]]
 name = "num-complex"
 version = "0.4.6"
@@ -3894,6 +4189,12 @@ dependencies = [
  "slash-formatter",
 ]
 
+[[package]]
+name = "path-clean"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
+
 [[package]]
 name = "path-dedot"
 version = "1.2.4"
@@ -4376,7 +4677,6 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.5.1",
  "serde",
@@ -4626,14 +4926,12 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
 
 [[package]]
 name = "regorus"
-version = "0.1.5"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77dd872918e5c172bd42ac49716f89a15e35be513bba3d902e355a531529a87f"
+checksum = "843c3d97f07e3b5ac0955d53ad0af4c91fe4a4f8525843ece5bf014f27829b73"
 dependencies = [
  "anyhow",
- "itertools 0.12.1",
  "lazy_static",
- "num",
  "rand",
  "regex",
  "scientific",
@@ -4666,6 +4964,7 @@ dependencies = [
  "bytes 1.6.1",
  "cookie",
  "cookie_store",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "http",
@@ -5919,7 +6218,7 @@ dependencies = [
  "backtrace",
  "bytes 1.6.1",
  "libc",
- "mio",
+ "mio 1.0.2",
  "parking_lot 0.12.3",
  "pin-project-lite",
  "signal-hook-registry",
@@ -6346,6 +6645,12 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
 [[package]]
 name = "uuid"
 version = "1.10.0"
@@ -7021,6 +7326,26 @@ dependencies = [
  "zvariant",
 ]
 
+[[package]]
+name = "zerocopy"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.71",
+]
+
 [[package]]
 name = "zerofrom"
 version = "0.1.4"

src/agent/Cargo.toml

@@ -80,10 +80,13 @@ strum_macros = "0.26.2"
 image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "v0.10.0", default-features = false, optional = true }
 
 # Agent Policy
-regorus = { version = "0.1.4", default-features = false, features = [
+regorus = { version = "0.2.6", default-features = false, features = [
     "arc",
     "regex",
+    "std",
 ], optional = true }
+cdi = { git = "https://github.com/cncf-tags/container-device-interface-rs", rev = "fba5677a8e7cc962fc6e495fcec98d7d765e332a" }
+json-patch = "2.0.0"
 
 [dev-dependencies]
 tempfile = "3.1.0"

src/agent/rustjail/src/container.rs

@@ -158,7 +158,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(1)
                 .minor(3)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()
@@ -168,7 +168,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(1)
                 .minor(5)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()
@@ -178,7 +178,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(1)
                 .minor(7)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()
@@ -188,7 +188,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(5)
                 .minor(0)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()
@@ -198,7 +198,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(1)
                 .minor(9)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()
@@ -208,7 +208,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(1)
                 .minor(8)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()

src/agent/src/cdh.rs

@@ -8,13 +8,15 @@
 // https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub
 
 use crate::AGENT_CONFIG;
-use crate::CDH_SOCKET_URI;
-use anyhow::{Context, Result};
+use anyhow::{bail, Context, Result};
 use derivative::Derivative;
 use protocols::{
     confidential_data_hub, confidential_data_hub_ttrpc_async,
     confidential_data_hub_ttrpc_async::{SealedSecretServiceClient, SecureMountServiceClient},
 };
+use std::fs;
+use std::os::unix::fs::symlink;
+use std::path::Path;
 use tokio::sync::OnceCell;
 
 // Nanoseconds
@@ -22,8 +24,14 @@ lazy_static! {
     static ref CDH_API_TIMEOUT: i64 = AGENT_CONFIG.cdh_api_timeout.as_nanos() as i64;
     pub static ref CDH_CLIENT: OnceCell<CDHClient> = OnceCell::new();
 }
+
 const SEALED_SECRET_PREFIX: &str = "sealed.";
 
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "cdh"))
+}
+
 #[derive(Derivative)]
 #[derivative(Clone, Debug)]
 pub struct CDHClient {
@@ -34,8 +42,8 @@ pub struct CDHClient {
 }
 
 impl CDHClient {
-    pub fn new() -> Result<Self> {
-        let client = ttrpc::asynchronous::Client::connect(CDH_SOCKET_URI)?;
+    pub fn new(cdh_socket_uri: &str) -> Result<Self> {
+        let client = ttrpc::asynchronous::Client::connect(cdh_socket_uri)?;
         let sealed_secret_client =
             confidential_data_hub_ttrpc_async::SealedSecretServiceClient::new(client.clone());
         let secure_mount_client =
@@ -78,9 +86,11 @@ impl CDHClient {
     }
 }
 
-pub async fn init_cdh_client() -> Result<()> {
+pub async fn init_cdh_client(cdh_socket_uri: &str) -> Result<()> {
     CDH_CLIENT
-        .get_or_try_init(|| async { CDHClient::new().context("Failed to create CDH Client") })
+        .get_or_try_init(|| async {
+            CDHClient::new(cdh_socket_uri).context("Failed to create CDH Client")
+        })
         .await?;
     Ok(())
 }
@@ -106,6 +116,74 @@ pub async fn unseal_env(env: &str) -> Result<String> {
     Ok((*env.to_owned()).to_string())
 }
 
+pub async fn unseal_file(path: &str) -> Result<()> {
+    let cdh_client = CDH_CLIENT
+        .get()
+        .expect("Confidential Data Hub not initialized");
+
+    if !Path::new(path).exists() {
+        bail!("sealed secret file {:?} does not exist", path);
+    }
+
+    // Iterate over all entries to handle the sealed secret file.
+    // For example, the directory is as follows:
+    // The secret directory in the guest: /run/kata-containers/shared/containers/21bbf0d932b70263d65d7052ecfd72ee46de03f766650cb378e93852ddb30a54-5063be11b6800f96-sealed-secret-target/:
+    // - ..2024_09_30_02_55_58.2237819815
+    // - ..data -> ..2024_09_30_02_55_58.2237819815
+    // - secret -> ..2024_09_30_02_55_58.2237819815/secret
+    //
+    // The directory "..2024_09_30_02_55_58.2237819815":
+    // - secret
+    for entry in fs::read_dir(path)? {
+        let entry = entry?;
+        let entry_type = entry.file_type()?;
+        if !entry_type.is_symlink() && !entry_type.is_file() {
+            debug!(
+                sl(),
+                "skipping sealed source entry {:?} because its file type is {:?}",
+                entry,
+                entry_type
+            );
+            continue;
+        }
+
+        let target_path = fs::canonicalize(&entry.path())?;
+        info!(sl(), "sealed source entry target path: {:?}", target_path);
+
+        // Skip if the target path is not a file (e.g., it's a symlink pointing to the secret file).
+        if !target_path.is_file() {
+            debug!(sl(), "sealed source is not a file: {:?}", target_path);
+            continue;
+        }
+
+        let secret_name = entry.file_name();
+        let contents = fs::read_to_string(&target_path)?;
+        if contents.starts_with(SEALED_SECRET_PREFIX) {
+            // Get the directory name of the sealed secret file
+            let dir_name = target_path
+                .parent()
+                .and_then(|p| p.file_name())
+                .map(|name| name.to_string_lossy().to_string())
+                .unwrap_or_default();
+
+            // Create the unsealed file name in the same directory, which will be written the unsealed data.
+            let unsealed_filename = format!("{}.unsealed", target_path.to_string_lossy());
+            // Create the unsealed file symlink, which is used for reading the unsealed data in the container.
+            let unsealed_filename_symlink =
+                format!("{}/{}.unsealed", dir_name, secret_name.to_string_lossy());
+
+            // Unseal the secret and write it to the unsealed file
+            let unsealed_value = cdh_client.unseal_secret_async(&contents).await?;
+            fs::write(&unsealed_filename, unsealed_value)?;
+
+            // Remove the original sealed symlink and create a symlink to the unsealed file
+            fs::remove_file(&entry.path())?;
+            symlink(unsealed_filename_symlink, &entry.path())?;
+        }
+    }
+    Ok(())
+}
+
 pub async fn secure_mount(
     volume_type: &str,
     options: &std::collections::HashMap<String, String>,
@@ -123,17 +201,15 @@ pub async fn secure_mount(
 }
 
 #[cfg(test)]
-#[cfg(feature = "sealed-secret")]
 mod tests {
-    use crate::cdh::CDHClient;
-    use crate::cdh::CDH_ADDR;
-    use anyhow::anyhow;
+    use super::*;
     use async_trait::async_trait;
-    use protocols::{confidential_data_hub, confidential_data_hub_ttrpc_async};
+    use std::fs::File;
+    use std::io::{Read, Write};
     use std::sync::Arc;
+    use tempfile::tempdir;
     use test_utils::skip_if_not_root;
     use tokio::signal::unix::{signal, SignalKind};
-
     struct TestService;
 
     #[async_trait]
@@ -161,17 +237,17 @@ mod tests {
         Ok(())
     }
 
-    fn start_ttrpc_server() {
+    fn start_ttrpc_server(cdh_socket_uri: String) {
         tokio::spawn(async move {
             let ss = Box::new(TestService {})
                 as Box<dyn confidential_data_hub_ttrpc_async::SealedSecretService + Send + Sync>;
             let ss = Arc::new(ss);
             let ss_service = confidential_data_hub_ttrpc_async::create_sealed_secret_service(ss);
 
-            remove_if_sock_exist(CDH_ADDR).unwrap();
+            remove_if_sock_exist(&cdh_socket_uri).unwrap();
 
             let mut server = ttrpc::asynchronous::Server::new()
-                .bind(CDH_ADDR)
+                .bind(&cdh_socket_uri)
                 .unwrap()
                 .register_service(ss_service);
 
@@ -187,23 +263,58 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_unseal_env() {
+    async fn test_sealed_secret() {
         skip_if_not_root!();
+        let test_dir = tempdir().expect("failed to create tmpdir");
+        let test_dir_path = test_dir.path();
+        let cdh_sock_uri = &format!(
+            "unix://{}",
+            test_dir_path.join("cdh.sock").to_str().unwrap()
+        );
 
         let rt = tokio::runtime::Runtime::new().unwrap();
         let _guard = rt.enter();
-        start_ttrpc_server();
+        start_ttrpc_server(cdh_sock_uri.to_string());
         std::thread::sleep(std::time::Duration::from_secs(2));
+        init_cdh_client(cdh_sock_uri).await.unwrap();
 
-        let cc = Some(CDHClient::new().unwrap());
-        let cdh_client = cc.as_ref().ok_or(anyhow!("get cdh_client failed")).unwrap();
+        // Test sealed secret as env vars
         let sealed_env = String::from("key=sealed.testdata");
-        let unsealed_env = cdh_client.unseal_env(&sealed_env).await.unwrap();
+        let unsealed_env = unseal_env(&sealed_env).await.unwrap();
         assert_eq!(unsealed_env, String::from("key=unsealed"));
         let normal_env = String::from("key=testdata");
-        let unchanged_env = cdh_client.unseal_env(&normal_env).await.unwrap();
+        let unchanged_env = unseal_env(&normal_env).await.unwrap();
         assert_eq!(unchanged_env, String::from("key=testdata"));
 
+        // Test sealed secret as files
+        let sealed_dir = test_dir_path.join("..test");
+        fs::create_dir(&sealed_dir).unwrap();
+        let sealed_filename = sealed_dir.join("secret");
+        let mut sealed_file = File::create(sealed_filename.clone()).unwrap();
+        sealed_file.write_all(b"sealed.testdata").unwrap();
+        let secret_symlink = test_dir_path.join("secret");
+        symlink(&sealed_filename, &secret_symlink).unwrap();
+
+        unseal_file(test_dir_path.to_str().unwrap()).await.unwrap();
+
+        let unsealed_filename = test_dir_path.join("secret");
+        let mut unsealed_file = File::open(unsealed_filename.clone()).unwrap();
+        let mut contents = String::new();
+        unsealed_file.read_to_string(&mut contents).unwrap();
+        assert_eq!(contents, String::from("unsealed"));
+        fs::remove_file(sealed_filename).unwrap();
+        fs::remove_file(unsealed_filename).unwrap();
+
+        let normal_filename = test_dir_path.join("secret");
+        let mut normal_file = File::create(normal_filename.clone()).unwrap();
+        normal_file.write_all(b"testdata").unwrap();
+        unseal_file(test_dir_path.to_str().unwrap()).await.unwrap();
+        let mut contents = String::new();
+        let mut normal_file = File::open(normal_filename.clone()).unwrap();
+        normal_file.read_to_string(&mut contents).unwrap();
+        assert_eq!(contents, String::from("testdata"));
+        fs::remove_file(normal_filename).unwrap();
+
         rt.shutdown_background();
         std::thread::sleep(std::time::Duration::from_secs(2));
     }

src/agent/src/config.rs

@@ -89,7 +89,7 @@ pub enum GuestComponentsFeatures {
     Resource,
 }
 
-#[derive(Clone, Copy, Debug, Default, Display, Deserialize, EnumString, PartialEq)]
+#[derive(Clone, Copy, Debug, Default, Display, Deserialize, EnumString, PartialEq, Eq)]
 /// Attestation-related processes that we want to spawn as children of the agent
 #[strum(serialize_all = "kebab-case")]
 #[serde(rename_all = "kebab-case")]
@@ -630,6 +630,7 @@ mod tests {
 
     use super::*;
     use anyhow::anyhow;
+    use rstest::*;
     use serial_test::serial;
     use std::fs::File;
     use std::io::Write;
@@ -1327,562 +1328,193 @@ mod tests {
         assert_eq!(expected.tracing, config.tracing);
     }
 
-    #[test]
-    fn test_logrus_to_slog_level() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            logrus_level: &'a str,
-            result: Result<slog::Level>,
-        }
-
-        let tests = &[
-            TestData {
-                logrus_level: "",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL)),
-            },
-            TestData {
-                logrus_level: "foo",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL)),
-            },
-            TestData {
-                logrus_level: "debugging",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL)),
-            },
-            TestData {
-                logrus_level: "xdebug",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL)),
-            },
-            TestData {
-                logrus_level: "trace",
-                result: Ok(slog::Level::Trace),
-            },
-            TestData {
-                logrus_level: "debug",
-                result: Ok(slog::Level::Debug),
-            },
-            TestData {
-                logrus_level: "info",
-                result: Ok(slog::Level::Info),
-            },
-            TestData {
-                logrus_level: "warn",
-                result: Ok(slog::Level::Warning),
-            },
-            TestData {
-                logrus_level: "warning",
-                result: Ok(slog::Level::Warning),
-            },
-            TestData {
-                logrus_level: "error",
-                result: Ok(slog::Level::Error),
-            },
-            TestData {
-                logrus_level: "critical",
-                result: Ok(slog::Level::Critical),
-            },
-            TestData {
-                logrus_level: "fatal",
-                result: Ok(slog::Level::Critical),
-            },
-            TestData {
-                logrus_level: "panic",
-                result: Ok(slog::Level::Critical),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = logrus_to_slog_level(d.logrus_level);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_LOG_LEVEL)))]
+    #[case("foo", Err(anyhow!(ERR_INVALID_LOG_LEVEL)))]
+    #[case("debugging", Err(anyhow!(ERR_INVALID_LOG_LEVEL)))]
+    #[case("xdebug", Err(anyhow!(ERR_INVALID_LOG_LEVEL)))]
+    #[case("trace", Ok(slog::Level::Trace))]
+    #[case("debug", Ok(slog::Level::Debug))]
+    #[case("info", Ok(slog::Level::Info))]
+    #[case("warn", Ok(slog::Level::Warning))]
+    #[case("warning", Ok(slog::Level::Warning))]
+    #[case("error", Ok(slog::Level::Error))]
+    #[case("critical", Ok(slog::Level::Critical))]
+    #[case("fatal", Ok(slog::Level::Critical))]
+    #[case("panic", Ok(slog::Level::Critical))]
+    fn test_logrus_to_slog_level(#[case] input: &str, #[case] expected: Result<slog::Level>) {
+        let result = logrus_to_slog_level(input);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_log_level() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<slog::Level>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)),
-            },
-            TestData {
-                param: "=",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "x=",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "=y",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "==",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)),
-            },
-            TestData {
-                param: "= =",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)),
-            },
-            TestData {
-                param: "x=y",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "agent=debug",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "agent.logg=debug",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "agent.log=trace",
-                result: Ok(slog::Level::Trace),
-            },
-            TestData {
-                param: "agent.log=debug",
-                result: Ok(slog::Level::Debug),
-            },
-            TestData {
-                param: "agent.log=info",
-                result: Ok(slog::Level::Info),
-            },
-            TestData {
-                param: "agent.log=warn",
-                result: Ok(slog::Level::Warning),
-            },
-            TestData {
-                param: "agent.log=warning",
-                result: Ok(slog::Level::Warning),
-            },
-            TestData {
-                param: "agent.log=error",
-                result: Ok(slog::Level::Error),
-            },
-            TestData {
-                param: "agent.log=critical",
-                result: Ok(slog::Level::Critical),
-            },
-            TestData {
-                param: "agent.log=fatal",
-                result: Ok(slog::Level::Critical),
-            },
-            TestData {
-                param: "agent.log=panic",
-                result: Ok(slog::Level::Critical),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_log_level(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    #[rstest]
+    #[case("",Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)))]
+    #[case("=",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("x=",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("=y",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("==",Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)))]
+    #[case("= =",Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)))]
+    #[case("x=y",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("agent=debug",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("agent.logg=debug",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("agent.log=trace", Ok(slog::Level::Trace))]
+    #[case("agent.log=debug", Ok(slog::Level::Debug))]
+    #[case("agent.log=info", Ok(slog::Level::Info))]
+    #[case("agent.log=warn", Ok(slog::Level::Warning))]
+    #[case("agent.log=warning", Ok(slog::Level::Warning))]
+    #[case("agent.log=error", Ok(slog::Level::Error))]
+    #[case("agent.log=critical", Ok(slog::Level::Critical))]
+    #[case("agent.log=fatal", Ok(slog::Level::Critical))]
+    #[case("agent.log=panic", Ok(slog::Level::Critical))]
+    fn test_get_log_level(#[case] input: &str, #[case] expected: Result<slog::Level>) {
+        let result = get_log_level(input);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_timeout() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<time::Duration>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_TIMEOUT)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout",
-                result: Err(anyhow!(ERR_INVALID_TIMEOUT)),
-            },
-            TestData {
-                param: "foo=bar",
-                result: Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)),
-            },
-            TestData {
-                param: "agent.hotplug_timeot=1",
-                result: Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)),
-            },
-            TestData {
-                param: "agent.chd_api_timeout=1",
-                result: Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=1",
-                result: Ok(time::Duration::from_secs(1)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=3",
-                result: Ok(time::Duration::from_secs(3)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=3600",
-                result: Ok(time::Duration::from_secs(3600)),
-            },
-            TestData {
-                param: "agent.cdh_api_timeout=600",
-                result: Ok(time::Duration::from_secs(600)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=0",
-                result: Ok(time::Duration::from_secs(0)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=-1",
-                result: Err(anyhow!(
-                    "unable to parse timeout
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_TIMEOUT)))]
+    #[case("agent.hotplug_timeout", Err(anyhow!(ERR_INVALID_TIMEOUT)))]
+    #[case("foo=bar", Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)))]
+    #[case("agent.hotplug_timeot=1", Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)))]
+    #[case("agent.hotplug_timeout=1", Ok(time::Duration::from_secs(1)))]
+    #[case("agent.hotplug_timeout=3", Ok(time::Duration::from_secs(3)))]
+    #[case("agent.hotplug_timeout=3600", Ok(time::Duration::from_secs(3600)))]
+    #[case("agent.hotplug_timeout=0", Ok(time::Duration::from_secs(0)))]
+    #[case("agent.hotplug_timeout=-1", Err(anyhow!(
+        "unable to parse timeout
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=4jbsdja",
-                result: Err(anyhow!(
-                    "unable to parse timeout
+    )))]
+    #[case("agent.hotplug_timeout=4jbsdja", Err(anyhow!(
+        "unable to parse timeout
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=foo",
-                result: Err(anyhow!(
-                    "unable to parse timeout
+    )))]
+    #[case("agent.hotplug_timeout=foo", Err(anyhow!(
+        "unable to parse timeout
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=j",
-                result: Err(anyhow!(
-                    "unable to parse timeout
+    )))]
+    #[case("agent.hotplug_timeout=j", Err(anyhow!(
+        "unable to parse timeout
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_timeout(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    )))]
+    #[case("agent.chd_api_timeout=1", Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)))]
+    #[case("agent.cdh_api_timeout=600", Ok(time::Duration::from_secs(600)))]
+    fn test_timeout(#[case] param: &str, #[case] expected: Result<time::Duration>) {
+        let result = get_timeout(param);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_container_pipe_size() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<i32>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE)),
-            },
-            TestData {
-                param: "agent.container_pipe_size",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE)),
-            },
-            TestData {
-                param: "foo=bar",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE_KEY)),
-            },
-            TestData {
-                param: "agent.container_pip_siz=1",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE_KEY)),
-            },
-            TestData {
-                param: "agent.container_pipe_size=1",
-                result: Ok(1),
-            },
-            TestData {
-                param: "agent.container_pipe_size=3",
-                result: Ok(3),
-            },
-            TestData {
-                param: "agent.container_pipe_size=2097152",
-                result: Ok(2097152),
-            },
-            TestData {
-                param: "agent.container_pipe_size=0",
-                result: Ok(0),
-            },
-            TestData {
-                param: "agent.container_pipe_size=-1",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_NEGATIVE)),
-            },
-            TestData {
-                param: "agent.container_pipe_size=foobar",
-                result: Err(anyhow!(
-                    "unable to parse container pipe size
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE)))]
+    #[case("agent.container_pipe_size", Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE)))]
+    #[case("foo=bar", Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE_KEY)))]
+    #[case("agent.container_pip_siz=1", Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE_KEY)))]
+    #[case("agent.container_pipe_size=1", Ok(1))]
+    #[case("agent.container_pipe_size=3", Ok(3))]
+    #[case("agent.container_pipe_size=2097152", Ok(2097152))]
+    #[case("agent.container_pipe_size=0", Ok(0))]
+    #[case("agent.container_pipe_size=-1", Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_NEGATIVE)))]
+    #[case("agent.container_pipe_size=foobar", Err(anyhow!(
+        "unable to parse container pipe size
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-            TestData {
-                param: "agent.container_pipe_size=j",
-                result: Err(anyhow!(
-                    "unable to parse container pipe size
+    )))]
+    #[case("agent.container_pipe_size=j", Err(anyhow!(
+        "unable to parse container pipe size
 
 Caused by:
     invalid digit found in string",
-                )),
-            },
-            TestData {
-                param: "agent.container_pipe_size=4jbsdja",
-                result: Err(anyhow!(
-                    "unable to parse container pipe size
+    )))]
+    #[case("agent.container_pipe_size=4jbsdja", Err(anyhow!(
+        "unable to parse container pipe size
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-            TestData {
-                param: "agent.container_pipe_size=4294967296",
-                result: Err(anyhow!(
-                    "unable to parse container pipe size
+    )))]
+    #[case("agent.container_pipe_size=4294967296", Err(anyhow!(
+        "unable to parse container pipe size
 
 Caused by:
     number too large to fit in target type"
-                )),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_container_pipe_size(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    )))]
+    fn test_get_container_pipe_size(#[case] param: &str, #[case] expected: Result<i32>) {
+        let result = get_container_pipe_size(param);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_string_value() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<String>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)),
-            },
-            TestData {
-                param: "=",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "==",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "x=",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_VALUE)),
-            },
-            TestData {
-                param: "x==",
-                result: Ok("=".into()),
-            },
-            TestData {
-                param: "x===",
-                result: Ok("==".into()),
-            },
-            TestData {
-                param: "x==x",
-                result: Ok("=x".into()),
-            },
-            TestData {
-                param: "x=x",
-                result: Ok("x".into()),
-            },
-            TestData {
-                param: "x=x=",
-                result: Ok("x=".into()),
-            },
-            TestData {
-                param: "x=x=x",
-                result: Ok("x=x".into()),
-            },
-            TestData {
-                param: "foo=bar",
-                result: Ok("bar".into()),
-            },
-            TestData {
-                param: "x= =",
-                result: Ok(" =".into()),
-            },
-            TestData {
-                param: "x= =",
-                result: Ok(" =".into()),
-            },
-            TestData {
-                param: "x= = ",
-                result: Ok(" = ".into()),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_string_value(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)))]
+    #[case("=", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("==", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("x=", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_VALUE)))]
+    #[case("x==", Ok("=".into()))]
+    #[case("x===", Ok("==".into()))]
+    #[case("x==x", Ok("=x".into()))]
+    #[case("x=x", Ok("x".into()))]
+    #[case("x=x=", Ok("x=".into()))]
+    #[case("x=x=x", Ok("x=x".into()))]
+    #[case("foo=bar", Ok("bar".into()))]
+    #[case("x= =", Ok(" =".into()))]
+    #[case("x= =", Ok(" =".into()))]
+    #[case("x= = ", Ok(" = ".into()))]
+    fn test_get_string_value(#[case] param: &str, #[case] expected: Result<String>) {
+        let result = get_string_value(param);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_guest_components_features_value() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<GuestComponentsFeatures>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)),
-            },
-            TestData {
-                param: "=",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "==",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "x=all",
-                result: Ok(GuestComponentsFeatures::All),
-            },
-            TestData {
-                param: "x=attestation",
-                result: Ok(GuestComponentsFeatures::Attestation),
-            },
-            TestData {
-                param: "x=resource",
-                result: Ok(GuestComponentsFeatures::Resource),
-            },
-            TestData {
-                param: "x===",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)),
-            },
-            TestData {
-                param: "x==x",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)),
-            },
-            TestData {
-                param: "x=x",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_guest_components_features_value(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)))]
+    #[case("=", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("==", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("x=all", Ok(GuestComponentsFeatures::All))]
+    #[case("x=attestation", Ok(GuestComponentsFeatures::Attestation))]
+    #[case("x=resource", Ok(GuestComponentsFeatures::Resource))]
+    #[case("x===", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)))]
+    #[case("x==x", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)))]
+    #[case("x=x", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)))]
+    fn test_get_guest_components_features_value(
+        #[case] input: &str,
+        #[case] expected: Result<GuestComponentsFeatures>,
+    ) {
+        let result = get_guest_components_features_value(input);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_guest_components_procs_value() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<GuestComponentsProcs>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)),
-            },
-            TestData {
-                param: "=",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "==",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "x=attestation-agent",
-                result: Ok(GuestComponentsProcs::AttestationAgent),
-            },
-            TestData {
-                param: "x=confidential-data-hub",
-                result: Ok(GuestComponentsProcs::ConfidentialDataHub),
-            },
-            TestData {
-                param: "x=none",
-                result: Ok(GuestComponentsProcs::None),
-            },
-            TestData {
-                param: "x=api-server-rest",
-                result: Ok(GuestComponentsProcs::ApiServerRest),
-            },
-            TestData {
-                param: "x===",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
-            },
-            TestData {
-                param: "x==x",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
-            },
-            TestData {
-                param: "x=x",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_guest_components_procs_value(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)))]
+    #[case("=", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("==", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("x=attestation-agent", Ok(GuestComponentsProcs::AttestationAgent))]
+    #[case(
+        "x=confidential-data-hub",
+        Ok(GuestComponentsProcs::ConfidentialDataHub)
+    )]
+    #[case("x=none", Ok(GuestComponentsProcs::None))]
+    #[case("x=api-server-rest", Ok(GuestComponentsProcs::ApiServerRest))]
+    #[case("x===", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)))]
+    #[case("x==x", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)))]
+    #[case("x=x", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)))]
+    fn test_get_guest_components_procs_value(
+        #[case] param: &str,
+        #[case] expected: Result<GuestComponentsProcs>,
+    ) {
+        let result = get_guest_components_procs_value(param);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
     #[test]

src/agent/src/device/mod.rs

@@ -11,6 +11,9 @@ use self::vfio_device_handler::{VfioApDeviceHandler, VfioPciDeviceHandler};
 use crate::pci;
 use crate::sandbox::Sandbox;
 use anyhow::{anyhow, Context, Result};
+use cdi::annotations::parse_annotations;
+use cdi::cache::{new_cache, with_auto_refresh, CdiOption};
+use cdi::spec_dirs::with_spec_dirs;
 use kata_types::device::DeviceHandlerManager;
 use nix::sys::stat;
 use oci::{LinuxDeviceCgroup, Spec};
@@ -25,6 +28,8 @@ use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
+use tokio::time;
+use tokio::time::Duration;
 use tracing::instrument;
 
 pub mod block_device_handler;
@@ -238,6 +243,69 @@ pub async fn add_devices(
     update_spec_devices(logger, spec, dev_updates)
 }
 
+#[instrument]
+pub async fn handle_cdi_devices(
+    logger: &Logger,
+    spec: &mut Spec,
+    spec_dir: &str,
+    cdi_timeout: u64,
+) -> Result<()> {
+    if let Some(container_type) = spec
+        .annotations()
+        .as_ref()
+        .and_then(|a| a.get("io.katacontainers.pkg.oci.container_type"))
+    {
+        if container_type == "pod_sandbox" {
+            return Ok(());
+        }
+    }
+
+    let (_, devices) = parse_annotations(spec.annotations().as_ref().unwrap())?;
+
+    if devices.is_empty() {
+        info!(logger, "no CDI annotations, no devices to inject");
+        return Ok(());
+    }
+    // Explicitly set the cache options to disable auto-refresh and
+    // to use the single spec dir "/var/run/cdi" for tests it can be overridden
+    let options: Vec<CdiOption> = vec![with_auto_refresh(false), with_spec_dirs(&[spec_dir])];
+    let cache: Arc<std::sync::Mutex<cdi::cache::Cache>> = new_cache(options);
+
+    for _ in 0..=cdi_timeout {
+        let inject_result = {
+            // Lock cache within this scope, std::sync::Mutex has no Send
+            // and await will not work with time::sleep
+            let mut cache = cache.lock().unwrap();
+            match cache.refresh() {
+                Ok(_) => {}
+                Err(e) => {
+                    return Err(anyhow!("error refreshing cache: {:?}", e));
+                }
+            }
+            cache.inject_devices(Some(spec), devices.clone())
+        };
+
+        match inject_result {
+            Ok(_) => {
+                info!(
+                    logger,
+                    "all devices injected successfully, modified CDI container spec: {:?}", &spec
+                );
+                return Ok(());
+            }
+            Err(e) => {
+                info!(logger, "error injecting devices: {:?}", e);
+                println!("error injecting devices: {:?}", e);
+            }
+        }
+        time::sleep(Duration::from_millis(1000)).await;
+    }
+    Err(anyhow!(
+        "failed to inject devices after CDI timeout of {} seconds",
+        cdi_timeout
+    ))
+}
+
 #[instrument]
 async fn validate_device(
     logger: &Logger,
@@ -1110,4 +1178,95 @@ mod tests {
         assert!(name.is_ok(), "{}", name.unwrap_err());
         assert_eq!(name.unwrap(), devname);
     }
+
+    #[tokio::test]
+    async fn test_handle_cdi_devices() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let mut spec = Spec::default();
+
+        let mut annotations = HashMap::new();
+        // cdi.k8s.io/vendor1_devices: vendor1.com/device=foo
+        annotations.insert(
+            "cdi.k8s.io/vfio17".to_string(),
+            "kata.com/gpu=0".to_string(),
+        );
+        spec.set_annotations(Some(annotations));
+
+        // create a file in /tmp/cdi with nvidia.json content
+        let cdi_dir = PathBuf::from("/tmp/cdi");
+        let cdi_file = cdi_dir.join("kata.json");
+
+        let cdi_version = "0.6.0";
+        let kind = "kata.com/gpu";
+        let device_name = "0";
+        let annotation_whatever = "false";
+        let annotation_whenever = "true";
+        let inner_env = "TEST_INNER_ENV=TEST_INNER_ENV_VALUE";
+        let outer_env = "TEST_OUTER_ENV=TEST_OUTER_ENV_VALUE";
+        let inner_device = "/dev/zero";
+        let outer_device = "/dev/null";
+
+        let cdi_content = format!(
+            r#"{{
+            "cdiVersion": "{cdi_version}",
+            "kind": "{kind}",
+            "devices": [
+                {{
+                    "name": "{device_name}",
+                    "annotations": {{
+                        "whatever": "{annotation_whatever}",
+                        "whenever": "{annotation_whenever}"
+                    }},
+                    "containerEdits": {{
+                        "env": [
+                            "{inner_env}"
+                        ],
+                        "deviceNodes": [
+                            {{
+                                "path": "{inner_device}"
+                            }}
+                        ]
+                    }}
+                }}
+            ],
+            "containerEdits": {{
+                "env": [
+                    "{outer_env}"
+                ],
+                "deviceNodes": [
+                    {{
+                        "path": "{outer_device}"
+                    }}
+                ]
+            }}
+        }}"#
+        );
+
+        fs::create_dir_all(&cdi_dir).unwrap();
+        fs::write(&cdi_file, cdi_content).unwrap();
+
+        let res = handle_cdi_devices(&logger, &mut spec, "/tmp/cdi", 0).await;
+        println!("modfied spec {:?}", spec);
+        assert!(res.is_ok(), "{}", res.err().unwrap());
+
+        let linux = spec.linux().as_ref().unwrap();
+        let devices = linux
+            .resources()
+            .as_ref()
+            .unwrap()
+            .devices()
+            .as_ref()
+            .unwrap();
+        assert_eq!(devices.len(), 2);
+
+        let env = spec.process().as_ref().unwrap().env().as_ref().unwrap();
+
+        // find string TEST_OUTER_ENV in evn
+        let outer_env = env.iter().find(|e| e.starts_with("TEST_OUTER_ENV"));
+        assert!(outer_env.is_some(), "TEST_OUTER_ENV not found in env");
+
+        // find TEST_INNER_ENV in env
+        let inner_env = env.iter().find(|e| e.starts_with("TEST_INNER_ENV"));
+        assert!(inner_env.is_some(), "TEST_INNER_ENV not found in env");
+    }
 }

src/agent/src/image.rs

@@ -21,6 +21,9 @@ use tokio::sync::Mutex;
 use crate::rpc::CONTAINER_BASE;
 use crate::AGENT_CONFIG;
 
+use kata_types::mount::KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL;
+use protocols::agent::Storage;
+
 pub const KATA_IMAGE_WORK_DIR: &str = "/run/kata-containers/image/";
 const CONFIG_JSON: &str = "config.json";
 const KATA_PAUSE_BUNDLE: &str = "/pause_bundle";
@@ -81,6 +84,28 @@ impl ImageService {
         Self { image_client }
     }
 
+    /// get guest pause image process specification
+    fn get_pause_image_process() -> Result<oci::Process> {
+        let guest_pause_bundle = Path::new(KATA_PAUSE_BUNDLE);
+        if !guest_pause_bundle.exists() {
+            bail!("Pause image not present in rootfs");
+        }
+        let guest_pause_config = scoped_join(guest_pause_bundle, CONFIG_JSON)?;
+
+        let image_oci = oci::Spec::load(guest_pause_config.to_str().ok_or_else(|| {
+            anyhow!(
+                "Failed to load the guest pause image config from {:?}",
+                guest_pause_config
+            )
+        })?)
+        .context("load image config file")?;
+
+        let image_oci_process = image_oci.process().as_ref().ok_or_else(|| {
+            anyhow!("The guest pause image config does not contain a process specification. Please check the pause image.")
+        })?;
+        Ok(image_oci_process.clone())
+    }
+
     /// pause image is packaged in rootfs
     fn unpack_pause_image(cid: &str) -> Result<String> {
         verify_id(cid).context("The guest pause image cid contains invalid characters.")?;
@@ -132,6 +157,20 @@ impl ImageService {
         Ok(pause_rootfs.display().to_string())
     }
 
+    /// check whether the image is for sandbox or for container.
+    fn is_sandbox(image_metadata: &HashMap<String, String>) -> bool {
+        let mut is_sandbox = false;
+        for key in K8S_CONTAINER_TYPE_KEYS.iter() {
+            if let Some(value) = image_metadata.get(key as &str) {
+                if value == "sandbox" {
+                    is_sandbox = true;
+                    break;
+                }
+            }
+        }
+        is_sandbox
+    }
+
     /// pull_image is used for call image-rs to pull image in the guest.
     /// # Parameters
     /// - `image`: Image name (exp: quay.io/prometheus/busybox:latest)
@@ -147,18 +186,7 @@ impl ImageService {
     ) -> Result<String> {
         info!(sl(), "image metadata: {image_metadata:?}");
 
-        //Check whether the image is for sandbox or for container.
-        let mut is_sandbox = false;
-        for key in K8S_CONTAINER_TYPE_KEYS.iter() {
-            if let Some(value) = image_metadata.get(key as &str) {
-                if value == "sandbox" {
-                    is_sandbox = true;
-                    break;
-                }
-            }
-        }
-
-        if is_sandbox {
+        if Self::is_sandbox(image_metadata) {
             let mount_path = Self::unpack_pause_image(cid)?;
             return Ok(mount_path);
         }
@@ -194,6 +222,32 @@ impl ImageService {
     }
 }
 
+/// get_process overrides the OCI process spec with pause image process spec if needed
+pub fn get_process(
+    ocip: &oci::Process,
+    oci: &oci::Spec,
+    storages: Vec<Storage>,
+) -> Result<oci::Process> {
+    let mut guest_pull = false;
+    for storage in storages {
+        if storage.driver == KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL {
+            guest_pull = true;
+            break;
+        }
+    }
+    if guest_pull {
+        match oci.annotations() {
+            Some(a) => {
+                if ImageService::is_sandbox(a) {
+                    return ImageService::get_pause_image_process();
+                }
+            }
+            None => {}
+        }
+    }
+    Ok(ocip.clone())
+}
+
 /// Set proxy environment from AGENT_CONFIG
 pub async fn set_proxy_env_vars() {
     if env::var("HTTPS_PROXY").is_err() {

src/agent/src/main.rs

@@ -21,7 +21,7 @@ extern crate slog;
 use anyhow::{anyhow, Context, Result};
 use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
-use const_format::concatcp;
+use const_format::{concatcp, formatcp};
 use nix::fcntl::OFlag;
 use nix::sys::reboot::{reboot, RebootMode};
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
@@ -29,7 +29,7 @@ use nix::unistd::{self, dup, sync, Pid};
 use std::env;
 use std::ffi::OsStr;
 use std::fs::{self, File};
-use std::os::unix::fs as unixfs;
+use std::os::unix::fs::{self as unixfs, FileTypeExt};
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
 use std::process::exit;
@@ -109,7 +109,18 @@ const CDH_SOCKET_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, CDH_SOCKET);
 const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest";
 
 /// Path of ocicrypt config file. This is used by image-rs when decrypting image.
-const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+const OCICRYPT_CONFIG_PATH: &str = "/run/confidential-containers/ocicrypt_config.json";
+
+const OCICRYPT_CONFIG: &str = formatcp!(
+    r#"{{
+    "key-providers": {{
+        "attestation-agent": {{
+            "ttrpc": "{}"
+        }}
+    }}
+}}"#,
+    CDH_SOCKET_URI
+);
 
 const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6;
 
@@ -408,15 +419,13 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     let gc_procs = config.guest_components_procs;
-    if gc_procs != GuestComponentsProcs::None {
-        if !attestation_binaries_available(logger, &gc_procs) {
-            warn!(
-                logger,
-                "attestation binaries requested for launch not available"
-            );
-        } else {
-            init_attestation_components(logger, config).await?;
-        }
+    if !attestation_binaries_available(logger, &gc_procs) {
+        warn!(
+            logger,
+            "attestation binaries requested for launch not available"
+        );
+    } else {
+        init_attestation_components(logger, config).await?;
     }
 
     // vsock:///dev/vsock, port
@@ -447,12 +456,7 @@ fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs)
     true
 }
 
-// Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
-// and the corresponding procs are enabled in the agent configuration. the process will be
-// launched in the background and the function will return immediately.
-// If the CDH is started, a CDH client will be instantiated and returned.
-async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> {
-    // skip launch of any guest-component
+async fn launch_guest_component_procs(logger: &Logger, config: &AgentConfig) -> Result<()> {
     if config.guest_components_procs == GuestComponentsProcs::None {
         return Ok(());
     }
@@ -472,17 +476,6 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
         return Ok(());
     }
 
-    let ocicrypt_config = serde_json::json!({
-        "key-providers": {
-            "attestation-agent":{
-                "ttrpc":CDH_SOCKET_URI
-            }
-        }
-    });
-
-    fs::write(OCICRYPT_CONFIG_PATH, ocicrypt_config.to_string().as_bytes())?;
-    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
-
     debug!(
         logger,
         "spawning confidential-data-hub process {}", CDH_PATH
@@ -497,9 +490,6 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
     )
     .map_err(|e| anyhow!("launch_process {} failed: {:?}", CDH_PATH, e))?;
 
-    // initialize cdh client
-    cdh::init_cdh_client().await?;
-
     // skip launch of api-server-rest
     if config.guest_components_procs == GuestComponentsProcs::ConfidentialDataHub {
         return Ok(());
@@ -522,6 +512,34 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
     Ok(())
 }
 
+// Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
+// and the corresponding procs are enabled in the agent configuration. the process will be
+// launched in the background and the function will return immediately.
+// If the CDH is started, a CDH client will be instantiated and returned.
+async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> {
+    launch_guest_component_procs(logger, config).await?;
+
+    fs::write(OCICRYPT_CONFIG_PATH, OCICRYPT_CONFIG.as_bytes())?;
+    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
+
+    // If a CDH socket exists, initialize the CDH client
+    match tokio::fs::metadata(CDH_SOCKET).await {
+        Ok(md) => {
+            if md.file_type().is_socket() {
+                cdh::init_cdh_client(CDH_SOCKET_URI).await?;
+            } else {
+                debug!(logger, "File {} is not a socket", CDH_SOCKET);
+            }
+        }
+        Err(err) => warn!(
+            logger,
+            "Failed to probe CDH socket file {}: {:?}", CDH_SOCKET, err
+        ),
+    }
+
+    Ok(())
+}
+
 fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> {
     let p = Path::new(path);
     let mut attempts = 0;

src/agent/src/policy.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::Result;
+use anyhow::{bail, Result};
 use protobuf::MessageDyn;
 use tokio::io::AsyncWriteExt;
 
@@ -68,6 +68,12 @@ pub struct AgentPolicy {
     engine: regorus::Engine,
 }
 
+#[derive(serde::Deserialize, Debug)]
+struct MetadataResponse {
+    allowed: bool,
+    ops: Option<json_patch::Patch>,
+}
+
 impl AgentPolicy {
     /// Create AgentPolicy object.
     pub fn new() -> Self {
@@ -82,6 +88,17 @@ impl AgentPolicy {
         let mut engine = regorus::Engine::new();
         engine.set_strict_builtin_errors(false);
         engine.set_gather_prints(true);
+        // assign a slice of the engine data "pstate" to be used as policy state
+        engine
+            .add_data(
+                regorus::Value::from_json_str(
+                    r#"{
+                        "pstate": {}
+                    }"#,
+                )
+                .unwrap(),
+            )
+            .unwrap();
         engine
     }
 
@@ -112,6 +129,23 @@ impl AgentPolicy {
         Ok(())
     }
 
+    async fn apply_patch_to_state(&mut self, patch: json_patch::Patch) -> Result<()> {
+        // Convert the current engine data to a JSON value
+        let mut state = serde_json::to_value(self.engine.get_data())?;
+
+        // Apply the patch to the state
+        json_patch::patch(&mut state, &patch)?;
+
+        // Clear the existing data in the engine
+        self.engine.clear_data();
+
+        // Add the patched state back to the engine
+        self.engine
+            .add_data(regorus::Value::from_json_str(&state.to_string())?)?;
+
+        Ok(())
+    }
+
     /// Ask regorus if an API call should be allowed or not.
     async fn allow_request(&mut self, ep: &str, ep_input: &str) -> Result<(bool, String)> {
         debug!(sl!(), "policy check: {ep}");
@@ -120,13 +154,56 @@ impl AgentPolicy {
         let query = format!("data.agent_policy.{ep}");
         self.engine.set_input_json(ep_input)?;
 
-        let mut allow = match self.engine.eval_bool_query(query, false) {
-            Ok(a) => a,
-            Err(e) => {
-                if !self.allow_failures {
-                    return Err(e);
+        let results = self.engine.eval_query(query, false)?;
+
+        let prints = match self.engine.take_prints() {
+            Ok(p) => p.join(" "),
+            Err(e) => format!("Failed to get policy log: {e}"),
+        };
+
+        if results.result.len() != 1 {
+            // Results are empty when AllowRequestsFailingPolicy is used to allow a Request that hasn't been defined in the policy
+            if self.allow_failures {
+                return Ok((true, prints));
+            }
+            bail!(
+                "policy check: unexpected eval_query result len {:?}",
+                results
+            );
+        }
+
+        if results.result[0].expressions.len() != 1 {
+            bail!(
+                "policy check: unexpected eval_query result expressions {:?}",
+                results
+            );
+        }
+
+        let mut allow = match &results.result[0].expressions[0].value {
+            regorus::Value::Bool(b) => *b,
+
+            // Match against a specific variant that could be interpreted as MetadataResponse
+            regorus::Value::Object(obj) => {
+                let json_str = serde_json::to_string(obj)?;
+
+                self.log_eval_input(ep, &json_str).await;
+
+                let metadata_response: MetadataResponse = serde_json::from_str(&json_str)?;
+
+                if metadata_response.allowed {
+                    if let Some(ops) = metadata_response.ops {
+                        self.apply_patch_to_state(ops).await?;
+                    }
                 }
-                false
+                metadata_response.allowed
+            }
+
+            _ => {
+                error!(sl!(), "allow_request: unexpected eval_query result type");
+                bail!(
+                    "policy check: unexpected eval_query result type {:?}",
+                    results
+                );
             }
         };
 
@@ -135,11 +212,6 @@ impl AgentPolicy {
             allow = true;
         }
 
-        let prints = match self.engine.take_prints() {
-            Ok(p) => p.join(" "),
-            Err(e) => format!("Failed to get policy log: {e}"),
-        };
-
         Ok((allow, prints))
     }
 

src/agent/src/rpc.rs

@@ -58,7 +58,7 @@ use rustjail::process::ProcessOperations;
 use crate::cdh;
 use crate::device::block_device_handler::get_virtio_blk_pci_device_name;
 use crate::device::network_device_handler::wait_for_net_interface;
-use crate::device::{add_devices, update_env_pci};
+use crate::device::{add_devices, handle_cdi_devices, update_env_pci};
 use crate::features::get_build_features;
 use crate::image::KATA_IMAGE_WORK_DIR;
 use crate::linux_abi::*;
@@ -130,6 +130,8 @@ const ERR_NO_SANDBOX_PIDNS: &str = "Sandbox does not have sandbox_pidns";
 // not available.
 const IPTABLES_RESTORE_WAIT_SEC: u64 = 5;
 
+const CDI_TIMEOUT_LIMIT: u64 = 100;
+
 // Convenience function to obtain the scope logger.
 fn sl() -> slog::Logger {
     slog_scope::logger()
@@ -224,54 +226,16 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&sl(), &req.devices, &mut oci, &self.sandbox).await?;
 
-        let process = oci
-            .process_mut()
-            .as_mut()
-            .ok_or_else(|| anyhow!("Spec didn't contain process field"))?;
-        if cdh::is_cdh_client_initialized().await {
-            if let Some(envs) = process.env_mut().as_mut() {
-                for env in envs.iter_mut() {
-                    match cdh::unseal_env(env).await {
-                        Ok(unsealed_env) => *env = unsealed_env.to_string(),
-                        Err(e) => {
-                            warn!(sl(), "Failed to unseal secret: {}", e)
-                        }
-                    }
-                }
-            }
-        }
+        // In guest-kernel mode some devices need extra handling. Taking the
+        // GPU as an example the shim will inject CDI annotations that will
+        // be used by the kata-agent to do containerEdits according to the
+        // CDI spec coming from a registry that is created on the fly by UDEV
+        // or other entities for a specifc device.
+        // In Kata we only consider the directory "/var/run/cdi", "/etc" may be
+        // readonly
+        handle_cdi_devices(&sl(), &mut oci, "/var/run/cdi", CDI_TIMEOUT_LIMIT).await?;
 
-        let linux = oci
-            .linux()
-            .as_ref()
-            .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?;
-
-        if cdh::is_cdh_client_initialized().await {
-            if let Some(devices) = linux.devices() {
-                for specdev in devices.iter() {
-                    if specdev.path().as_path().to_str() == Some(TRUSTED_IMAGE_STORAGE_DEVICE) {
-                        let dev_major_minor = format!("{}:{}", specdev.major(), specdev.minor());
-                        let secure_storage_integrity =
-                            AGENT_CONFIG.secure_storage_integrity.to_string();
-                        info!(
-                            sl(),
-                            "trusted_store device major:min {}, enable data integrity {}",
-                            dev_major_minor,
-                            secure_storage_integrity
-                        );
-
-                        let options = std::collections::HashMap::from([
-                            ("deviceId".to_string(), dev_major_minor),
-                            ("encryptType".to_string(), "LUKS".to_string()),
-                            ("dataIntegrity".to_string(), secure_storage_integrity),
-                        ]);
-                        cdh::secure_mount("BlockDevice", &options, vec![], KATA_IMAGE_WORK_DIR)
-                            .await?;
-                        break;
-                    }
-                }
-            }
-        }
+        cdh_handler(&mut oci).await?;
 
         // Both rootfs and volumes (invoked with --volume for instance) will
         // be processed the same way. The idea is to always mount any provided
@@ -280,7 +244,13 @@ impl AgentService {
         // After all those storages have been processed, no matter the order
         // here, the agent will rely on rustjail (using the oci.Mounts
         // list) to bind mount all of them inside the container.
-        let m = add_storages(sl(), req.storages, &self.sandbox, Some(req.container_id)).await?;
+        let m = add_storages(
+            sl(),
+            req.storages.clone(),
+            &self.sandbox,
+            Some(req.container_id),
+        )
+        .await?;
 
         let mut s = self.sandbox.lock().await;
         s.container_mounts.insert(cid.clone(), m);
@@ -335,6 +305,13 @@ impl AgentService {
         let pipe_size = AGENT_CONFIG.container_pipe_size;
 
         let p = if let Some(p) = oci.process() {
+            #[cfg(feature = "guest-pull")]
+            {
+                let new_p = image::get_process(p, &oci, req.storages.clone())?;
+                Process::new(&sl(), &new_p, cid.as_str(), true, pipe_size, proc_io)?
+            }
+
+            #[cfg(not(feature = "guest-pull"))]
             Process::new(&sl(), p, cid.as_str(), true, pipe_size, proc_io)?
         } else {
             info!(sl(), "no process configurations!");
@@ -1726,13 +1703,19 @@ fn update_container_namespaces(
     if let Some(namespaces) = linux.namespaces_mut() {
         for namespace in namespaces.iter_mut() {
             if namespace.typ().to_string() == NSTYPEIPC {
-                namespace.set_path(Some(PathBuf::from(&sandbox.shared_ipcns.path.clone())));
-                namespace.set_path(None);
+                namespace.set_path(if !sandbox.shared_ipcns.path.is_empty() {
+                    Some(PathBuf::from(&sandbox.shared_ipcns.path))
+                } else {
+                    None
+                });
                 continue;
             }
             if namespace.typ().to_string() == NSTYPEUTS {
-                namespace.set_path(Some(PathBuf::from(&sandbox.shared_utsns.path.clone())));
-                namespace.set_path(None);
+                namespace.set_path(if !sandbox.shared_utsns.path.is_empty() {
+                    Some(PathBuf::from(&sandbox.shared_utsns.path))
+                } else {
+                    None
+                });
                 continue;
             }
         }
@@ -1750,7 +1733,7 @@ fn update_container_namespaces(
                 if !pidns.path.is_empty() {
                     pid_ns.set_path(Some(PathBuf::from(&pidns.path)));
                 }
-            } else {
+            } else if !sandbox.containers.is_empty() {
                 return Err(anyhow!(ERR_NO_SANDBOX_PIDNS));
             }
         }
@@ -2093,6 +2076,76 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> {
     }
 }
 
+async fn cdh_handler(oci: &mut Spec) -> Result<()> {
+    if !cdh::is_cdh_client_initialized().await {
+        return Ok(());
+    }
+    let process = oci
+        .process_mut()
+        .as_mut()
+        .ok_or_else(|| anyhow!("Spec didn't contain process field"))?;
+    if let Some(envs) = process.env_mut().as_mut() {
+        for env in envs.iter_mut() {
+            match cdh::unseal_env(env).await {
+                Ok(unsealed_env) => *env = unsealed_env.to_string(),
+                Err(e) => {
+                    warn!(sl(), "Failed to unseal secret: {}", e)
+                }
+            }
+        }
+    }
+
+    let mounts = oci
+        .mounts_mut()
+        .as_mut()
+        .ok_or_else(|| anyhow!("Spec didn't contain mounts field"))?;
+
+    for m in mounts.iter_mut() {
+        if m.destination().starts_with("/sealed") {
+            info!(
+                sl(),
+                "sealed mount destination: {:?} source: {:?}",
+                m.destination(),
+                m.source()
+            );
+            if let Some(source_str) = m.source().as_ref().and_then(|p| p.to_str()) {
+                cdh::unseal_file(source_str).await?;
+            } else {
+                warn!(sl(), "Failed to unseal: Mount source is None or invalid");
+            }
+        }
+    }
+
+    let linux = oci
+        .linux()
+        .as_ref()
+        .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?;
+
+    if let Some(devices) = linux.devices() {
+        for specdev in devices.iter() {
+            if specdev.path().as_path().to_str() == Some(TRUSTED_IMAGE_STORAGE_DEVICE) {
+                let dev_major_minor = format!("{}:{}", specdev.major(), specdev.minor());
+                let secure_storage_integrity = AGENT_CONFIG.secure_storage_integrity.to_string();
+                info!(
+                    sl(),
+                    "trusted_store device major:min {}, enable data integrity {}",
+                    dev_major_minor,
+                    secure_storage_integrity
+                );
+
+                let options = std::collections::HashMap::from([
+                    ("deviceId".to_string(), dev_major_minor),
+                    ("encryptType".to_string(), "LUKS".to_string()),
+                    ("dataIntegrity".to_string(), secure_storage_integrity),
+                ]);
+                cdh::secure_mount("BlockDevice", &options, vec![], KATA_IMAGE_WORK_DIR).await?;
+                break;
+            }
+        }
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 #[allow(dead_code)]
 mod tests {
@@ -2527,14 +2580,6 @@ mod tests {
                     .unwrap()],
                 ..Default::default()
             },
-            TestData {
-                namespaces: vec![],
-                sandbox_pidns_path: None,
-                use_sandbox_pidns: true,
-                result: Err(anyhow!(ERR_NO_SANDBOX_PIDNS)),
-                expected_namespaces: vec![],
-                ..Default::default()
-            },
             TestData {
                 has_linux_in_spec: false,
                 result: Err(anyhow!(ERR_NO_LINUX_FIELD)),

src/agent/src/storage/ephemeral_handler.rs

@@ -170,7 +170,7 @@ impl EphemeralHandler {
         let size = size_str
             .unwrap()
             .parse::<u64>()
-            .context(format!("parse size: {:?}", &pagesize_str))?;
+            .context(format!("parse size: {:?}", &size_str))?;
 
         Ok((pagesize, size))
     }

src/libs/Cargo.lock

@@ -240,19 +240,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "cgroups-rs"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
-dependencies = [
- "libc",
- "log",
- "nix 0.25.1",
- "regex",
- "thiserror",
-]
-
 [[package]]
 name = "chrono"
 version = "0.4.20"
@@ -814,7 +801,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -975,18 +961,6 @@ dependencies = [
  "memoffset 0.6.5",
 ]
 
-[[package]]
-name = "nix"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
-dependencies = [
- "autocfg",
- "bitflags",
- "cfg-if",
- "libc",
-]
-
 [[package]]
 name = "nix"
 version = "0.26.4"
@@ -1316,7 +1290,6 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.2.0",
  "serde",

src/libs/kata-sys-util/Cargo.toml

@@ -13,7 +13,6 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0.31"
 byteorder = "1.4.3"
-cgroups = { package = "cgroups-rs", version = "0.3.2" }
 chrono = "0.4.0"
 common-path = "=1.0.0"
 fail = "0.5.0"

src/libs/kata-sys-util/README.md

@@ -1,10 +1,9 @@
-# kata-sys-util
+# `kata-sys-util`
 
 This crate is a collection of utilities and helpers for 
 [Kata Containers](https://github.com/kata-containers/kata-containers/) components to access system services.
 
 It provides safe wrappers over system services, such as:
-- cgroups
 - file systems
 - mount
 - NUMA

src/libs/kata-sys-util/src/spec.rs

@@ -97,11 +97,3 @@ pub fn load_oci_spec() -> Result<oci::Spec, OciSpecError> {
 
     oci::Spec::load(spec_file.to_str().unwrap_or_default())
 }
-
-/// handle string parsing for input possibly be JSON string.
-pub fn parse_json_string(input: &str) -> &str {
-    let json_str: &str = serde_json::from_str(input).unwrap_or(input);
-    let stripped_str = json_str.strip_prefix("CAP_").unwrap_or(json_str);
-
-    stripped_str
-}

src/libs/kata-types/src/annotations/cri_containerd.rs

@@ -11,7 +11,12 @@ pub const CONTAINER_NAME_LABEL_KEY: &str = "io.kubernetes.cri.container-name";
 pub const SANDBOX: &str = "sandbox";
 pub const CONTAINER: &str = "container";
 
+// SandboxID is the sandbox ID annotation
 pub const SANDBOX_ID_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-id";
+// SandboxName is the name of the sandbox (pod)
+pub const SANDBOX_NAME_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-name";
+// SandboxNamespace is the name of the namespace of the sandbox (pod)
+pub const SANDBOX_NAMESPACE_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-namespace";
 
 // Ref: https://pkg.go.dev/github.com/containerd/containerd@v1.6.7/pkg/cri/annotations
 // SandboxCPU annotations are based on the initial CPU configuration for the sandbox. This is calculated as the

src/libs/kata-types/src/annotations/mod.rs

@@ -88,12 +88,6 @@ pub const KATA_ANNO_CFG_HYPERVISOR_PREFIX: &str = "io.katacontainers.config.hype
 pub const KATA_ANNO_CFG_HYPERVISOR_PATH: &str = "io.katacontainers.config.hypervisor.path";
 /// A sandbox annotation for passing a container hypervisor binary SHA-512 hash value.
 pub const KATA_ANNO_CFG_HYPERVISOR_HASH: &str = "io.katacontainers.config.hypervisor.path_hash";
-/// A sandbox annotation for passing a per container path pointing at the hypervisor control binary
-/// that will run the container VM.
-pub const KATA_ANNO_CFG_HYPERVISOR_CTLPATH: &str = "io.katacontainers.config.hypervisor.ctlpath";
-/// A sandbox annotation for passing a container hypervisor control binary SHA-512 hash value.
-pub const KATA_ANNO_CFG_HYPERVISOR_CTLHASH: &str =
-    "io.katacontainers.config.hypervisor.hypervisorctl_hash";
 /// A sandbox annotation for passing a per container path pointing at the jailer that will constrain
 /// the container VM.
 pub const KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH: &str =
@@ -506,10 +500,6 @@ impl Annotation {
                         hv.validate_hypervisor_path(value)?;
                         hv.path = value.to_string();
                     }
-                    KATA_ANNO_CFG_HYPERVISOR_CTLPATH => {
-                        hv.validate_hypervisor_ctlpath(value)?;
-                        hv.ctlpath = value.to_string();
-                    }
 
                     KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH => {
                         hv.validate_jailer_path(value)?;

src/libs/kata-types/src/config/default.rs

@@ -98,3 +98,11 @@ pub const DEFAULT_FIRECRACKER_GUEST_KERNEL_IMAGE: &str = "vmlinux";
 pub const DEFAULT_FIRECRACKER_GUEST_KERNEL_PARAMS: &str = "";
 pub const MAX_FIRECRACKER_VCPUS: u32 = 32;
 pub const MIN_FIRECRACKER_MEMORY_SIZE_MB: u32 = 128;
+
+// Default configuration for remote
+pub const DEFAULT_REMOTE_HYPERVISOR_SOCKET: &str = "/run/peerpod/hypervisor.sock";
+pub const DEFAULT_REMOTE_HYPERVISOR_TIMEOUT: i32 = 600; // 600 Seconds
+pub const MAX_REMOTE_VCPUS: u32 = 32;
+pub const MIN_REMOTE_MEMORY_SIZE_MB: u32 = 64;
+pub const DEFAULT_REMOTE_MEMORY_SIZE_MB: u32 = 128;
+pub const DEFAULT_REMOTE_MEMORY_SLOTS: u32 = 128;

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -44,6 +44,9 @@ pub use self::qemu::{QemuConfig, HYPERVISOR_NAME_QEMU};
 mod ch;
 pub use self::ch::{CloudHypervisorConfig, HYPERVISOR_NAME_CH};
 
+mod remote;
+pub use self::remote::{RemoteConfig, HYPERVISOR_NAME_REMOTE};
+
 /// Virtual PCI block device driver.
 pub const VIRTIO_BLK_PCI: &str = "virtio-blk-pci";
 
@@ -540,6 +543,7 @@ impl TopologyConfigInfo {
             HYPERVISOR_NAME_CH,
             HYPERVISOR_NAME_DRAGONBALL,
             HYPERVISOR_NAME_FIRECRACKER,
+            HYPERVISOR_NAME_REMOTE,
         ];
         let hypervisor_name = toml_config.runtime.hypervisor_name.as_str();
         if !hypervisor_names.contains(&hypervisor_name) {
@@ -1040,6 +1044,18 @@ impl SharedFsInfo {
     }
 }
 
+/// Configuration information for remote hypervisor type.
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub struct RemoteInfo {
+    /// Remote hypervisor socket path
+    #[serde(default)]
+    pub hypervisor_socket: String,
+
+    /// Remote hyperisor timeout of creating (in seconds)
+    #[serde(default)]
+    pub hypervisor_timeout: i32,
+}
+
 /// Common configuration information for hypervisors.
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct Hypervisor {
@@ -1123,6 +1139,10 @@ pub struct Hypervisor {
     #[serde(default, flatten)]
     pub shared_fs: SharedFsInfo,
 
+    /// Remote hypervisor configuration information.
+    #[serde(default, flatten)]
+    pub remote_info: RemoteInfo,
+
     /// A sandbox annotation used to specify prefetch_files.list host path container image
     /// being used, and runtime will pass it to Hypervisor to  search for corresponding
     /// prefetch list file:
@@ -1164,6 +1184,10 @@ impl ConfigOps for Hypervisor {
     fn adjust_config(conf: &mut TomlConfig) -> Result<()> {
         HypervisorVendor::adjust_config(conf)?;
         let hypervisors: Vec<String> = conf.hypervisor.keys().cloned().collect();
+        info!(
+            sl!(),
+            "Adjusting hypervisor configuration {:?}", hypervisors
+        );
         for hypervisor in hypervisors.iter() {
             if let Some(plugin) = get_hypervisor_plugin(hypervisor) {
                 plugin.adjust_config(conf)?;

src/libs/kata-types/src/config/hypervisor/remote.rs

@@ -0,0 +1,116 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use byte_unit::{Byte, Unit};
+use std::io::Result;
+use std::path::Path;
+use std::sync::Arc;
+use sysinfo::System;
+
+use crate::{
+    config::{
+        default::{self, MAX_REMOTE_VCPUS, MIN_REMOTE_MEMORY_SIZE_MB},
+        ConfigPlugin,
+    }, device::DRIVER_NVDIMM_TYPE, eother, resolve_path
+};
+
+use super::register_hypervisor_plugin;
+
+/// Hypervisor name for remote, used to index `TomlConfig::hypervisor`.
+pub const HYPERVISOR_NAME_REMOTE: &str = "remote";
+
+/// Configuration information for remote.
+#[derive(Default, Debug)]
+pub struct RemoteConfig {}
+
+impl RemoteConfig {
+    /// Create a new instance of `RemoteConfig`
+    pub fn new() -> Self {
+        RemoteConfig {}
+    }
+
+    /// Register the remote plugin.
+    pub fn register(self) {
+        let plugin = Arc::new(self);
+        register_hypervisor_plugin(HYPERVISOR_NAME_REMOTE, plugin);
+    }
+}
+
+impl ConfigPlugin for RemoteConfig {
+    fn name(&self) -> &str {
+        HYPERVISOR_NAME_REMOTE
+    }
+
+    /// Adjust the configuration information after loading from configuration file.
+    fn adjust_config(&self, conf: &mut crate::config::TomlConfig) -> Result<()> {
+        if let Some(remote) = conf.hypervisor.get_mut(HYPERVISOR_NAME_REMOTE) {
+            if remote.remote_info.hypervisor_socket.is_empty() {
+                remote.remote_info.hypervisor_socket =
+                    default::DEFAULT_REMOTE_HYPERVISOR_SOCKET.to_string();
+            }
+            resolve_path!(
+                remote.remote_info.hypervisor_socket,
+                "Remote hypervisor socket `{}` is invalid: {}"
+            )?;
+            if remote.remote_info.hypervisor_timeout == 0 {
+                remote.remote_info.hypervisor_timeout = default::DEFAULT_REMOTE_HYPERVISOR_TIMEOUT;
+            }
+            if remote.memory_info.default_memory == 0 {
+                remote.memory_info.default_memory = default::MIN_REMOTE_MEMORY_SIZE_MB;
+            }
+            if remote.memory_info.memory_slots == 0 {
+                remote.memory_info.memory_slots = default::DEFAULT_REMOTE_MEMORY_SLOTS
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Validate the configuration information.
+    fn validate(&self, conf: &crate::config::TomlConfig) -> Result<()> {
+        if let Some(remote) = conf.hypervisor.get(HYPERVISOR_NAME_REMOTE) {
+            let s = System::new_all();
+            let total_memory = Byte::from_u64(s.total_memory())
+                .get_adjusted_unit(Unit::MiB)
+                .get_value() as u32;
+            if remote.memory_info.default_maxmemory != total_memory {
+                return Err(eother!(
+                    "Remote hypervisor does not support memory hotplug, default_maxmemory must be equal to the total system memory",
+                ));
+            }
+            let cpus = num_cpus::get() as u32;
+            if remote.cpu_info.default_maxvcpus != cpus {
+                return Err(eother!(
+                    "Remote hypervisor does not support CPU hotplug, default_maxvcpus must be equal to the total system CPUs",
+                ));
+            }
+            if !remote.boot_info.initrd.is_empty() {
+                return Err(eother!("Remote hypervisor does not support initrd"));
+            }
+            if !remote.boot_info.rootfs_type.is_empty() {
+                return Err(eother!("Remote hypervisor does not support rootfs_type"));
+            }
+            if remote.blockdev_info.block_device_driver.as_str() == DRIVER_NVDIMM_TYPE {
+                return Err(eother!("Remote hypervisor does not support nvdimm"));
+            }
+            if remote.memory_info.default_memory < MIN_REMOTE_MEMORY_SIZE_MB {
+                return Err(eother!(
+                    "Remote hypervisor has minimal memory limitation {}",
+                    MIN_REMOTE_MEMORY_SIZE_MB
+                ));
+            }
+        }
+
+        Ok(())
+    }
+
+    fn get_min_memory(&self) -> u32 {
+        MIN_REMOTE_MEMORY_SIZE_MB
+    }
+
+    fn get_max_cpus(&self) -> u32 {
+        MAX_REMOTE_VCPUS
+    }
+}

src/libs/kata-types/src/config/mod.rs

@@ -26,7 +26,7 @@ pub use self::agent::Agent;
 use self::default::DEFAULT_AGENT_DBG_CONSOLE_PORT;
 pub use self::hypervisor::{
     BootInfo, CloudHypervisorConfig, DragonballConfig, FirecrackerConfig, Hypervisor, QemuConfig,
-    HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_FIRECRACKER, HYPERVISOR_NAME_QEMU,
+    RemoteConfig, HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_FIRECRACKER, HYPERVISOR_NAME_QEMU,
 };
 
 mod runtime;

src/libs/protocols/Cargo.toml

@@ -7,19 +7,17 @@ license = "Apache-2.0"
 
 [features]
 default = []
-with-serde = [ "serde", "serde_json" ]
+with-serde = []
 async = ["ttrpc/async", "async-trait"]
 
 [dependencies]
 ttrpc = "0.8"
 async-trait = { version = "0.1.42", optional = true }
 protobuf = { version = "3.2.0" }
-serde = { version = "1.0.130", features = ["derive"], optional = true }
-serde_json = { version = "1.0.68", optional = true }
+serde = { version = "1.0.130", features = ["derive"] }
+serde_json = "1.0.68"
 oci-spec = { version = "0.6.8", features = ["runtime"] }
 
-kata-sys-util = { path = "../kata-sys-util" }
-
 [build-dependencies]
 ttrpc-codegen = "0.4.2"
 protobuf = { version = "3.2.0" }

src/libs/protocols/build.rs

@@ -204,6 +204,7 @@ fn real_main() -> Result<(), std::io::Error> {
                 "protos/agent.proto",
                 "protos/health.proto",
                 "protos/confidential_data_hub.proto",
+                "protos/remote.proto",
             ],
             true,
         )?;
@@ -214,6 +215,7 @@ fn real_main() -> Result<(), std::io::Error> {
             "src/confidential_data_hub_ttrpc.rs",
             "src/confidential_data_hub_ttrpc_async.rs",
         )?;
+        fs::rename("src/remote_ttrpc.rs", "src/remote_ttrpc_async.rs")?;
     }
 
     codegen(
@@ -222,6 +224,7 @@ fn real_main() -> Result<(), std::io::Error> {
             "protos/agent.proto",
             "protos/health.proto",
             "protos/confidential_data_hub.proto",
+            "protos/remote.proto",
         ],
         false,
     )?;

src/libs/protocols/protos/remote.proto

@@ -0,0 +1,47 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+syntax = "proto3";
+
+package remote;
+
+service Hypervisor {
+	rpc CreateVM(CreateVMRequest) returns (CreateVMResponse) {}
+	rpc StartVM(StartVMRequest) returns (StartVMResponse) {}
+	rpc StopVM(StopVMRequest) returns (StopVMResponse) {}
+	rpc Version(VersionRequest) returns (VersionResponse) {}
+}
+
+message VersionRequest {
+	string version = 1;
+}
+
+message VersionResponse {
+	string version = 1;
+}
+
+message CreateVMRequest {
+	string id = 1;
+	map<string, string> annotations = 2;
+	string networkNamespacePath = 3;
+}
+
+message CreateVMResponse {
+	string agentSocketPath = 1;
+}
+
+message StartVMRequest {
+	string id = 1;
+}
+
+message StartVMResponse {
+}
+
+message StopVMRequest {
+	string id = 1;
+}
+
+message StopVMResponse {
+}

src/libs/protocols/src/lib.rs

@@ -21,6 +21,10 @@ pub mod oci;
 mod serde_config;
 pub mod trans;
 pub mod types;
+pub mod remote;
+pub mod remote_ttrpc;
+#[cfg(feature = "async")]
+pub mod remote_ttrpc_async;
 
 #[cfg(feature = "with-serde")]
 pub use serde_config::{

src/libs/protocols/src/trans.rs

@@ -10,7 +10,6 @@ use std::convert::TryFrom;
 use std::path::PathBuf;
 
 use crate::oci as grpc;
-use kata_sys_util::spec::parse_json_string;
 use oci_spec::runtime as oci;
 
 // translate from interface to ttprc tools
@@ -41,8 +40,9 @@ fn cap_hashset2vec(hash_set: &Option<HashSet<oci::Capability>>) -> Vec<String> {
 fn cap_vec2hashset(caps: Vec<String>) -> HashSet<oci::Capability> {
     caps.iter()
         .map(|cap: &String| {
-            let cap_str = parse_json_string(cap);
-            cap_str
+            // cap might be JSON-encoded
+            let decoded: &str = serde_json::from_str(cap).unwrap_or(cap);
+            decoded.strip_prefix("CAP_").unwrap_or(decoded)
                 .parse::<oci::Capability>()
                 .unwrap_or_else(|_| panic!("Failed to parse {:?} to Enum Capability", cap))
         })
@@ -97,6 +97,8 @@ impl From<oci::LinuxCapabilities> for grpc::LinuxCapabilities {
     }
 }
 
+// TODO(burgerdev): remove condition here and below after upgrading to oci_spec > 0.7.
+#[cfg(target_os = "linux")]
 impl From<oci::PosixRlimit> for grpc::POSIXRlimit {
     fn from(from: oci::PosixRlimit) -> Self {
         grpc::POSIXRlimit {
@@ -118,6 +120,7 @@ impl From<oci::Process> for grpc::Process {
             Env: option_vec_to_vec(from.env()),
             Cwd: from.cwd().display().to_string(),
             Capabilities: from_option(from.capabilities().clone()),
+            #[cfg(target_os = "linux")]
             Rlimits: from_option_vec(from.rlimits().clone()),
             NoNewPrivileges: from.no_new_privileges().unwrap_or_default(),
             ApparmorProfile: from
@@ -993,6 +996,7 @@ impl From<grpc::Linux> for oci::Linux {
     }
 }
 
+#[cfg(target_os = "linux")]
 impl From<grpc::POSIXRlimit> for oci::PosixRlimit {
     fn from(proto: grpc::POSIXRlimit) -> Self {
         oci::PosixRlimitBuilder::default()
@@ -1078,6 +1082,8 @@ impl From<grpc::Process> for oci::Process {
         } else {
             process.set_capabilities(None);
         }
+
+        #[cfg(target_os = "linux")]
         if !from.Rlimits().is_empty() {
             process.set_rlimits(Some(
                 from.Rlimits().iter().cloned().map(|r| r.into()).collect(),
@@ -1238,6 +1244,11 @@ impl From<grpc::LinuxIntelRdt> for oci::LinuxIntelRdt {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashSet;
+
+    use super::cap_vec2hashset;
+    use super::oci;
+
     fn from_vec<F: Sized, T: From<F>>(from: Vec<F>) -> Vec<T> {
         let mut to: Vec<T> = vec![];
         for data in from {
@@ -1289,4 +1300,26 @@ mod tests {
         assert_eq!(from.len(), to.len());
         assert_eq!(from[0].from, to[0].to);
     }
+
+    #[test]
+    fn test_cap_vec2hashset_good() {
+        let expected: HashSet<oci::Capability> =
+            vec![oci::Capability::NetAdmin, oci::Capability::Mknod]
+                .into_iter()
+                .collect();
+        let actual = cap_vec2hashset(vec![
+            "CAP_NET_ADMIN".to_string(),
+            "\"CAP_MKNOD\"".to_string(),
+        ]);
+
+        assert_eq!(expected, actual);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_cap_vec2hashset_bad() {
+        cap_vec2hashset(vec![
+            "CAP_DOES_NOT_EXIST".to_string(),
+        ]);
+    }
 }

src/runtime-rs/Cargo.lock

@@ -1685,8 +1685,11 @@ dependencies = [
  "libc",
  "logging",
  "nix 0.24.3",
+ "oci-spec",
  "path-clean",
  "persist",
+ "protobuf 3.2.0",
+ "protocols",
  "qapi",
  "qapi-qmp",
  "qapi-spec",
@@ -1706,6 +1709,8 @@ dependencies = [
  "thiserror",
  "tokio",
  "tracing",
+ "ttrpc",
+ "ttrpc-codegen",
  "vmm-sys-util 0.11.1",
 ]
 
@@ -1839,7 +1844,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -2994,9 +2998,10 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.2.0",
+ "serde",
+ "serde_json",
  "ttrpc",
  "ttrpc-codegen",
 ]

src/runtime-rs/Makefile

@@ -83,16 +83,18 @@ LOCALSTATEDIR := /var
 CONFIG_FILE = configuration.toml
 RUNTIMENAME := virt_container
 HYPERVISOR_DB = dragonball
-HYPERVISOR_ACRN = acrn
 HYPERVISOR_FC = firecracker
 HYPERVISOR_QEMU = qemu
 HYPERVISOR_CLH = cloud-hypervisor
+HYPERVISOR_REMOTE = remote
 
+# When set to true, builds the built-in Dragonball hypervisor
+USE_BUILDIN_DB := true
 
-DEFAULT_HYPERVISOR ?= $(HYPERVISOR_DB)
+HYPERVISOR ?= $(HYPERVISOR_DB)
 
 ##VAR HYPERVISOR=<hypervisor_name> List of hypervisors this build system can generate configuration for.
-HYPERVISORS := $(HYPERVISOR_DB) $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
+HYPERVISORS := $(HYPERVISOR_DB) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_REMOTE)
 
 CLHPATH := $(CLHBINDIR)/$(CLHCMD)
 CLHVALIDHYPERVISORPATHS := [\"$(CLHPATH)\"]
@@ -187,8 +189,6 @@ CONFIG_PATHS =
 SYSCONFIG_PATHS =
 # List of hypervisors known for the current architecture
 KNOWN_HYPERVISORS =
-# List of hypervisors known for the current architecture
-KNOWN_HYPERVISORS =
 
 CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR)/runtime-rs
 SYSCONFDIR := $(SYSCONFDIR)/$(PROJECT_DIR)
@@ -318,16 +318,33 @@ ifneq (,$(FCCMD))
     DEFSTATICRESOURCEMGMT_FC := true
 endif
 
-ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_DB))
+ifneq (,$(REMOTE))
+    KNOWN_HYPERVISORS += $(HYPERVISOR_REMOTE)
+    CONFIG_FILE_REMOTE = configuration-remote.toml
+    CONFIG_REMOTE = config/$(CONFIG_FILE_REMOTE)
+    CONFIG_REMOTE_IN = $(CONFIG_REMOTE).in
+    CONFIG_PATH_REMOTE = $(abspath $(CONFDIR)/$(CONFIG_FILE_REMOTE))
+    CONFIG_PATHS += $(CONFIG_PATH_REMOTE)
+    SYSCONFDIR_REMOTE = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_REMOTE))
+    SYSCONFIG_PATHS += $(SYSCONFDIR_REMOTE)
+    CONFIGS += $(CONFIG_REMOTE)
+    # remote-specific options (all should be suffixed by "_REMOTE")
+    DEFSANDBOXCGROUPONLY_REMOTE := false
+endif
+
+ifeq ($(HYPERVISOR),$(HYPERVISOR_DB))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_DB)
 endif
 
-ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_QEMU))
+ifeq ($(HYPERVISOR),$(HYPERVISOR_QEMU))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_QEMU)
 endif
-ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_FC))
+ifeq ($(HYPERVISOR),$(HYPERVISOR_FC))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_FC)
 endif
+ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_REMOTE))
+    DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_REMOTE)
+endif
 # list of variables the user may wish to override
 USER_VARS += ARCH
 USER_VARS += BINDIR
@@ -335,8 +352,10 @@ USER_VARS += CONFIG_DB_IN
 USER_VARS += CONFIG_FC_IN
 USER_VARS += CONFIG_PATH
 USER_VARS += CONFIG_QEMU_IN
+USER_VARS += CONFIG_REMOTE_IN
 USER_VARS += DESTDIR
-USER_VARS += DEFAULT_HYPERVISOR
+USER_VARS += HYPERVISOR
+USER_VARS += USE_BUILDIN_DB
 USER_VARS += DBCMD
 USER_VARS += DBCTLCMD
 USER_VARS += FCCTLCMD
@@ -399,7 +418,6 @@ USER_VARS += SYSCONFDIR
 USER_VARS += DEFVCPUS
 USER_VARS += DEFVCPUS_QEMU
 USER_VARS += DEFMAXVCPUS
-USER_VARS += DEFMAXVCPUS_ACRN
 USER_VARS += DEFMAXVCPUS_DB
 USER_VARS += DEFMAXVCPUS_QEMU
 USER_VARS += DEFMEMSZ
@@ -444,6 +462,7 @@ USER_VARS += DEFSANDBOXCGROUPONLY_QEMU
 USER_VARS += DEFSANDBOXCGROUPONLY_DB
 USER_VARS += DEFSANDBOXCGROUPONLY_FC
 USER_VARS += DEFSANDBOXCGROUPONLY_CLH
+USER_VARS += DEFSANDBOXCGROUPONLY_REMOTE
 USER_VARS += DEFSTATICRESOURCEMGMT_DB
 USER_VARS += DEFSTATICRESOURCEMGMT_FC
 USER_VARS += DEFSTATICRESOURCEMGMT_CLH
@@ -475,6 +494,11 @@ COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown)
 
 EXTRA_RUSTFEATURES :=
 
+# if use dragonball hypervisor, add the feature to build dragonball in runtime
+ifeq ($(USE_BUILDIN_DB),true)
+    EXTRA_RUSTFEATURES +=  dragonball
+endif
+
 ifneq ($(EXTRA_RUSTFEATURES),)
     override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES)
 endif
@@ -614,7 +638,7 @@ show-summary: show-header
 	@printf "  %s\n" "$(call get_toolchain_version)"
 	@printf "\n"
 	@printf "• Hypervisors:\n"
-	@printf "\tDefault: $(DEFAULT_HYPERVISOR)\n"
+	@printf "\tDefault: $(HYPERVISOR)\n"
 	@printf "\tKnown: $(sort $(HYPERVISORS))\n"
 	@printf "\tAvailable for this architecture: $(sort $(KNOWN_HYPERVISORS))\n"
 	@printf "\n"
@@ -634,7 +658,7 @@ show-summary: show-header
 	@printf "\talternate config paths (SYSCONFIG_PATHS) : %s\n"
 	@printf \
 	  "$(foreach c,$(sort $(SYSCONFIG_PATHS)),$(shell printf "\\t - $(c)\\\n"))"
-	@printf "\tdefault install path for $(DEFAULT_HYPERVISOR) (CONFIG_PATH) : %s\n" $(abspath $(CONFIG_PATH))
+	@printf "\tdefault install path for $(HYPERVISOR) (CONFIG_PATH) : %s\n" $(abspath $(CONFIG_PATH))
 	@printf "\tdefault alternate config path (SYSCONFIG) : %s\n" $(abspath $(SYSCONFIG))
 ifneq (,$(findstring $(HYPERVISOR_QEMU),$(KNOWN_HYPERVISORS)))
 	@printf "\t$(HYPERVISOR_QEMU) hypervisor path (QEMUPATH) : %s\n" $(abspath $(QEMUPATH))
@@ -647,9 +671,6 @@ ifneq (,$(findstring $(HYPERVISOR_CLH),$(KNOWN_HYPERVISORS)))
 endif
 ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS)))
 	@printf "\t$(HYPERVISOR_FC) hypervisor path (FCPATH) : %s\n" $(abspath $(FCPATH))
-endif
-ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS)))
-	@printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH))
 endif
 	@printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR))
 	@printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR))

src/runtime-rs/arch/x86_64-options.mk

@@ -20,3 +20,5 @@ CLHCMD := cloud-hypervisor
 # firecracker binary (vmm and jailer)
 FCCMD := firecracker
 FCJAILERCMD := jailer
+
+REMOTE := remote

src/runtime-rs/config/configuration-remote.toml.in

@@ -1,43 +1,47 @@
-# Copyright (c) 2017-2019 Intel Corporation
-# Copyright (c) 2021 Adobe Inc.
+# Copyright 2024 Kata Contributors
 #
 # SPDX-License-Identifier: Apache-2.0
 #
 
+
 # XXX: WARNING: this file is auto-generated.
 # XXX:
-# XXX: Source file: "@CONFIG_ACRN_IN@"
+# XXX: Source file: "@CONFIG_REMOTE_IN@"
 # XXX: Project:
 # XXX:   Name: @PROJECT_NAME@
 # XXX:   Type: @PROJECT_TYPE@
 
-[hypervisor.acrn]
-path = "@ACRNPATH@"
-ctlpath = "@ACRNCTLPATH@"
-kernel = "@KERNELPATH_ACRN@"
-image = "@IMAGEPATH@"
 
-# rootfs filesystem type:
-#   - ext4 (default)
-#   - xfs
-#   - erofs
-rootfs_type=@DEFROOTFSTYPE@
+[hypervisor.remote]
+# Default VM information query service unix domain socket, created by cloud-api-adaptor
+# Ref: https://github.com/confidential-containers/cloud-api-adaptor/blob/main/src/cloud-api-adaptor/docs/vminfo.md
+remote_hypervisor_socket = "/run/peerpod/hypervisor.sock"
+# Timeout in seconds for creating a remote hypervisor, 600s(10min) by default
+remote_hypervisor_timeout = 600
+
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+# confidential_guest = true
+
 
 # List of valid annotation names for the hypervisor
 # Each member of the list is a regular expression, which is the base name
 # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
-enable_annotations = @DEFENABLEANNOTATIONS@
-
-# List of valid annotations values for the hypervisor
-# Each member of the list is a path pattern as described by glob(3).
-# The default if not set is empty (all annotations rejected.)
-# Your distribution recommends: @ACRNVALIDHYPERVISORPATHS@
-valid_hypervisor_paths = @ACRNVALIDHYPERVISORPATHS@
-
-# List of valid annotations values for ctlpath
-# The default if not set is empty (all annotations rejected.)
-# Your distribution recommends: @ACRNVALIDCTLPATHS@
-valid_ctlpaths = @ACRNVALIDCTLPATHS@
+# Note: Remote hypervisor is only handling the following annotations
+enable_annotations = ["machine_type", "default_memory", "default_vcpus"]
 
 # Optional space-separated list of options to pass to the guest kernel.
 # For example, use `kernel_params = "vsyscall=emulate"` if you are having
@@ -49,12 +53,20 @@ valid_ctlpaths = @ACRNVALIDCTLPATHS@
 # may stop the virtual machine from booting.
 # To see the list of default parameters, enable hypervisor debug, create a
 # container and look for 'default-kernel-parameters' log entries.
-kernel_params = "@KERNELPARAMS@"
+# NOTE: kernel_params are not currently passed over in remote hypervisor
+# kernel_params = ""
 
 # Path to the firmware.
-# If you want that acrn uses the default firmware leave this option empty
+# If you want that qemu uses the default firmware leave this option empty
 firmware = "@FIRMWAREPATH@"
 
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+# default_vcpus = 1
+
 # Default maximum number of vCPUs per SB/VM:
 # unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
 #                                     of vCPUs supported by KVM if that number is exceeded
@@ -69,14 +81,15 @@ firmware = "@FIRMWAREPATH@"
 # `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
 # vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
 # unless you know what are you doing.
-default_maxvcpus = @DEFMAXVCPUS_ACRN@
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+# default_maxvcpus = @DEFMAXVCPUS@
 
 # Bridges can be used to hot plug devices.
 # Limitations:
 # * Currently only pci bridges are supported
 # * Until 30 devices per bridge can be hot plugged.
 # * Until 5 PCI bridges can be cold plugged per VM.
-#   This limitation could be a bug in the kernel
+#   This limitation could be a bug in qemu or in the kernel
 # Default number of bridges per SB/VM:
 # unspecified or 0   --> will be set to @DEFBRIDGES@
 # > 1 <= 5           --> will be set to the specified number
@@ -85,27 +98,20 @@ default_bridges = @DEFBRIDGES@
 
 # Default memory size in MiB for SB/VM.
 # If unspecified then it will be set @DEFMEMSZ@ MiB.
-default_memory = @DEFMEMSZ@
-
-# Block storage driver to be used for the hypervisor in case the container
-# rootfs is backed by a block device. ACRN only supports virtio-blk.
-block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@"
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+# default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+#memory_slots = @DEFMEMSLOTS@
 
 # This option changes the default hypervisor and kernel parameters
-# to enable debug output where available.
+# to enable debug output where available. And Debug also enable the hmp socket.
 #
 # Default false
-#enable_debug = true
-
-# Disable the customizations done in the runtime when it detects
-# that it is running on top a VMM. This will result in the runtime
-# behaving as it would when running on bare metal.
-#
-#disable_nesting_checks = true
-
-# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics.
-# Default false
-#disable_vhost_net = true
+# enable_debug = true
 
 # Path to OCI hook binaries in the *guest rootfs*.
 # This does not affect host-side hooks which must instead be added to
@@ -127,10 +133,18 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@"
 # disable applying SELinux on the VMM process (default false)
 disable_selinux=@DEFDISABLESELINUX@
 
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+# Note: The remote hypervisor has a different guest, so currently requires this to be disabled
+disable_guest_selinux = true
+
 [agent.@PROJECT_TYPE@]
 # If enabled, make the agent display debug-level messages.
 # (default: disabled)
-#enable_debug = true
+# enable_debug = true
 
 # Enable agent tracing.
 #
@@ -144,7 +158,7 @@ disable_selinux=@DEFDISABLESELINUX@
 #   increasing the container shutdown time slightly.
 #
 # (default: disabled)
-#enable_tracing = true
+# enable_tracing = true
 
 # Enable debug console.
 
@@ -154,31 +168,20 @@ disable_selinux=@DEFDISABLESELINUX@
 #debug_console_enabled = true
 
 # Agent connection dialing timeout value in seconds
-# (default: 45)
-dial_timeout = 45
-
-# Confidential Data Hub API timeout value in seconds
-# (default: 50)
-#cdh_api_timeout = 50
+# (default: 30)
+#dial_timeout = 30
 
 [runtime]
 # If enabled, the runtime will log additional debug messages to the
 # system log
 # (default: disabled)
-#enable_debug = true
+# enable_debug = true
 #
 # Internetworking model
 # Determines how the VM should be connected to the
 # the container network interface
 # Options:
 #
-#   - bridged (Deprecated)
-#     Uses a linux bridge to interconnect the container interface to
-#     the VM. Works for most cases except macvlan and ipvlan.
-#     ***NOTE: This feature has been deprecated with plans to remove this
-#     feature in the future. Please use other network models listed below.
-#
-#
 #   - macvtap
 #     Used when the Container network interface can be bridged using
 #     macvtap.
@@ -190,14 +193,29 @@ dial_timeout = 45
 #     Uses tc filter rules to redirect traffic from the network interface
 #     provided by plugin to a tap interface connected to the VM.
 #
-internetworking_model="@DEFNETWORKMODEL_ACRN@"
+# Note: The remote hypervisor, uses it's own network, so "none" is required
+internetworking_model="none"
+
+name="virt_container"
+hypervisor_name="remote"
+agent_name="kata"
 
 # disable guest seccomp
 # Determines whether container seccomp profiles are passed to the virtual
 # machine and applied by the kata agent. If set to true, seccomp is not applied
 # within the guest
 # (default: true)
-disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+# Note: The remote hypervisor has a different guest, so currently requires this to be set to true
+disable_guest_seccomp=true
+
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
 
 # If enabled, the runtime will create opentracing.io traces and spans.
 # (See https://www.jaegertracing.io/docs/getting-started).
@@ -216,11 +234,12 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 
 # If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
 # This option may have some potential impacts to your host. It should only be used when you know what you're doing.
-# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
 # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
 # (like OVS) directly.
 # (default: false)
-#disable_new_netns = true
+# Note: The remote hypervisor has a different networking model, which requires true 
+disable_new_netns = false
 
 # if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
 # The container cgroups in the host are not created, just one single cgroup per sandbox.
@@ -228,11 +247,43 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 # The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
 # The sandbox cgroup is constrained if there is no container type annotation.
 # See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
-sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY_REMOTE@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+# Note: the remote hypervisor uses the peer pod config to determine the sandbox size, so requires this to be set to true
+static_sandbox_resource_mgmt=true
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
 
 # If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
 # be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
-disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+# Note: remote hypervisor has no sharing of emptydir mounts from host to guest
+disable_guest_empty_dir=false
 
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
@@ -244,20 +295,3 @@ experimental=@DEFAULTEXPFEATURES@
 # If enabled, user can run pprof tools with shim v2 process through kata-monitor.
 # (default: false)
 # enable_pprof = true
-
-# Indicates the CreateContainer request timeout needed for the workload(s)
-# It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
-# In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
-create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
-
-# Base directory of directly attachable network config.
-# Network devices for VM-based containers are allowed to be placed in the
-# host netns to eliminate as many hops as possible, which is what we
-# called a "Directly Attachable Network". The config, set by special CNI
-# plugins, is used to tell the Kata containers what devices are attached
-# to the hypervisor.
-# (default: /run/kata-containers/dans)
-dan_conf = "@DEFDANCONF@"

src/runtime-rs/crates/agent/src/sock/mod.rs

@@ -8,6 +8,8 @@ mod hybrid_vsock;
 pub use hybrid_vsock::HybridVsock;
 mod vsock;
 pub use vsock::Vsock;
+mod remote;
+pub use remote::Remote;
 
 use std::{
     pin::Pin,
@@ -28,6 +30,7 @@ use url::Url;
 
 const VSOCK_SCHEME: &str = "vsock";
 const HYBRID_VSOCK_SCHEME: &str = "hvsock";
+const REMOTE_SCHEME: &str = "remote";
 
 /// Socket stream
 pub enum Stream {
@@ -98,6 +101,7 @@ impl ConnectConfig {
 enum SockType {
     Vsock(Vsock),
     HybridVsock(HybridVsock),
+    Remote(Remote),
 }
 
 #[async_trait]
@@ -114,6 +118,7 @@ pub fn new(address: &str, port: u32) -> Result<Arc<dyn Sock>> {
     match parse(address, port).context("parse url")? {
         SockType::Vsock(sock) => Ok(Arc::new(sock)),
         SockType::HybridVsock(sock) => Ok(Arc::new(sock)),
+        SockType::Remote(sock) => Ok(Arc::new(sock)),
     }
 }
 
@@ -136,6 +141,13 @@ fn parse(address: &str, port: u32) -> Result<SockType> {
             let uds = path[0];
             Ok(SockType::HybridVsock(HybridVsock::new(uds, port)))
         }
+        REMOTE_SCHEME => {
+            let path: Vec<&str> = url.path().split(':').collect();
+            if path.len() != 1 {
+                return Err(anyhow!("invalid path {:?}", path));
+            }
+            Ok(SockType::Remote(Remote::new(path[0].to_string())))
+        }
         _ => Err(anyhow!("Unsupported scheme")),
     }
 }

src/runtime-rs/crates/agent/src/sock/remote.rs

@@ -0,0 +1,61 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::{os::unix::prelude::AsRawFd, path::Path};
+
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use tokio::{io::Interest, net::UnixStream};
+
+use super::{ConnectConfig, Sock, Stream};
+
+#[derive(Debug, PartialEq)]
+pub struct Remote {
+    path: String,
+}
+
+impl Remote {
+    pub fn new(path: String) -> Self {
+        Self { path }
+    }
+}
+
+#[async_trait]
+impl Sock for Remote {
+    async fn connect(&self, config: &ConnectConfig) -> Result<Stream> {
+        let retry_times = config.reconnect_timeout_ms / config.dial_timeout_ms;
+        for i in 0..retry_times {
+            match connect_helper(&self.path).await {
+                Ok(stream) => {
+                    info!(
+                        sl!(),
+                        "remote connect success on {} current client fd {}",
+                        i,
+                        stream.as_raw_fd()
+                    );
+                    return Ok(Stream::Unix(stream));
+                }
+                Err(err) => {
+                    debug!(sl!(), "remote connect on {} err : {:?}", i, err);
+                    tokio::time::sleep(std::time::Duration::from_millis(config.dial_timeout_ms))
+                        .await;
+                    continue;
+                }
+            }
+        }
+        Err(anyhow!("cannot connect to agent ttrpc server {:?}", config))
+    }
+}
+
+async fn connect_helper(address: &str) -> Result<UnixStream> {
+    let stream = UnixStream::connect(Path::new(&address))
+        .await
+        .context("failed to create UnixAddr")?;
+    stream
+        .ready(Interest::READABLE | Interest::WRITABLE)
+        .await?;
+    Ok(stream)
+}

src/runtime-rs/crates/agent/src/types.rs

@@ -42,7 +42,7 @@ pub struct StringUser {
     pub additional_gids: Vec<String>,
 }
 
-#[derive(PartialEq, Clone, Default)]
+#[derive(PartialEq, Clone, Debug, Default)]
 pub struct Device {
     pub id: String,
     pub field_type: String,

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -28,12 +28,15 @@ rand = "0.8.4"
 path-clean = "1.0.1"
 lazy_static = "1.4"
 tracing = "0.1.36"
+ttrpc = {version = "0.8.1", features = ["async"] }
+protobuf = "3.1.0"
 
-dbs-utils = { path = "../../../dragonball/src/dbs_utils" }
 kata-sys-util = { path = "../../../libs/kata-sys-util" }
 kata-types = { path = "../../../libs/kata-types" }
 logging = { path = "../../../libs/logging" }
+protocols = { path = "../../../libs/protocols", features = ["async"] }
 shim-interface = { path = "../../../libs/shim-interface" }
+oci-spec = { version = "0.6.8", features = ["runtime"] }
 
 ch-config = { path = "ch-config", optional = true }
 tests_utils = { path = "../../tests/utils" }
@@ -48,7 +51,7 @@ qapi-spec = "0.3.1"
 qapi-qmp = "0.14.0"
 
 [target.'cfg(not(target_arch = "s390x"))'.dependencies]
-dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall", "virtio-mem", "virtio-balloon", "vhost-user-net", "host-device"] }
+dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall", "virtio-mem", "virtio-balloon", "vhost-user-net", "host-device"], optional = true }
 dbs-utils = { path = "../../../dragonball/src/dbs_utils" }
 hyperlocal = "0.8.0"
 hyper = {version = "0.14.18", features = ["client"]}
@@ -56,6 +59,7 @@ hyper = {version = "0.14.18", features = ["client"]}
 [features]
 default = []
 
+dragonball = ["dep:dragonball"]
 # Feature is not yet complete, so not enabled by default.
 # See https://github.com/kata-containers/kata-containers/issues/6264.
 cloud-hypervisor = ["ch-config"]
@@ -68,3 +72,7 @@ hypervisor = { path = ".", features = ["cloud-hypervisor"] }
 test-utils = { path = "../../../libs/test-utils" }
 
 serial_test = "2.0.0"
+
+
+[build-dependencies]
+ttrpc-codegen = "0.4.2"

src/runtime-rs/crates/hypervisor/src/ch/inner.rs

@@ -16,7 +16,6 @@ use persist::sandbox_persist::Persist;
 use std::collections::HashMap;
 use std::os::unix::net::UnixStream;
 use tokio::sync::watch::{channel, Receiver, Sender};
-use tokio::sync::Mutex;
 use tokio::task::JoinHandle;
 use tokio::{process::Child, sync::mpsc};
 
@@ -79,13 +78,12 @@ pub struct CloudHypervisorInner {
     pub(crate) _guest_memory_block_size_mb: u32,
 
     pub(crate) exit_notify: Option<mpsc::Sender<i32>>,
-    pub(crate) exit_waiter: Mutex<(mpsc::Receiver<i32>, i32)>,
 }
 
 const CH_DEFAULT_TIMEOUT_SECS: u32 = 10;
 
 impl CloudHypervisorInner {
-    pub fn new() -> Self {
+    pub fn new(exit_notify: Option<mpsc::Sender<i32>>) -> Self {
         let mut capabilities = Capabilities::new();
         capabilities.set(
             CapabilityBits::BlockDeviceSupport
@@ -95,7 +93,6 @@ impl CloudHypervisorInner {
         );
 
         let (tx, rx) = channel(true);
-        let (exit_notify, exit_waiter) = mpsc::channel(1);
 
         Self {
             api_socket: None,
@@ -122,8 +119,7 @@ impl CloudHypervisorInner {
             ch_features: None,
             _guest_memory_block_size_mb: 0,
 
-            exit_notify: Some(exit_notify),
-            exit_waiter: Mutex::new((exit_waiter, 0)),
+            exit_notify,
         }
     }
 
@@ -138,14 +134,14 @@ impl CloudHypervisorInner {
 
 impl Default for CloudHypervisorInner {
     fn default() -> Self {
-        Self::new()
+        Self::new(None)
     }
 }
 
 #[async_trait]
 impl Persist for CloudHypervisorInner {
     type State = HypervisorState;
-    type ConstructorArgs = ();
+    type ConstructorArgs = mpsc::Sender<i32>;
 
     // Return a state object that will be saved by the caller.
     async fn save(&self) -> Result<Self::State> {
@@ -166,11 +162,10 @@ impl Persist for CloudHypervisorInner {
 
     // Set the hypervisor state to the specified state
     async fn restore(
-        _hypervisor_args: Self::ConstructorArgs,
+        exit_notify: mpsc::Sender<i32>,
         hypervisor_state: Self::State,
     ) -> Result<Self> {
         let (tx, rx) = channel(true);
-        let (exit_notify, exit_waiter) = mpsc::channel(1);
 
         let mut ch = Self {
             config: Some(hypervisor_state.config),
@@ -190,7 +185,6 @@ impl Persist for CloudHypervisorInner {
             jailer_root: String::default(),
             ch_features: None,
             exit_notify: Some(exit_notify),
-            exit_waiter: Mutex::new((exit_waiter, 0)),
 
             ..Default::default()
         };
@@ -207,7 +201,9 @@ mod tests {
 
     #[actix_rt::test]
     async fn test_save_clh() {
-        let mut clh = CloudHypervisorInner::new();
+        let (exit_notify, _exit_waiter) = mpsc::channel(1);
+
+        let mut clh = CloudHypervisorInner::new(Some(exit_notify.clone()));
         clh.id = String::from("123456");
         clh.netns = Some(String::from("/var/run/netns/testnet"));
         clh.vm_path = String::from("/opt/kata/bin/cloud-hypervisor");
@@ -229,7 +225,7 @@ mod tests {
         assert!(!state.jailed);
         assert_eq!(state.hypervisor_type, HYPERVISOR_NAME_CH.to_string());
 
-        let clh = CloudHypervisorInner::restore((), state.clone())
+        let clh = CloudHypervisorInner::restore(exit_notify, state.clone())
             .await
             .unwrap();
         assert_eq!(clh.id, state.id);

src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs

@@ -19,7 +19,6 @@ use ch_config::ch_api::{
 };
 use ch_config::{guest_protection_is_tdx, NamedHypervisorConfig, VmConfig};
 use core::future::poll_fn;
-use futures::executor::block_on;
 use futures::future::join_all;
 use kata_sys_util::protection::{available_guest_protection, GuestProtection};
 use kata_types::capabilities::{Capabilities, CapabilityBits};
@@ -640,7 +639,7 @@ impl CloudHypervisorInner {
         Ok(())
     }
 
-    pub(crate) fn stop_vm(&mut self) -> Result<()> {
+    pub(crate) async fn stop_vm(&mut self) -> Result<()> {
         // If the container workload exits, this method gets called. However,
         // the container manager always makes a ShutdownContainer request,
         // which results in this method being called potentially a second
@@ -652,19 +651,14 @@ impl CloudHypervisorInner {
 
         self.state = VmmState::NotReady;
 
-        block_on(self.cloud_hypervisor_shutdown()).map_err(|e| anyhow!(e))?;
+        self.cloud_hypervisor_shutdown().await?;
 
         Ok(())
     }
 
+    #[allow(dead_code)]
     pub(crate) async fn wait_vm(&self) -> Result<i32> {
-        debug!(sl!(), "Waiting CH vmm");
-        let mut waiter = self.exit_waiter.lock().await;
-        if let Some(exitcode) = waiter.0.recv().await {
-            waiter.1 = exitcode;
-        }
-
-        Ok(waiter.1)
+        Ok(0)
     }
 
     pub(crate) fn pause_vm(&self) -> Result<()> {

src/runtime-rs/crates/hypervisor/src/ch/mod.rs

@@ -11,8 +11,9 @@ use async_trait::async_trait;
 use kata_types::capabilities::{Capabilities, CapabilityBits};
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 use persist::sandbox_persist::Persist;
+use std::collections::HashMap;
 use std::sync::Arc;
-use tokio::sync::RwLock;
+use tokio::sync::{mpsc, Mutex, RwLock};
 
 // Convenience macro to obtain the scope logger
 #[macro_export]
@@ -29,15 +30,19 @@ mod utils;
 
 use inner::CloudHypervisorInner;
 
-#[derive(Debug, Default, Clone)]
+#[derive(Debug)]
 pub struct CloudHypervisor {
     inner: Arc<RwLock<CloudHypervisorInner>>,
+    exit_waiter: Mutex<(mpsc::Receiver<i32>, i32)>,
 }
 
 impl CloudHypervisor {
     pub fn new() -> Self {
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
         Self {
-            inner: Arc::new(RwLock::new(CloudHypervisorInner::new())),
+            inner: Arc::new(RwLock::new(CloudHypervisorInner::new(Some(exit_notify)))),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
         }
     }
 
@@ -47,9 +52,20 @@ impl CloudHypervisor {
     }
 }
 
+impl Default for CloudHypervisor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 #[async_trait]
 impl Hypervisor for CloudHypervisor {
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }
@@ -61,12 +77,17 @@ impl Hypervisor for CloudHypervisor {
 
     async fn stop_vm(&self) -> Result<()> {
         let mut inner = self.inner.write().await;
-        inner.stop_vm()
+        inner.stop_vm().await
     }
 
     async fn wait_vm(&self) -> Result<i32> {
-        let inner = self.inner.read().await;
-        inner.wait_vm().await
+        debug!(sl!(), "Waiting CH vmm");
+        let mut waiter = self.exit_waiter.lock().await;
+        if let Some(exitcode) = waiter.0.recv().await {
+            waiter.1 = exitcode;
+        }
+
+        Ok(waiter.1)
     }
 
     async fn pause_vm(&self) -> Result<()> {
@@ -204,12 +225,15 @@ impl Persist for CloudHypervisor {
     }
 
     async fn restore(
-        hypervisor_args: Self::ConstructorArgs,
+        _hypervisor_args: Self::ConstructorArgs,
         hypervisor_state: Self::State,
     ) -> Result<Self> {
-        let inner = CloudHypervisorInner::restore(hypervisor_args, hypervisor_state).await?;
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
+        let inner = CloudHypervisorInner::restore(exit_notify, hypervisor_state).await?;
         Ok(Self {
             inner: Arc::new(RwLock::new(inner)),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
         })
     }
 }

src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs

@@ -115,12 +115,12 @@ pub enum VfioDeviceType {
     Mediated,
 }
 
-// DeviceVendor represents a PCI device's device id and vendor id
-// DeviceVendor: (device, vendor)
+// DeviceVendorClass represents a PCI device's deviceID, vendorID and classID
+// DeviceVendorClass: (device, vendor, class)
 #[derive(Clone, Debug)]
-pub struct DeviceVendor(String, String);
+pub struct DeviceVendorClass(String, String, String);
 
-impl DeviceVendor {
+impl DeviceVendorClass {
     pub fn get_device_vendor(&self) -> Result<(u32, u32)> {
         // default value is 0 when vendor_id or device_id is empty
         if self.0.is_empty() || self.1.is_empty() {
@@ -142,6 +142,10 @@ impl DeviceVendor {
         Ok((device, vendor))
     }
 
+    pub fn get_vendor_class_id(&self) -> Result<(&str, &str)> {
+        Ok((&self.1, &self.2))
+    }
+
     pub fn get_device_vendor_id(&self) -> Result<u32> {
         let (device, vendor) = self
             .get_device_vendor()
@@ -163,8 +167,8 @@ pub struct HostDevice {
     /// PCI device information (BDF): "bus:slot:function"
     pub bus_slot_func: String,
 
-    /// device_vendor: device id and vendor id
-    pub device_vendor: Option<DeviceVendor>,
+    /// device_vendor_class: (device, vendor, class)
+    pub device_vendor_class: Option<DeviceVendorClass>,
 
     /// type of vfio device
     pub vfio_type: VfioDeviceType,
@@ -336,13 +340,14 @@ impl VfioDevice {
     }
 
     // read vendor and deviceor from /sys/bus/pci/devices/BDF/X
-    fn get_vfio_device_vendor(&self, bdf: &str) -> Result<DeviceVendor> {
+    fn get_vfio_device_vendor_class(&self, bdf: &str) -> Result<DeviceVendorClass> {
         let device =
             get_device_property(bdf, "device").context("get device from syspath failed")?;
         let vendor =
             get_device_property(bdf, "vendor").context("get vendor from syspath failed")?;
+        let class = get_device_property(bdf, "class").context("get class from syspath failed")?;
 
-        Ok(DeviceVendor(device, vendor))
+        Ok(DeviceVendorClass(device, vendor, class))
     }
 
     fn set_vfio_config(
@@ -356,13 +361,13 @@ impl VfioDevice {
 
         // It's safe as BDF really exists.
         let dev_bdf = vfio_dev_details.0.unwrap();
-        let dev_vendor = self
-            .get_vfio_device_vendor(&dev_bdf)
+        let dev_vendor_class = self
+            .get_vfio_device_vendor_class(&dev_bdf)
             .context("get property device and vendor failed")?;
 
         let vfio_dev = HostDevice {
             bus_slot_func: dev_bdf.clone(),
-            device_vendor: Some(dev_vendor),
+            device_vendor_class: Some(dev_vendor_class),
             sysfs_path: vfio_dev_details.1,
             vfio_type: vfio_dev_details.2,
             ..Default::default()

src/runtime-rs/crates/hypervisor/src/device/mod.rs

@@ -20,6 +20,8 @@ use self::topology::PCIeTopology;
 pub mod device_manager;
 pub mod driver;
 pub mod pci_path;
+mod tap;
+pub use self::tap::{Error as TapError, Tap};
 pub mod topology;
 pub mod util;
 

src/runtime-rs/crates/hypervisor/src/device/tap.rs

@@ -0,0 +1,264 @@
+// Copyright 2024 Kata Contributors
+// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the THIRD-PARTY file.
+
+use std::ffi::CStr;
+use std::fs::File;
+use std::io::{Error as IoError, Read, Result as IoResult, Write};
+use std::net::UdpSocket;
+use std::os::raw::*;
+use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
+
+use libc::ifreq;
+use vmm_sys_util::ioctl::{ioctl_with_mut_ref, ioctl_with_ref, ioctl_with_val};
+use vmm_sys_util::{ioctl_ioc_nr, ioctl_iow_nr};
+// As defined in the Linux UAPI:
+// https://elixir.bootlin.com/linux/v4.17/source/include/uapi/linux/if.h#L33
+pub(crate) const IFACE_NAME_MAX_LEN: usize = 16;
+
+/// List of errors the tap implementation can throw.
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    /// Failed to create a socket.
+    #[error("cannot create socket. {0}")]
+    CreateSocket(#[source] IoError),
+
+    /// Unable to create tap interface.
+    #[error("cannot create tap device. {0}")]
+    CreateTap(IoError),
+
+    /// Invalid interface name.
+    #[error("invalid network interface name")]
+    InvalidIfname,
+
+    /// ioctl failed.
+    #[error("failure while issue Tap ioctl command. {0}")]
+    IoctlError(#[source] IoError),
+
+    /// Couldn't open /dev/net/tun.
+    #[error("cannot open tap device. {0}")]
+    OpenTun(#[source] IoError),
+}
+
+pub type Result<T> = ::std::result::Result<T, Error>;
+
+const TUNTAP: ::std::os::raw::c_uint = 84;
+ioctl_iow_nr!(TUNSETIFF, TUNTAP, 202, ::std::os::raw::c_int);
+ioctl_iow_nr!(TUNSETOFFLOAD, TUNTAP, 208, ::std::os::raw::c_uint);
+ioctl_iow_nr!(TUNSETVNETHDRSZ, TUNTAP, 216, ::std::os::raw::c_int);
+
+/// Handle for a network tap interface.
+///
+/// For now, this simply wraps the file descriptor for the tap device so methods
+/// can run ioctls on the interface. The tap interface fd will be closed when
+/// Tap goes out of scope, and the kernel will clean up the interface automatically.
+#[derive(Debug)]
+pub struct Tap {
+    /// tap device file handle
+    pub tap_file: File,
+    pub(crate) if_name: [std::os::raw::c_char; IFACE_NAME_MAX_LEN],
+    pub(crate) if_flags: std::os::raw::c_short,
+}
+
+impl PartialEq for Tap {
+    fn eq(&self, other: &Tap) -> bool {
+        self.if_name == other.if_name
+    }
+}
+fn create_socket() -> Result<UdpSocket> {
+    // This is safe since we check the return value.
+    let sock = unsafe { libc::socket(libc::AF_INET, libc::SOCK_DGRAM, 0) };
+    if sock < 0 {
+        return Err(Error::CreateSocket(IoError::last_os_error()));
+    }
+
+    // This is safe; nothing else will use or hold onto the raw sock fd.
+    Ok(unsafe { UdpSocket::from_raw_fd(sock) })
+}
+
+// Returns an array representing the contents of a null-terminated C string
+// containing if_name.
+pub fn build_terminated_if_name(if_name: &str) -> Result<[c_char; IFACE_NAME_MAX_LEN]> {
+    // Convert the string slice to bytes, and shadow the variable,
+    // since we no longer need the &str version.
+    let if_name_bytes = if_name.as_bytes();
+
+    if if_name_bytes.len() >= IFACE_NAME_MAX_LEN {
+        return Err(Error::InvalidIfname);
+    }
+
+    let mut terminated_if_name = [0 as c_char; IFACE_NAME_MAX_LEN];
+    for (i, &byte) in if_name_bytes.iter().enumerate() {
+        terminated_if_name[i] = byte as c_char;
+    }
+
+    // 0 is the null terminator for c_char type
+    terminated_if_name[if_name_bytes.len()] = 0 as c_char;
+
+    Ok(terminated_if_name)
+}
+
+impl Tap {
+    /// Create a TUN/TAP device given the interface name.
+    /// # Arguments
+    ///
+    /// * `if_name` - the name of the interface.
+    pub fn open_named(if_name: &str, multi_vq: bool) -> Result<Tap> {
+        let terminated_if_name = build_terminated_if_name(if_name)?;
+
+        // Initialize an `ifreq` structure with the given interface name
+        // and configure its flags for setting up a network interface.
+        let mut ifr = ifreq {
+            ifr_name: terminated_if_name,
+            ifr_ifru: libc::__c_anonymous_ifr_ifru {
+                ifru_flags: (libc::IFF_TAP
+                    | libc::IFF_NO_PI
+                    | libc::IFF_VNET_HDR
+                    | if multi_vq { libc::IFF_MULTI_QUEUE } else { 0 })
+                    as c_short,
+            },
+        };
+        Tap::create_tap_with_ifreq(&mut ifr)
+    }
+
+    fn create_tap_with_ifreq(ifr: &mut ifreq) -> Result<Tap> {
+        let fd = unsafe {
+            let dev_net_tun = CStr::from_bytes_with_nul(b"/dev/net/tun\0").unwrap_or_else(|_| {
+                unreachable!("The string is guaranteed to be null-terminated and valid.")
+            });
+
+            // Open calls are safe because we use a CStr, which guarantees a
+            // constant null-terminated string.
+            libc::open(
+                dev_net_tun.as_ptr(),
+                libc::O_RDWR | libc::O_NONBLOCK | libc::O_CLOEXEC,
+            )
+        };
+        if fd < 0 {
+            return Err(Error::OpenTun(IoError::last_os_error()));
+        }
+
+        // We just checked that the fd is valid.
+        let tuntap = unsafe { File::from_raw_fd(fd) };
+
+        // ioctl is safe since we call it with a valid tap fd and check the return
+        // value.
+        let ret = unsafe { ioctl_with_mut_ref(&tuntap, TUNSETIFF(), ifr) };
+        if ret < 0 {
+            return Err(Error::CreateTap(IoError::last_os_error()));
+        }
+
+        Ok(Tap {
+            tap_file: tuntap,
+            if_name: ifr.ifr_name,
+            // This is safe since ifru_flags was correctly initialized earlier.
+            if_flags: unsafe { ifr.ifr_ifru.ifru_flags },
+        })
+    }
+
+    /// Change the origin tap into multiqueue taps.
+    pub fn into_mq_taps(self, vq_pairs: usize) -> Result<Vec<Tap>> {
+        let mut taps = Vec::with_capacity(vq_pairs);
+
+        if vq_pairs == 1 {
+            // vq_pairs cannot be less than 1, so only handle the case where it equals 1.
+            taps.push(self);
+            return Ok(taps);
+        }
+
+        // Add other socket into the origin tap interface.
+        for _ in 0..vq_pairs - 1 {
+            let mut ifr: ifreq = self.get_ifreq();
+            let tap = Tap::create_tap_with_ifreq(&mut ifr)?;
+
+            tap.enable()?;
+
+            taps.push(tap);
+        }
+
+        taps.insert(0, self);
+        Ok(taps)
+    }
+
+    /// Set the offload flags for the tap interface.
+    pub fn set_offload(&self, flags: c_uint) -> Result<()> {
+        // ioctl is safe. Called with a valid tap fd, and we check the return.
+        let ret = unsafe { ioctl_with_val(&self.tap_file, TUNSETOFFLOAD(), c_ulong::from(flags)) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
+    }
+
+    /// Enable the tap interface.
+    pub fn enable(&self) -> Result<()> {
+        let sock = create_socket()?;
+
+        let mut ifr = self.get_ifreq();
+        ifr.ifr_ifru.ifru_flags = (libc::IFF_UP | libc::IFF_RUNNING) as i16;
+
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        let ret = unsafe { ioctl_with_ref(&sock, c_ulong::from(libc::SIOCSIFFLAGS), &ifr) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
+    }
+
+    /// Set the size of the vnet hdr.
+    pub fn set_vnet_hdr_size(&self, size: c_int) -> Result<()> {
+        // ioctl is safe. Called with a valid tap fd, and we check the return.
+        let ret = unsafe { ioctl_with_ref(&self.tap_file, TUNSETVNETHDRSZ(), &size) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
+    }
+
+    fn get_ifreq(&self) -> ifreq {
+        let mut ifr_name = [0 as c_char; libc::IFNAMSIZ];
+        ifr_name[..self.if_name.len()].copy_from_slice(&self.if_name);
+
+        // Return an `ifreq` structure with the interface name and flags.
+        ifreq {
+            ifr_name,
+            ifr_ifru: libc::__c_anonymous_ifr_ifru {
+                ifru_flags: self.if_flags,
+            },
+        }
+    }
+
+    /// Get the origin flags when interface was created.
+    pub fn if_flags(&self) -> u32 {
+        self.if_flags as u32
+    }
+}
+
+impl Read for Tap {
+    fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
+        self.tap_file.read(buf)
+    }
+}
+
+impl Write for Tap {
+    fn write(&mut self, buf: &[u8]) -> IoResult<usize> {
+        self.tap_file.write(buf)
+    }
+
+    fn flush(&mut self) -> IoResult<()> {
+        Ok(())
+    }
+}
+
+impl AsRawFd for Tap {
+    fn as_raw_fd(&self) -> RawFd {
+        self.tap_file.as_raw_fd()
+    }
+}

src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs

@@ -147,8 +147,8 @@ impl DragonballInner {
         // And the the first one is Primary device.
         // safe here, devices is not empty.
         let primary_device = device.devices.first_mut().unwrap();
-        let vendor_device_id = if let Some(vd) = primary_device.device_vendor.as_ref() {
-            vd.get_device_vendor_id()?
+        let vendor_device_id = if let Some(vdc) = primary_device.device_vendor_class.as_ref() {
+            vdc.get_device_vendor_id()?
         } else {
             0
         };

src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs

@@ -12,6 +12,7 @@ use inner::DragonballInner;
 use persist::sandbox_persist::Persist;
 pub mod vmm_instance;
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use anyhow::{Context, Result};
@@ -69,7 +70,12 @@ impl Dragonball {
 #[async_trait]
 impl Hypervisor for Dragonball {
     #[instrument]
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }

src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs

@@ -107,6 +107,11 @@ impl FcInner {
             .get_resource(&self.config.boot_info.image, FC_ROOT_FS)
             .context("get resource ROOTFS")?;
 
+        let body_config: String = json!({
+            "mem_size_mib": self.config.memory_info.default_memory,
+            "vcpu_count": self.config.cpu_info.default_vcpus,
+        })
+        .to_string();
         let body_kernel: String = json!({
             "kernel_image_path": kernel,
             "boot_args": parameters,
@@ -124,6 +129,8 @@ impl FcInner {
         info!(sl(), "Before first request");
         self.request_with_retry(Method::PUT, "/boot-source", body_kernel)
             .await?;
+        self.request_with_retry(Method::PUT, "/machine-config", body_config)
+            .await?;
         self.request_with_retry(Method::PUT, "/drives/rootfs", body_rootfs)
             .await?;
 

src/runtime-rs/crates/hypervisor/src/firecracker/inner.rs

@@ -46,14 +46,12 @@ pub struct FcInner {
     pub(crate) capabilities: Capabilities,
     pub(crate) fc_process: Mutex<Option<Child>>,
     pub(crate) exit_notify: Option<mpsc::Sender<()>>,
-    pub(crate) exit_waiter: Mutex<(mpsc::Receiver<()>, i32)>,
 }
 
 impl FcInner {
-    pub fn new() -> FcInner {
+    pub fn new(exit_notify: mpsc::Sender<()>) -> FcInner {
         let mut capabilities = Capabilities::new();
         capabilities.set(CapabilityBits::BlockDeviceSupport);
-        let (exit_notify, exit_waiter) = mpsc::channel(1);
 
         FcInner {
             id: String::default(),
@@ -71,7 +69,6 @@ impl FcInner {
             capabilities,
             fc_process: Mutex::new(None),
             exit_notify: Some(exit_notify),
-            exit_waiter: Mutex::new((exit_waiter, 0)),
         }
     }
 
@@ -124,11 +121,10 @@ impl FcInner {
         let mut child = cmd.stderr(Stdio::piped()).spawn()?;
 
         let stderr = child.stderr.take().unwrap();
-        let exit_notify: mpsc::Sender<()> = self
+        let exit_notify = self
             .exit_notify
             .take()
             .ok_or_else(|| anyhow!("no exit notify"))?;
-
         tokio::spawn(log_fc_stderr(stderr, exit_notify));
 
         match child.id() {
@@ -216,7 +212,7 @@ async fn log_fc_stderr(stderr: ChildStderr, exit_notify: mpsc::Sender<()>) -> Re
 #[async_trait]
 impl Persist for FcInner {
     type State = HypervisorState;
-    type ConstructorArgs = ();
+    type ConstructorArgs = mpsc::Sender<()>;
 
     async fn save(&self) -> Result<Self::State> {
         Ok(HypervisorState {
@@ -231,12 +227,7 @@ impl Persist for FcInner {
             ..Default::default()
         })
     }
-    async fn restore(
-        _hypervisor_args: Self::ConstructorArgs,
-        hypervisor_state: Self::State,
-    ) -> Result<Self> {
-        let (exit_notify, exit_waiter) = mpsc::channel(1);
-
+    async fn restore(exit_notify: mpsc::Sender<()>, hypervisor_state: Self::State) -> Result<Self> {
         Ok(FcInner {
             id: hypervisor_state.id,
             asock_path: String::default(),
@@ -253,7 +244,6 @@ impl Persist for FcInner {
             capabilities: Capabilities::new(),
             fc_process: Mutex::new(None),
             exit_notify: Some(exit_notify),
-            exit_waiter: Mutex::new((exit_waiter, 0)),
         })
     }
 }

src/runtime-rs/crates/hypervisor/src/firecracker/inner_hypervisor.rs

@@ -102,21 +102,14 @@ impl FcInner {
     }
 
     pub(crate) async fn wait_vm(&self) -> Result<i32> {
-        debug!(sl(), "Wait fc sandbox");
-        let mut waiter = self.exit_waiter.lock().await;
-
-        //wait until the fc process exited.
-        waiter.0.recv().await;
-
         let mut fc_process = self.fc_process.lock().await;
 
         if let Some(mut fc_process) = fc_process.take() {
-            if let Ok(status) = fc_process.wait().await {
-                waiter.1 = status.code().unwrap_or(0);
-            }
+            let status = fc_process.wait().await?;
+            Ok(status.code().unwrap_or(0))
+        } else {
+            Err(anyhow!("the process has been reaped"))
         }
-
-        Ok(waiter.1)
     }
 
     pub(crate) fn pause_vm(&self) -> Result<()> {

src/runtime-rs/crates/hypervisor/src/firecracker/mod.rs

@@ -18,12 +18,16 @@ use inner::FcInner;
 use kata_types::capabilities::Capabilities;
 use kata_types::capabilities::CapabilityBits;
 use persist::sandbox_persist::Persist;
+use std::collections::HashMap;
 use std::sync::Arc;
+use tokio::sync::mpsc;
+use tokio::sync::Mutex;
 use tokio::sync::RwLock;
 
 #[derive(Debug)]
 pub struct Firecracker {
     inner: Arc<RwLock<FcInner>>,
+    exit_waiter: Mutex<(mpsc::Receiver<()>, i32)>,
 }
 
 // Convenience function to set the scope.
@@ -39,8 +43,11 @@ impl Default for Firecracker {
 
 impl Firecracker {
     pub fn new() -> Self {
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
         Self {
-            inner: Arc::new(RwLock::new(FcInner::new())),
+            inner: Arc::new(RwLock::new(FcInner::new(exit_notify))),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
         }
     }
 
@@ -52,7 +59,12 @@ impl Firecracker {
 
 #[async_trait]
 impl Hypervisor for Firecracker {
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }
@@ -68,8 +80,18 @@ impl Hypervisor for Firecracker {
     }
 
     async fn wait_vm(&self) -> Result<i32> {
+        debug!(sl(), "Wait fc sandbox");
+        let mut waiter = self.exit_waiter.lock().await;
+
+        //wait until the fc process exited.
+        waiter.0.recv().await;
+
         let inner = self.inner.read().await;
-        inner.wait_vm().await
+        if let Ok(exit_code) = inner.wait_vm().await {
+            waiter.1 = exit_code;
+        }
+
+        Ok(waiter.1)
     }
 
     async fn pause_vm(&self) -> Result<()> {
@@ -209,12 +231,15 @@ impl Persist for Firecracker {
     }
     /// Restore a component from a specified state.
     async fn restore(
-        hypervisor_args: Self::ConstructorArgs,
+        _hypervisor_args: Self::ConstructorArgs,
         hypervisor_state: Self::State,
     ) -> Result<Self> {
-        let inner = FcInner::restore(hypervisor_args, hypervisor_state).await?;
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+        let inner = FcInner::restore(exit_notify, hypervisor_state).await?;
+
         Ok(Self {
             inner: Arc::new(RwLock::new(inner)),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
         })
     }
 }

src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs

@@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 #[derive(Serialize, Deserialize, Default, Clone, Debug)]
 pub struct HypervisorState {
-    // Type of hypervisor, E.g. dragonball/qemu/firecracker/acrn.
+    // Type of hypervisor, E.g. dragonball/qemu/firecracker.
     pub hypervisor_type: String,
     pub pid: Option<i32>,
     pub uuid: String,

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -13,12 +13,13 @@ pub mod device;
 pub mod hypervisor_persist;
 pub use device::driver::*;
 use device::DeviceType;
-#[cfg(not(target_arch = "s390x"))]
+#[cfg(all(feature = "dragonball", not(target_arch = "s390x")))]
 pub mod dragonball;
 #[cfg(not(target_arch = "s390x"))]
 pub mod firecracker;
 mod kernel_param;
 pub mod qemu;
+pub mod remote;
 pub use kernel_param::Param;
 pub mod utils;
 use std::collections::HashMap;
@@ -53,17 +54,20 @@ const VM_ROOTFS_FILESYSTEM_EROFS: &str = "erofs";
 // /dev/hugepages will be the mount point
 // mkdir -p /dev/hugepages
 // mount -t hugetlbfs none /dev/hugepages
-#[cfg(not(target_arch = "s390x"))]
-const DEV_HUGEPAGES: &str = "/dev/hugepages";
 pub const HUGETLBFS: &str = "hugetlbfs";
-#[cfg(not(target_arch = "s390x"))]
+// Constants required for Dragonball VMM when enabled and not on s390x.
+// Not needed when the built-in VMM is not used.
+#[cfg(all(feature = "dragonball", not(target_arch = "s390x")))]
+const DEV_HUGEPAGES: &str = "/dev/hugepages";
+#[cfg(all(feature = "dragonball", not(target_arch = "s390x")))]
 const SHMEM: &str = "shmem";
-#[cfg(not(target_arch = "s390x"))]
+#[cfg(all(feature = "dragonball", not(target_arch = "s390x")))]
 const HUGE_SHMEM: &str = "hugeshmem";
 
 pub const HYPERVISOR_DRAGONBALL: &str = "dragonball";
 pub const HYPERVISOR_QEMU: &str = "qemu";
 pub const HYPERVISOR_FIRECRACKER: &str = "firecracker";
+pub const HYPERVISOR_REMOTE: &str = "remote";
 
 pub const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock";
 pub const JAILER_ROOT: &str = "root";
@@ -93,7 +97,12 @@ pub struct MemoryConfig {
 #[async_trait]
 pub trait Hypervisor: std::fmt::Debug + Send + Sync {
     // vm manager
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()>;
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        annotations: &HashMap<String, String>,
+    ) -> Result<()>;
     async fn start_vm(&self, timeout: i32) -> Result<()>;
     async fn stop_vm(&self) -> Result<()>;
     async fn wait_vm(&self) -> Result<i32>;

src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs

@@ -8,6 +8,7 @@ use crate::{kernel_param::KernelParams, Address, HypervisorConfig};
 
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
+use kata_types::config::hypervisor::VIRTIO_SCSI;
 use std::collections::HashMap;
 use std::fmt::Display;
 use std::fs::{read_to_string, File};
@@ -49,7 +50,7 @@ trait ToQemuParams: Send + Sync {
     async fn qemu_params(&self) -> Result<Vec<String>>;
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone, Copy)]
 enum VirtioBusType {
     Pci,
     Ccw,
@@ -70,6 +71,14 @@ impl Display for VirtioBusType {
     }
 }
 
+fn bus_type(config: &HypervisorConfig) -> VirtioBusType {
+    if config.machine_info.machine_type.contains("-ccw-") {
+        VirtioBusType::Ccw
+    } else {
+        VirtioBusType::Pci
+    }
+}
+
 // Conventions used in qemu command line generation
 // ================================================
 //
@@ -975,7 +984,7 @@ fn format_fds(files: &[File]) -> String {
 }
 
 #[derive(Debug)]
-struct Netdev {
+pub struct Netdev {
     id: String,
 
     // File descriptors for vhost multi-queue support.
@@ -1013,6 +1022,18 @@ impl Netdev {
         self.disable_vhost_net = disable_vhost_net;
         self
     }
+
+    pub fn get_id(&self) -> &String {
+        &self.id
+    }
+
+    pub fn get_fds(&self) -> &Vec<File> {
+        &self.fds["fds"]
+    }
+
+    pub fn get_vhostfds(&self) -> &Vec<File> {
+        &self.fds["vhostfds"]
+    }
 }
 
 #[async_trait]
@@ -1081,6 +1102,26 @@ impl DeviceVirtioNet {
         self.iommu_platform = iommu_platform;
         self
     }
+
+    pub fn get_netdev_id(&self) -> &String {
+        &self.netdev_id
+    }
+
+    pub fn get_device_driver(&self) -> &String {
+        &self.device_driver
+    }
+
+    pub fn get_mac_addr(&self) -> String {
+        format!("{:?}", self.mac_address)
+    }
+
+    pub fn get_num_queues(&self) -> u32 {
+        self.num_queues
+    }
+
+    pub fn get_disable_modern(&self) -> bool {
+        self.disable_modern
+    }
 }
 
 #[async_trait]
@@ -1290,6 +1331,74 @@ impl ToQemuParams for DeviceIntelIommu {
     }
 }
 
+#[derive(Debug)]
+struct DevicePciBridge {
+    driver: String,
+    bus: String,
+    id: String,
+    chassis_nr: u32,
+    shpc: bool,
+    addr: u32,
+    io_reserve: String,
+    mem_reserve: String,
+    pref64_reserve: String,
+}
+
+impl DevicePciBridge {
+    fn new(config: &HypervisorConfig, bridge_idx: u32) -> DevicePciBridge {
+        DevicePciBridge {
+            // The go runtime doesn't support bridges other than PCI although
+            // PCIe should also be available.  Stick with the legacy behaviour
+            // of ignoring PCIe since it's not clear to me how to decide
+            // between the two.
+            driver: "pci-bridge".to_owned(),
+            bus: match config.machine_info.machine_type.as_str() {
+                "q35" | "virt" => "pcie.0",
+                _ => "pci.0",
+            }
+            .to_owned(),
+            id: format!("pci-bridge-{}", bridge_idx),
+            // Each bridge is required to be assigned a unique chassis id > 0.
+            chassis_nr: bridge_idx + 1,
+            shpc: false,
+            // 2 is documented by the go runtime as the first slot available
+            // for a bridge (on x86_64)
+            // (https://github.com/kata-containers/kata-containers/blob/99730256a2899c82d111400024621519d17ea15d/src/runtime/virtcontainers/qemu_arch_base.go#L212)
+            addr: 2 + bridge_idx,
+            // Values taken from the go runtime implementation which comments
+            // the choices as follows:
+            // Certain guest BIOS versions think !SHPC means no hotplug, and
+            // won't reserve the IO and memory windows that will be needed for
+            // devices added underneath this bridge.  This will only break for
+            // certain combinations of exact qemu, BIOS and guest kernel
+            // versions, but for consistency, just hint the usual default
+            // windows for a bridge (as the BIOS would use with SHPC) so that
+            // we can do ACPI hotplug.
+            // (https://github.com/kata-containers/kata-containers/blob/99730256a2899c82d111400024621519d17ea15d/src/runtime/virtcontainers/qemu.go#L2474)
+            io_reserve: "4k".to_owned(),
+            mem_reserve: "1m".to_owned(),
+            pref64_reserve: "1m".to_owned(),
+        }
+    }
+}
+
+#[async_trait]
+impl ToQemuParams for DevicePciBridge {
+    async fn qemu_params(&self) -> Result<Vec<String>> {
+        let mut params = Vec::new();
+        params.push(self.driver.clone());
+        params.push(format!("bus={}", self.bus));
+        params.push(format!("id={}", self.id));
+        params.push(format!("chassis_nr={}", self.chassis_nr));
+        params.push(format!("shpc={}", if self.shpc { "on" } else { "off" }));
+        params.push(format!("addr={}", self.addr));
+        params.push(format!("io-reserve={}", self.io_reserve));
+        params.push(format!("mem-reserve={}", self.mem_reserve));
+        params.push(format!("pref64-reserve={}", self.pref64_reserve));
+        Ok(vec!["-device".to_owned(), params.join(",")])
+    }
+}
+
 // Qemu provides methods and types for managing QEMU instances.
 // To manage a qemu instance after it has been launched you need
 // to pass the -qmp option during launch requesting the qemu instance
@@ -1405,6 +1514,66 @@ impl ToQemuParams for QmpSocket {
     }
 }
 
+#[derive(Debug)]
+struct DeviceVirtioScsi {
+    bus_type: VirtioBusType,
+    id: String,
+    disable_modern: bool,
+    iothread: String,
+}
+
+impl DeviceVirtioScsi {
+    fn new(id: &str, disable_modern: bool, bus_type: VirtioBusType) -> Self {
+        DeviceVirtioScsi {
+            bus_type,
+            id: id.to_owned(),
+            disable_modern,
+            iothread: "".to_owned(),
+        }
+    }
+
+    fn set_iothread(&mut self, iothread: &str) {
+        self.iothread = iothread.to_owned();
+    }
+}
+
+#[async_trait]
+impl ToQemuParams for DeviceVirtioScsi {
+    async fn qemu_params(&self) -> Result<Vec<String>> {
+        let mut params = Vec::new();
+        params.push(format!("virtio-scsi-{}", self.bus_type));
+        params.push(format!("id={}", self.id));
+        if self.disable_modern {
+            params.push("disable-modern=true".to_owned());
+        }
+        if !self.iothread.is_empty() {
+            params.push(format!("iothread={}", self.iothread));
+        }
+        Ok(vec!["-device".to_owned(), params.join(",")])
+    }
+}
+
+#[derive(Debug)]
+struct ObjectIoThread {
+    id: String,
+}
+
+impl ObjectIoThread {
+    fn new(id: &str) -> Self {
+        ObjectIoThread { id: id.to_owned() }
+    }
+}
+
+#[async_trait]
+impl ToQemuParams for ObjectIoThread {
+    async fn qemu_params(&self) -> Result<Vec<String>> {
+        let mut params = Vec::new();
+        params.push("iothread".to_owned());
+        params.push(format!("id={}", self.id));
+        Ok(vec!["-object".to_owned(), params.join(",")])
+    }
+}
+
 fn is_running_in_vm() -> Result<bool> {
     let res = read_to_string("/proc/cpuinfo")?
         .lines()
@@ -1480,10 +1649,18 @@ impl<'a> QemuCmdLine<'a> {
 
         qemu_cmd_line.add_rtc();
 
-        if qemu_cmd_line.bus_type() != VirtioBusType::Ccw {
+        if bus_type(config) != VirtioBusType::Ccw {
             qemu_cmd_line.add_rng();
         }
 
+        if bus_type(config) != VirtioBusType::Ccw && config.device_info.default_bridges > 0 {
+            qemu_cmd_line.add_bridges(config.device_info.default_bridges);
+        }
+
+        if config.blockdev_info.block_device_driver == VIRTIO_SCSI {
+            qemu_cmd_line.add_scsi_controller();
+        }
+
         Ok(qemu_cmd_line)
     }
 
@@ -1507,14 +1684,6 @@ impl<'a> QemuCmdLine<'a> {
         self.devices.push(Box::new(rng_device));
     }
 
-    fn bus_type(&self) -> VirtioBusType {
-        if self.config.machine_info.machine_type.contains("-ccw-") {
-            VirtioBusType::Ccw
-        } else {
-            VirtioBusType::Pci
-        }
-    }
-
     fn add_iommu(&mut self) {
         let dev_iommu = DeviceIntelIommu::new();
         self.devices.push(Box::new(dev_iommu));
@@ -1526,6 +1695,25 @@ impl<'a> QemuCmdLine<'a> {
         self.machine.set_kernel_irqchip("split");
     }
 
+    fn add_bridges(&mut self, count: u32) {
+        for idx in 0..count {
+            let bridge = DevicePciBridge::new(self.config, idx);
+            self.devices.push(Box::new(bridge));
+        }
+    }
+
+    fn add_scsi_controller(&mut self) {
+        let mut virtio_scsi =
+            DeviceVirtioScsi::new("scsi0", should_disable_modern(), bus_type(self.config));
+        if self.config.enable_iothreads {
+            let iothread_id = "scsi-io-thread";
+            let iothread = ObjectIoThread::new(iothread_id);
+            virtio_scsi.set_iothread(iothread_id);
+            self.devices.push(Box::new(iothread));
+        }
+        self.devices.push(Box::new(virtio_scsi));
+    }
+
     pub fn add_virtiofs_share(
         &mut self,
         virtiofsd_socket_path: &str,
@@ -1542,9 +1730,11 @@ impl<'a> QemuCmdLine<'a> {
 
         self.devices.push(Box::new(virtiofsd_socket_chardev));
 
-        let mut virtiofs_device = DeviceVhostUserFs::new(chardev_name, mount_tag, self.bus_type());
+        let bus_type = bus_type(self.config);
+
+        let mut virtiofs_device = DeviceVhostUserFs::new(chardev_name, mount_tag, bus_type);
         virtiofs_device.set_queue_size(queue_size);
-        if self.config.device_info.enable_iommu_platform && self.bus_type() == VirtioBusType::Ccw {
+        if self.config.device_info.enable_iommu_platform && bus_type == VirtioBusType::Ccw {
             virtiofs_device.set_iommu_platform(true);
         }
         self.devices.push(Box::new(virtiofs_device));
@@ -1558,7 +1748,7 @@ impl<'a> QemuCmdLine<'a> {
         //self.devices.push(Box::new(mem_file));
         self.memory.set_memory_backend_file(&mem_file);
 
-        match self.bus_type() {
+        match bus_type {
             VirtioBusType::Pci => {
                 self.machine.set_nvdimm(true);
                 self.devices.push(Box::new(NumaNode::new(&mem_file.id)));
@@ -1572,7 +1762,7 @@ impl<'a> QemuCmdLine<'a> {
     pub fn add_vsock(&mut self, vhostfd: tokio::fs::File, guest_cid: u32) -> Result<()> {
         clear_cloexec(vhostfd.as_raw_fd()).context("clearing O_CLOEXEC failed on vsock fd")?;
 
-        let mut vhost_vsock_pci = VhostVsock::new(vhostfd, guest_cid, self.bus_type());
+        let mut vhost_vsock_pci = VhostVsock::new(vhostfd, guest_cid, bus_type(self.config));
 
         if !self.config.disable_nesting_checks && should_disable_modern() {
             vhost_vsock_pci.set_disable_modern(true);
@@ -1619,8 +1809,10 @@ impl<'a> QemuCmdLine<'a> {
     pub fn add_block_device(&mut self, device_id: &str, path: &str) -> Result<()> {
         self.devices
             .push(Box::new(BlockBackend::new(device_id, path)));
-        self.devices
-            .push(Box::new(DeviceVirtioBlk::new(device_id, self.bus_type())));
+        self.devices.push(Box::new(DeviceVirtioBlk::new(
+            device_id,
+            bus_type(self.config),
+        )));
         Ok(())
     }
 
@@ -1634,32 +1826,9 @@ impl<'a> QemuCmdLine<'a> {
         ));
     }
 
-    pub fn add_network_device(
-        &mut self,
-        dev_index: u64,
-        host_dev_name: &str,
-        guest_mac: Address,
-    ) -> Result<()> {
-        let mut netdev = Netdev::new(
-            &format!("network-{}", dev_index),
-            host_dev_name,
-            self.config.network_info.network_queues,
-        )?;
-        if self.config.network_info.disable_vhost_net {
-            netdev.set_disable_vhost_net(true);
-        }
-
-        let mut virtio_net_device = DeviceVirtioNet::new(&netdev.id, guest_mac);
-
-        if should_disable_modern() {
-            virtio_net_device.set_disable_modern(true);
-        }
-        if self.config.device_info.enable_iommu_platform && self.bus_type() == VirtioBusType::Ccw {
-            virtio_net_device.set_iommu_platform(true);
-        }
-        if self.config.network_info.network_queues > 1 {
-            virtio_net_device.set_num_queues(self.config.network_info.network_queues);
-        }
+    pub fn add_network_device(&mut self, host_dev_name: &str, guest_mac: Address) -> Result<()> {
+        let (netdev, virtio_net_device) =
+            get_network_device(self.config, host_dev_name, guest_mac)?;
 
         self.devices.push(Box::new(netdev));
         self.devices.push(Box::new(virtio_net_device));
@@ -1667,8 +1836,10 @@ impl<'a> QemuCmdLine<'a> {
     }
 
     pub fn add_console(&mut self, console_socket_path: &str) {
-        let mut serial_dev = DeviceVirtioSerial::new("serial0", self.bus_type());
-        if self.config.device_info.enable_iommu_platform && self.bus_type() == VirtioBusType::Ccw {
+        let mut serial_dev = DeviceVirtioSerial::new("serial0", bus_type(self.config));
+        if self.config.device_info.enable_iommu_platform
+            && bus_type(self.config) == VirtioBusType::Ccw
+        {
             serial_dev.set_iommu_platform(true);
         }
         self.devices.push(Box::new(serial_dev));
@@ -1709,3 +1880,32 @@ impl<'a> QemuCmdLine<'a> {
         Ok(result)
     }
 }
+
+pub fn get_network_device(
+    config: &HypervisorConfig,
+    host_dev_name: &str,
+    guest_mac: Address,
+) -> Result<(Netdev, DeviceVirtioNet)> {
+    let mut netdev = Netdev::new(
+        &format!("network-{}", host_dev_name),
+        host_dev_name,
+        config.network_info.network_queues,
+    )?;
+    if config.network_info.disable_vhost_net {
+        netdev.set_disable_vhost_net(true);
+    }
+
+    let mut virtio_net_device = DeviceVirtioNet::new(&netdev.id, guest_mac);
+
+    if should_disable_modern() {
+        virtio_net_device.set_disable_modern(true);
+    }
+    if config.device_info.enable_iommu_platform && bus_type(config) == VirtioBusType::Ccw {
+        virtio_net_device.set_iommu_platform(true);
+    }
+    if config.network_info.network_queues > 1 {
+        virtio_net_device.set_num_queues(config.network_info.network_queues);
+    }
+
+    Ok((netdev, virtio_net_device))
+}

src/runtime-rs/crates/hypervisor/src/qemu/inner.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use super::cmdline_generator::{QemuCmdLine, QMP_SOCKET_FILE};
+use super::cmdline_generator::{get_network_device, QemuCmdLine, QMP_SOCKET_FILE};
 use super::qmp::Qmp;
 use crate::{
     hypervisor_persist::HypervisorState, utils::enter_netns, HypervisorConfig, MemoryConfig,
@@ -43,13 +43,10 @@ pub struct QemuInner {
     netns: Option<String>,
 
     exit_notify: Option<mpsc::Sender<()>>,
-    exit_waiter: Mutex<(mpsc::Receiver<()>, i32)>,
 }
 
 impl QemuInner {
-    pub fn new() -> QemuInner {
-        let (exit_notify, exit_waiter) = mpsc::channel(1);
-
+    pub fn new(exit_notify: mpsc::Sender<()>) -> QemuInner {
         QemuInner {
             id: "".to_string(),
             qemu_process: Mutex::new(None),
@@ -59,7 +56,6 @@ impl QemuInner {
             netns: None,
 
             exit_notify: Some(exit_notify),
-            exit_waiter: Mutex::new((exit_waiter, 0)),
         }
     }
 
@@ -124,7 +120,6 @@ impl QemuInner {
                     let _netns_guard = NetnsGuard::new(&netns).context("new netns guard")?;
 
                     cmdline.add_network_device(
-                        network.config.index,
                         &network.config.host_dev_name,
                         network.config.guest_mac.clone().unwrap(),
                     )?;
@@ -202,22 +197,14 @@ impl QemuInner {
     }
 
     pub(crate) async fn wait_vm(&self) -> Result<i32> {
-        info!(sl!(), "Wait QEMU VM");
-
-        let mut waiter = self.exit_waiter.lock().await;
-
-        //wait until the qemu process exited.
-        waiter.0.recv().await;
-
         let mut qemu_process = self.qemu_process.lock().await;
 
         if let Some(mut qemu_process) = qemu_process.take() {
-            if let Ok(status) = qemu_process.wait().await {
-                waiter.1 = status.code().unwrap_or(0);
-            }
+            let status = qemu_process.wait().await?;
+            Ok(status.code().unwrap_or(0))
+        } else {
+            Err(anyhow!("the process has been reaped"))
         }
-
-        Ok(waiter.1)
     }
 
     pub(crate) fn pause_vm(&self) -> Result<()> {
@@ -552,9 +539,16 @@ use crate::device::DeviceType;
 
 // device manager part of Hypervisor
 impl QemuInner {
-    pub(crate) async fn add_device(&mut self, device: DeviceType) -> Result<DeviceType> {
+    pub(crate) async fn add_device(&mut self, mut device: DeviceType) -> Result<DeviceType> {
         info!(sl!(), "QemuInner::add_device() {}", device);
-        self.devices.push(device.clone());
+        let is_qemu_ready_to_hotplug = self.qmp.is_some();
+        if is_qemu_ready_to_hotplug {
+            // hypervisor is running already
+            device = self.hotplug_device(device)?;
+        } else {
+            // store the device to coldplug it later, on hypervisor launch
+            self.devices.push(device.clone());
+        }
         Ok(device)
     }
 
@@ -565,6 +559,26 @@ impl QemuInner {
             device
         ))
     }
+
+    fn hotplug_device(&mut self, device: DeviceType) -> Result<DeviceType> {
+        let qmp = match self.qmp {
+            Some(ref mut qmp) => qmp,
+            None => return Err(anyhow!("QMP not initialized")),
+        };
+
+        match device {
+            DeviceType::Network(ref network_device) => {
+                let (netdev, virtio_net_device) = get_network_device(
+                    &self.config,
+                    &network_device.config.host_dev_name,
+                    network_device.config.guest_mac.clone().unwrap(),
+                )?;
+                qmp.hotplug_network_device(&netdev, &virtio_net_device)?
+            }
+            _ => info!(sl!(), "hotplugging of {:#?} is unsupported", device),
+        }
+        Ok(device)
+    }
 }
 
 // private helpers
@@ -589,7 +603,7 @@ impl QemuInner {
 #[async_trait]
 impl Persist for QemuInner {
     type State = HypervisorState;
-    type ConstructorArgs = ();
+    type ConstructorArgs = mpsc::Sender<()>;
 
     /// Save a state of hypervisor
     async fn save(&self) -> Result<Self::State> {
@@ -602,12 +616,7 @@ impl Persist for QemuInner {
     }
 
     /// Restore hypervisor
-    async fn restore(
-        _hypervisor_args: Self::ConstructorArgs,
-        hypervisor_state: Self::State,
-    ) -> Result<Self> {
-        let (exit_notify, exit_waiter) = mpsc::channel(1);
-
+    async fn restore(exit_notify: mpsc::Sender<()>, hypervisor_state: Self::State) -> Result<Self> {
         Ok(QemuInner {
             id: hypervisor_state.id,
             qemu_process: Mutex::new(None),
@@ -617,7 +626,6 @@ impl Persist for QemuInner {
             netns: None,
 
             exit_notify: Some(exit_notify),
-            exit_waiter: Mutex::new((exit_waiter, 0)),
         })
     }
 }

src/runtime-rs/crates/hypervisor/src/qemu/mod.rs

@@ -18,12 +18,15 @@ use persist::sandbox_persist::Persist;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 
+use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::RwLock;
+use tokio::sync::{mpsc, Mutex};
 
 #[derive(Debug)]
 pub struct Qemu {
     inner: Arc<RwLock<QemuInner>>,
+    exit_waiter: Mutex<(mpsc::Receiver<()>, i32)>,
 }
 
 impl Default for Qemu {
@@ -34,8 +37,11 @@ impl Default for Qemu {
 
 impl Qemu {
     pub fn new() -> Self {
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
         Self {
-            inner: Arc::new(RwLock::new(QemuInner::new())),
+            inner: Arc::new(RwLock::new(QemuInner::new(exit_notify))),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
         }
     }
 
@@ -47,7 +53,12 @@ impl Qemu {
 
 #[async_trait]
 impl Hypervisor for Qemu {
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }
@@ -63,8 +74,19 @@ impl Hypervisor for Qemu {
     }
 
     async fn wait_vm(&self) -> Result<i32> {
+        info!(sl!(), "Wait QEMU VM");
+
+        let mut waiter = self.exit_waiter.lock().await;
+
+        //wait until the qemu process exited.
+        waiter.0.recv().await;
+
         let inner = self.inner.read().await;
-        inner.wait_vm().await
+        if let Ok(exit_code) = inner.wait_vm().await {
+            waiter.1 = exit_code;
+        }
+
+        Ok(waiter.1)
     }
 
     async fn pause_vm(&self) -> Result<()> {
@@ -204,12 +226,15 @@ impl Persist for Qemu {
 
     /// Restore a component from a specified state.
     async fn restore(
-        hypervisor_args: Self::ConstructorArgs,
+        _hypervisor_args: Self::ConstructorArgs,
         hypervisor_state: Self::State,
     ) -> Result<Self> {
-        let inner = QemuInner::restore(hypervisor_args, hypervisor_state).await?;
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
+        let inner = QemuInner::restore(exit_notify, hypervisor_state).await?;
         Ok(Self {
             inner: Arc::new(RwLock::new(inner)),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
         })
     }
 }

src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs

@@ -3,9 +3,13 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use crate::qemu::cmdline_generator::{DeviceVirtioNet, Netdev};
+
 use anyhow::{anyhow, Result};
+use nix::sys::socket::{sendmsg, ControlMessage, MsgFlags};
 use std::fmt::{Debug, Error, Formatter};
 use std::io::BufReader;
+use std::os::fd::{AsRawFd, RawFd};
 use std::os::unix::net::UnixStream;
 use std::time::Duration;
 
@@ -291,6 +295,178 @@ impl Qmp {
         }
         Ok(())
     }
+
+    fn find_free_slot(&mut self) -> Result<(String, i64)> {
+        let pci = self.qmp.execute(&qapi_qmp::query_pci {})?;
+        for pci_info in &pci {
+            for pci_dev in &pci_info.devices {
+                let pci_bridge = match &pci_dev.pci_bridge {
+                    Some(bridge) => bridge,
+                    None => continue,
+                };
+
+                info!(sl!(), "found PCI bridge: {}", pci_dev.qdev_id);
+
+                if let Some(bridge_devices) = &pci_bridge.devices {
+                    let occupied_slots = bridge_devices
+                        .iter()
+                        .map(|pci_dev| pci_dev.slot)
+                        .collect::<Vec<_>>();
+
+                    info!(
+                        sl!(),
+                        "already occupied slots on bridge {}: {:#?}",
+                        pci_dev.qdev_id,
+                        occupied_slots
+                    );
+
+                    // from virtcontainers' bridges.go
+                    let pci_bridge_max_capacity = 30;
+                    for slot in 0..pci_bridge_max_capacity {
+                        if !occupied_slots.iter().any(|elem| *elem == slot) {
+                            info!(
+                                sl!(),
+                                "found free slot on bridge {}: {}", pci_dev.qdev_id, slot
+                            );
+                            return Ok((pci_dev.qdev_id.clone(), slot));
+                        }
+                    }
+                }
+            }
+        }
+        Err(anyhow!("no free slots on PCI bridges"))
+    }
+
+    fn pass_fd(&mut self, fd: RawFd, fdname: &str) -> Result<()> {
+        info!(sl!(), "passing fd {:?} as {}", fd, fdname);
+
+        // Put the QMP 'getfd' command itself into the message payload.
+        let getfd_cmd = format!(
+            "{{ \"execute\": \"getfd\", \"arguments\": {{ \"fdname\": \"{}\" }} }}",
+            fdname
+        );
+        let buf = getfd_cmd.as_bytes();
+        let bufs = &mut [std::io::IoSlice::new(buf)][..];
+
+        debug!(sl!(), "bufs: {:?}", bufs);
+
+        let fds = [fd];
+        let cmsg = [ControlMessage::ScmRights(&fds)];
+
+        let result = sendmsg::<()>(
+            self.qmp.inner_mut().get_mut_write().as_raw_fd(),
+            bufs,
+            &cmsg,
+            MsgFlags::empty(),
+            None,
+        );
+        info!(sl!(), "sendmsg() result: {:#?}", result);
+
+        let result = self.qmp.read_response::<&qmp::getfd>();
+
+        match result {
+            Ok(_) => {
+                info!(sl!(), "successfully passed {} ({})", fdname, fd);
+                Ok(())
+            }
+            Err(err) => Err(anyhow!("failed to pass {} ({}): {}", fdname, fd, err)),
+        }
+    }
+
+    pub fn hotplug_network_device(
+        &mut self,
+        netdev: &Netdev,
+        virtio_net_device: &DeviceVirtioNet,
+    ) -> Result<()> {
+        debug!(
+            sl!(),
+            "hotplug_network_device(): PCI before {}: {:#?}",
+            virtio_net_device.get_netdev_id(),
+            self.qmp.execute(&qapi_qmp::query_pci {})?
+        );
+
+        let (bus, slot) = self.find_free_slot()?;
+
+        let mut fd_names = vec![];
+        for (idx, fd) in netdev.get_fds().iter().enumerate() {
+            let fdname = format!("fd{}", idx);
+            self.pass_fd(fd.as_raw_fd(), fdname.as_ref())?;
+            fd_names.push(fdname);
+        }
+
+        let mut vhostfd_names = vec![];
+        for (idx, fd) in netdev.get_vhostfds().iter().enumerate() {
+            let vhostfdname = format!("vhostfd{}", idx);
+            self.pass_fd(fd.as_raw_fd(), vhostfdname.as_ref())?;
+            vhostfd_names.push(vhostfdname);
+        }
+
+        self.qmp
+            .execute(&qapi_qmp::netdev_add(qapi_qmp::Netdev::tap {
+                id: netdev.get_id().clone(),
+                tap: qapi_qmp::NetdevTapOptions {
+                    br: None,
+                    downscript: None,
+                    fd: None,
+                    // Logic in cmdline_generator::Netdev::new() seems to
+                    // guarantee that there will always be at least one fd.
+                    fds: Some(fd_names.join(",")),
+                    helper: None,
+                    ifname: None,
+                    poll_us: None,
+                    queues: None,
+                    script: None,
+                    sndbuf: None,
+                    vhost: if vhostfd_names.is_empty() {
+                        None
+                    } else {
+                        Some(true)
+                    },
+                    vhostfd: None,
+                    vhostfds: if vhostfd_names.is_empty() {
+                        None
+                    } else {
+                        Some(vhostfd_names.join(","))
+                    },
+                    vhostforce: None,
+                    vnet_hdr: None,
+                },
+            }))?;
+
+        let mut netdev_frontend_args = Dictionary::new();
+        netdev_frontend_args.insert(
+            "netdev".to_owned(),
+            virtio_net_device.get_netdev_id().clone().into(),
+        );
+        netdev_frontend_args.insert("addr".to_owned(), format!("{:02}", slot).into());
+        netdev_frontend_args.insert("mac".to_owned(), virtio_net_device.get_mac_addr().into());
+        netdev_frontend_args.insert("mq".to_owned(), "on".into());
+        // As the golang runtime documents the vectors computation, it's
+        // 2N+2 vectors, N for tx queues, N for rx queues, 1 for config, and one for possible control vq
+        netdev_frontend_args.insert(
+            "vectors".to_owned(),
+            (2 * virtio_net_device.get_num_queues() + 2).into(),
+        );
+        if virtio_net_device.get_disable_modern() {
+            netdev_frontend_args.insert("disable-modern".to_owned(), true.into());
+        }
+
+        self.qmp.execute(&qmp::device_add {
+            bus: Some(bus),
+            id: Some(format!("frontend-{}", virtio_net_device.get_netdev_id())),
+            driver: virtio_net_device.get_device_driver().clone(),
+            arguments: netdev_frontend_args,
+        })?;
+
+        debug!(
+            sl!(),
+            "hotplug_network_device(): PCI after {}: {:#?}",
+            virtio_net_device.get_netdev_id(),
+            self.qmp.execute(&qapi_qmp::query_pci {})?
+        );
+
+        Ok(())
+    }
 }
 
 fn vcpu_id_from_core_id(core_id: i64) -> String {

src/runtime-rs/crates/hypervisor/src/remote/inner.rs

@@ -0,0 +1,387 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::{
+    device::DeviceType, hypervisor_persist::HypervisorState, HypervisorConfig, HYPERVISOR_REMOTE,
+};
+use crate::{MemoryConfig, VcpuThreadIds};
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+use kata_types::{
+    annotations::{
+        cri_containerd::{SANDBOX_NAMESPACE_LABEL_KEY, SANDBOX_NAME_LABEL_KEY},
+        KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS,
+        KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH, KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE,
+    },
+    capabilities::{Capabilities, CapabilityBits},
+};
+use persist::sandbox_persist::Persist;
+use protocols::{
+    remote::{CreateVMRequest, StartVMRequest, StopVMRequest},
+    remote_ttrpc_async::HypervisorClient,
+};
+use std::{collections::HashMap, time};
+use tokio::sync::{mpsc, Mutex};
+use ttrpc::context::{self};
+use ttrpc::r#async::Client;
+
+const REMOTE_SCHEME: &str = "remote";
+const DEFAULT_MIN_TIMEOUT: i32 = time::Duration::from_secs(60).as_millis() as i32;
+
+pub struct RemoteInner {
+    /// sandbox id
+    pub(crate) id: String,
+    /// hypervisor config
+    pub(crate) config: HypervisorConfig,
+    /// agent socket path
+    pub(crate) agent_socket_path: String,
+    /// netns path
+    pub(crate) netns: Option<String>,
+    /// hypervisor unix client
+    pub(crate) client: Option<Client>,
+
+    exit_notify: Option<mpsc::Sender<i32>>,
+    exit_waiter: Mutex<(mpsc::Receiver<i32>, i32)>,
+}
+
+impl std::fmt::Debug for RemoteInner {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RemoteInner")
+            .field("id", &self.id)
+            .field("config", &self.config)
+            .field("agent_socket_path", &self.agent_socket_path)
+            .field("netns", &self.netns)
+            .finish()
+    }
+}
+
+impl RemoteInner {
+    pub fn new() -> Self {
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
+        Self {
+            id: "".to_string(),
+            config: HypervisorConfig::default(),
+            agent_socket_path: "".to_string(),
+            netns: None,
+            client: None,
+
+            exit_notify: Some(exit_notify),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
+        }
+    }
+
+    fn get_ttrpc_client(&mut self) -> Result<HypervisorClient> {
+        match self.client {
+            Some(ref c) => Ok(HypervisorClient::new(c.clone())),
+            None => {
+                let c = Client::connect(&format!(
+                    "unix://{}",
+                    &self.config.remote_info.hypervisor_socket
+                ))
+                .context("connect to ")?;
+                self.client = Some(c.clone());
+                Ok(HypervisorClient::new(c))
+            }
+        }
+    }
+
+    fn prepare_annotations(
+        &self,
+        oci_annotations: &HashMap<String, String>,
+    ) -> HashMap<String, String> {
+        let mut annotations: HashMap<String, String> = HashMap::new();
+        let config = &self.config;
+        annotations.insert(
+            SANDBOX_NAME_LABEL_KEY.to_string(),
+            oci_annotations
+                .get(SANDBOX_NAME_LABEL_KEY)
+                .cloned()
+                .unwrap_or_default(),
+        );
+        annotations.insert(
+            SANDBOX_NAMESPACE_LABEL_KEY.to_string(),
+            oci_annotations
+                .get(SANDBOX_NAMESPACE_LABEL_KEY)
+                .cloned()
+                .unwrap_or_default(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE.to_string(),
+            config.machine_info.machine_type.to_string(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS.to_string(),
+            config.cpu_info.default_vcpus.to_string(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY.to_string(),
+            config.memory_info.default_memory.to_string(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH.to_string(),
+            config.boot_info.image.to_string(),
+        );
+        annotations
+    }
+
+    pub(crate) async fn prepare_vm(
+        &mut self,
+        id: &str,
+        netns: Option<String>,
+        annotations: &HashMap<String, String>,
+    ) -> Result<()> {
+        info!(sl!(), "Preparing REMOTE VM");
+        self.id = id.to_string();
+
+        if let Some(netns_path) = &netns {
+            debug!(sl!(), "set netns for vmm master {:?}", &netns_path);
+            std::fs::metadata(netns_path).context("check netns path")?;
+        }
+
+        let client = self.get_ttrpc_client()?;
+
+        let ctx = context::Context::default();
+        let req = CreateVMRequest {
+            id: id.to_string(),
+            annotations: self.prepare_annotations(annotations),
+            networkNamespacePath: netns.clone().unwrap_or_default(),
+            ..Default::default()
+        };
+        info!(sl!(), "Preparing REMOTE VM req: {:?}", req.clone());
+        let resp = client.create_vm(ctx, &req).await?;
+        info!(sl!(), "Preparing REMOTE VM resp: {:?}", resp.clone());
+        self.agent_socket_path = resp.agentSocketPath;
+        self.netns = netns;
+        Ok(())
+    }
+
+    pub(crate) async fn start_vm(&mut self, timeout: i32) -> Result<()> {
+        info!(sl!(), "Starting REMOTE VM");
+
+        let mut min_timeout = DEFAULT_MIN_TIMEOUT;
+        if self.config.remote_info.hypervisor_timeout > 0 {
+            min_timeout = self.config.remote_info.hypervisor_timeout.min(timeout);
+        }
+        let timeout = min_timeout;
+
+        let client = self.get_ttrpc_client()?;
+
+        let req = StartVMRequest {
+            id: self.id.clone(),
+            ..Default::default()
+        };
+        let ctx =
+            context::with_timeout(time::Duration::from_secs(timeout as u64).as_nanos() as i64);
+        let _resp = client.start_vm(ctx, &req).await?;
+
+        Ok(())
+    }
+
+    pub(crate) async fn stop_vm(&mut self) -> Result<()> {
+        info!(sl!(), "Stopping REMOTE VM");
+
+        let client = self.get_ttrpc_client()?;
+
+        let ctx = context::with_timeout(time::Duration::from_secs(1).as_nanos() as i64);
+        let req = StopVMRequest {
+            id: self.id.clone(),
+            ..Default::default()
+        };
+        let _resp = client.stop_vm(ctx, &req).await?;
+
+        self.exit_notify.take().unwrap().send(1).await?;
+        Ok(())
+    }
+
+    pub(crate) async fn pause_vm(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::pause_vm(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn wait_vm(&self) -> Result<i32> {
+        info!(sl!(), "Wait Remote VM");
+        let mut waiter = self.exit_waiter.lock().await;
+        if let Some(exitcode) = waiter.0.recv().await {
+            waiter.1 = exitcode;
+        }
+
+        Ok(waiter.1)
+    }
+
+    pub(crate) async fn resume_vm(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::resume_vm(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn save_vm(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::save_vm(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn add_device(&self, device: DeviceType) -> Result<DeviceType> {
+        warn!(sl!(), "RemoteInner::add_device(): NOT YET IMPLEMENTED");
+        Ok(device)
+    }
+
+    pub(crate) async fn remove_device(&self, _device: DeviceType) -> Result<()> {
+        warn!(sl!(), "RemoteInner::remove_device(): NOT YET IMPLEMENTED");
+        Ok(())
+    }
+
+    pub(crate) async fn update_device(&self, _device: DeviceType) -> Result<()> {
+        warn!(sl!(), "RemoteInner::update_device(): NOT YET IMPLEMENTED");
+        Ok(())
+    }
+
+    pub(crate) async fn get_agent_socket(&self) -> Result<String> {
+        Ok(format!("{}://{}", REMOTE_SCHEME, &self.agent_socket_path))
+    }
+
+    pub(crate) async fn disconnect(&mut self) {
+        warn!(sl!(), "RemoteInner::disconnect(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub fn hypervisor_config(&self) -> HypervisorConfig {
+        info!(
+            sl!(),
+            "RemoteInner::hypervisor_config(): {:?}",
+            self.config.clone()
+        );
+        self.config.clone()
+    }
+
+    pub(crate) async fn get_thread_ids(&self) -> Result<VcpuThreadIds> {
+        warn!(sl!(), "RemoteInner::get_thread_ids(): NOT YET IMPLEMENTED");
+        let vcpu_thread_ids: VcpuThreadIds = VcpuThreadIds {
+            vcpus: HashMap::new(),
+        };
+        Ok(vcpu_thread_ids)
+    }
+
+    pub(crate) async fn get_vmm_master_tid(&self) -> Result<u32> {
+        warn!(sl!(), "RemoteInner::get_vmm_master_tid()");
+        let tid = nix::unistd::gettid().as_raw();
+        Ok(tid as u32)
+    }
+
+    pub(crate) async fn get_ns_path(&self) -> Result<String> {
+        info!(sl!(), "RemoteInner::get_ns_path()");
+        Ok(self.netns.clone().unwrap_or_default())
+    }
+
+    pub(crate) async fn cleanup(&self) -> Result<()> {
+        info!(sl!(), "RemoteInner::cleanup(): NOT YET IMPLEMENTED");
+        Ok(())
+    }
+
+    pub(crate) async fn resize_vcpu(
+        &mut self,
+        _old_vcpus: u32,
+        _new_vcpus: u32,
+    ) -> Result<(u32, u32)> {
+        info!(sl!(), "RemoteInner::resize_vcpu(): NOT YET IMPLEMENTED");
+        Ok((_old_vcpus, _new_vcpus))
+    }
+
+    pub(crate) async fn get_pids(&self) -> Result<Vec<u32>> {
+        warn!(sl!(), "RemoteInner::get_pids(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn check(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::check(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn get_jailer_root(&self) -> Result<String> {
+        warn!(sl!(), "RemoteInner::get_jailer_root(): NOT YET IMPLEMENTED");
+        Ok("".into())
+    }
+
+    pub(crate) async fn capabilities(&self) -> Result<Capabilities> {
+        Ok(Capabilities::default())
+    }
+
+    pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) {
+        self.config = config;
+    }
+
+    pub(crate) async fn get_hypervisor_metrics(&self) -> Result<String> {
+        warn!(
+            sl!(),
+            "RemoteInner::get_hypervisor_metrics(): NOT YET IMPLEMENTED"
+        );
+        todo!()
+    }
+
+    pub(crate) fn set_capabilities(&mut self, _flag: CapabilityBits) {
+        warn!(
+            sl!(),
+            "RemoteInner::set_capabilities(): NOT YET IMPLEMENTED"
+        );
+        todo!()
+    }
+
+    pub(crate) fn set_guest_memory_block_size(&mut self, _size: u32) {
+        info!(
+            sl!(),
+            "RemoteInner::set_guest_memory_block_size(): NOT YET IMPLEMENTED"
+        )
+    }
+
+    pub(crate) fn guest_memory_block_size_mb(&self) -> u32 {
+        warn!(
+            sl!(),
+            "RemoteInner::guest_memory_block_size_mb(): NOT YET IMPLEMENTED"
+        );
+        0
+    }
+
+    pub(crate) fn resize_memory(&self, _new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
+        Ok((
+            _new_mem_mb,
+            MemoryConfig {
+                ..Default::default()
+            },
+        ))
+    }
+}
+
+#[async_trait]
+impl Persist for RemoteInner {
+    type State = HypervisorState;
+    type ConstructorArgs = ();
+
+    /// Save a state of hypervisor
+    async fn save(&self) -> Result<Self::State> {
+        Ok(HypervisorState {
+            hypervisor_type: HYPERVISOR_REMOTE.to_string(),
+            id: self.id.clone(),
+            config: self.config.clone(),
+            netns: self.netns.clone(),
+            ..Default::default()
+        })
+    }
+
+    /// Restore hypervisor
+    async fn restore(
+        _hypervisor_args: Self::ConstructorArgs,
+        hypervisor_state: Self::State,
+    ) -> Result<Self> {
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
+        Ok(RemoteInner {
+            id: hypervisor_state.id,
+            config: hypervisor_state.config,
+            agent_socket_path: "".to_string(),
+            netns: hypervisor_state.netns,
+            client: None,
+            exit_notify: Some(exit_notify),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
+        })
+    }
+}

src/runtime-rs/crates/hypervisor/src/remote/mod.rs

@@ -0,0 +1,216 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::HypervisorState;
+use crate::{device::DeviceType, Hypervisor, HypervisorConfig, MemoryConfig, VcpuThreadIds};
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+use inner::RemoteInner;
+use kata_types::capabilities::{Capabilities, CapabilityBits};
+use persist::sandbox_persist::Persist;
+use std::collections::HashMap;
+
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+mod inner;
+
+#[derive(Debug)]
+pub struct Remote {
+    inner: Arc<RwLock<RemoteInner>>,
+}
+
+impl Default for Remote {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Remote {
+    pub fn new() -> Self {
+        Self {
+            inner: Arc::new(RwLock::new(RemoteInner::new())),
+        }
+    }
+
+    pub async fn set_hypervisor_config(&mut self, config: HypervisorConfig) {
+        let mut inner = self.inner.write().await;
+        inner.set_hypervisor_config(config)
+    }
+}
+
+#[async_trait]
+impl Hypervisor for Remote {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        annotations: &HashMap<String, String>,
+    ) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.prepare_vm(id, netns, annotations).await
+    }
+
+    async fn start_vm(&self, timeout: i32) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.start_vm(timeout).await
+    }
+
+    async fn stop_vm(&self) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.stop_vm().await
+    }
+
+    async fn wait_vm(&self) -> Result<i32> {
+        let inner = self.inner.read().await;
+        inner.wait_vm().await
+    }
+
+    async fn pause_vm(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.pause_vm().await
+    }
+
+    async fn resume_vm(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.resume_vm().await
+    }
+
+    async fn save_vm(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.save_vm().await
+    }
+
+    async fn add_device(&self, device: DeviceType) -> Result<DeviceType> {
+        let inner = self.inner.write().await;
+        inner.add_device(device).await
+    }
+
+    async fn remove_device(&self, device: DeviceType) -> Result<()> {
+        let inner = self.inner.write().await;
+        inner.remove_device(device).await
+    }
+
+    async fn update_device(&self, device: DeviceType) -> Result<()> {
+        let inner = self.inner.write().await;
+        inner.update_device(device).await
+    }
+
+    async fn get_agent_socket(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_agent_socket().await
+    }
+
+    async fn disconnect(&self) {
+        let mut inner = self.inner.write().await;
+        inner.disconnect().await
+    }
+
+    async fn hypervisor_config(&self) -> HypervisorConfig {
+        let inner = self.inner.read().await;
+        inner.hypervisor_config()
+    }
+
+    async fn get_thread_ids(&self) -> Result<VcpuThreadIds> {
+        let inner = self.inner.read().await;
+        inner.get_thread_ids().await
+    }
+
+    async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let inner = self.inner.read().await;
+        inner.get_vmm_master_tid().await
+    }
+
+    async fn get_ns_path(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_ns_path().await
+    }
+
+    async fn cleanup(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.cleanup().await
+    }
+
+    async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> {
+        let mut inner = self.inner.write().await;
+        inner.resize_vcpu(old_vcpus, new_vcpus).await
+    }
+
+    async fn get_pids(&self) -> Result<Vec<u32>> {
+        let inner = self.inner.read().await;
+        inner.get_pids().await
+    }
+
+    async fn check(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.check().await
+    }
+
+    async fn get_jailer_root(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_jailer_root().await
+    }
+
+    async fn save_state(&self) -> Result<HypervisorState> {
+        self.save().await
+    }
+
+    async fn capabilities(&self) -> Result<Capabilities> {
+        let inner = self.inner.read().await;
+        inner.capabilities().await
+    }
+
+    async fn get_hypervisor_metrics(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_hypervisor_metrics().await
+    }
+
+    async fn set_capabilities(&self, flag: CapabilityBits) {
+        let mut inner = self.inner.write().await;
+        inner.set_capabilities(flag)
+    }
+
+    async fn set_guest_memory_block_size(&self, size: u32) {
+        let mut inner = self.inner.write().await;
+        inner.set_guest_memory_block_size(size);
+    }
+
+    async fn guest_memory_block_size(&self) -> u32 {
+        let inner = self.inner.read().await;
+        inner.guest_memory_block_size_mb()
+    }
+
+    async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
+        let inner = self.inner.read().await;
+        inner.resize_memory(new_mem_mb)
+    }
+
+    async fn get_passfd_listener_addr(&self) -> Result<(String, u32)> {
+        Err(anyhow::anyhow!("Not yet supported"))
+    }
+}
+
+#[async_trait]
+impl Persist for Remote {
+    type State = HypervisorState;
+    type ConstructorArgs = ();
+
+    /// Save a state of the component.
+    async fn save(&self) -> Result<Self::State> {
+        let inner = self.inner.read().await;
+        inner.save().await.context("save remote hypervisor state")
+    }
+
+    /// Restore a component from a specified state.
+    async fn restore(
+        hypervisor_args: Self::ConstructorArgs,
+        hypervisor_state: Self::State,
+    ) -> Result<Self> {
+        let inner = RemoteInner::restore(hypervisor_args, hypervisor_state).await?;
+        Ok(Self {
+            inner: Arc::new(RwLock::new(inner)),
+        })
+    }
+}

src/runtime-rs/crates/hypervisor/src/utils.rs

@@ -11,13 +11,14 @@ use std::{
 };
 
 use anyhow::{anyhow, Context, Result};
-use dbs_utils::net::Tap;
 use kata_types::config::KATA_PATH;
 use nix::{
     fcntl,
     sched::{setns, CloneFlags},
 };
 
+use crate::device::Tap;
+
 use crate::{DEFAULT_HYBRID_VSOCK_NAME, JAILER_ROOT};
 
 pub fn get_child_threads(pid: u32) -> HashSet<u32> {

src/runtime-rs/crates/resource/src/cdi_devices/container_device.rs

@@ -0,0 +1,475 @@
+//
+// Copyright (c) 2024 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::collections::HashMap;
+use std::path::Path;
+
+use anyhow::Result;
+use oci_spec::runtime::Spec;
+
+use super::{resolve_cdi_device_kind, ContainerDevice};
+use agent::types::Device;
+
+const CDI_PREFIX: &str = "cdi.k8s.io";
+
+// Sort the devices based on the first element's PCI_Guest_Path in the PCI bus according to options.
+fn sort_devices_by_guest_pcipath(devices: &mut [ContainerDevice]) {
+    // Extract first guest_pcipath from device_options
+    let extract_first_guest_pcipath = |options: &[String]| -> Option<String> {
+        options
+            .first()
+            .and_then(|option| option.split('=').nth(1))
+            .map(|path| path.to_string())
+    };
+
+    devices.sort_by(|a, b| {
+        let guest_path_a = extract_first_guest_pcipath(&a.device.options);
+        let guest_path_b = extract_first_guest_pcipath(&b.device.options);
+
+        guest_path_a.cmp(&guest_path_b)
+    });
+}
+
+// Annotate container devices with CDI annotations in OCI Spec
+pub fn annotate_container_devices(
+    spec: &mut Spec,
+    container_devices: Vec<ContainerDevice>,
+) -> Result<Vec<Device>> {
+    let mut devices_agent: Vec<Device> = Vec::new();
+    // Make sure that annotations is Some().
+    if spec.annotations().is_none() {
+        spec.set_annotations(Some(HashMap::new()));
+    }
+
+    // Step 1: Extract all devices and filter out devices without device_info for vfio_devices
+    let vfio_devices: Vec<ContainerDevice> = container_devices
+        .into_iter()
+        .map(|device| {
+            // push every device's Device to agent_devices
+            devices_agent.push(device.device.clone());
+            device
+        })
+        .filter(|device| device.device_info.is_some())
+        .collect();
+
+    // Step 2: Group devices by vendor_id-class_id
+    let mut grouped_devices: HashMap<String, Vec<ContainerDevice>> = HashMap::new();
+    for device in vfio_devices {
+        // Extract the vendor/class key and insert into the map if both are present
+        if let Some(key) = device
+            .device_info
+            .as_ref()
+            .and_then(|info| resolve_cdi_device_kind(&info.vendor_id, &info.class_id))
+        {
+            grouped_devices
+                .entry(key.to_owned())
+                .or_default()
+                .push(device);
+        }
+    }
+
+    // Step 3: Sort devices within each group by guest_pcipath
+    grouped_devices
+        .iter_mut()
+        .for_each(|(vendor_class, container_devices)| {
+            // The *offset* is a monotonically increasing counter that keeps track of the number of devices
+            // within an IOMMU group. It increments by total_of whenever a new IOMMU group is processed.
+            let offset: &mut usize = &mut 0;
+
+            sort_devices_by_guest_pcipath(container_devices);
+            container_devices
+                .iter()
+                .enumerate()
+                .for_each(|(base, container_device)| {
+                    let total_of = container_device.device.options.len();
+                    // annotate device with cdi information in OCI Spec.
+                    for index in 0..total_of {
+                        if let Some(iommu_grpid) =
+                            Path::new(&container_device.device.container_path)
+                                .file_name()
+                                .and_then(|name| name.to_str())
+                        {
+                            spec.annotations_mut().as_mut().unwrap().insert(
+                                format!("{}/vfio{}.{}", CDI_PREFIX, iommu_grpid, index), // cdi.k8s.io/vfioX.y
+                                format!("{}={}", vendor_class, base + *offset), // vendor/class=name
+                            );
+                        }
+                    }
+
+                    // update the offset with *total_of*.
+                    *offset += total_of - 1;
+                });
+        });
+
+    Ok(devices_agent)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use crate::cdi_devices::DeviceInfo;
+    use agent::types::Device;
+    use oci_spec::runtime::SpecBuilder;
+
+    use super::*;
+
+    #[test]
+    fn test_sort_devices_by_guest_pcipath() {
+        let mut devices = vec![
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0xffff".to_string(),
+                    class_id: "0x030x".to_string(),
+                    host_path: PathBuf::from("/dev/device3"),
+                }),
+                device: Device {
+                    options: vec!["pci_host_path03=BB:DD03.F03".to_string()],
+                    ..Default::default()
+                },
+            },
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0xffff".to_string(),
+                    class_id: "0x030x".to_string(),
+                    host_path: PathBuf::from("/dev/device1"),
+                }),
+                device: Device {
+                    options: vec!["pci_host_path01=BB:DD01.F01".to_string()],
+                    ..Default::default()
+                },
+            },
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0xffff".to_string(),
+                    class_id: "0x030x".to_string(),
+                    host_path: PathBuf::from("/dev/device2"),
+                }),
+                device: Device {
+                    options: vec!["pci_host_path02=BB:DD02.F02".to_string()],
+                    ..Default::default()
+                },
+            },
+        ];
+
+        sort_devices_by_guest_pcipath(&mut devices);
+
+        let expected_devices_order = vec![
+            "/dev/device1".to_string(),
+            "/dev/device2".to_string(),
+            "/dev/device3".to_string(),
+        ];
+        let actual_devices_order: Vec<String> = devices
+            .iter()
+            .map(|cd| {
+                cd.device_info
+                    .as_ref()
+                    .unwrap()
+                    .host_path
+                    .display()
+                    .to_string()
+            })
+            .collect();
+
+        assert_eq!(actual_devices_order, expected_devices_order);
+    }
+
+    #[test]
+    fn test_sort_devices_with_empty_options() {
+        let mut devices = vec![
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0xffff".to_string(),
+                    class_id: "0x030x".to_string(),
+                    host_path: PathBuf::from("/dev/device1"),
+                }),
+                device: Device {
+                    options: vec![], // empty
+                    ..Default::default()
+                },
+            },
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0xffff".to_string(),
+                    class_id: "0x030x".to_string(),
+                    host_path: PathBuf::from("/dev/device2"),
+                }),
+                device: Device {
+                    options: vec!["pci_host_path02=BB:DD02.F02".to_string()],
+                    ..Default::default()
+                },
+            },
+        ];
+
+        sort_devices_by_guest_pcipath(&mut devices);
+
+        // As the first device has no options, ignore it.
+        let expected_devices_order = vec!["BB:DD02.F02".to_string()];
+
+        let actual_devices_order: Vec<String> = devices
+            .iter()
+            .filter_map(|d| d.device.options.first())
+            .map(|option| option.split('=').nth(1).unwrap_or("").to_string())
+            .collect();
+
+        assert_eq!(actual_devices_order, expected_devices_order);
+    }
+
+    #[test]
+    fn test_annotate_container_devices() {
+        let devices = vec![
+            ContainerDevice {
+                device_info: None,
+                device: Device {
+                    id: "test0000x".to_string(),
+                    container_path: "/dev/xvdx".to_string(),
+                    field_type: "virtio-blk".to_string(),
+                    vm_path: "/dev/vdx".to_string(),
+                    options: vec![],
+                },
+            },
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0x1002".to_string(),
+                    class_id: "0x0302".to_string(),
+                    host_path: PathBuf::from("/dev/device2"),
+                }),
+                device: Device {
+                    container_path: "/dev/device2".to_string(),
+                    options: vec!["pci_host_path02=BB:DD02.F02".to_string()],
+                    ..Default::default()
+                },
+            },
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0x1002".to_string(),
+                    class_id: "0x0302".to_string(),
+                    host_path: PathBuf::from("/dev/device3"),
+                }),
+                device: Device {
+                    container_path: "/dev/device3".to_string(),
+                    options: vec!["pci_host_path03=BB:DD03.F03".to_string()],
+                    ..Default::default()
+                },
+            },
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0x1002".to_string(),
+                    class_id: "0x0302".to_string(),
+                    host_path: PathBuf::from("/dev/device1"),
+                }),
+                device: Device {
+                    container_path: "/dev/device1".to_string(),
+                    options: vec!["pci_host_path01=BB:DD01.F01".to_string()],
+                    ..Default::default()
+                },
+            },
+            ContainerDevice {
+                device_info: None,
+                device: Device {
+                    id: "test0000yx".to_string(),
+                    container_path: "/dev/xvdyx".to_string(),
+                    field_type: "virtio-blk".to_string(),
+                    vm_path: "/dev/vdyx".to_string(),
+                    options: vec![],
+                },
+            },
+        ];
+
+        let annotations = HashMap::new();
+        let mut spec = SpecBuilder::default()
+            .annotations(annotations)
+            .build()
+            .unwrap();
+
+        // do annotate container devices
+        let _devices = annotate_container_devices(&mut spec, devices);
+
+        let expected_annotations: HashMap<String, String> = vec![
+            (
+                "cdi.k8s.io/vfiodevice3.0".to_owned(),
+                "amd.com/gpu=2".to_owned(),
+            ),
+            (
+                "cdi.k8s.io/vfiodevice1.0".to_owned(),
+                "amd.com/gpu=0".to_owned(),
+            ),
+            (
+                "cdi.k8s.io/vfiodevice2.0".to_owned(),
+                "amd.com/gpu=1".to_owned(),
+            ),
+        ]
+        .into_iter()
+        .collect();
+
+        assert_eq!(Some(expected_annotations), spec.annotations().clone());
+    }
+
+    #[test]
+    fn test_annotate_container_multi_vendor_devices() {
+        let devices = vec![
+            ContainerDevice {
+                device_info: None,
+                device: Device {
+                    id: "test0000x".to_string(),
+                    container_path: "/dev/xvdx".to_string(),
+                    field_type: "virtio-blk".to_string(),
+                    vm_path: "/dev/vdx".to_string(),
+                    options: vec![],
+                },
+            },
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0x10de".to_string(),
+                    class_id: "0x0302".to_string(),
+                    host_path: PathBuf::from("/dev/device2"),
+                }),
+                device: Device {
+                    container_path: "/dev/device2".to_string(),
+                    options: vec!["pci_host_path02=BB:DD02.F02".to_string()],
+                    ..Default::default()
+                },
+            },
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0x10de".to_string(),
+                    class_id: "0x0302".to_string(),
+                    host_path: PathBuf::from("/dev/device3"),
+                }),
+                device: Device {
+                    container_path: "/dev/device3".to_string(),
+                    options: vec!["pci_host_path03=BB:DD03.F03".to_string()],
+                    ..Default::default()
+                },
+            },
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0x8086".to_string(),
+                    class_id: "0x0302".to_string(),
+                    host_path: PathBuf::from("/dev/device1"),
+                }),
+                device: Device {
+                    container_path: "/dev/device1".to_string(),
+                    options: vec!["pci_host_path01=BB:DD01.F01".to_string()],
+                    ..Default::default()
+                },
+            },
+            ContainerDevice {
+                device_info: Some(DeviceInfo {
+                    vendor_id: "0x8086".to_string(),
+                    class_id: "0x0302".to_string(),
+                    host_path: PathBuf::from("/dev/device4"),
+                }),
+                device: Device {
+                    container_path: "/dev/device4".to_string(),
+                    options: vec!["pci_host_path04=BB:DD01.F04".to_string()],
+                    ..Default::default()
+                },
+            },
+            ContainerDevice {
+                device_info: None,
+                device: Device {
+                    id: "test0000yx".to_string(),
+                    container_path: "/dev/xvdyx".to_string(),
+                    field_type: "virtio-blk".to_string(),
+                    vm_path: "/dev/vdyx".to_string(),
+                    options: vec![],
+                },
+            },
+        ];
+
+        let annotations = HashMap::new();
+        let mut spec = SpecBuilder::default()
+            .annotations(annotations)
+            .build()
+            .unwrap();
+
+        let _devices = annotate_container_devices(&mut spec, devices);
+
+        let expected_annotations: HashMap<String, String> = vec![
+            (
+                "cdi.k8s.io/vfiodevice1.0".to_owned(),
+                "intel.com/gpu=0".to_owned(),
+            ),
+            (
+                "cdi.k8s.io/vfiodevice2.0".to_owned(),
+                "nvidia.com/gpu=0".to_owned(),
+            ),
+            (
+                "cdi.k8s.io/vfiodevice3.0".to_owned(),
+                "nvidia.com/gpu=1".to_owned(),
+            ),
+            (
+                "cdi.k8s.io/vfiodevice4.0".to_owned(),
+                "intel.com/gpu=1".to_owned(),
+            ),
+        ]
+        .into_iter()
+        .collect();
+
+        assert_eq!(Some(expected_annotations), spec.annotations().clone());
+    }
+
+    #[test]
+    fn test_annotate_container_without_vfio_devices() {
+        let devices = vec![
+            ContainerDevice {
+                device_info: None,
+                device: Device {
+                    id: "test0000x".to_string(),
+                    container_path: "/dev/xvdx".to_string(),
+                    field_type: "virtio-blk".to_string(),
+                    vm_path: "/dev/vdx".to_string(),
+                    options: vec![],
+                },
+            },
+            ContainerDevice {
+                device_info: None,
+                device: Device {
+                    id: "test0000y".to_string(),
+                    container_path: "/dev/yvdy".to_string(),
+                    field_type: "virtio-blk".to_string(),
+                    vm_path: "/dev/vdy".to_string(),
+                    options: vec![],
+                },
+            },
+            ContainerDevice {
+                device_info: None,
+                device: Device {
+                    id: "test0000z".to_string(),
+                    container_path: "/dev/zvdz".to_string(),
+                    field_type: "virtio-blk".to_string(),
+                    vm_path: "/dev/zvdz".to_string(),
+                    options: vec![],
+                },
+            },
+        ];
+
+        let annotations = HashMap::from([(
+            "cdi.k8s.io/vfiodeviceX".to_owned(),
+            "katacontainer.com/device=Y".to_owned(),
+        )]);
+        let mut spec = SpecBuilder::default()
+            .annotations(annotations)
+            .build()
+            .unwrap();
+
+        // do annotate container devices
+        let annotated_devices = annotate_container_devices(&mut spec, devices.clone()).unwrap();
+
+        let actual_devices = devices
+            .iter()
+            .map(|d| d.device.clone())
+            .collect::<Vec<Device>>();
+        let expected_annotations: HashMap<String, String> = HashMap::from([(
+            "cdi.k8s.io/vfiodeviceX".to_owned(),
+            "katacontainer.com/device=Y".to_owned(),
+        )]);
+
+        assert_eq!(Some(expected_annotations), spec.annotations().clone());
+        assert_eq!(annotated_devices, actual_devices);
+    }
+}

src/runtime-rs/crates/resource/src/cdi_devices/mod.rs

@@ -0,0 +1,63 @@
+//
+// Copyright (c) 2024 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub mod container_device;
+
+use agent::types::Device;
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+#[derive(Clone, Default)]
+pub struct DeviceInfo {
+    pub class_id: String,
+    pub vendor_id: String,
+    pub host_path: PathBuf,
+}
+
+#[derive(Clone, Default)]
+pub struct ContainerDevice {
+    pub device_info: Option<DeviceInfo>,
+    pub device: Device,
+}
+
+lazy_static! {
+    // *CDI_DEVICE_KIND_TABLE* is static hash map to store a mapping between device vendor and class
+    // identifiers and their corresponding CDI vendor and class strings. This mapping is essentially a
+    // lookup table that allows the system to determine the appropriate CDI for a given device based on
+    // its vendor and class information.
+    // Note: Our device mapping is designed to be flexible and responsive to user needs. The current list
+    // is not exhaustive and will be updated as required.
+    pub static ref CDI_DEVICE_KIND_TABLE: HashMap<&'static str, &'static str> = {
+        let mut m = HashMap::new();
+        m.insert("0x10de-0x030", "nvidia.com/gpu");
+        m.insert("0x8086-0x030", "intel.com/gpu");
+        m.insert("0x1002-0x030", "amd.com/gpu");
+        m.insert("0x15b3-0x020", "nvidia.com/nic");
+        // TODO:  it will be updated as required.
+        m
+    };
+}
+
+// Sort devices by guest_pcipath
+pub fn sort_options_by_pcipath(mut device_options: Vec<String>) -> Vec<String> {
+    device_options.sort_by(|a, b| {
+        let extract_path = |s: &str| s.split('=').nth(1).map(|path| path.to_string());
+        let guest_path_a = extract_path(a);
+        let guest_path_b = extract_path(b);
+
+        guest_path_a.cmp(&guest_path_b)
+    });
+    device_options
+}
+
+// Resolve the CDI vendor ID/device Class by a lookup table based on the provided vendor and class.
+pub fn resolve_cdi_device_kind<'a>(vendor_id: &'a str, class_id: &'a str) -> Option<&'a str> {
+    let vendor_class = format!("{}-{}", vendor_id, class_id);
+    // The first 12 characters of the string ("0x10de-0x030") provide a concise
+    // and clear identification of both the manufacturer and the device category.
+    // it returns "nvidia.com/gpu", "amd.com/gpu" or others.
+    CDI_DEVICE_KIND_TABLE.get(&vendor_class[..12]).copied()
+}

src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs

@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::convert::TryFrom;
+use std::{collections::HashMap, convert::TryFrom};
 
 use anyhow::{Context, Result};
 use kata_types::{
@@ -22,6 +22,34 @@ struct InitialSize {
     orig_toml_default_mem: u32,
 }
 
+// generate initial resource(vcpu and memory in MiB) from annotations
+impl TryFrom<&HashMap<String, String>> for InitialSize {
+    type Error = anyhow::Error;
+    fn try_from(an: &HashMap<String, String>) -> Result<Self> {
+        let mut vcpu: u32 = 0;
+
+        let annotation = Annotation::new(an.clone());
+        let (period, quota, memory) =
+            get_sizing_info(annotation).context("failed to get sizing info")?;
+        let mut cpu = oci::LinuxCpu::default();
+        cpu.set_period(Some(period));
+        cpu.set_quota(Some(quota));
+
+        // although it may not be actually a linux container, we are only using the calculation inside
+        // LinuxContainerCpuResources::try_from to generate our vcpu number
+        if let Ok(cpu_resource) = LinuxContainerCpuResources::try_from(&cpu) {
+            vcpu = get_nr_vcpu(&cpu_resource);
+        }
+        let mem_mb = convert_memory_to_mb(memory);
+
+        Ok(Self {
+            vcpu,
+            mem_mb,
+            orig_toml_default_mem: 0,
+        })
+    }
+}
+
 // generate initial resource(vcpu and memory in MiB) from spec's information
 impl TryFrom<&oci::Spec> for InitialSize {
     type Error = anyhow::Error;
@@ -32,19 +60,7 @@ impl TryFrom<&oci::Spec> for InitialSize {
             // podsandbox, from annotation
             ContainerType::PodSandbox => {
                 let spec_annos = spec.annotations().clone().unwrap_or_default();
-                let annotation = Annotation::new(spec_annos);
-                let (period, quota, memory) =
-                    get_sizing_info(annotation).context("failed to get sizing info")?;
-                let mut cpu = oci::LinuxCpu::default();
-                cpu.set_period(Some(period));
-                cpu.set_quota(Some(quota));
-
-                // although it may not be actually a linux container, we are only using the calculation inside
-                // LinuxContainerCpuResources::try_from to generate our vcpu number
-                if let Ok(cpu_resource) = LinuxContainerCpuResources::try_from(&cpu) {
-                    vcpu = get_nr_vcpu(&cpu_resource);
-                }
-                mem_mb = convert_memory_to_mb(memory);
+                return InitialSize::try_from(&spec_annos);
             }
             // single container, from container spec
             _ => {
@@ -107,6 +123,13 @@ impl InitialSizeManager {
         })
     }
 
+    pub fn new_from(annotation: &HashMap<String, String>) -> Result<Self> {
+        Ok(Self {
+            resource: InitialSize::try_from(annotation)
+                .context("failed to construct static resource")?,
+        })
+    }
+
     pub fn setup_config(&mut self, config: &mut TomlConfig) -> Result<()> {
         // update this data to the hypervisor config for later use by hypervisor
         let hypervisor_name = &config.runtime.hypervisor_name;

src/runtime-rs/crates/resource/src/lib.rs

@@ -23,6 +23,7 @@ pub mod rootfs;
 pub mod share_fs;
 pub mod volume;
 pub use manager::ResourceManager;
+pub mod cdi_devices;
 pub mod cpu_mem;
 
 use kata_types::config::hypervisor::SharedFsInfo;

src/runtime-rs/crates/resource/src/manager.rs

@@ -6,7 +6,6 @@
 
 use std::sync::Arc;
 
-use agent::types::Device;
 use agent::{Agent, Storage};
 use anyhow::Result;
 use async_trait::async_trait;
@@ -20,6 +19,7 @@ use persist::sandbox_persist::Persist;
 use tokio::sync::RwLock;
 use tracing::instrument;
 
+use crate::cdi_devices::ContainerDevice;
 use crate::cpu_mem::initial_size::InitialSizeManager;
 use crate::network::NetworkConfig;
 use crate::resource_persist::ResourceState;
@@ -116,7 +116,7 @@ impl ResourceManager {
         inner.handler_volumes(cid, spec).await
     }
 
-    pub async fn handler_devices(&self, cid: &str, linux: &Linux) -> Result<Vec<Device>> {
+    pub async fn handler_devices(&self, cid: &str, linux: &Linux) -> Result<Vec<ContainerDevice>> {
         let inner = self.inner.read().await;
         inner.handler_devices(cid, linux).await
     }

src/runtime-rs/crates/resource/src/manager_inner.rs

@@ -25,6 +25,7 @@ use persist::sandbox_persist::Persist;
 use tokio::{runtime, sync::RwLock};
 
 use crate::{
+    cdi_devices::{sort_options_by_pcipath, ContainerDevice, DeviceInfo},
     cgroups::{CgroupArgs, CgroupsResource},
     cpu_mem::{cpu::CpuResource, initial_size::InitialSizeManager, mem::MemResource},
     manager::ManagerArgs,
@@ -292,7 +293,7 @@ impl ResourceManagerInner {
             .await
     }
 
-    pub async fn handler_devices(&self, _cid: &str, linux: &Linux) -> Result<Vec<Device>> {
+    pub async fn handler_devices(&self, _cid: &str, linux: &Linux) -> Result<Vec<ContainerDevice>> {
         let mut devices = vec![];
 
         let linux_devices = linux.devices().clone().unwrap_or_default();
@@ -329,7 +330,10 @@ impl ResourceManagerInner {
                             vm_path: device.config.virt_path,
                             ..Default::default()
                         };
-                        devices.push(agent_device);
+                        devices.push(ContainerDevice {
+                            device_info: None,
+                            device: agent_device,
+                        });
                     }
                 }
                 LinuxDeviceType::C => {
@@ -361,14 +365,33 @@ impl ResourceManagerInner {
 
                     // create agent device
                     if let DeviceType::Vfio(device) = device_info {
+                        let device_options = sort_options_by_pcipath(device.device_options);
                         let agent_device = Device {
                             id: device.device_id, // just for kata-agent
                             container_path: d.path().display().to_string().clone(),
                             field_type: vfio_mode,
-                            options: device.device_options,
+                            options: device_options,
                             ..Default::default()
                         };
-                        devices.push(agent_device);
+
+                        let vendor_class = device
+                            .devices
+                            .first()
+                            .unwrap()
+                            .device_vendor_class
+                            .as_ref()
+                            .unwrap()
+                            .get_vendor_class_id()
+                            .context("get vendor class failed")?;
+                        let device_info = Some(DeviceInfo {
+                            vendor_id: vendor_class.0.to_owned(),
+                            class_id: vendor_class.1.to_owned(),
+                            host_path: d.path().clone(),
+                        });
+                        devices.push(ContainerDevice {
+                            device_info,
+                            device: agent_device,
+                        });
                     }
                 }
                 _ => {

src/runtime-rs/crates/runtimes/common/src/error.rs

@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::types::{ContainerProcess, Response};
+use crate::types::{ContainerProcess, TaskResponse};
 
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
@@ -13,5 +13,5 @@ pub enum Error {
     #[error("failed to find process {0}")]
     ProcessNotFound(ContainerProcess),
     #[error("unexpected response {0} to shim {1}")]
-    UnexpectedResponse(Response, String),
+    UnexpectedResponse(TaskResponse, String),
 }