Merge pull request #7418 from wainersm/release_0.7.0-fix

workflows: fix malformed CC payload s390x yaml

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -1,9 +1,20 @@
-name: CI | Publish kata-deploy payload for amd64
+name: CI | Build kata-static tarball for amd64
 on:
   workflow_call:
     inputs:
-      target-arch:
-        required: true
+      stage:
+        required: false
+        type: string
+        default: test
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+      commit-hash:
+        required: false
         type: string
 
 jobs:
@@ -13,15 +24,37 @@ jobs:
       matrix:
         asset:
           - cloud-hypervisor
+          - cloud-hypervisor-glibc
           - firecracker
           - kernel
+          - kernel-sev
+          - kernel-dragonball-experimental
+          - kernel-tdx-experimental
+          - kernel-nvidia-gpu
+          - kernel-nvidia-gpu-snp
+          - kernel-nvidia-gpu-tdx-experimental
           - nydus
+          - ovmf
+          - ovmf-sev
           - qemu
+          - qemu-snp-experimental
+          - qemu-tdx-experimental
           - rootfs-image
+          - rootfs-image-tdx
           - rootfs-initrd
+          - rootfs-initrd-mariner
+          - rootfs-initrd-sev
+          - shim-v2
+          - tdvf
           - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
+        exclude:
+          - asset: cloud-hypervisor-glibc
+            stage: release
     steps:
       - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
         uses: docker/login-action@v2
         with:
           registry: quay.io
@@ -30,7 +63,9 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
+          ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -40,12 +75,12 @@ jobs:
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: yes
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
         with:
-          name: kata-artifacts-amd64
+          name: kata-artifacts-amd64${{ inputs.tarball-suffix }}
           path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
           retention-days: 1
           if-no-files-found: error
@@ -55,10 +90,12 @@ jobs:
     needs: build-asset
     steps:
       - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:
-          name: kata-artifacts-amd64
+          name: kata-artifacts-amd64${{ inputs.tarball-suffix }}
           path: kata-artifacts
       - name: merge-artifacts
         run: |
@@ -66,31 +103,7 @@ jobs:
       - name: store-artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: kata-static-tarball-amd64
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
           path: kata-static.tar.xz
           retention-days: 1
           if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -1,9 +1,16 @@
-name: CI | Publish kata-deploy payload for arm64
+name: CI | Build kata-static tarball for arm64
 on:
   workflow_call:
     inputs:
-      target-arch:
-        required: true
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+      commit-hash:
+        required: false
         type: string
 
 jobs:
@@ -15,25 +22,29 @@ jobs:
           - cloud-hypervisor
           - firecracker
           - kernel
+          - kernel-dragonball-experimental
           - nydus
           - qemu
           - rootfs-image
           - rootfs-initrd
+          - shim-v2
           - virtiofsd
     steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
       - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
         uses: docker/login-action@v2
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
       - uses: actions/checkout@v3
         with:
+          ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
       - name: Build ${{ matrix.asset }}
         run: |
@@ -44,12 +55,12 @@ jobs:
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: yes
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
         with:
-          name: kata-artifacts-arm64
+          name: kata-artifacts-arm64${{ inputs.tarball-suffix }}
           path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
           retention-days: 1
           if-no-files-found: error
@@ -63,10 +74,12 @@ jobs:
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
       - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:
-          name: kata-artifacts-arm64
+          name: kata-artifacts-arm64${{ inputs.tarball-suffix }}
           path: kata-artifacts
       - name: merge-artifacts
         run: |
@@ -74,35 +87,7 @@ jobs:
       - name: store-artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: kata-static-tarball-arm64
+          name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
           path: kata-static.tar.xz
           retention-days: 1
           if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: arm64
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-arm64
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -1,9 +1,16 @@
-name: CI | Publish kata-deploy payload for s390x
+name: CI | Build kata-static tarball for s390x
 on:
   workflow_call:
     inputs:
-      target-arch:
-        required: true
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+      commit-hash:
+        required: false
         type: string
 
 jobs:
@@ -13,25 +20,27 @@ jobs:
       matrix:
         asset:
           - kernel
-          - shim-v2
           - qemu
           - rootfs-image
           - rootfs-initrd
+          - shim-v2
           - virtiofsd
     steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
       - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
         uses: docker/login-action@v2
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
       - uses: actions/checkout@v3
         with:
+          ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
       - name: Build ${{ matrix.asset }}
         run: |
@@ -43,12 +52,12 @@ jobs:
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: yes
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
         with:
-          name: kata-artifacts-s390x
+          name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
           path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
           retention-days: 1
           if-no-files-found: error
@@ -62,10 +71,12 @@ jobs:
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
       - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:
-          name: kata-artifacts-s390x
+          name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
           path: kata-artifacts
       - name: merge-artifacts
         run: |
@@ -73,35 +84,7 @@ jobs:
       - name: store-artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: kata-static-tarball-s390x
+          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
           path: kata-static.tar.xz
           retention-days: 1
           if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: s390x
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push-amd64.yaml

@@ -11,22 +11,29 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
+        measured_rootfs:
+          - no
         asset:
           - cc-cloud-hypervisor
-          - cc-kernel
           - cc-qemu
-          - cc-rootfs-image
           - cc-virtiofsd
           - cc-sev-kernel
           - cc-sev-ovmf
           - cc-x86_64-ovmf
           - cc-snp-qemu
           - cc-sev-rootfs-initrd
-          - cc-tdx-kernel
-          - cc-tdx-rootfs-image
           - cc-tdx-qemu
           - cc-tdx-td-shim
           - cc-tdx-tdvf
+        include:
+          - measured_rootfs: yes
+            asset: cc-kernel
+          - measured_rootfs: yes
+            asset: cc-tdx-kernel
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
+          - measured_rootfs: yes
+            asset: cc-tdx-rootfs-image
     steps:
       - name: Login to Kata Containers quay.io
         uses: docker/login-action@v2
@@ -48,6 +55,7 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -106,6 +114,7 @@ jobs:
           sudo cp -r "${build_dir}" "kata-build"
         env:
           PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: yes
 
       - name: store-artifact cc-shim-v2
         uses: actions/upload-artifact@v3

.github/workflows/cc-payload-after-push-s390x.yaml

@@ -11,13 +11,18 @@ jobs:
     runs-on: s390x
     strategy:
       matrix:
+        measured_rootfs:
+          - no
         asset:
-          - cc-kernel
           - cc-qemu
-          - cc-rootfs-image
           - cc-rootfs-initrd
           - cc-se-image
           - cc-virtiofsd
+        include:
+          - measured_rootfs: yes
+            asset: cc-kernel
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
     steps:
       - name: Login to Kata Containers quay.io
         uses: docker/login-action@v2
@@ -52,6 +57,7 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
           HKD_PATH: "host-key-document"
 
       - name: store-artifact ${{ matrix.asset }}
@@ -101,6 +107,7 @@ jobs:
           sudo cp -r "${build_dir}" "kata-build"
         env:
           PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: yes
 
       - name: store-artifact cc-shim-v2
         uses: actions/upload-artifact@v3

.github/workflows/cc-payload-after-push.yaml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - CCv0
+  workflow_dispatch:
 
 jobs:
   build-assets-amd64:
@@ -31,7 +32,14 @@ jobs:
           username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
 
-      - name: Push multi-arch manifest
+      - name: Push commit multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA} \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}
+
+      - name: Push latest multi-arch manifest
         run: |
           docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest \
           --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-amd64 \

.github/workflows/cc-payload-amd64.yaml

@@ -11,22 +11,29 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
+        measured_rootfs:
+          - no
         asset:
           - cc-cloud-hypervisor
-          - cc-kernel
           - cc-qemu
-          - cc-rootfs-image
           - cc-virtiofsd
           - cc-sev-kernel
           - cc-sev-ovmf
           - cc-x86_64-ovmf
           - cc-snp-qemu
           - cc-sev-rootfs-initrd
-          - cc-tdx-kernel
-          - cc-tdx-rootfs-image
           - cc-tdx-qemu
           - cc-tdx-td-shim
           - cc-tdx-tdvf
+        include:
+          - measured_rootfs: yes
+            asset: cc-kernel
+          - measured_rootfs: yes
+            asset: cc-tdx-kernel
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
+          - measured_rootfs: yes
+            asset: cc-tdx-rootfs-image
     steps:
       - uses: actions/checkout@v3
       - name: Build ${{ matrix.asset }}
@@ -38,6 +45,7 @@ jobs:
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -87,6 +95,8 @@ jobs:
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
           sudo cp -r "${build_dir}" "kata-build"
+        env:
+          MEASURED_ROOTFS: yes
 
       - name: store-artifact cc-shim-v2
         uses: actions/upload-artifact@v3

.github/workflows/cc-payload-s390x.yaml

@@ -11,11 +11,16 @@ jobs:
     runs-on: s390x
     strategy:
       matrix:
+        measured_rootfs:
+          - no
         asset:
-          - cc-kernel
           - cc-qemu
-          - cc-rootfs-image
           - cc-virtiofsd
+        include:
+          - measured_rootfs: yes
+            asset: cc-kernel
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
     steps:
       - name: Adjust a permission for repo
         run: |
@@ -31,6 +36,7 @@ jobs:
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -70,6 +76,8 @@ jobs:
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
           sudo cp -r "${build_dir}" "kata-build"
+        env:
+          MEASURED_ROOTFS: yes
 
       - name: store-artifact cc-shim-v2
         uses: actions/upload-artifact@v3

.github/workflows/cc-payload.yaml

@@ -31,7 +31,14 @@ jobs:
           username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
 
-      - name: Push multi-arch manifest
+      - name: Push commit multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA} \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}
+
+      - name: Push latest multi-arch manifest
         run: |
           docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-latest \
           --amend quay.io/confidential-containers/runtime-payload:kata-containers-amd64 \

.github/workflows/ci-nightly.yaml

@@ -0,0 +1,14 @@
+name: Kata Containers Nightly CI
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+jobs:
+  kata-containers-ci-on-push:
+    uses: ./.github/workflows/ci.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      pr-number: "nightly"
+      tag: ${{ github.sha }}-nightly
+    secrets: inherit

.github/workflows/ci-on-push.yaml

@@ -0,0 +1,25 @@
+name: Kata Containers CI
+on:
+  pull_request_target:
+    branches:
+      - 'main'
+    types:
+      # Adding 'labeled' to the list of activity types that trigger this event
+      # (default: opened, synchronize, reopened) so that we can run this
+      # workflow when the 'ok-to-test' label is added.
+      # Reference: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+    paths-ignore:
+      - 'docs/**'
+jobs:
+  kata-containers-ci-on-push:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
+    uses: ./.github/workflows/ci.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      pr-number: ${{ github.event.pull_request.number }}
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
+    secrets: inherit

.github/workflows/ci.yaml

@@ -0,0 +1,76 @@
+name: Run the Kata Containers CI
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+
+  publish-kata-deploy-payload-amd64:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+    secrets: inherit
+
+  run-k8s-tests-on-aks:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+    secrets: inherit
+
+  run-k8s-tests-on-sev:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-sev.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+
+  run-k8s-tests-on-snp:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-snp.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+
+  run-k8s-tests-on-tdx:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+
+  run-metrics-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-metrics.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}

.github/workflows/commit-message-check.yaml

@@ -62,6 +62,9 @@ jobs:
         #   to be specified at the start of the regex as the action is passed
         #   the entire commit message.
         #
+        # - This check will pass if the commit message only contains a subject
+        #   line, as other body message properties are enforced elsewhere.
+        #
         # - Body lines *can* be longer than the maximum if they start
         #   with a non-alphabetic character or if there is no whitespace in
         #   the line.
@@ -75,7 +78,7 @@ jobs:
         #
         # - A SoB comment can be any length (as it is unreasonable to penalise
         #   people with long names/email addresses :)
-        pattern: '^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$'
+        pattern: '(^[^\n]+$|^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$)'
         error: 'Body line too long (max 150)'
         post_error: ${{ env.error_msg }}
 

.github/workflows/kata-deploy-push.yaml

@@ -1,84 +0,0 @@
-name: kata deploy build
-
-on: 
-  pull_request:
-    types:
-      - opened
-      - edited
-      - reopened
-      - synchronize
-    paths:
-      - tools/**
-      - versions.yaml
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - kernel
-          - kernel-dragonball-experimental
-          - shim-v2
-          - qemu
-          - cloud-hypervisor
-          - firecracker
-          - rootfs-image
-          - rootfs-initrd
-          - virtiofsd
-          - nydus
-    steps:
-      - uses: actions/checkout@v2
-      - name: Build ${{ matrix.asset }}
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r --preserve=all "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: build
-      - name: merge-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make merge-builds
-      - name: store-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  make-kata-tarball:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Free disk space
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-      - uses: actions/checkout@v2
-      - name: make kata-tarball
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make kata-tarball
-          sudo make install-tarball

.github/workflows/kata-deploy-test.yaml

@@ -1,164 +0,0 @@
-on:
-  workflow_dispatch: # this is used to trigger the workflow on non-main branches
-    inputs:
-      pr:
-        description: 'PR number from the selected branch to test'
-        type: string
-        required: true
-  issue_comment:
-    types: [created, edited]
-
-name: test-kata-deploy
-
-jobs:
-  check-comment-and-membership:
-    runs-on: ubuntu-latest
-    if: |
-      github.event.issue.pull_request
-      && github.event_name == 'issue_comment'
-      && github.event.action == 'created'
-      && startsWith(github.event.comment.body, '/test_kata_deploy')
-      || github.event_name == 'workflow_dispatch'
-    steps:
-      - name: Check membership on comment or dispatch
-        uses: kata-containers/is-organization-member@1.0.1
-        id: is_organization_member
-        with:
-          organization: kata-containers
-          username: ${{ github.event.comment.user.login || github.event.sender.login }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Fail if not member
-        run: |
-          result=${{ steps.is_organization_member.outputs.result }}
-          if [ $result == false ]; then
-              user=${{ github.event.comment.user.login || github.event.sender.login }}
-              echo Either ${user} is not part of the kata-containers organization
-              echo or ${user} has its Organization Visibility set to Private at
-              echo https://github.com/orgs/kata-containers/people?query=${user}
-              echo 
-              echo Ensure you change your Organization Visibility to Public and
-              echo trigger the test again.
-              exit 1
-          fi
-
-  build-asset:
-    runs-on: ubuntu-latest
-    needs: check-comment-and-membership
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - kernel-dragonball-experimental
-          - nydus
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-          - virtiofsd
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            if [ ${{ github.event_name }} == 'issue_comment' ]; then
-                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            else # workflow_dispatch
-                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
-            fi
-            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
-            echo "pr-ref=${ref}" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            if [ ${{ github.event_name }} == 'issue_comment' ]; then
-                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            else # workflow_dispatch
-                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
-            fi
-            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
-            echo "pr-ref=${ref}" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            if [ ${{ github.event_name }} == 'issue_comment' ]; then
-                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            else # workflow_dispatch
-                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
-            fi
-            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
-            echo "pr-ref=${ref}" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          PR_SHA=$(git log --format=format:%H -n1)
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$PR_SHA $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "PKG_SHA=${PR_SHA}" >> $GITHUB_OUTPUT
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}

.github/workflows/payload-after-push.yaml

@@ -7,26 +7,59 @@ on:
 
 jobs:
   build-assets-amd64:
-    uses: ./.github/workflows/payload-after-push-amd64.yaml
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
     with:
-      target-arch: amd64
+      commit-hash: ${{ github.sha }}
+      push-to-registry: yes
     secrets: inherit
 
   build-assets-arm64:
-    uses: ./.github/workflows/payload-after-push-arm64.yaml
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
     with:
-      target-arch: arm64
+      commit-hash: ${{ github.sha }}
+      push-to-registry: yes
     secrets: inherit
 
   build-assets-s390x:
-    uses: ./.github/workflows/payload-after-push-s390x.yaml
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
     with:
-      target-arch: s390x
+      commit-hash: ${{ github.sha }}
+      push-to-registry: yes
     secrets: inherit
 
-  publish:
+  publish-kata-deploy-payload-amd64:
+    needs: build-assets-amd64
+    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-amd64
+    secrets: inherit
+
+  publish-kata-deploy-payload-arm64:
+    needs: build-assets-arm64
+    uses: ./.github/workflows/publish-kata-deploy-payload-arm64.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-arm64
+    secrets: inherit
+
+  publish-kata-deploy-payload-s390x:
+    needs: build-assets-s390x
+    uses: ./.github/workflows/publish-kata-deploy-payload-s390x.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-s390x
+    secrets: inherit
+
+  publish-manifest:
     runs-on: ubuntu-latest
-    needs: [build-assets-amd64, build-assets-arm64, build-assets-s390x]
+    needs: [publish-kata-deploy-payload-amd64, publish-kata-deploy-payload-arm64, publish-kata-deploy-payload-s390x]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -0,0 +1,55 @@
+name: CI | Publish kata-deploy payload for amd64
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  kata-payload:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -0,0 +1,60 @@
+name: CI | Publish kata-deploy payload for arm64
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  kata-payload:
+    runs-on: arm64
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}
+

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -0,0 +1,59 @@
+name: CI | Publish kata-deploy payload for s390x
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  kata-payload:
+    runs-on: s390x
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/release-amd64.yaml

@@ -0,0 +1,53 @@
+name: Publish Kata release artifacts for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      stage: release
+
+  kata-deploy:
+    needs: build-kata-static-tarball-amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64
+
+      - name: build-and-push-kata-deploy-ci-amd64
+        id: build-and-push-kata-deploy-ci-amd64
+        run: |
+          # We need to do such trick here as the format of the $GITHUB_REF 
+          # is "refs/tags/<tag>"
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+          done

.github/workflows/release-arm64.yaml

@@ -0,0 +1,53 @@
+name: Publish Kata release artifacts for arm64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-arm64:
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+    with:
+      stage: release
+
+  kata-deploy:
+    needs: build-kata-static-tarball-arm64
+    runs-on: arm64
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64
+
+      - name: build-and-push-kata-deploy-ci-arm64
+        id: build-and-push-kata-deploy-ci-arm64
+        run: |
+          # We need to do such trick here as the format of the $GITHUB_REF 
+          # is "refs/tags/<tag>"
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+          done

.github/workflows/release-s390x.yaml

@@ -0,0 +1,53 @@
+name: Publish Kata release artifacts for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-s390x:
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      stage: release
+
+  kata-deploy:
+    needs: build-kata-static-tarball-s390x
+    runs-on: s390x
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-deploy-ci-s390x
+        id: build-and-push-kata-deploy-ci-s390x
+        run: |
+          # We need to do such trick here as the format of the $GITHUB_REF 
+          # is "refs/tags/<tag>"
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+          done

.github/workflows/release.yaml

@@ -5,127 +5,112 @@ on:
       - '[0-9]+.[0-9]+.[0-9]+*'
 
 jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - kernel-dragonball-experimental
-          - nydus
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-          - virtiofsd
-    steps:
-      - uses: actions/checkout@v2
-      - name: Build ${{ matrix.asset }}
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+  build-and-push-assets-amd64:
+    uses: ./.github/workflows/release-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
 
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
+  build-and-push-assets-arm64:
+    uses: ./.github/workflows/release-arm64.yaml
+    with:
+      target-arch: arm64
+    secrets: inherit
 
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
+  build-and-push-assets-s390x:
+    uses: ./.github/workflows/release-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
 
-  kata-deploy:
-    needs: create-kata-tarball
+  publish-multi-arch-images:
     runs-on: ubuntu-latest
+    needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x]
     steps:
-      - uses: actions/checkout@v2
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v2
         with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push katadocker/kata-deploy-ci:$pkg_sha
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "PKG_SHA=${pkg_sha}" >> $GITHUB_OUTPUT
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
         with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
-      - name: push-tarball
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
         run: |
           # tag the container image we created and push to DockerHub
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tags=($tag)
           tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
-          for tag in ${tags[@]}; do \
-            docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag} && \
-            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} quay.io/kata-containers/kata-deploy:${tag} && \
-            docker push katadocker/kata-deploy:${tag} && \
-            docker push quay.io/kata-containers/kata-deploy:${tag}; \
+          # push to quay.io and docker.io
+          for tag in ${tags[@]}; do
+            docker manifest create quay.io/kata-containers/kata-deploy:${tag} \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-amd64 \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-arm64 \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-s390x
+
+            docker manifest create docker.io/katadocker/kata-deploy:${tag} \
+              --amend docker.io/katadocker/kata-deploy:${tag}-amd64 \
+              --amend docker.io/katadocker/kata-deploy:${tag}-arm64 \
+              --amend docker.io/katadocker/kata-deploy:${tag}-s390x
+
+            docker manifest push quay.io/kata-containers/kata-deploy:${tag}
+            docker manifest push docker.io/katadocker/kata-deploy:${tag}
           done
 
-  upload-static-tarball:
-    needs: kata-deploy
+  upload-multi-arch-static-tarball:
+    needs: publish-multi-arch-images
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: download-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
+      - uses: actions/checkout@v3
       - name: install hub
         run: |
-          HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
+          wget -q -O- https://github.com/mislav/hub/releases/download/v2.14.2/hub-linux-amd64-2.14.2.tgz | \
           tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
-      - name: push static tarball to github
+
+      - name: download-artifacts-amd64
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64
+      - name: push amd64 static tarball to github
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-x86_64.tar.xz"
+          tarball="kata-static-$tag-amd64.tar.xz"
+          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
+          pushd $GITHUB_WORKSPACE
+          echo "uploading asset '${tarball}' for tag: ${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          popd
+
+      - name: download-artifacts-arm64
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64
+      - name: push arm64 static tarball to github
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tarball="kata-static-$tag-arm64.tar.xz"
+          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
+          pushd $GITHUB_WORKSPACE
+          echo "uploading asset '${tarball}' for tag: ${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          popd
+
+      - name: download-artifacts-s390x
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+      - name: push s390x static tarball to github
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tarball="kata-static-$tag-s390x.tar.xz"
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
@@ -133,10 +118,10 @@ jobs:
           popd
 
   upload-cargo-vendored-tarball:
-    needs: upload-static-tarball
+    needs: upload-multi-arch-static-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: generate-and-upload-tarball
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
@@ -150,7 +135,7 @@ jobs:
     needs: upload-cargo-vendored-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: download-and-upload-tarball
         env:
           GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -0,0 +1,76 @@
+name: CI | Run kubernetes tests on AKS
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        host_os:
+          - ubuntu
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+        include:
+          - host_os: cbl-mariner
+            vmm: clh
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: Download Azure CLI
+        run: bash tests/integration/gha-run.sh install-azure-cli
+
+      - name: Log into the Azure account
+        run: bash tests/integration/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+
+      - name: Create AKS cluster
+        run: bash tests/integration/gha-run.sh create-cluster
+
+      - name: Install `bats`
+        run: bash tests/integration/gha-run.sh install-bats
+
+      - name: Install `kubectl`
+        run: bash tests/integration/gha-run.sh install-kubectl
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: bash tests/integration/gha-run.sh get-cluster-credentials
+
+      - name: Run tests
+        timeout-minutes: 60
+        run: bash tests/integration/gha-run.sh run-tests-aks
+
+      - name: Delete AKS cluster
+        if: always()
+        run: bash tests/integration/gha-run.sh delete-cluster

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -0,0 +1,43 @@
+name: CI | Run kubernetes tests on SEV
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-sev
+    runs-on: sev
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/gha-run.sh run-tests-sev
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/gha-run.sh cleanup-sev

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -0,0 +1,43 @@
+name: CI | Run kubernetes tests on SEV-SNP
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-snp
+    runs-on: sev-snp
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/gha-run.sh run-tests-snp
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/gha-run.sh cleanup-snp

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -0,0 +1,43 @@
+name: CI | Run kubernetes tests on TDX
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/gha-run.sh run-tests-tdx
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/gha-run.sh cleanup-tdx

.github/workflows/run-metrics.yaml

@@ -0,0 +1,58 @@
+name: CI | Run test metrics
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  run-metrics:
+    strategy:
+      fail-fast: true
+      matrix:
+        vmm: ['clh', 'qemu']
+      max-parallel: 1
+    runs-on: metrics
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/metrics/gha-run.sh install-kata kata-artifacts
+
+      - name: run launch times test
+        run: bash tests/metrics/gha-run.sh run-test-launchtimes
+
+      - name: run memory foot print test
+        run:  bash tests/metrics/gha-run.sh run-test-memory-usage
+
+      - name: run memory usage inside container test
+        run:  bash tests/metrics/gha-run.sh run-test-memory-usage-inside-container
+
+      - name: run blogbench test
+        run:  bash tests/metrics/gha-run.sh run-test-blogbench
+
+      - name: make metrics tarball ${{ matrix.vmm }}
+        run: bash tests/metrics/gha-run.sh make-tarball-results
+          
+      - name: archive metrics results ${{ matrix.vmm }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: metrics-artifacts-${{ matrix.vmm }}
+          path: results-${{ matrix.vmm }}.tar.gz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/snap-release.yaml

@@ -1,52 +0,0 @@
-name: Release Kata in snapcraft store
-on:
-  push:
-    tags:
-      - '[0-9]+.[0-9]+.[0-9]+*'
-
-env:
-  SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.snapcraft_token }}
-
-jobs:
-  release-snap:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Install Snapcraft
-        run: |
-          # Required to avoid snapcraft install failure
-          sudo chown root:root /
-
-          # "--classic" is needed for the GitHub action runner
-          # environment.
-          sudo snap install snapcraft --classic
-
-          # Allow other parts to access snap binaries
-          echo /snap/bin >> "$GITHUB_PATH"
-
-      - name: Build snap
-        run: |
-          # Removing man-db, workflow kept failing, fixes: #4480
-          sudo apt -y remove --purge man-db
-          sudo apt-get install -y git git-extras
-          kata_url="https://github.com/kata-containers/kata-containers"
-          latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
-          current_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          # Check semantic versioning format (x.y.z) and if the current tag is the latest tag
-          if echo "${current_version}" | grep -q "^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$" && echo -e "$latest_version\n$current_version" | sort -C -V; then
-            # Current version is the latest version, build it
-            snapcraft snap --debug --destructive-mode
-          fi
-
-      - name: Upload snap
-        run: |
-          snap_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          snap_file="kata-containers_${snap_version}_amd64.snap"
-          # Upload the snap if it exists
-          if [ -f ${snap_file} ]; then
-            snapcraft upload --release=stable ${snap_file}
-          fi

.github/workflows/snap.yaml

@@ -1,37 +0,0 @@
-name: snap CI
-on:
-  pull_request:
-    types:
-      - opened
-      - synchronize
-      - reopened
-      - edited
-    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
-
-jobs:
-  test:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Install Snapcraft
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          # Required to avoid snapcraft install failure
-          sudo chown root:root /
-
-          # "--classic" is needed for the GitHub action runner
-          # environment.
-          sudo snap install snapcraft --classic
-
-          # Allow other parts to access snap binaries
-          echo /snap/bin >> "$GITHUB_PATH"
-
-      - name: Build snap
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          snapcraft snap --debug --destructive-mode

.github/workflows/static-checks-dragonball.yaml

@@ -23,7 +23,7 @@ jobs:
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
           ./ci/install_rust.sh
-          PATH=$PATH:"$HOME/.cargo/bin"
+          echo PATH="$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
       - name: Run Unit Test
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |

.gitignore

@@ -6,6 +6,8 @@
 **/.vscode
 **/.idea
 **/.fleet
+**/*.swp
+**/*.swo
 pkg/logging/Cargo.lock
 src/agent/src/version.rs
 src/agent/kata-agent.service

Makefile

@@ -18,6 +18,7 @@ TOOLS =
 TOOLS += agent-ctl
 TOOLS += kata-ctl
 TOOLS += log-parser
+TOOLS += log-parser-rs
 TOOLS += runk
 TOOLS += trace-forwarder
 

README.md

@@ -1,4 +1,6 @@
-<img src="https://www.openstack.org/assets/kata/kata-vertical-on-white.png" width="150">
+<img src="https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/kata/SVG/kata-1.svg" width="900">
+
+[![CI | Publish Kata Containers payload](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml) [![Kata Containers Nightly CI](https://github.com/kata-containers/kata-containers/actions/workflows/ci-nightly.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/ci-nightly.yaml)
 
 # Kata Containers
 
@@ -134,6 +136,7 @@ The table below lists the remaining parts of the project:
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
 | [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
+| [`log-parser-rs`](src/tools/log-parser-rs) | utility | Tool that aid in analyzing logs from the kata runtime. |
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
 | [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |
@@ -143,8 +146,10 @@ The table below lists the remaining parts of the project:
 
 Kata Containers is now
 [available natively for most distributions](docs/install/README.md#packaged-installation-methods).
-However, packaging scripts and metadata are still used to generate [snap](snap/local) and GitHub releases. See
-the [components](#components) section for further details.
+
+## Metrics tests
+
+See the [metrics documentation](tests/metrics/README.md).
 
 ## Glossary of Terms
 

VERSION

@@ -1 +1 @@
-3.1.0-rc0
+3.2.0-alpha3

docs/Developer-Guide.md

@@ -587,10 +587,15 @@ $ sudo kata-monitor
 
 #### Connect to debug console
 
-Command `kata-runtime exec` is used to connect to the debug console.
+You need to start a container for example:
+```bash
+$ sudo ctr run  --runtime io.containerd.kata.v2 -d docker.io/library/ubuntu:latest testdebug
+```
+
+Then, you can use the command `kata-runtime exec <sandbox id>` to connect to the debug console.
 
 ```
-$ kata-runtime exec 1a9ab65be63b8b03dfd0c75036d27f0ed09eab38abb45337fea83acd3cd7bacd
+$ kata-runtime exec testdebug
 bash-4.2# id
 uid=0(root) gid=0(root) groups=0(root)
 bash-4.2# pwd

docs/Limitations.md

@@ -147,7 +147,8 @@ these commands is potentially challenging.
 See issue https://github.com/clearcontainers/runtime/issues/341 and [the constraints challenge](#the-constraints-challenge) for more information.
 
 For CPUs resource management see
-[CPU constraints](design/vcpu-handling.md).
+[CPU constraints(in runtime-go)](design/vcpu-handling-runtime-go.md).
+[CPU constraints(in runtime-rs)](design/vcpu-handling-runtime-rs.md).
 
 # Architectural limitations
 

docs/design/README.md

@@ -6,7 +6,8 @@ Kata Containers design documents:
 - [API Design of Kata Containers](kata-api-design.md)
 - [Design requirements for Kata Containers](kata-design-requirements.md)
 - [VSocks](VSocks.md)
-- [VCPU handling](vcpu-handling.md)
+- [VCPU handling(in runtime-go)](vcpu-handling-runtime-go.md)
+- [VCPU handling(in runtime-rs)](vcpu-handling-runtime-rs.md)
 - [VCPU threads pinning](vcpu-threads-pinning.md)
 - [Host cgroups](host-cgroups.md)
 - [Agent systemd cgroup](agent-systemd-cgroup.md)
@@ -16,6 +17,7 @@ Kata Containers design documents:
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 - [Design for direct-assigned volume](direct-blk-device-assignment.md)
 - [Design for core-scheduling](core-scheduling.md)
+- [Virtualization Reference Architecture](kata-vra.md)
 ---
 
 - [Design proposals](proposals)

docs/design/VSocks.md

@@ -78,4 +78,4 @@ with the containers is if the VM itself or the `containerd-shim-kata-v2` dies, i
 the containers are removed automatically.
 
 [1]: https://wiki.qemu.org/Features/VirtioVsock
-[2]: ./vcpu-handling.md#virtual-cpus-and-kubernetes-pods
+[2]: ./vcpu-handling-runtime-go.md#virtual-cpus-and-kubernetes-pods

docs/design/architecture/networking.md

@@ -36,7 +36,7 @@ compatibility, and performance on par with MACVTAP.
 Kata Containers has deprecated support for bridge due to lacking performance relative to TC-filter and MACVTAP.
 
 Kata Containers supports both
-[CNM](https://github.com/docker/libnetwork/blob/master/docs/design.md#the-container-network-model)
+[CNM](https://github.com/moby/libnetwork/blob/master/docs/design.md#the-container-network-model)
 and [CNI](https://github.com/containernetworking/cni) for networking management.
 
 ## Network Hotplug

docs/design/kata-vra.md

@@ -0,0 +1,434 @@
+# Virtualization Reference Architecture
+
+## Subject to Change | © 2022 by NVIDIA Corporation. All rights reserved. | For test and development only_
+
+Before digging deeper into the virtualization reference architecture, let's
+first look at the various GPUDirect use cases in the following table. We’re
+distinguishing between two top-tier use cases where the devices are (1)
+passthrough and (2) virtualized, where a VM gets assigned a virtual function
+(VF) and not the physical function (PF). A combination of PF and VF would also
+be possible.
+
+| Device #1  (passthrough)  | Device #2 (passthrough) | P2P Compatibility and Mode                   |
+| ------------------------- | ----------------------- | -------------------------------------------- |
+| GPU PF                    | GPU PF                  | GPUDirect P2P                                |
+| GPU PF                    | NIC PF                  | GPUDirect RDMA                               |
+| MIG-slice                 | MIG-slice               | _No GPUDirect P2P_                           |
+| MIG-slice                 | NIC PF                  | GPUDirect RDMA                               |
+| **PDevice #1  (virtualized)** | **Device #2 (virtualized)** | **P2P Compatibility and   Mode**     |
+| Time-slice vGPU VF        | Time-slice vGPU VF      | _No GPUDirect P2P  but NVLINK P2P available_ |
+| Time-slice vGPU VF        | NIC VF                  | GPUDirect RDMA                               |
+| MIG-slice vGPU            | MIG-slice vGPU          | _No GPUDirect P2P_                           |
+| MIG-slice vGPU            | NIC VF                  | GPUDirect RDMA                               |
+
+In a virtualized environment we have several distinct features that may prevent
+Peer-to-peer (P2P) communication of two endpoints in a PCI Express topology. The
+IOMMU translates IO virtual addresses (IOVA) to physical addresses (PA). Each
+device behind an IOMMU has its own IOVA memory space, usually, no two devices
+share the same IOVA memory space but it’s up to the hypervisor or OS how it
+chooses to map devices to IOVA spaces.  Any PCI Express DMA transactions will
+use IOVAs, which the IOMMU must translate. By default, all the traffic is routed
+to the root complex and not issued directly to the peer device.
+
+An IOMMU can be used to isolate and protect devices even if virtualization is
+not used; since devices can only access memory regions that are mapped for it, a
+DMA from one device to another is not possible. DPDK uses the IOMMU to have
+better isolation between devices, another benefit is that IOVA space can be
+represented as a contiguous memory even if the PA space is heavily scattered.
+
+In the case of virtualization, the IOMMU is responsible for isolating the device
+and memory between VMs for safe device assignment without compromising the host
+and other guest OSes. Without an IOMMU, any device can access the entire system
+and perform DMA transactions _anywhere_.
+
+The second feature is ACS (Access Control Services), which controls which
+devices are allowed to communicate with one another and thus avoids improper
+routing of packets irrespectively of whether IOMMU is enabled or not.
+
+When IOMMU is enabled, ACS is normally configured to force all PCI Express DMA
+to go through the root complex so IOMMU can translate it, impacting performance
+between peers with higher latency and reduced bandwidth.
+
+A way to avoid the performance hit is to enable Address Translation Services
+(ATS). ATS-capable endpoints can prefetch IOVA -> PA translations from the IOMMU
+and then perform DMA transactions directly to another endpoint. Hypervisors
+enable this by enabling ATS in such endpoints, configuring ACS to enable Direct
+Translated P2P, and configuring the IOMMU to allow Address Translation requests.
+
+Another important factor is that the NVIDIA driver stack will use the PCI
+Express topology of the system it is running on to determine whether the
+hardware is capable of supporting P2P. The driver stack qualifies specific
+chipsets, and PCI Express switches for use with GPUDirect P2P. In virtual
+environments, the PCI Express topology is flattened and obfuscated to present a
+uniform environment to the software inside the VM, which breaks the GPUDirect
+P2P use case.
+
+On a bare metal machine, the driver stack groups GPUs into cliques that can
+perform GPUDirect P2P communication, excluding peer mappings where P2P
+communication is not possible, prominently if GPUs are attached to multiple CPU
+sockets.  
+
+CPUs and local memory banks are referred to as NUMA nodes. In a two-socket
+server, each of the CPUs has a local memory bank for a total of two NUMA nodes.
+Some servers provide the ability to configure additional NUMA nodes per CPU,
+which means a CPU socket can have two NUMA nodes  (some servers support four
+NUMA nodes per socket) with local memory banks and L3 NUMA domains for improved
+performance.
+
+One of the current solutions is that the hypervisor provides additional topology
+information that the driver stack can pick up and enable GPUDirect P2P between
+GPUs, even if the virtualized environment does not directly expose it. The PCI
+Express virtual P2P approval capability structure in the PCI configuration space
+is entirely emulated by the hypervisor of passthrough GPU devices.
+
+A clique ID is provided where GPUs with the same clique ID belong to a group of
+GPUs capable of P2P communication
+
+On vSphere, Azure, and other CPSs,  the hypervisor lays down a `topologies.xml`
+which NCCL can pick up and deduce the right P2P level[^1]. NCCL is leveraging
+Infiniband (IB) and/or Unified Communication X (UCX) for communication, and
+GPUDirect P2P and GPUDirect RDMA should just work in this case. The only culprit
+is that software or applications that do not use the XML file to deduce the
+topology will fail and not enable GPUDirect ( [`nccl-p2p-level`](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html#nccl-p2p-level) )
+
+## Hypervisor PCI Express Topology
+
+To enable every part of the accelerator stack, we propose a virtualized
+reference architecture to enable GPUDirect P2P and GPUDirect RDMA for any
+hypervisor. The idea is split into two parts to enable the right PCI Express
+topology. The first part builds upon extending the PCI Express virtual P2P
+approval capability structure to every device that wants to do P2P in some way
+and groups devices by clique ID. The other part involves replicating a subset of
+the host topology so that applications running in the VM do not need to read
+additional information and enable the P2P capability like in the bare-metal use
+case described above. The driver stack can then deduce automatically if the
+topology presented in the VM is capable of P2P communication.
+
+We will work with the following host topology for the following sections. It is
+a system with two converged DPUs, each having an `A100X` GPU and two `ConnectX-6`
+network ports connected to the downstream ports of a PCI Express switch.
+
+```sh
++-00.0-[d8-df]----00.0-[d9-df]--+-00.0-[da-db]--+-00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               +-00.1  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               \-00.2  Mellanox Tech MT42822 BlueField-2 SoC Management Interface
+                                 \-01.0-[dc-df]----00.0-[dd-df]----08.0-[de-df]----00.0  NVIDIA Corporation GA100 [A100X]
+
++-00.0-[3b-42]----00.0-[3c-42]--+-00.0-[3d-3e]--+-00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               +-00.1  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               \-00.2  Mellanox Tech MT42822 BlueField-2 SoC Management Interface
+                                 \-01.0-[3f-42]----00.0-[40-42]----08.0-[41-42]----00.0  NVIDIA Corporation GA100 [A100X]
+```
+
+The green path highlighted above is the optimal and preferred path for
+efficient P2P communication.
+
+## PCI Express Virtual P2P Approval Capability
+
+Most of the time, the PCI Express topology is flattened and obfuscated to ensure
+easy migration of the VM image between different physical hardware topologies.
+In Kata, we can configure the hypervisor to use PCI Express root ports to
+hotplug the VFIO  devices one is passing through. A user can select how many PCI
+Express root ports to allocate depending on how many devices are passed through.
+A recent addition to Kata will detect the right amount of PCI Express devices
+that need hotplugging and bail out if the number of root ports is insufficient.
+In Kata, we do not automatically increase the number of root ports, we want the
+user to be in full control of the topology.
+
+```toml
+# /etc/kata-containers/configuration.toml
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hot-plugging on the root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hot-plugging on
+# a bridge.
+# Default “bridge-port”
+hotplug_vfio = "root-port"
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as NVIDIA GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+pcie_root_port = 8
+```
+
+VFIO devices are hotplugged on a PCIe-PCI bridge by default. Hotplug of PCI
+Express devices is only supported on PCI Express root or downstream ports. With
+this configuration set, if we start up a Kata container, we can inspect our
+topology and see the allocated PCI Express root ports and the hotplugged
+devices.
+
+```sh
+$ lspci -tv
+ -[0000:00]-+-00.0  Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
+           +-01.0  Red Hat, Inc. Virtio console
+           +-02.0  Red Hat, Inc. Virtio SCSI
+           +-03.0  Red Hat, Inc. Virtio RNG
+           +-04.0-[01]----00.0  Mellanox Technologies MT42822 BlueField-2 integrated ConnectX-6
+           +-05.0-[02]----00.0  Mellanox Technologies MT42822 BlueField-2 integrated ConnectX-6
+           +-06.0-[03]----00.0  NVIDIA Corporation Device 20b8
+           +-07.0-[04]----00.0  NVIDIA Corporation Device 20b8
+           +-08.0-[05]--
+           +-09.0-[06]--
+           +-0a.0-[07]--
+           +-0b.0-[08]--
+           +-0c.0  Red Hat, Inc. Virtio socket
+           +-0d.0  Red Hat, Inc. Virtio file system
+           +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+           +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller
+           \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+For devices with huge BARs (Base Address Registers) like the GPU (we need to
+configure the PCI Express root port properly and allocate enough memory for
+mapping), we have added a heuristic to Kata to deduce the right settings. Hence,
+the BARs can be mapped correctly. This functionality is added to
+[`nvidia/go-nvlib1](https://gitlab.com/nvidia/cloud-native/go-nvlib) which is part
+of Kata now.
+
+```sh
+$ sudo dmesg | grep BAR
+[    0.179960] pci 0000:00:04.0: BAR 7: assigned [io  0x1000-0x1fff]
+[    0.179962] pci 0000:00:05.0: BAR 7: assigned [io  0x2000-0x2fff]
+[    0.179963] pci 0000:00:06.0: BAR 7: assigned [io  0x3000-0x3fff]
+[    0.179964] pci 0000:00:07.0: BAR 7: assigned [io  0x4000-0x4fff]
+[    0.179966] pci 0000:00:08.0: BAR 7: assigned [io  0x5000-0x5fff]
+[    0.179967] pci 0000:00:09.0: BAR 7: assigned [io  0x6000-0x6fff]
+[    0.179968] pci 0000:00:0a.0: BAR 7: assigned [io  0x7000-0x7fff]
+[    0.179969] pci 0000:00:0b.0: BAR 7: assigned [io  0x8000-0x8fff]
+[    2.115912] pci 0000:01:00.0: BAR 0: assigned [mem 0x13000000000-0x13001ffffff 64bit pref]
+[    2.116203] pci 0000:01:00.0: BAR 2: assigned [mem 0x13002000000-0x130027fffff 64bit pref]
+[    2.683132] pci 0000:02:00.0: BAR 0: assigned [mem 0x12000000000-0x12001ffffff 64bit pref]
+[    2.683419] pci 0000:02:00.0: BAR 2: assigned [mem 0x12002000000-0x120027fffff 64bit pref]
+[    2.959155] pci 0000:03:00.0: BAR 1: assigned [mem 0x11000000000-0x117ffffffff 64bit pref]
+[    2.959345] pci 0000:03:00.0: BAR 3: assigned [mem 0x11800000000-0x11801ffffff 64bit pref]
+[    2.959523] pci 0000:03:00.0: BAR 0: assigned [mem 0xf9000000-0xf9ffffff]
+[    2.966119] pci 0000:04:00.0: BAR 1: assigned [mem 0x10000000000-0x107ffffffff 64bit pref]
+[    2.966295] pci 0000:04:00.0: BAR 3: assigned [mem 0x10800000000-0x10801ffffff 64bit pref]
+[    2.966472] pci 0000:04:00.0: BAR 0: assigned [mem 0xf7000000-0xf7ffffff]
+```
+
+The NVIDIA driver stack in this case would refuse to do P2P communication since
+(1) the topology is not what it expects, (2)  we do not have a qualified
+chipset. Since our P2P devices are not connected to a PCI Express switch port,
+we need to provide additional information to support the P2P functionality. One
+way of providing such meta information would be to annotate the container; most
+of the settings in Kata's configuration file can be overridden via annotations,
+but this limits the flexibility, and a user would need to update all the
+containers that he wants to run with Kata. The goal is to make such things as
+transparent as possible, so we also introduced
+[CDI](https://github.com/container-orchestrated-devices/container-device-interface)
+(Container Device Interface) to Kata. CDI is a[
+specification](https://github.com/container-orchestrated-devices/container-device-interface/blob/master/SPEC.md)
+for container runtimes to support third-party devices.
+
+As written before, we can provide a clique ID for the devices that belong
+together and are capable of doing P2P. This information is provided to the
+hypervisor, which will set up things in the VM accordingly. Let's suppose the
+user wanted to do GPUDirect RDMA with the first GPU and the NIC that reside on
+the same DPU, one could provide the specification telling the hypervisor that
+they belong to the same clique.
+
+```yaml
+# /etc/cdi/nvidia.yaml
+cdiVersion: 0.4.0
+kind: nvidia.com/gpu
+devices:
+- name: gpu0
+  annotations:
+    bdf: “41:00.0”
+    clique-id: “0”
+  containerEdits:
+    deviceNodes:
+    - path: “/dev/vfio/71"
+
+# /etc/cdi/mellanox.yaml
+cdiVersion: 0.4.0
+kind: mellanox.com/nic
+devices:
+- name: nic0
+  annotations:
+    bdf: “3d:00.0”
+    clique-id: “0”
+    attach-pci: “true”
+  containerEdits:
+    deviceNodes:
+    - path: "/dev/vfio/66"
+```
+
+Since this setting is bound to the device and not the container we do not need
+to alter the container just allocate the right resource and GPUDirect RDMA would
+be set up correctly. Rather than exposing them separately, an idea would be to
+expose a GPUDirect RDMA device via NFD (Node Feature Discovery) that combines
+both of them; this way, we could make sure that the right pair is allocated and
+used more on  Kubernetes deployment in the next section.
+
+The GPU driver stack is leveraging the PCI Express virtual P2P approval
+capability, but the NIC stack does not use this now. One of the action items is
+to enable MOFED to read the P2P approval capability and enable ATS and ACS
+settings as described above.
+
+This way, we could enable GPUDirect P2P and GPUDirect RDMA on any topology
+presented to the VM application. It is the responsibility of the administrator
+or infrastructure engineer to provide the right information either via
+annotations or a CDI specification.
+
+## Host Topology Replication
+
+The other way to represent the PCI Express topology in the VM is to replicate a
+subset of the topology needed to support the P2P use case inside the VM. Similar
+to the configuration for the root ports, we can easily configure the usage of
+PCI Express switch ports to hotplug the devices.
+
+```toml
+# /etc/kata-containers/configuration.toml
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hot plugging on the root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hot plugging on
+# a bridge.
+# Default “bridge-port”
+hotplug_vfio = "switch-port"
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+pcie_switch_port = 8
+```
+
+Each device that is passed through is attached to a PCI Express downstream port
+as illustrated below. We can even replicate the host’s two DPUs topologies with
+added metadata through the CDI. Most of the time, a container only needs one
+pair of GPU and NIC for GPUDirect RDMA. This is more of a showcase of what we
+can do with the power of Kata and CDI. One could even think of adding groups of
+devices that support P2P, even from different CPU sockets or NUMA nodes, into
+one container; indeed, the first group is NUMA node 0 (red), and the second
+group is NUMA node 1 (green). Since they are grouped correctly, P2P would be
+enabled naturally inside a group, aka clique ID.
+
+```sh
+$ lspci -tv
+ -[0000:00]-+-00.0  Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
+            +-01.0  Red Hat, Inc. Virtio console
+            +-02.0  Red Hat, Inc. Virtio SCSI
+            +-03.0  Red Hat, Inc. Virtio RNG
+            +-04.0-[01-04]----00.0-[02-04]--+-00.0-[03]----00.0  NVIDIA Corporation Device 20b8
+            |                               \-01.0-[04]----00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx
+            +-05.0-[05-08]----00.0-[06-08]--+-00.0-[07]----00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx
+            |                               \-01.0-[08]----00.0  NVIDIA Corporation Device 20b8
+            +-06.0  Red Hat, Inc. Virtio socket
+            +-07.0  Red Hat, Inc. Virtio file system
+            +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+            +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller [AHCI mode]
+            \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+            \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+The configuration of using either the root port or switch port can be applied on
+a per Container or Pod basis, meaning we can switch PCI Express topologies on
+each run of an application.
+
+## Hypervisor Resource Limits
+
+Every hypervisor will have resource limits in terms of how many PCI Express root
+ports, switch ports, or bridge ports can be created, especially with devices
+that need to reserve a 4K IO range per PCI specification. Each instance of root
+or switch port will consume 4K IO of very limited capacity, 64k is the maximum.
+
+Simple math brings us to the conclusion that we can have a maximum of 16 PCI
+Express root ports or 16 PCI Express switch ports in QEMU if devices with IO
+BARs are used in the PCI Express hierarchy.
+
+Additionally, one can have 32 slots on the PCI root bus and a maximum of 256
+slots for the complete PCI(e) topology.
+
+Per default, QEMU will attach a multi-function device in the last slot on the
+PCI root bus,
+
+```sh
+ +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+ +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller [AHCI mode]
+ \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+Kata will additionally add `virtio-xxx-pci` devices consuming (5 slots) plus a
+PCIe-PCI-bridge (1 slot) and a DRAM controller (1 slot), meaning per default, we
+have already eight slots used. This leaves us 24 slots for adding other devices
+to the root bus.
+
+The problem that arises here is one use-case from a customer that uses recent
+RTX GPUs with Kata. The user wanted to pass through eight of these GPUs into one
+container and ran into issues. The problem is that those cards often consist of
+four individual device nodes: GPU, Audio, and two USB controller devices (some
+cards have a USB-C output).
+
+These devices are grouped into one IOMMU group. Since one needs to pass through
+the complete IOMMU group into the VM, we need to allocate 32 PCI Express root
+ports or 32 PCI Express switch ports, which is technically impossible due to the
+resource limits outlined above. Since all the devices appear as PCI Express
+devices, we need to hotplug those into a root or switch port.
+
+The solution to this problem is leveraging CDI. For each device, add the
+information if it is going to be hotplugged as a PCI Express or PCI device,
+which results in either using a PCI Express root/switch port or an ordinary PCI
+bridge. PCI bridges are not affected by the limited IO range. This way, the GPU
+is attached as a PCI Express device to a root/switch port and the other three
+PCI devices to a PCI bridge, leaving enough resources to create the needed PCI
+Express root/switch ports.  For example, we’re going to attach the GPUs to a PCI
+Express root port and the NICs to a PCI bridge.
+
+```jsonld
+# /etc/cdi/mellanox.json
+cdiVersion: 0.4.0
+kind: mellanox.com/nic
+devices:
+- name: nic0
+  annotations:
+    bdf: “3d:00.0”
+    clique-id: “0”
+    attach-pci: “true”
+  containerEdits:
+    deviceNodes:
+    - path: "/dev/vfio/66"
+- name: nic1
+  annotations:
+    bdf: “3d:00.1”
+    clique-id: “1”
+    attach-pci: “true”
+  containerEdits:
+    deviceNodes:
+    - path: "/dev/vfio/67”
+```
+
+The configuration is set to use eight root ports for the GPUs and attach the
+NICs to a PCI bridge which is connected to a PCI Express-PCI bridge which is the
+preferred way of introducing a PCI topology in a PCI Express machine.
+
+```sh
+$ lspci -tv
+-[0000:00]-+-00.0  Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
+           +-01.0  Red Hat, Inc. Virtio console
+           +-02.0  Red Hat, Inc. Virtio SCSI
+           +-03.0  Red Hat, Inc. Virtio RNG
+           +-04.0-[01]----00.0  NVIDIA Corporation Device 20b8
+           +-05.0-[02]----00.0  NVIDIA Corporation Device 20b8
+           +-06.0-[03]--
+           +-07.0-[04]--
+           +-08.0-[05]--
+           +-09.0-[06]--
+           +-0a.0-[07]--
+           +-0b.0-[08]--
+           +-0c.0-[09-0a]----00.0-[0a]--+-00.0  Mellanox Tech MT42822 BlueField-2 ConnectX-6
+           |                             \-01.0  Mellanox Tech MT42822 BlueField-2 ConnectX-6
+           +-0d.0  Red Hat, Inc. Virtio socket
+           +-0e.0  Red Hat, Inc. Virtio file system
+           +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+           +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller
+           \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+The PCI devices will consume a slot of which we have 256 in the PCI(e) topology
+and leave scarce resources for the needed PCI Express devices.

docs/design/vcpu-handling-runtime-rs.md

@@ -0,0 +1,51 @@
+# Virtual machine vCPU sizing in Kata Containers 3.0
+
+> Preview: 
+> [Kubernetes(since 1.23)][1] and [Containerd(since 1.6.0-beta4)][2] will help calculate `Sandbox Size` info and pass it to Kata Containers through annotations.
+> In order to adapt to this beneficial change and be compatible with the past, we have implemented the new vCPUs handling way in `runtime-rs`, which is slightly different from the original `runtime-go`'s design.
+
+## When do we need to handle vCPUs size?
+vCPUs sizing should be determined by the container workloads. So throughout the life cycle of Kata Containers, there are several points in time when we need to think about how many vCPUs should be at the time. Mainly including the time points of `CreateVM`, `CreateContainer`, `UpdateContainer`, and `DeleteContainer`.
+* `CreateVM`: When creating a sandbox, we need to know how many vCPUs to start the VM with.
+* `CreateContainer`: When creating a new container in the VM, we may need to hot-plug the vCPUs according to the requirements in container's spec.
+* `UpdateContainer`: When receiving the `UpdateContainer` request, we may need to update the vCPU resources according to the new requirements of the container.
+* `DeleteContainer`: When a container is removed from the VM, we may need to hot-unplug the vCPUs to reclaim the vCPU resources introduced by the container.
+
+## On what basis do we calculate the number of vCPUs?
+When Kata calculate the number of vCPUs, We have three data sources, the `default_vcpus` and `default_maxvcpus` specified in the configuration file (named `TomlConfig` later in the doc), the `io.kubernetes.cri.sandbox-cpu-quota` and `io.kubernetes.cri.sandbox-cpu-period` annotations passed by the upper layer runtime, and the corresponding CPU resource part in the container's spec for the container when `CreateContainer`/`UpdateContainer`/`DeleteContainer` is requested.
+
+Our understanding and priority of these resources are as follows, which will affect how we calculate the number of vCPUs later.
+
+* From `TomlConfig`:
+  * `default_vcpus`: default number of vCPUs when starting a VM.
+  * `default_maxvcpus`: maximum number of vCPUs.
+* From `Annotation`:
+  * `InitialSize`: we call the size of the resource passed from the annotations as `InitialSize`. Kubernetes will calculate the sandbox size according to the Pod's statement, which is the `InitialSize` here. This size should be the size we want to prioritize. 
+* From `Container Spec`:
+  * The amount of CPU resources that the Container wants to use will be declared through the spec. Including the aforementioned annotations, we mainly consider `cpu quota` and `cpuset` when calculating the number of vCPUs.
+  * `cpu quota`: `cpu quota` is the most common way to declare the amount of CPU resources. The number of vCPUs introduced by `cpu quota` declared in a container's spec is: `vCPUs = ceiling( quota / period )`.
+  * `cpuset`: `cpuset` is often used to bind the CPUs that tasks can run on. The number of vCPUs may introduced by `cpuset` declared in a container's spec is the number of CPUs specified in the set that do not overlap with other containers.
+
+
+## How to calculate and adjust the vCPUs size:
+There are two types of vCPUs that we need to consider, one is the number of vCPUs when starting the VM (named `Boot Size` in the doc). The second is the number of vCPUs when `CreateContainer`/`UpdateContainer`/`DeleteContainer` request is received (`Real-time Size` in the doc).
+
+### `Boot Size`
+The main considerations are `InitialSize` and `default_vcpus`. There are the following principles:
+`InitialSize` has priority over `default_vcpus` declared in `TomlConfig`.
+1. When there is such an annotation statement, the originally `default_vcpus` will be modified to the number of vCPUs in the `InitialSize` as the `Boot Size`. (Because not all runtimes support this annotation for the time being, we still keep the `default_cpus` in `TomlConfig`.)
+2. When the specs of all containers are aggregated for sandbox size calculation, the method is consistent with the calculation method of `InitialSize` here.
+
+### `Real-time Size`
+When we receive an OCI request, it may be for a single container. But what we have to consider is the number of vCPUs for the entire VM. So we will maintain a list. Every time there is a demand for adjustment, the entire list will be traversed to calculate a value for the number of vCPUs. In addition, there are the following principles:
+1. Do not cut computing power and try to keep the number of vCPUs specified by `InitialSize`.
+   * So the number of vCPUs after will not be less than the `Boot Size`.
+2. `cpu quota` takes precedence over `cpuset` and the setting history are took into account.
+   * We think quota describes the CPU time slice that a cgroup can use, and `cpuset` describes the actual CPU number that a cgroup can use. Quota can better describe the size of the CPU time slice that a cgroup actually wants to use. The `cpuset` only describes which CPUs the cgroup can use, but the cgroup can use the specified CPU but consumes a smaller time slice, so the quota takes precedence over the `cpuset`.
+   * On the one hand, when both `cpu quota` and `cpuset` are specified, we will calculate the number of vCPUs based on `cpu quota` and ignore `cpuset`. On the other hand, if `cpu quota` was used to control the number of vCPUs in the past, and only `cpuset` was updated during `UpdateContainer`, we will not adjust the number of vCPUs at this time.
+3. `StaticSandboxResourceMgmt` controls hotplug.
+   * Some VMMs and kernels of some architectures do not support hotplugging. We can accommodate this situation through `StaticSandboxResourceMgmt`. When `StaticSandboxResourceMgmt = true` is set, we don't make any further attempts to update the number of vCPUs after booting.
+
+
+[1]: https://github.com/kubernetes/kubernetes/pull/104886
+[2]: https://github.com/containerd/containerd/pull/6155

docs/how-to/README.md

@@ -45,6 +45,8 @@
 - [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
 - [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
 - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
+- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)
+
 ## Confidential Containers
 - [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
 - [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md)

docs/how-to/ccv0.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 #
-# Copyright (c) 2021, 2022 IBM Corporation
+# Copyright (c) 2021, 2023 IBM Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -24,6 +24,10 @@ export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
 export KUBERNETES=${KUBERNETES:-"no"}
 export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}"
 export AA_KBC="${AA_KBC:-offline_fs_kbc}"
+export KATA_BUILD_CC=${KATA_BUILD_CC:-"yes"}
+export TEE_TYPE=${TEE_TYPE:-}
+export PREFIX="${PREFIX:-/opt/confidential-containers}"
+export RUNTIME_CONFIG_PATH="${RUNTIME_CONFIG_PATH:-${PREFIX}/share/defaults/kata-containers/configuration.toml}"
 
 # Allow the user to overwrite the default repo and branch names if they want to build from a fork
 export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}"
@@ -66,8 +70,6 @@ fi
 
 [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
 
-export RUNTIME_CONFIG_PATH=/etc/kata-containers/configuration.toml
-
 usage() {
     exit_code="$1"
     cat <<EOF
@@ -210,13 +212,14 @@ checkout_kata_containers_repo() {
 }
 
 build_and_install_kata_runtime() {
-    pushd ${katacontainers_repo_dir}/src/runtime
-    make clean && make DEFAULT_HYPERVISOR=${KATA_HYPERVISOR} && sudo -E PATH=$PATH make DEFAULT_HYPERVISOR=${KATA_HYPERVISOR} install
-    popd
+    export DEFAULT_HYPERVISOR=${KATA_HYPERVISOR}
+    ${tests_repo_dir}/.ci/install_runtime.sh
 }
 
 configure() {
-    configure_kata_to_use_rootfs
+    # configure kata to use rootfs, not initrd
+    sudo sed -i 's/^\(initrd =.*\)/# \1/g' ${RUNTIME_CONFIG_PATH}
+
     enable_full_debug
     enable_agent_console
 
@@ -226,12 +229,9 @@ configure() {
     configure_cc_containerd
     # From crictl v1.24.1 the default timoout leads to the pod creation failing, so update it
     sudo crictl config --set timeout=10
-}
 
-configure_kata_to_use_rootfs() {
-    sudo mkdir -p /etc/kata-containers/
-    sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
-    sudo sed -i 's/^\(initrd =.*\)/# \1/g' ${RUNTIME_CONFIG_PATH}
+    # Verity checks aren't working locally, as we aren't re-genning the hash maybe? so remove it from the kernel parameters
+    remove_kernel_param "cc_rootfs_verity.scheme"
 }
 
 build_and_add_agent_to_rootfs() {
@@ -306,20 +306,15 @@ install_rootfs_image() {
     local commit=$(git log --format=%h -1 HEAD)
     local date=$(date +%Y-%m-%d-%T.%N%z)
     local image="kata-containers-${date}-${commit}"
-    sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
-    (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
-    echo "Built Rootfs from ${ROOTFS_DIR} to /usr/share/kata-containers/${image}"
-    ls -al /usr/share/kata-containers/
+    sudo install -o root -g root -m 0640 -D kata-containers.img "${PREFIX}/share/kata-containers/${image}"
+    (cd ${PREFIX}/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+    echo "Built Rootfs from ${ROOTFS_DIR} to ${PREFIX}/share/kata-containers/${image}"
+    ls -al ${PREFIX}/share/kata-containers
     popd
 }
 
 install_guest_kernel_image() {
-    pushd ${katacontainers_repo_dir}/tools/packaging/kernel
-    sudo -E PATH=$PATH ./build-kernel.sh setup
-    sudo -E PATH=$PATH ./build-kernel.sh build
-    sudo chmod u+wrx /usr/share/kata-containers/ # Give user permission to install kernel
-    sudo -E PATH=$PATH ./build-kernel.sh install
-    popd
+   ${tests_repo_dir}/.ci/install_kata_kernel.sh
 }
 
 build_qemu() {
@@ -428,7 +423,7 @@ crictl_delete_cc() {
 test_kata_runtime() {
     echo "Running ctr with the kata runtime..."
     local test_image="quay.io/kata-containers/confidential-containers:signed"
-    if [ -z $(ctr images ls -q name=="${test_image}") ]; then
+    if [ -z $(sudo ctr images ls -q name=="${test_image}") ]; then
         sudo ctr image pull "${test_image}"
     fi
     sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -44,12 +44,11 @@ $ popd
 - Build a custom QEMU
 ```bash
 $ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu.snp.url")"
-$ qemu_branch="$(get_from_kata_deps "assets.hypervisor.qemu.snp.branch")"
-$ qemu_commit="$(get_from_kata_deps "assets.hypervisor.qemu.snp.commit")"
-$ git clone -b "${qemu_branch}" "${qemu_url}"
+$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")"
+$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")"
+$ git clone "${qemu_url}"
 $ pushd qemu
-$ git checkout "${qemu_commit}"
+$ git checkout "${qemu_tag}"
 $ ./configure --enable-virtfs --target-list=x86_64-softmmu --enable-debug
 $ make -j "$(nproc)"
 $ popd

docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md

@@ -0,0 +1,226 @@
+# A new way for Kata Containers to use Kinds of Block Volumes
+
+> **Note:** This guide is only available for runtime-rs with default Hypervisor Dragonball.
+> Now, other hypervisors are still ongoing, and it'll be updated when they're ready.
+
+
+## Background
+
+Currently, there is no widely applicable and convenient method available for users to use some kinds of backend storages, such as File on host based block volume, SPDK based volume or VFIO device based volume for Kata Containers, so we adopt [Proposal: Direct Block Device Assignment](https://github.com/kata-containers/kata-containers/blob/main/docs/design/direct-blk-device-assignment.md) to address it.
+
+## Solution
+
+According to the proposal, it requires to use the `kata-ctl direct-volume` command to add a direct assigned block volume device to the Kata Containers runtime. 
+
+And then with the help of method [get_volume_mount_info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L95), get information from JSON file: `(mountinfo.json)` and parse them into structure [Direct Volume Info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L70) which is used to save device-related information. 
+
+We only fill the `mountinfo.json`, such as `device` ,`volume_type`, `fs_type`, `metadata` and `options`, which correspond to the fields in [Direct Volume Info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L70), to describe a device. 
+
+The JSON file `mountinfo.json` placed in a sub-path `/kubelet/kata-test-vol-001/volume001` which under fixed path `/run/kata-containers/shared/direct-volumes/`. 
+And the full path looks like: `/run/kata-containers/shared/direct-volumes/kubelet/kata-test-vol-001/volume001`, But for some security reasons. it is 
+encoded as `/run/kata-containers/shared/direct-volumes/L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx`.
+
+Finally, when running a Kata Containers with `ctr run --mount type=X, src=Y, dst=Z,,options=rbind:rw`, the `type=X` should be specified a proprietary type specifically designed for some kind of volume. 
+
+Now, supported types: 
+
+- `directvol` for direct volume
+- `vfiovol` for VFIO device based volume
+- `spdkvol` for SPDK/vhost-user based volume
+
+
+## Setup Device and Run a Kata-Containers
+
+### Direct Block Device Based Volume
+
+#### create raw block based backend storage
+
+> **Tips:** raw block based backend storage MUST be formatted with `mkfs`.
+
+```bash
+$ sudo dd if=/dev/zero of=/tmp/stor/rawdisk01.20g bs=1M count=20480
+$ sudo mkfs.ext4 /tmp/stor/rawdisk01.20g
+```
+
+#### setup direct block device for kata-containers
+
+```json
+{
+  "device": "/tmp/stor/rawdisk01.20g", 
+  "volume_type": "directvol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+```bash
+$ sudo kata-ctl direct-volume add /kubelet/kata-direct-vol-002/directvol002 "{\"device\": \"/tmp/stor/rawdisk01.20g\", \"volume_type\": \"directvol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}"
+$# /kubelet/kata-direct-vol-002/directvol002 <==> /run/kata-containers/shared/direct-volumes/W1lMa2F0ZXQva2F0YS10a2F0DAxvbC0wMDEvdm9sdW1lMDAx
+$ cat W1lMa2F0ZXQva2F0YS10a2F0DAxvbC0wMDEvdm9sdW1lMDAx/mountInfo.json 
+{"volume_type":"directvol","device":"/tmp/stor/rawdisk01.20g","fs_type":"ext4","metadata":{},"options":[]}
+```
+
+#### Run a Kata container with direct block device volume
+
+```bash
+$ # type=disrectvol,src=/kubelet/kata-direct-vol-002/directvol002,dst=/disk002,options=rbind:rw
+$ sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=directvol,src=/kubelet/kata-direct-vol-002/directvol002,dst=/disk002,options=rbind:rw "$image" kata-direct-vol-xx05302045 /bin/bash
+```
+
+
+### VFIO Device Based Block Volume
+
+#### create VFIO device based backend storage
+
+> **Tip:** It only supports `vfio-pci` based PCI device passthrough mode.
+
+In this scenario, the device's host kernel driver will be replaced by `vfio-pci`, and IOMMU group ID generated.
+And either device's BDF or its VFIO IOMMU group ID in `/dev/vfio/` is fine for "device" in `mountinfo.json`.
+
+```bash
+$ lspci -nn -k -s 45:00.1 
+45:00.1 SCSI storage controller 
+...
+Kernel driver in use: vfio-pci
+...
+$ ls /dev/vfio/110
+/dev/vfio/110
+$ ls /sys/kernel/iommu_groups/110/devices/
+0000:45:00.1
+```
+
+#### setup VFIO device for kata-containers
+
+First, configure the `mountinfo.json`, as below:
+
+- (1) device with `BB:DD:F`
+
+```json
+{
+  "device": "45:00.1",
+  "volume_type": "vfiovol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+- (2) device with `DDDD:BB:DD:F`
+
+```json
+{
+  "device": "0000:45:00.1",
+  "volume_type": "vfiovol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+- (3) device with `/dev/vfio/X`
+
+```json
+{
+  "device": "/dev/vfio/110", 
+  "volume_type": "vfiovol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+Second, run kata-containers with device(`/dev/vfio/110`) as an example:
+
+```bash
+$ sudo kata-ctl direct-volume add /kubelet/kata-vfio-vol-003/vfiovol003 "{\"device\": \"/dev/vfio/110\", \"volume_type\": \"vfiovol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}"
+$ # /kubelet/kata-vfio-vol-003/directvol003 <==> /run/kata-containers/shared/direct-volumes/F0va22F0ZvaS12F0YS10a2F0DAxvbC0F0ZXvdm9sdF0Z0YSx
+$ cat F0va22F0ZvaS12F0YS10a2F0DAxvbC0F0ZXvdm9sdF0Z0YSx/mountInfo.json 
+{"volume_type":"vfiovol","device":"/dev/vfio/110","fs_type":"ext4","metadata":{},"options":[]}
+```
+
+#### Run a Kata container with VFIO block device based volume
+
+```bash
+$ # type=disrectvol,src=/kubelet/kata-vfio-vol-003/vfiovol003,dst=/disk003,options=rbind:rw
+$ sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=vfiovol,src=/kubelet/kata-vfio-vol-003/vfiovol003,dst=/disk003,options=rbind:rw "$image" kata-vfio-vol-xx05302245 /bin/bash
+```
+
+
+### SPDK Device Based Block Volume
+
+SPDK vhost-user devices in runtime-rs, unlike runtime (golang version), there is no need to `mknod` device node under `/dev/` any more.
+Just using the `kata-ctl direct-volume add ..` to make a mount info config is enough.
+
+#### Run SPDK vhost target and Expose vhost block device
+
+Run a SPDK vhost target and get vhost-user block controller as an example:
+
+First, run SPDK vhost target:
+
+> **Tips:** If driver `vfio-pci` supported, you can run SPDK with `DRIVER_OVERRIDE=vfio-pci`
+> Otherwise, Just run without it `sudo HUGEMEM=4096 ./scripts/setup.sh`.
+
+```bash
+$ SPDK_DEVEL=/xx/spdk
+$ VHU_UDS_PATH=/tmp/vhu-targets
+$ RAW_DISKS=/xx/rawdisks
+$ # Reset first
+$ ${SPDK_DEVEL}/scripts/setup.sh reset
+$ sudo sysctl -w vm.nr_hugepages=2048
+$ #4G Huge Memory for spdk
+$ sudo HUGEMEM=4096 DRIVER_OVERRIDE=vfio-pci ${SPDK_DEVEL}/scripts/setup.sh
+$ sudo ${SPDK_DEVEL}/build/bin/spdk_tgt -S $VHU_UDS_PATH -s 1024 -m 0x3 &
+```
+
+Second, create a vhost controller:
+
+```bash
+$ sudo dd if=/dev/zero of=${RAW_DISKS}/rawdisk01.20g bs=1M count=20480
+$ sudo ${SPDK_DEVEL}/scripts/rpc.py bdev_aio_create ${RAW_DISKS}/rawdisk01.20g vhu-rawdisk01.20g 512
+$ sudo ${SPDK_DEVEL}/scripts/rpc.py vhost_create_blk_controller vhost-blk-rawdisk01.sock vhu-rawdisk01.20g
+```
+
+Here, a vhost controller `vhost-blk-rawdisk01.sock` is created, and the controller will
+be passed to Hypervisor, such as Dragonball, Cloud-Hypervisor, Firecracker or QEMU.
+
+
+#### setup vhost-user block device for kata-containers
+
+
+First, `mkdir` a sub-path `kubelet/kata-test-vol-001/` under `/run/kata-containers/shared/direct-volumes/`.
+
+Second, fill fields in `mountinfo.json`, it looks like as below:
+```json
+{
+  "device": "/tmp/vhu-targets/vhost-blk-rawdisk01.sock",
+  "volume_type": "spdkvol",
+  "fs_type": "ext4",
+  "metadata":"{}",
+  "options": []
+}
+```
+
+Third, with the help of `kata-ctl direct-volume` to add block device to generate `mountinfo.json`, and run a kata container with `--mount`.
+
+```bash
+$ # kata-ctl direct-volume add
+$ sudo kata-ctl direct-volume add /kubelet/kata-test-vol-001/volume001 "{\"device\": \"/tmp/vhu-targets/vhost-blk-rawdisk01.sock\", \"volume_type\":\"spdkvol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}"
+$ # /kubelet/kata-test-vol-001/volume001 <==> /run/kata-containers/shared/direct-volumes/L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx
+$ cat L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx/mountInfo.json
+$ {"volume_type":"spdkvol","device":"/tmp/vhu-targets/vhost-blk-rawdisk01.sock","fs_type":"ext4","metadata":{},"options":[]}
+```
+
+As `/run/kata-containers/shared/direct-volumes/` is a fixed path , we will be able to run a kata pod with `--mount` and set
+`src` sub-path. And the `--mount` argument looks like: `--mount type=spdkvol,src=/kubelet/kata-test-vol-001/volume001,dst=/disk001`.
+
+
+#### Run a Kata container with SPDK vhost-user block device
+
+
+In the case, `ctr run --mount type=X, src=source, dst=dest`, the X will be set `spdkvol` which is a proprietary type specifically designed for SPDK volumes.
+
+```bash
+$ # ctr run with --mount type=spdkvol,src=/kubelet/kata-test-vol-001/volume001,dst=/disk001
+$ sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=spdkvol,src=/kubelet/kata-test-vol-001/volume001,dst=/disk001,options=rbind:rw "$image" kata-spdk-vol-xx0530 /bin/bash
+```

docs/how-to/how-to-run-rootless-vmm.md

@@ -1,5 +1,5 @@
 ## Introduction
-To improve security, Kata Container supports running the VMM process (currently only QEMU) as a non-`root` user.
+To improve security, Kata Container supports running the VMM process (QEMU and cloud-hypervisor) as a non-`root` user. 
 This document describes how to enable the rootless VMM mode and its limitations.
 
 ## Pre-requisites
@@ -27,7 +27,7 @@ Another necessary change is to move the hypervisor runtime files (e.g. `vhost-fs
 ## Limitations
 
 1. Only the VMM process is running as a non-root user. Other processes such as Kata Container shimv2 and `virtiofsd` still run as the root user.
-2. Currently, this feature is only supported in QEMU. Still need to bring it to Firecracker and Cloud Hypervisor (see https://github.com/kata-containers/kata-containers/issues/2567).
+2. Currently, this feature is only supported in QEMU and cloud-hypervisor. For firecracker, you can use jailer to run the VMM process with a non-root user.
 3. Certain features will not work when rootless VMM is enabled, including:
    1. Passing devices to the guest (`virtio-blk`, `virtio-scsi`) will not work if the non-privileged user does not have permission to access it (leading to a permission denied error). A more permissive permission (e.g. 666) may overcome this issue. However, you need to be aware of the potential security implications of reducing the security on such devices.
    2. `vfio` device will also not work because of permission denied error.

docs/install/README.md

@@ -19,7 +19,6 @@ Packaged installation methods uses your distribution's native package format (su
 |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|
 | [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | 
 | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users.                                                                   |
-| [Using snap](#snap-installation)                     | Easy to install                                                                              | yes               | Good alternative to official distro packages.                                                 |
 | [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 |
 | [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system                                      | **No!**           | For those who want the latest release with more control.                                      |
 | [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.                                                              |
@@ -42,12 +41,6 @@ Kata packages are provided by official distribution repositories for:
 | [CentOS](centos-installation-guide.md)                   | 8                                                                              |
 | [Fedora](fedora-installation-guide.md)                   | 34                                                                             |
 
-### Snap Installation
-
-The snap installation is available for all distributions which support `snapd`.
-
-[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io.
-
 ### Automatic Installation
 
 [Use `kata-manager`](/utils/README.md) to automatically install a working Kata Containers system.

docs/install/kata-containers-3.0-rust-runtime-installation-guide.md

@@ -26,7 +26,6 @@ architectures:
 |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|----------- |
 | [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | Yes |
 | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users. | No |                                                                   
-| [Using snap](#snap-installation)                     | Easy to install                                                                              | yes               | Good alternative to official distro packages.                                                 | No |
 | [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 | No |
 | [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system                                      | **No!**           | For those who want the latest release with more control.                                      | No |
 | [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.  | Yes |              
@@ -36,8 +35,6 @@ architectures:
 Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md).
 ### Official packages
 `ToDo`
-### Snap Installation
-`ToDo`
 ### Automatic Installation
 `ToDo`
 ### Manual Installation
@@ -49,14 +46,14 @@ Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md).
 
 * Download `Rustup` and install  `Rust`
     > **Notes:**
-    > Rust version 1.62.0 is needed
+    > For Rust version, please set `RUST_VERSION` to the value of `languages.rust.meta.newest-version key` in [`versions.yaml`](../../versions.yaml) or, if `yq` is available on your system, run `export RUST_VERSION=$(yq read versions.yaml languages.rust.meta.newest-version)`.
 
     Example for `x86_64`
     ```
     $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
     $ source $HOME/.cargo/env
-    $ rustup install 1.62.0
-    $ rustup default 1.62.0-x86_64-unknown-linux-gnu
+    $ rustup install ${RUST_VERSION}
+    $ rustup default ${RUST_VERSION}-x86_64-unknown-linux-gnu
     ```
 
 * Musl support for fully static binary

docs/install/snap-installation-guide.md

@@ -1,52 +0,0 @@
-# Kata Containers snap package
-
-## Install Kata Containers
-
-Kata Containers can be installed in any Linux distribution that supports
-[snapd](https://docs.snapcraft.io/installing-snapd).
-
-Run the following command to install **Kata Containers**:
-
-```sh
-$ sudo snap install kata-containers --stable --classic
-```
-
-## Configure Kata Containers
-
-By default Kata Containers snap image is mounted at `/snap/kata-containers` as a
-read-only file system, therefore default configuration file can not be edited.
-Fortunately Kata Containers supports loading a configuration file from another
-path than the default.
-
-```sh
-$ sudo mkdir -p /etc/kata-containers
-$ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/
-$ $EDITOR /etc/kata-containers/configuration.toml
-```
-
-## Integration with shim v2 Container Engines
-
-The Container engine daemon (`cri-o`, `containerd`, etc) needs to be able to find the
-`containerd-shim-kata-v2` binary to allow Kata Containers to be created.
-Run the following command to create a symbolic link to the shim v2 binary.
-
-```sh
-$ sudo ln -sf /snap/kata-containers/current/usr/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2
-```
-
-Once the symbolic link has been created and the engine daemon configured, `io.containerd.kata.v2`
-can be used as runtime.
-
-Read the following documents to know how to run Kata Containers 2.x with `containerd`.
-
-* [How to use Kata Containers and Containerd](../how-to/containerd-kata.md)
-* [Install Kata Containers with containerd](./container-manager/containerd/containerd-install.md)
-
-
-## Remove Kata Containers snap package
-
-Run the following command to remove the Kata Containers snap:
-
-```sh
-$ sudo snap remove kata-containers
-```

snap/local/README.md

@@ -1,101 +0,0 @@
-# Kata Containers snap image
-
-This directory contains the resources needed to build the Kata Containers
-[snap][1] image.
-
-## Initial setup
-
-Kata Containers can be installed in any Linux distribution that supports
-[snapd](https://docs.snapcraft.io/installing-snapd). For this example, we
-assume Ubuntu as your base distro.
-```sh
-$ sudo apt-get --no-install-recommends install -y apt-utils ca-certificates snapd snapcraft
-```
-
-## Install snap
-
-You can install the Kata Containers snap from the [snapcraft store][8] or by running the following command:
-
-```sh
-$ sudo snap install kata-containers --classic
-```
-
-## Build and install snap image
-
-Run the command below which will use the packaging Makefile to build the snap image:
-
-```sh
-$ make -C tools/packaging snap
-```
-
-> **Warning:**
->
-> By default, `snapcraft` will create a clean virtual machine
-> environment to build the snap in using the `multipass` tool.
->
-> However, `multipass` is silently disabled when `--destructive-mode` is
-> used.
->
-> Since building the Kata Containers package currently requires
-> `--destructive-mode`, the snap will be built using the host
-> environment. To avoid parts of the build auto-detecting additional
-> features to enable (for example for QEMU), we recommend that you
-> only run the snap build in a minimal host environment.
-
-To install the resulting snap image, snap must be put in [classic mode][3] and the
-security confinement must be disabled (`--classic`). Also since the resulting snap
-has not been signed the verification of signature must be omitted (`--dangerous`).
-
-```sh
-$ sudo snap install --classic --dangerous "kata-containers_${version}_${arch}.snap"
-```
-
-Replace `${version}` with the current version of Kata Containers and `${arch}` with
-the system architecture.
-
-## Configure Kata Containers
-
-By default Kata Containers snap image is mounted at `/snap/kata-containers` as a
-read-only file system, therefore default configuration file can not be edited.
-Fortunately [`kata-runtime`][4] supports loading a configuration file from another
-path than the default.
-
-```sh
-$ sudo mkdir -p /etc/kata-containers
-$ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/
-$ $EDITOR /etc/kata-containers/configuration.toml
-```
-
-## Integration with docker and Kubernetes
-
-The path to the runtime provided by the Kata Containers snap image is
-`/snap/kata-containers/current/usr/bin/kata-runtime`. You should use it to
-run Kata Containers with [docker][9] and [Kubernetes][10].
-
-## Remove snap
-
-You can remove the Kata Containers snap by running the following command:
-
-```sh
-$ sudo snap remove kata-containers
-```
-
-## Limitations
-
-The [miniOS image][2] is not included in the snap image as it is not possible for
-QEMU to open a guest RAM backing store on a read-only filesystem. Fortunately,
-you can start Kata Containers with a Linux initial RAM disk (initrd) that is
-included in the snap image. If you want to use the miniOS image instead of initrd,
-then a new configuration file can be [created](#configure-kata-containers)
-and [configured][7].
-
-[1]: https://docs.snapcraft.io/snaps/intro
-[2]: ../../docs/design/architecture/README.md#root-filesystem-image
-[3]: https://docs.snapcraft.io/reference/confinement#classic
-[4]: https://github.com/kata-containers/kata-containers/tree/main/src/runtime#configuration
-[5]: https://docs.docker.com/engine/reference/commandline/dockerd
-[6]: ../../docs/install/docker/ubuntu-docker-install.md
-[7]: ../../docs/Developer-Guide.md#configure-to-use-initrd-or-rootfs-image
-[8]: https://snapcraft.io/kata-containers
-[9]: ../../docs/Developer-Guide.md#run-kata-containers-with-docker
-[10]: ../../docs/Developer-Guide.md#run-kata-containers-with-kubernetes

snap/local/snap-common.sh

@@ -1,114 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2022 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-# Description: Idempotent script to be sourced by all parts in a
-#   snapcraft config file.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# XXX: Bash-specific code. zsh doesn't support this option and that *does*
-# matter if this script is run sourced... since it'll be using zsh! ;)
-[ -n "$BASH_VERSION" ] && set -o errtrace
-
-[ -n "${DEBUG:-}" ] && set -o xtrace
-
-die()
-{
-	echo >&2 "ERROR: $0: $*"
-}
-
-[ -n "${SNAPCRAFT_STAGE:-}" ] ||\
-	die "must be sourced from a snapcraft config file"
-
-snap_yq_version=3.4.1
-
-snap_common_install_yq()
-{
-      export yq="${SNAPCRAFT_STAGE}/bin/yq"
-
-      local yq_pkg
-      yq_pkg="github.com/mikefarah/yq"
-
-      local yq_url
-      yq_url="https://${yq_pkg}/releases/download/${snap_yq_version}/yq_${goos}_${goarch}"
-      curl -o "${yq}" -L "${yq_url}"
-      chmod +x "${yq}"
-}
-
-# Function that should be called for each snap "part" in
-# snapcraft.yaml.
-snap_common_main()
-{
-	# Architecture
-	arch="$(uname -m)"
-
-	case "${arch}" in
-		aarch64)
-			goarch="arm64"
-			qemu_arch="${arch}"
-			;;
-
-		ppc64le)
-			goarch="ppc64le"
-			qemu_arch="ppc64"
-			;;
-
-		s390x)
-			goarch="${arch}"
-			qemu_arch="${arch}"
-			;;
-
-		x86_64)
-			goarch="amd64"
-			qemu_arch="${arch}"
-			;;
-
-		*) die "unsupported architecture: ${arch}" ;;
-	esac
-
-	dpkg_arch=$(dpkg --print-architecture)
-
-	# golang
-	#
-	# We need the O/S name in golang format, but since we don't
-	# know if the godeps part has run, we don't know if golang is
-	# available yet, hence fall back to a standard system command.
-	goos="$(go env GOOS &>/dev/null || true)"
-	[ -z "$goos" ] && goos=$(uname -s|tr '[A-Z]' '[a-z]')
-
-	export GOROOT="${SNAPCRAFT_STAGE}"
-	export GOPATH="${GOROOT}/gopath"
-	export GO111MODULE="auto"
-
-	mkdir -p "${GOPATH}/bin"
-	export PATH="${GOPATH}/bin:${PATH}"
-
-	# Proxy
-	export http_proxy="${http_proxy:-}"
-	export https_proxy="${https_proxy:-}"
-
-	# Binaries
-	mkdir -p "${SNAPCRAFT_STAGE}/bin"
-
-	export PATH="$PATH:${SNAPCRAFT_STAGE}/bin"
-
-	# YAML query tool
-	export yq="${SNAPCRAFT_STAGE}/bin/yq"
-
-	# Kata paths
-	export kata_dir=$(printf "%s/src/github.com/%s/%s" \
-		"${GOPATH}" \
-		"${SNAPCRAFT_PROJECT_NAME}" \
-		"${SNAPCRAFT_PROJECT_NAME}")
-
-	export versions_file="${kata_dir}/versions.yaml"
-
-	[ -n "${yq:-}" ] && [ -x "${yq:-}" ] || snap_common_install_yq
-}
-
-snap_common_main

snap/snapcraft.yaml

@@ -1,371 +0,0 @@
-name: kata-containers
-website: https://github.com/kata-containers/kata-containers
-summary: Build lightweight VMs that seamlessly plug into the containers ecosystem
-description: |
-  Kata Containers is an open source project and community working to build a
-  standard implementation of lightweight Virtual Machines (VMs) that feel and
-  perform like containers, but provide the workload isolation and security
-  advantages of VMs
-confinement: classic
-adopt-info: metadata
-base: core20
-
-parts:
-  metadata:
-    plugin: nil
-    prime:
-      - -*
-    build-packages:
-      - git
-      - git-extras
-    override-pull: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      version="9999"
-
-      if echo "${GITHUB_REF:-}" | grep -q -E "^refs/tags"; then
-        version=$(echo ${GITHUB_REF:-} | cut -d/ -f3)
-        git checkout ${version}
-      fi
-
-      snapcraftctl set-grade "stable"
-      snapcraftctl set-version "${version}"
-
-      mkdir -p $(dirname ${kata_dir})
-      ln -sf $(realpath "${SNAPCRAFT_STAGE}/..") ${kata_dir}
-
-  godeps:
-    after: [metadata]
-    plugin: nil
-    prime:
-      - -*
-    build-packages:
-      - curl
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      # put everything in stage
-      cd "${SNAPCRAFT_STAGE}"
-
-      version="$(${yq} r ${kata_dir}/versions.yaml languages.golang.meta.newest-version)"
-      tarfile="go${version}.${goos}-${goarch}.tar.gz"
-      curl -LO https://golang.org/dl/${tarfile}
-      tar -xf ${tarfile} --strip-components=1
-
-  rustdeps:
-    after: [metadata]
-    plugin: nil
-    prime:
-      - -*
-    build-packages:
-      - curl
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      # put everything in stage
-      cd "${SNAPCRAFT_STAGE}"
-
-      version="$(${yq} r ${kata_dir}/versions.yaml languages.rust.meta.newest-version)"
-      if ! command -v rustup > /dev/null; then
-        curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${version}
-      fi
-
-      export PATH=${PATH}:${HOME}/.cargo/bin
-      rustup toolchain install ${version}
-      rustup default ${version}
-      if [ "${arch}" == "ppc64le" ] || [ "${arch}" == "s390x" ] ; then
-        [ "${arch}" == "ppc64le" ] && arch="powerpc64le"
-        rustup target add ${arch}-unknown-linux-gnu
-      else
-        rustup target add ${arch}-unknown-linux-musl
-        $([ "$(whoami)" != "root" ] && echo sudo) ln -sf /usr/bin/g++ /bin/musl-g++
-      fi
-      rustup component add rustfmt
-
-  docker:
-    after: [metadata]
-    plugin: nil
-    prime:
-      - -*
-    build-packages:
-      - ca-certificates
-      - containerd
-      - curl
-      - gnupg
-      - lsb-release
-      - runc
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\
-          sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-      distro_codename=$(lsb_release -cs)
-      echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\
-          sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-      sudo apt-get -y update
-      sudo apt-get -y install docker-ce docker-ce-cli containerd.io
-
-      echo "Unmasking docker service"
-      sudo -E systemctl unmask docker.service || true
-      sudo -E systemctl unmask docker.socket || true
-      echo "Adding $USER into docker group"
-      sudo -E gpasswd -a $USER docker
-      echo "Starting docker"
-      sudo -E systemctl start docker || true
-
-  image:
-    after: [godeps, docker, qemu, kernel]
-    plugin: nil
-    build-packages:
-      - docker.io
-      - cpio
-      - git
-      - iptables
-      - software-properties-common
-      - uidmap
-      - gnupg2
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      [ "${arch}" = "ppc64le" ] || [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler
-
-      if [ -n "$http_proxy" ]; then
-        echo "Setting proxy $http_proxy"
-        sudo -E systemctl set-environment http_proxy="$http_proxy" || true
-        sudo -E systemctl set-environment https_proxy="$https_proxy" || true
-      fi
-
-      # Copy yq binary. It's used in the container
-      cp -a "${yq}" "${GOPATH}/bin/"
-
-      cd "${kata_dir}/tools/osbuilder"
-
-      # build image
-      export AGENT_INIT=yes
-      export USE_DOCKER=1
-      export DEBUG=1
-      initrd_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.initrd.architecture.${arch}.name)
-      image_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.image.architecture.${arch}.name)
-      case "$arch" in
-        x86_64)
-          # In some build systems it's impossible to build a rootfs image, try with the initrd image
-          sudo -E PATH=$PATH make image DISTRO="${image_distro}" || sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
-        ;;
-
-        aarch64|ppc64le|s390x)
-          sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
-        ;;
-
-        *) die "unsupported architecture: ${arch}" ;;
-      esac
-
-      # Install image
-      kata_image_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
-      mkdir -p "${kata_image_dir}"
-      cp kata-containers*.img "${kata_image_dir}"
-
-  runtime:
-    after: [godeps, image, cloud-hypervisor]
-    plugin: nil
-    build-attributes: [no-patchelf]
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      cd "${kata_dir}/src/runtime"
-
-      qemu_cmd="qemu-system-${qemu_arch}"
-
-      # build and install runtime
-      make \
-        PREFIX="/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr" \
-        SKIP_GO_VERSION_CHECK=1 \
-        QEMUCMD="${qemu_cmd}"
-
-      make install \
-        PREFIX=/usr \
-        DESTDIR="${SNAPCRAFT_PART_INSTALL}" \
-        SKIP_GO_VERSION_CHECK=1 \
-        QEMUCMD="${qemu_cmd}"
-
-      if [ ! -f ${SNAPCRAFT_PART_INSTALL}/../../image/install/usr/share/kata-containers/kata-containers.img ]; then
-        sed -i -e "s|^image =.*|initrd = \"/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr/share/kata-containers/kata-containers-initrd.img\"|" \
-          ${SNAPCRAFT_PART_INSTALL}/usr/share/defaults/${SNAPCRAFT_PROJECT_NAME}/configuration.toml
-      fi
-
-  kernel:
-    after: [godeps]
-    plugin: nil
-    build-packages:
-      - libelf-dev
-      - curl
-      - build-essential
-      - bison
-      - flex
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      kernel_version="$(${yq} r $versions_file assets.kernel.version)"
-      #Remove extra 'v'
-      kernel_version="${kernel_version#v}"
-
-      [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
-
-      cd "${kata_dir}/tools/packaging/kernel"
-      kernel_dir_prefix="kata-linux-"
-
-      # Setup and build kernel
-      ./build-kernel.sh -v "${kernel_version}" -d setup
-      cd ${kernel_dir_prefix}*
-      make -j $(nproc ${CI:+--ignore 1}) EXTRAVERSION=".container"
-
-      kernel_suffix="${kernel_version}.container"
-      kata_kernel_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
-      mkdir -p "${kata_kernel_dir}"
-
-      # Install bz kernel
-      make install INSTALL_PATH="${kata_kernel_dir}" EXTRAVERSION=".container" || true
-      vmlinuz_name="vmlinuz-${kernel_suffix}"
-      ln -sf "${vmlinuz_name}" "${kata_kernel_dir}/vmlinuz.container"
-
-      # Install raw kernel
-      vmlinux_path="vmlinux"
-      [ "${arch}" = "s390x" ] && vmlinux_path="arch/s390/boot/vmlinux"
-      vmlinux_name="vmlinux-${kernel_suffix}"
-      cp "${vmlinux_path}" "${kata_kernel_dir}/${vmlinux_name}"
-      ln -sf "${vmlinux_name}" "${kata_kernel_dir}/vmlinux.container"
-
-  qemu:
-    plugin: make
-    after: [godeps]
-    build-packages:
-      - gcc
-      - python3
-      - zlib1g-dev
-      - libcap-ng-dev
-      - libglib2.0-dev
-      - libpixman-1-dev
-      - libnuma-dev
-      - libltdl-dev
-      - libcap-dev
-      - libattr1-dev
-      - libfdt-dev
-      - curl
-      - libcapstone-dev
-      - bc
-      - libblkid-dev
-      - libffi-dev
-      - libmount-dev
-      - libseccomp-dev
-      - libselinux1-dev
-      - ninja-build
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
-      url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
-      commit=""
-      patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
-      patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
-
-      # download source
-      qemu_dir="${SNAPCRAFT_STAGE}/qemu"
-      rm -rf "${qemu_dir}"
-      git clone --depth 1 --branch ${branch} --single-branch ${url} "${qemu_dir}"
-      cd "${qemu_dir}"
-      [ -z "${commit}" ] || git checkout "${commit}"
-
-      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone --depth 1 https://github.com/qemu/keycodemapdb ui/keycodemapdb/
-      [ -n "$(ls -A capstone)" ] || git clone --depth 1 https://github.com/qemu/capstone capstone
-
-      # Apply branch patches
-      [ -d "${patches_version_dir}" ] || mkdir "${patches_version_dir}"
-      ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_dir}"
-      ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_version_dir}"
-
-      # Only x86_64 supports libpmem
-      [ "${arch}" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev
-
-      configure_hypervisor="${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh"
-      chmod +x "${configure_hypervisor}"
-      # static build. The --prefix, --libdir, --libexecdir, --datadir arguments are
-      # based on PREFIX and set by configure-hypervisor.sh
-      echo "$(PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr ${configure_hypervisor} -s kata-qemu) \
-        --disable-rbd " \
-        | xargs ./configure
-
-      # Copy QEMU configurations (Kconfigs)
-      case "${branch}" in
-      "v5.1.0")
-        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* default-configs
-        ;;
-
-      *)
-        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* configs/devices/
-        ;;
-      esac
-
-      # build and install
-      make -j $(nproc ${CI:+--ignore 1})
-      make install DESTDIR="${SNAPCRAFT_PART_INSTALL}"
-    prime:
-      - -snap/
-      - -usr/bin/qemu-ga
-      - -usr/bin/qemu-pr-helper
-      - -usr/bin/virtfs-proxy-helper
-      - -usr/include/
-      - -usr/share/applications/
-      - -usr/share/icons/
-      - -usr/var/
-      - usr/*
-      - lib/*
-    organize:
-      # Hack: move qemu to /
-      "snap/kata-containers/current/": "./"
-
-  virtiofsd:
-    plugin: nil
-    after: [godeps, rustdeps, docker]
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      echo "INFO: Building rust version of virtiofsd"
-
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      # Clean-up build dir in case it already exists
-      sudo -E NO_TTY=true make virtiofsd-tarball
-
-      sudo install \
-        --owner='root' \
-        --group='root' \
-        --mode=0755 \
-        -D \
-        --target-directory="${SNAPCRAFT_PART_INSTALL}/usr/libexec/" \
-        build/virtiofsd/builddir/virtiofsd/virtiofsd
-
-  cloud-hypervisor:
-    plugin: nil
-    after: [godeps, docker]
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      if [ "${arch}" == "aarch64" ] || [ "${arch}" == "x86_64" ]; then
-          cd "${SNAPCRAFT_PROJECT_DIR}"
-          sudo -E NO_TTY=true make cloud-hypervisor-tarball
-
-          tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-cloud-hypervisor.tar.xz"
-          tmpdir=$(mktemp -d)
-
-          tar -xvJpf "${tarfile}" -C "${tmpdir}"
-
-          install -D "${tmpdir}/opt/kata/bin/cloud-hypervisor" "${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor"
-
-          rm -rf "${tmpdir}"
-      fi
-
-apps:
-  runtime:
-    command: usr/bin/kata-runtime
-  shim:
-    command: usr/bin/containerd-shim-kata-v2
-  collect-data:
-    command: usr/bin/kata-collect-data.sh

src/agent/Cargo.toml

@@ -10,8 +10,8 @@ oci = { path = "../libs/oci" }
 rustjail = { path = "rustjail" }
 protocols = { path = "../libs/protocols", features = ["async"] }
 lazy_static = "1.3.0"
-ttrpc = { version = "0.6.0", features = ["async"], default-features = false }
-protobuf = "2.27.0"
+ttrpc = { version = "0.7.1", features = ["async"], default-features = false }
+protobuf = "3.2.0"
 libc = "0.2.58"
 nix = "0.24.2"
 capctl = "0.2.0"
@@ -28,10 +28,10 @@ url = "2.2.2"
 # Async helpers
 async-trait = "0.1.42"
 async-recursion = "0.3.2"
-futures = "0.3.17"
+futures = "0.3.28"
 
 # Async runtime
-tokio = { version = "1.21.2", features = ["full"] }
+tokio = { version = "1.28.1", features = ["full"] }
 tokio-vsock = "0.3.1"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
@@ -71,11 +71,7 @@ clap = { version = "3.0.1", features = ["derive"] }
 openssl = { version = "0.10.38", features = ["vendored"] }
 
 # Image pull/decrypt
-[target.'cfg(target_arch = "s390x")'.dependencies]
-image-rs = { git = "https://github.com/confidential-containers/image-rs", tag = "v0.5.1", default-features = false, features = ["kata-cc-s390x"] }
-
-[target.'cfg(not(target_arch = "s390x"))'.dependencies]
-image-rs = { git = "https://github.com/confidential-containers/image-rs", tag = "v0.5.1", default-features = false, features = ["kata-cc"] }
+image-rs = { git = "https://github.com/confidential-containers/guest-components", tag = "v0.7.0", default-features = false, features = ["kata-cc-native-tls"] }
 
 [patch.crates-io]
 oci-distribution = { git = "https://github.com/krustlet/oci-distribution.git", rev = "f44124c" }

src/agent/Makefile

@@ -33,6 +33,12 @@ ifeq ($(SECCOMP),yes)
     override EXTRA_RUSTFEATURES += seccomp
 endif
 
+include ../../utils.mk
+
+ifeq ($(ARCH), ppc64le)
+    override ARCH = powerpc64le
+endif
+
 ##VAR STANDARD_OCI_RUNTIME=yes|no define if agent enables standard oci runtime feature
 STANDARD_OCI_RUNTIME := no
 
@@ -45,8 +51,6 @@ ifneq ($(EXTRA_RUSTFEATURES),)
     override EXTRA_RUSTFEATURES := --features "$(EXTRA_RUSTFEATURES)"
 endif
 
-include ../../utils.mk
-
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
 ##VAR DESTDIR=<path> is a directory prepended to each installed target file

src/agent/rustjail/Cargo.toml

@@ -18,7 +18,7 @@ scopeguard = "1.0.0"
 capctl = "0.2.0"
 lazy_static = "1.3.0"
 libc = "0.2.58"
-protobuf = "2.27.0"
+protobuf = "3.2.0"
 slog = "2.5.2"
 slog-scope = "4.1.2"
 scan_fmt = "0.2.6"
@@ -29,7 +29,7 @@ cgroups = { package = "cgroups-rs", version = "0.3.2" }
 rlimit = "0.5.3"
 cfg-if = "0.1.0"
 
-tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
+tokio = { version = "1.28.1", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
 futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -27,7 +27,7 @@ use oci::{
     LinuxNetwork, LinuxPids, LinuxResources,
 };
 
-use protobuf::{CachedSize, RepeatedField, SingularPtrField, UnknownFields};
+use protobuf::MessageField;
 use protocols::agent::{
     BlkioStats, BlkioStatsEntry, CgroupStats, CpuStats, CpuUsage, HugetlbStats, MemoryData,
     MemoryStats, PidsStats, ThrottlingData,
@@ -39,18 +39,16 @@ use std::path::Path;
 
 const GUEST_CPUS_PATH: &str = "/sys/devices/system/cpu/online";
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger().new(o!("subsystem" => "cgroups"))
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "cgroups"))
 }
 
 macro_rules! get_controller_or_return_singular_none {
     ($cg:ident) => {
         match $cg.controller_of() {
             Some(c) => c,
-            None => return SingularPtrField::none(),
+            None => return MessageField::none(),
         }
     };
 }
@@ -82,7 +80,7 @@ impl CgroupManager for Manager {
 
     fn set(&self, r: &LinuxResources, update: bool) -> Result<()> {
         info!(
-            sl!(),
+            sl(),
             "cgroup manager set resources for container. Resources input {:?}", r
         );
 
@@ -120,7 +118,7 @@ impl CgroupManager for Manager {
 
         // set devices resources
         set_devices_resources(&self.cgroup, &r.devices, res);
-        info!(sl!(), "resources after processed {:?}", res);
+        info!(sl(), "resources after processed {:?}", res);
 
         // apply resources
         self.cgroup.apply(res)?;
@@ -134,11 +132,10 @@ impl CgroupManager for Manager {
 
         let throttling_data = get_cpu_stats(&self.cgroup);
 
-        let cpu_stats = SingularPtrField::some(CpuStats {
+        let cpu_stats = MessageField::some(CpuStats {
             cpu_usage,
             throttling_data,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
+            ..Default::default()
         });
 
         // Memorystats
@@ -160,8 +157,7 @@ impl CgroupManager for Manager {
             pids_stats,
             blkio_stats,
             hugetlb_stats,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
+            ..Default::default()
         })
     }
 
@@ -199,7 +195,7 @@ impl CgroupManager for Manager {
         if guest_cpuset.is_empty() {
             return Ok(());
         }
-        info!(sl!(), "update_cpuset_path to: {}", guest_cpuset);
+        info!(sl(), "update_cpuset_path to: {}", guest_cpuset);
 
         let h = cgroups::hierarchies::auto();
         let root_cg = h.root_control_group();
@@ -207,12 +203,12 @@ impl CgroupManager for Manager {
         let root_cpuset_controller: &CpuSetController = root_cg.controller_of().unwrap();
         let path = root_cpuset_controller.path();
         let root_path = Path::new(path);
-        info!(sl!(), "root cpuset path: {:?}", &path);
+        info!(sl(), "root cpuset path: {:?}", &path);
 
         let container_cpuset_controller: &CpuSetController = self.cgroup.controller_of().unwrap();
         let path = container_cpuset_controller.path();
         let container_path = Path::new(path);
-        info!(sl!(), "container cpuset path: {:?}", &path);
+        info!(sl(), "container cpuset path: {:?}", &path);
 
         let mut paths = vec![];
         for ancestor in container_path.ancestors() {
@@ -221,7 +217,7 @@ impl CgroupManager for Manager {
             }
             paths.push(ancestor);
         }
-        info!(sl!(), "parent paths to update cpuset: {:?}", &paths);
+        info!(sl(), "parent paths to update cpuset: {:?}", &paths);
 
         let mut i = paths.len();
         loop {
@@ -235,7 +231,7 @@ impl CgroupManager for Manager {
                 .to_str()
                 .unwrap()
                 .trim_start_matches(root_path.to_str().unwrap());
-            info!(sl!(), "updating cpuset for parent path {:?}", &r_path);
+            info!(sl(), "updating cpuset for parent path {:?}", &r_path);
             let cg = new_cgroup(cgroups::hierarchies::auto(), r_path)?;
             let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
             cpuset_controller.set_cpus(guest_cpuset)?;
@@ -243,7 +239,7 @@ impl CgroupManager for Manager {
 
         if !container_cpuset.is_empty() {
             info!(
-                sl!(),
+                sl(),
                 "updating cpuset for container path: {:?} cpuset: {}",
                 &container_path,
                 container_cpuset
@@ -278,7 +274,7 @@ fn set_network_resources(
     network: &LinuxNetwork,
     res: &mut cgroups::Resources,
 ) {
-    info!(sl!(), "cgroup manager set network");
+    info!(sl(), "cgroup manager set network");
 
     // set classid
     // description can be found at https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/net_cls.html
@@ -305,7 +301,7 @@ fn set_devices_resources(
     device_resources: &[LinuxDeviceCgroup],
     res: &mut cgroups::Resources,
 ) {
-    info!(sl!(), "cgroup manager set devices");
+    info!(sl(), "cgroup manager set devices");
     let mut devices = vec![];
 
     for d in device_resources.iter() {
@@ -334,7 +330,7 @@ fn set_hugepages_resources(
     hugepage_limits: &[LinuxHugepageLimit],
     res: &mut cgroups::Resources,
 ) {
-    info!(sl!(), "cgroup manager set hugepage");
+    info!(sl(), "cgroup manager set hugepage");
     let mut limits = vec![];
     let hugetlb_controller = cg.controller_of::<HugeTlbController>();
 
@@ -348,7 +344,7 @@ fn set_hugepages_resources(
             limits.push(hr);
         } else {
             warn!(
-                sl!(),
+                sl(),
                 "{} page size support cannot be verified, dropping requested limit", l.page_size
             );
         }
@@ -361,7 +357,7 @@ fn set_block_io_resources(
     blkio: &LinuxBlockIo,
     res: &mut cgroups::Resources,
 ) {
-    info!(sl!(), "cgroup manager set block io");
+    info!(sl(), "cgroup manager set block io");
 
     res.blkio.weight = blkio.weight;
     res.blkio.leaf_weight = blkio.leaf_weight;
@@ -389,13 +385,13 @@ fn set_block_io_resources(
 }
 
 fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCpu) -> Result<()> {
-    info!(sl!(), "cgroup manager set cpu");
+    info!(sl(), "cgroup manager set cpu");
 
     let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
 
     if !cpu.cpus.is_empty() {
         if let Err(e) = cpuset_controller.set_cpus(&cpu.cpus) {
-            warn!(sl!(), "write cpuset failed: {:?}", e);
+            warn!(sl(), "write cpuset failed: {:?}", e);
         }
     }
 
@@ -426,7 +422,7 @@ fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCpu) -> Result<()> {
 }
 
 fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool) -> Result<()> {
-    info!(sl!(), "cgroup manager set memory");
+    info!(sl(), "cgroup manager set memory");
     let mem_controller: &MemController = cg.controller_of().unwrap();
 
     if !update {
@@ -446,14 +442,14 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
         let memstat = get_memory_stats(cg)
             .into_option()
             .ok_or_else(|| anyhow!("failed to get the cgroup memory stats"))?;
-        let memusage = memstat.get_usage();
+        let memusage = memstat.usage();
 
         // When update memory limit, the kernel would check the current memory limit
         // set against the new swap setting, if the current memory limit is large than
         // the new swap, then set limit first, otherwise the kernel would complain and
         // refused to set; on the other hand, if the current memory limit is smaller than
         // the new swap, then we should set the swap first and then set the memor limit.
-        if swap == -1 || memusage.get_limit() < swap as u64 {
+        if swap == -1 || memusage.limit() < swap as u64 {
             mem_controller.set_memswap_limit(swap)?;
             set_resource!(mem_controller, set_limit, memory, limit);
         } else {
@@ -495,7 +491,7 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
 }
 
 fn set_pids_resources(cg: &cgroups::Cgroup, pids: &LinuxPids) -> Result<()> {
-    info!(sl!(), "cgroup manager set pids");
+    info!(sl(), "cgroup manager set pids");
     let pid_controller: &PidController = cg.controller_of().unwrap();
     let v = if pids.limit > 0 {
         MaxValue::Value(pids.limit)
@@ -660,21 +656,20 @@ lazy_static! {
     };
 }
 
-fn get_cpu_stats(cg: &cgroups::Cgroup) -> SingularPtrField<ThrottlingData> {
+fn get_cpu_stats(cg: &cgroups::Cgroup) -> MessageField<ThrottlingData> {
     let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
     let stat = cpu_controller.cpu().stat;
     let h = lines_to_map(&stat);
 
-    SingularPtrField::some(ThrottlingData {
+    MessageField::some(ThrottlingData {
         periods: *h.get("nr_periods").unwrap_or(&0),
         throttled_periods: *h.get("nr_throttled").unwrap_or(&0),
         throttled_time: *h.get("throttled_time").unwrap_or(&0),
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     })
 }
 
-fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
+fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> MessageField<CpuUsage> {
     if let Some(cpuacct_controller) = cg.controller_of::<CpuAcctController>() {
         let cpuacct = cpuacct_controller.cpuacct();
 
@@ -688,13 +683,12 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
 
         let percpu_usage = line_to_vec(&cpuacct.usage_percpu);
 
-        return SingularPtrField::some(CpuUsage {
+        return MessageField::some(CpuUsage {
             total_usage,
             percpu_usage,
             usage_in_kernelmode,
             usage_in_usermode,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
+            ..Default::default()
         });
     }
 
@@ -707,17 +701,16 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
     let total_usage = *h.get("usage_usec").unwrap_or(&0);
     let percpu_usage = vec![];
 
-    SingularPtrField::some(CpuUsage {
+    MessageField::some(CpuUsage {
         total_usage,
         percpu_usage,
         usage_in_kernelmode,
         usage_in_usermode,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     })
 }
 
-fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
+fn get_memory_stats(cg: &cgroups::Cgroup) -> MessageField<MemoryStats> {
     let memory_controller: &MemController = get_controller_or_return_singular_none!(cg);
 
     // cache from memory stat
@@ -729,52 +722,48 @@ fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
     let use_hierarchy = value == 1;
 
     // get memory data
-    let usage = SingularPtrField::some(MemoryData {
+    let usage = MessageField::some(MemoryData {
         usage: memory.usage_in_bytes,
         max_usage: memory.max_usage_in_bytes,
         failcnt: memory.fail_cnt,
         limit: memory.limit_in_bytes as u64,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     });
 
     // get swap usage
     let memswap = memory_controller.memswap();
 
-    let swap_usage = SingularPtrField::some(MemoryData {
+    let swap_usage = MessageField::some(MemoryData {
         usage: memswap.usage_in_bytes,
         max_usage: memswap.max_usage_in_bytes,
         failcnt: memswap.fail_cnt,
         limit: memswap.limit_in_bytes as u64,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     });
 
     // get kernel usage
     let kmem_stat = memory_controller.kmem_stat();
 
-    let kernel_usage = SingularPtrField::some(MemoryData {
+    let kernel_usage = MessageField::some(MemoryData {
         usage: kmem_stat.usage_in_bytes,
         max_usage: kmem_stat.max_usage_in_bytes,
         failcnt: kmem_stat.fail_cnt,
         limit: kmem_stat.limit_in_bytes as u64,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     });
 
-    SingularPtrField::some(MemoryStats {
+    MessageField::some(MemoryStats {
         cache,
         usage,
         swap_usage,
         kernel_usage,
         use_hierarchy,
         stats: memory.stat.raw,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     })
 }
 
-fn get_pids_stats(cg: &cgroups::Cgroup) -> SingularPtrField<PidsStats> {
+fn get_pids_stats(cg: &cgroups::Cgroup) -> MessageField<PidsStats> {
     let pid_controller: &PidController = get_controller_or_return_singular_none!(cg);
 
     let current = pid_controller.get_pid_current().unwrap_or(0);
@@ -788,11 +777,10 @@ fn get_pids_stats(cg: &cgroups::Cgroup) -> SingularPtrField<PidsStats> {
         },
     } as u64;
 
-    SingularPtrField::some(PidsStats {
+    MessageField::some(PidsStats {
         current,
         limit,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     })
 }
 
@@ -828,8 +816,8 @@ https://github.com/opencontainers/runc/blob/a5847db387ae28c0ca4ebe4beee1a76900c8
     Total 0
 */
 
-fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> RepeatedField<BlkioStatsEntry> {
-    let mut m = RepeatedField::new();
+fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> Vec<BlkioStatsEntry> {
+    let mut m = Vec::new();
     if blkiodata.is_empty() {
         return m;
     }
@@ -842,16 +830,15 @@ fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> RepeatedField<BlkioStats
             minor: d.minor as u64,
             op: op.clone(),
             value: d.data,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
+            ..Default::default()
         });
     }
 
     m
 }
 
-fn get_blkio_stat_ioservice(services: &[IoService]) -> RepeatedField<BlkioStatsEntry> {
-    let mut m = RepeatedField::new();
+fn get_blkio_stat_ioservice(services: &[IoService]) -> Vec<BlkioStatsEntry> {
+    let mut m = Vec::new();
 
     if services.is_empty() {
         return m;
@@ -875,17 +862,16 @@ fn build_blkio_stats_entry(major: i16, minor: i16, op: &str, value: u64) -> Blki
         minor: minor as u64,
         op: op.to_string(),
         value,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     }
 }
 
-fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
+fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> MessageField<BlkioStats> {
     let blkio_controller: &BlkIoController = get_controller_or_return_singular_none!(cg);
     let blkio = blkio_controller.blkio();
 
     let mut resp = BlkioStats::new();
-    let mut blkio_stats = RepeatedField::new();
+    let mut blkio_stats = Vec::new();
 
     let stat = blkio.io_stat;
     for s in stat {
@@ -901,10 +887,10 @@ fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
 
     resp.io_service_bytes_recursive = blkio_stats;
 
-    SingularPtrField::some(resp)
+    MessageField::some(resp)
 }
 
-fn get_blkio_stats(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
+fn get_blkio_stats(cg: &cgroups::Cgroup) -> MessageField<BlkioStats> {
     if cg.v2() {
         return get_blkio_stats_v2(cg);
     }
@@ -937,7 +923,7 @@ fn get_blkio_stats(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
         m.sectors_recursive = get_blkio_stat_blkiodata(&blkio.sectors_recursive);
     }
 
-    SingularPtrField::some(m)
+    MessageField::some(m)
 }
 
 fn get_hugetlb_stats(cg: &cgroups::Cgroup) -> HashMap<String, HugetlbStats> {
@@ -961,8 +947,7 @@ fn get_hugetlb_stats(cg: &cgroups::Cgroup) -> HashMap<String, HugetlbStats> {
                 usage,
                 max_usage,
                 failcnt,
-                unknown_fields: UnknownFields::default(),
-                cached_size: CachedSize::default(),
+                ..Default::default()
             },
         );
     }
@@ -978,7 +963,7 @@ pub fn get_paths() -> Result<HashMap<String, String>> {
     for l in fs::read_to_string(PATHS)?.lines() {
         let fl: Vec<&str> = l.split(':').collect();
         if fl.len() != 3 {
-            info!(sl!(), "Corrupted cgroup data!");
+            info!(sl(), "Corrupted cgroup data!");
             continue;
         }
 
@@ -999,7 +984,7 @@ pub fn get_mounts(paths: &HashMap<String, String>) -> Result<HashMap<String, Str
         let post: Vec<&str> = p[1].split(' ').collect();
 
         if post.len() != 3 {
-            warn!(sl!(), "can't parse {} line {:?}", MOUNTS, l);
+            warn!(sl(), "can't parse {} line {:?}", MOUNTS, l);
             continue;
         }
 

src/agent/rustjail/src/cgroups/mock.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use protobuf::{CachedSize, SingularPtrField, UnknownFields};
+use protobuf::MessageField;
 
 use crate::cgroups::Manager as CgroupManager;
 use crate::protocols::agent::{BlkioStats, CgroupStats, CpuStats, MemoryStats, PidsStats};
@@ -33,13 +33,12 @@ impl CgroupManager for Manager {
 
     fn get_stats(&self) -> Result<CgroupStats> {
         Ok(CgroupStats {
-            cpu_stats: SingularPtrField::some(CpuStats::default()),
-            memory_stats: SingularPtrField::some(MemoryStats::new()),
-            pids_stats: SingularPtrField::some(PidsStats::new()),
-            blkio_stats: SingularPtrField::some(BlkioStats::new()),
+            cpu_stats: MessageField::some(CpuStats::default()),
+            memory_stats: MessageField::some(MemoryStats::new()),
+            pids_stats: MessageField::some(PidsStats::new()),
+            blkio_stats: MessageField::some(BlkioStats::new()),
             hugetlb_stats: HashMap::new(),
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
+            ..Default::default()
         })
     }
 

src/agent/rustjail/src/cgroups/notifier.rs

@@ -16,11 +16,9 @@ use inotify::{Inotify, WatchMask};
 use tokio::io::AsyncReadExt;
 use tokio::sync::mpsc::{channel, Receiver};
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger().new(o!("subsystem" => "cgroups_notifier"))
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "cgroups_notifier"))
 }
 
 pub async fn notify_oom(cid: &str, cg_dir: String) -> Result<Receiver<String>> {
@@ -38,7 +36,7 @@ pub async fn notify_oom(cid: &str, cg_dir: String) -> Result<Receiver<String>> {
 fn get_value_from_cgroup(path: &Path, key: &str) -> Result<i64> {
     let content = fs::read_to_string(path)?;
     info!(
-        sl!(),
+        sl(),
         "get_value_from_cgroup file: {:?}, content: {}", &path, &content
     );
 
@@ -67,11 +65,11 @@ async fn register_memory_event_v2(
     let event_control_path = Path::new(&cg_dir).join(memory_event_name);
     let cgroup_event_control_path = Path::new(&cg_dir).join(cgroup_event_name);
     info!(
-        sl!(),
+        sl(),
         "register_memory_event_v2 event_control_path: {:?}", &event_control_path
     );
     info!(
-        sl!(),
+        sl(),
         "register_memory_event_v2 cgroup_event_control_path: {:?}", &cgroup_event_control_path
     );
 
@@ -82,8 +80,8 @@ async fn register_memory_event_v2(
     // Because no `unix.IN_DELETE|unix.IN_DELETE_SELF` event for cgroup file system, so watching all process exited
     let cg_wd = inotify.add_watch(&cgroup_event_control_path, WatchMask::MODIFY)?;
 
-    info!(sl!(), "ev_wd: {:?}", ev_wd);
-    info!(sl!(), "cg_wd: {:?}", cg_wd);
+    info!(sl(), "ev_wd: {:?}", ev_wd);
+    info!(sl(), "cg_wd: {:?}", cg_wd);
 
     let (sender, receiver) = channel(100);
     let containere_id = containere_id.to_string();
@@ -97,17 +95,17 @@ async fn register_memory_event_v2(
         while let Some(event_or_error) = stream.next().await {
             let event = event_or_error.unwrap();
             info!(
-                sl!(),
+                sl(),
                 "container[{}] get event for container: {:?}", &containere_id, &event
             );
             // info!("is1: {}", event.wd == wd1);
-            info!(sl!(), "event.wd: {:?}", event.wd);
+            info!(sl(), "event.wd: {:?}", event.wd);
 
             if event.wd == ev_wd {
                 let oom = get_value_from_cgroup(&event_control_path, "oom_kill");
                 if oom.unwrap_or(0) > 0 {
                     let _ = sender.send(containere_id.clone()).await.map_err(|e| {
-                        error!(sl!(), "send containere_id failed, error: {:?}", e);
+                        error!(sl(), "send containere_id failed, error: {:?}", e);
                     });
                     return;
                 }
@@ -171,13 +169,13 @@ async fn register_memory_event(
             let mut buf = [0u8; 8];
             match eventfd_stream.read(&mut buf).await {
                 Err(err) => {
-                    warn!(sl!(), "failed to read from eventfd: {:?}", err);
+                    warn!(sl(), "failed to read from eventfd: {:?}", err);
                     return;
                 }
                 Ok(_) => {
                     let content = fs::read_to_string(path.clone());
                     info!(
-                        sl!(),
+                        sl(),
                         "cgroup event for container: {}, path: {:?}, content: {:?}",
                         &containere_id,
                         &path,
@@ -193,7 +191,7 @@ async fn register_memory_event(
             }
 
             let _ = sender.send(containere_id.clone()).await.map_err(|e| {
-                error!(sl!(), "send containere_id failed, error: {:?}", e);
+                error!(sl(), "send containere_id failed, error: {:?}", e);
             });
         }
     });

src/agent/rustjail/src/cgroups/systemd/dbus_client.rs

@@ -26,7 +26,7 @@ pub trait SystemdInterface {
 
     fn get_version(&self) -> Result<String>;
 
-    fn unit_exist(&self, unit_name: &str) -> Result<bool>;
+    fn unit_exists(&self, unit_name: &str) -> Result<bool>;
 
     fn add_process(&self, pid: i32, unit_name: &str) -> Result<()>;
 }
@@ -36,8 +36,9 @@ pub struct DBusClient {}
 
 impl DBusClient {
     fn build_proxy(&self) -> Result<SystemManager<'static>> {
-        let connection = zbus::blocking::Connection::system()?;
-        let proxy = SystemManager::new(&connection)?;
+        let connection =
+            zbus::blocking::Connection::system().context("Establishing a D-Bus connection")?;
+        let proxy = SystemManager::new(&connection).context("Building a D-Bus proxy manager")?;
         Ok(proxy)
     }
 }
@@ -108,8 +109,10 @@ impl SystemdInterface for DBusClient {
         Ok(systemd_version)
     }
 
-    fn unit_exist(&self, unit_name: &str) -> Result<bool> {
-        let proxy = self.build_proxy()?;
+    fn unit_exists(&self, unit_name: &str) -> Result<bool> {
+        let proxy = self
+            .build_proxy()
+            .with_context(|| format!("Checking if systemd unit {} exists", unit_name))?;
 
         Ok(proxy.get_unit(unit_name).is_ok())
     }

src/agent/rustjail/src/cgroups/systemd/manager.rs

@@ -41,7 +41,7 @@ pub struct Manager {
 impl CgroupManager for Manager {
     fn apply(&self, pid: pid_t) -> Result<()> {
         let unit_name = self.unit_name.as_str();
-        if self.dbus_client.unit_exist(unit_name).unwrap() {
+        if self.dbus_client.unit_exists(unit_name)? {
             self.dbus_client.add_process(pid, self.unit_name.as_str())?;
         } else {
             self.dbus_client.start_unit(

src/agent/rustjail/src/cgroups/systemd/subsystem/cpu.rs

@@ -71,7 +71,7 @@ impl Cpu {
     }
 
     // v2:
-    // cpu.shares <-> CPUShares
+    // cpu.shares <-> CPUWeight
     // cpu.period <-> CPUQuotaPeriodUSec
     // cpu.period & cpu.quota <-> CPUQuotaPerSecUSec
     fn unified_apply(
@@ -80,8 +80,8 @@ impl Cpu {
         systemd_version: &str,
     ) -> Result<()> {
         if let Some(shares) = cpu_resources.shares {
-            let unified_shares = get_unified_cpushares(shares);
-            properties.push(("CPUShares", Value::U64(unified_shares)));
+            let weight = shares_to_weight(shares);
+            properties.push(("CPUWeight", Value::U64(weight)));
         }
 
         if let Some(period) = cpu_resources.period {
@@ -104,7 +104,7 @@ impl Cpu {
 
 // ref: https://github.com/containers/crun/blob/main/crun.1.md#cgroup-v2
 // [2-262144] to [1-10000]
-fn get_unified_cpushares(shares: u64) -> u64 {
+fn shares_to_weight(shares: u64) -> u64 {
     if shares == 0 {
         return 100;
     }

src/agent/rustjail/src/container.rs

@@ -48,7 +48,7 @@ use nix::unistd::{self, fork, ForkResult, Gid, Pid, Uid, User};
 use std::os::unix::fs::MetadataExt;
 use std::os::unix::io::AsRawFd;
 
-use protobuf::SingularPtrField;
+use protobuf::MessageField;
 
 use oci::State as OCIState;
 use regex::Regex;
@@ -875,7 +875,7 @@ impl BaseContainer for LinuxContainer {
         // what about network interface stats?
 
         Ok(StatsContainerResponse {
-            cgroup_stats: SingularPtrField::some(self.cgroup_manager.as_ref().get_stats()?),
+            cgroup_stats: MessageField::some(self.cgroup_manager.as_ref().get_stats()?),
             ..Default::default()
         })
     }
@@ -1596,10 +1596,8 @@ mod tests {
     use tempfile::tempdir;
     use test_utils::skip_if_not_root;
 
-    macro_rules! sl {
-        () => {
-            slog_scope::logger()
-        };
+    fn sl() -> slog::Logger {
+        slog_scope::logger()
     }
 
     #[test]
@@ -1854,7 +1852,7 @@ mod tests {
         let _ = new_linux_container_and_then(|mut c: LinuxContainer| {
             c.processes.insert(
                 1,
-                Process::new(&sl!(), &oci::Process::default(), "123", true, 1).unwrap(),
+                Process::new(&sl(), &oci::Process::default(), "123", true, 1).unwrap(),
             );
             let p = c.get_process("123");
             assert!(p.is_ok(), "Expecting Ok, Got {:?}", p);
@@ -1881,7 +1879,7 @@ mod tests {
         let (c, _dir) = new_linux_container();
         let ret = c
             .unwrap()
-            .start(Process::new(&sl!(), &oci::Process::default(), "123", true, 1).unwrap())
+            .start(Process::new(&sl(), &oci::Process::default(), "123", true, 1).unwrap())
             .await;
         assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
     }
@@ -1891,7 +1889,7 @@ mod tests {
         let (c, _dir) = new_linux_container();
         let ret = c
             .unwrap()
-            .run(Process::new(&sl!(), &oci::Process::default(), "123", true, 1).unwrap())
+            .run(Process::new(&sl(), &oci::Process::default(), "123", true, 1).unwrap())
             .await;
         assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
     }

src/agent/rustjail/src/lib.rs

@@ -82,11 +82,11 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
         let cap = p.Capabilities.as_ref().unwrap();
 
         Some(oci::LinuxCapabilities {
-            bounding: cap.Bounding.clone().into_vec(),
-            effective: cap.Effective.clone().into_vec(),
-            inheritable: cap.Inheritable.clone().into_vec(),
-            permitted: cap.Permitted.clone().into_vec(),
-            ambient: cap.Ambient.clone().into_vec(),
+            bounding: cap.Bounding.clone(),
+            effective: cap.Effective.clone(),
+            inheritable: cap.Inheritable.clone(),
+            permitted: cap.Permitted.clone(),
+            ambient: cap.Ambient.clone(),
         })
     } else {
         None
@@ -108,8 +108,8 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
         terminal: p.Terminal,
         console_size,
         user,
-        args: p.Args.clone().into_vec(),
-        env: p.Env.clone().into_vec(),
+        args: p.Args.clone(),
+        env: p.Env.clone(),
         cwd: p.Cwd.clone(),
         capabilities,
         rlimits,
@@ -130,9 +130,9 @@ fn root_grpc_to_oci(root: &grpc::Root) -> oci::Root {
 fn mount_grpc_to_oci(m: &grpc::Mount) -> oci::Mount {
     oci::Mount {
         destination: m.destination.clone(),
-        r#type: m.field_type.clone(),
+        r#type: m.type_.clone(),
         source: m.source.clone(),
-        options: m.options.clone().into_vec(),
+        options: m.options.clone(),
     }
 }
 
@@ -143,8 +143,8 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
     for e in h.iter() {
         r.push(oci::Hook {
             path: e.Path.clone(),
-            args: e.Args.clone().into_vec(),
-            env: e.Env.clone().into_vec(),
+            args: e.Args.clone(),
+            env: e.Env.clone(),
             timeout: Some(e.Timeout as i32),
         });
     }
@@ -365,7 +365,7 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
             let mut args = Vec::new();
 
             let errno_ret: u32 = if sys.has_errnoret() {
-                sys.get_errnoret()
+                sys.errnoret()
             } else {
                 libc::EPERM as u32
             };
@@ -380,7 +380,7 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
             }
 
             r.push(oci::LinuxSyscall {
-                names: sys.Names.clone().into_vec(),
+                names: sys.Names.clone(),
                 action: sys.Action.clone(),
                 errno_ret,
                 args,
@@ -391,8 +391,8 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
 
     oci::LinuxSeccomp {
         default_action: sec.DefaultAction.clone(),
-        architectures: sec.Architectures.clone().into_vec(),
-        flags: sec.Flags.clone().into_vec(),
+        architectures: sec.Architectures.clone(),
+        flags: sec.Flags.clone(),
         syscalls,
     }
 }
@@ -462,8 +462,8 @@ fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
         devices,
         seccomp,
         rootfs_propagation: l.RootfsPropagation.clone(),
-        masked_paths: l.MaskedPaths.clone().into_vec(),
-        readonly_paths: l.ReadonlyPaths.clone().into_vec(),
+        masked_paths: l.MaskedPaths.clone(),
+        readonly_paths: l.ReadonlyPaths.clone(),
         mount_label: l.MountLabel.clone(),
         intel_rdt,
     }
@@ -564,35 +564,30 @@ mod tests {
                 // All fields specified
                 grpcproc: grpc::Process {
                     Terminal: true,
-                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::some(grpc::Box {
+                    ConsoleSize: protobuf::MessageField::<grpc::Box>::some(grpc::Box {
                         Height: 123,
                         Width: 456,
                         ..Default::default()
                     }),
-                    User: protobuf::SingularPtrField::<grpc::User>::some(grpc::User {
+                    User: protobuf::MessageField::<grpc::User>::some(grpc::User {
                         UID: 1234,
                         GID: 5678,
                         AdditionalGids: Vec::from([910, 1112]),
                         Username: String::from("username"),
                         ..Default::default()
                     }),
-                    Args: protobuf::RepeatedField::from(Vec::from([
-                        String::from("arg1"),
-                        String::from("arg2"),
-                    ])),
-                    Env: protobuf::RepeatedField::from(Vec::from([String::from("env")])),
+                    Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                    Env: Vec::from([String::from("env")]),
                     Cwd: String::from("cwd"),
-                    Capabilities: protobuf::SingularPtrField::some(grpc::LinuxCapabilities {
-                        Bounding: protobuf::RepeatedField::from(Vec::from([String::from("bnd")])),
-                        Effective: protobuf::RepeatedField::from(Vec::from([String::from("eff")])),
-                        Inheritable: protobuf::RepeatedField::from(Vec::from([String::from(
-                            "inher",
-                        )])),
-                        Permitted: protobuf::RepeatedField::from(Vec::from([String::from("perm")])),
-                        Ambient: protobuf::RepeatedField::from(Vec::from([String::from("amb")])),
+                    Capabilities: protobuf::MessageField::some(grpc::LinuxCapabilities {
+                        Bounding: Vec::from([String::from("bnd")]),
+                        Effective: Vec::from([String::from("eff")]),
+                        Inheritable: Vec::from([String::from("inher")]),
+                        Permitted: Vec::from([String::from("perm")]),
+                        Ambient: Vec::from([String::from("amb")]),
                         ..Default::default()
                     }),
-                    Rlimits: protobuf::RepeatedField::from(Vec::from([
+                    Rlimits: Vec::from([
                         grpc::POSIXRlimit {
                             Type: String::from("r#type"),
                             Hard: 123,
@@ -605,7 +600,7 @@ mod tests {
                             Soft: 1011,
                             ..Default::default()
                         },
-                    ])),
+                    ]),
                     NoNewPrivileges: true,
                     ApparmorProfile: String::from("apparmor profile"),
                     OOMScoreAdj: 123456,
@@ -655,7 +650,7 @@ mod tests {
             TestData {
                 // None ConsoleSize
                 grpcproc: grpc::Process {
-                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::none(),
+                    ConsoleSize: protobuf::MessageField::<grpc::Box>::none(),
                     OOMScoreAdj: 0,
                     ..Default::default()
                 },
@@ -668,7 +663,7 @@ mod tests {
             TestData {
                 // None User
                 grpcproc: grpc::Process {
-                    User: protobuf::SingularPtrField::<grpc::User>::none(),
+                    User: protobuf::MessageField::<grpc::User>::none(),
                     OOMScoreAdj: 0,
                     ..Default::default()
                 },
@@ -686,7 +681,7 @@ mod tests {
             TestData {
                 // None Capabilities
                 grpcproc: grpc::Process {
-                    Capabilities: protobuf::SingularPtrField::none(),
+                    Capabilities: protobuf::MessageField::none(),
                     OOMScoreAdj: 0,
                     ..Default::default()
                 },
@@ -787,99 +782,57 @@ mod tests {
             TestData {
                 // All specified
                 grpchooks: grpc::Hooks {
-                    Prestart: protobuf::RepeatedField::from(Vec::from([
+                    Prestart: Vec::from([
                         grpc::Hook {
                             Path: String::from("prestartpath"),
-                            Args: protobuf::RepeatedField::from(Vec::from([
-                                String::from("arg1"),
-                                String::from("arg2"),
-                            ])),
-                            Env: protobuf::RepeatedField::from(Vec::from([
-                                String::from("env1"),
-                                String::from("env2"),
-                            ])),
+                            Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                            Env: Vec::from([String::from("env1"), String::from("env2")]),
                             Timeout: 10,
                             ..Default::default()
                         },
                         grpc::Hook {
                             Path: String::from("prestartpath2"),
-                            Args: protobuf::RepeatedField::from(Vec::from([
-                                String::from("arg3"),
-                                String::from("arg4"),
-                            ])),
-                            Env: protobuf::RepeatedField::from(Vec::from([
-                                String::from("env3"),
-                                String::from("env4"),
-                            ])),
+                            Args: Vec::from([String::from("arg3"), String::from("arg4")]),
+                            Env: Vec::from([String::from("env3"), String::from("env4")]),
                             Timeout: 25,
                             ..Default::default()
                         },
-                    ])),
-                    Poststart: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    ]),
+                    Poststart: Vec::from([grpc::Hook {
                         Path: String::from("poststartpath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
-                    Poststop: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    }]),
+                    Poststop: Vec::from([grpc::Hook {
                         Path: String::from("poststoppath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
-                    CreateRuntime: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    }]),
+                    CreateRuntime: Vec::from([grpc::Hook {
                         Path: String::from("createruntimepath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
-                    CreateContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    }]),
+                    CreateContainer: Vec::from([grpc::Hook {
                         Path: String::from("createcontainerpath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
-                    StartContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    }]),
+                    StartContainer: Vec::from([grpc::Hook {
                         Path: String::from("startcontainerpath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
+                    }]),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -932,72 +885,42 @@ mod tests {
             TestData {
                 // Prestart empty
                 grpchooks: grpc::Hooks {
-                    Prestart: protobuf::RepeatedField::from(Vec::from([])),
-                    Poststart: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    Prestart: Vec::from([]),
+                    Poststart: Vec::from([grpc::Hook {
                         Path: String::from("poststartpath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
-                    Poststop: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    }]),
+                    Poststop: Vec::from([grpc::Hook {
                         Path: String::from("poststoppath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
-                    CreateRuntime: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    }]),
+                    CreateRuntime: Vec::from([grpc::Hook {
                         Path: String::from("createruntimepath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
-                    CreateContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    }]),
+                    CreateContainer: Vec::from([grpc::Hook {
                         Path: String::from("createcontainerpath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
-                    StartContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    }]),
+                    StartContainer: Vec::from([grpc::Hook {
                         Path: String::from("startcontainerpath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
+                    }]),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -1069,11 +992,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::from("destination"),
                     source: String::from("source"),
-                    field_type: String::from("fieldtype"),
-                    options: protobuf::RepeatedField::from(Vec::from([
-                        String::from("option1"),
-                        String::from("option2"),
-                    ])),
+                    type_: String::from("fieldtype"),
+                    options: Vec::from([String::from("option1"), String::from("option2")]),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -1087,8 +1007,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::from("destination"),
                     source: String::from("source"),
-                    field_type: String::from("fieldtype"),
-                    options: protobuf::RepeatedField::from(Vec::new()),
+                    type_: String::from("fieldtype"),
+                    options: Vec::new(),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -1102,8 +1022,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::new(),
                     source: String::from("source"),
-                    field_type: String::from("fieldtype"),
-                    options: protobuf::RepeatedField::from(Vec::from([String::from("option1")])),
+                    type_: String::from("fieldtype"),
+                    options: Vec::from([String::from("option1")]),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -1117,8 +1037,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::from("destination"),
                     source: String::from("source"),
-                    field_type: String::new(),
-                    options: protobuf::RepeatedField::from(Vec::from([String::from("option1")])),
+                    type_: String::new(),
+                    options: Vec::from([String::from("option1")]),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -1178,27 +1098,15 @@ mod tests {
                 grpchook: &[
                     grpc::Hook {
                         Path: String::from("path"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
                     },
                     grpc::Hook {
                         Path: String::from("path2"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg3"),
-                            String::from("arg4"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env3"),
-                            String::from("env4"),
-                        ])),
+                        Args: Vec::from([String::from("arg3"), String::from("arg4")]),
+                        Env: Vec::from([String::from("env3"), String::from("env4")]),
                         Timeout: 20,
                         ..Default::default()
                     },

src/agent/rustjail/src/mount.rs

@@ -35,7 +35,7 @@ use crate::log_child;
 // struct is populated from the content in the /proc/<pid>/mountinfo file.
 #[derive(std::fmt::Debug, PartialEq)]
 pub struct Info {
-    mount_point: String,
+    pub mount_point: String,
     optional: String,
     fstype: String,
 }
@@ -553,7 +553,7 @@ fn rootfs_parent_mount_private(path: &str) -> Result<()> {
 
 // Parse /proc/self/mountinfo because comparing Dev and ino does not work from
 // bind mounts
-fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
+pub fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
     let file = File::open(mountinfo_path)?;
     let reader = BufReader::new(file);
     let mut infos = Vec::new();

src/agent/src/config.rs

@@ -31,6 +31,9 @@ const HTTPS_PROXY: &str = "agent.https_proxy";
 const NO_PROXY: &str = "agent.no_proxy";
 const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity";
 const ENABLE_SIGNATURE_VERIFICATION: &str = "agent.enable_signature_verification";
+const IMAGE_POLICY_FILE: &str = "agent.image_policy";
+const IMAGE_REGISTRY_AUTH_FILE: &str = "agent.image_registry_auth";
+const SIMPLE_SIGNING_SIGSTORE_CONFIG: &str = "agent.simple_signing_sigstore_config";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
@@ -89,6 +92,9 @@ pub struct AgentConfig {
     pub no_proxy: String,
     pub data_integrity: bool,
     pub enable_signature_verification: bool,
+    pub image_policy_file: String,
+    pub image_registry_auth_file: String,
+    pub simple_signing_sigstore_config: String,
 }
 
 #[derive(Debug, Deserialize)]
@@ -110,6 +116,9 @@ pub struct AgentConfigBuilder {
     pub no_proxy: Option<String>,
     pub data_integrity: Option<bool>,
     pub enable_signature_verification: Option<bool>,
+    pub image_policy_file: Option<String>,
+    pub image_registry_auth_file: Option<String>,
+    pub simple_signing_sigstore_config: Option<String>,
 }
 
 macro_rules! config_override {
@@ -177,6 +186,9 @@ impl Default for AgentConfig {
             no_proxy: String::from(""),
             data_integrity: false,
             enable_signature_verification: true,
+            image_policy_file: String::from(""),
+            image_registry_auth_file: String::from(""),
+            simple_signing_sigstore_config: String::from(""),
         }
     }
 }
@@ -215,6 +227,13 @@ impl FromStr for AgentConfig {
             agent_config,
             enable_signature_verification
         );
+        config_override!(agent_config_builder, agent_config, image_policy_file);
+        config_override!(agent_config_builder, agent_config, image_registry_auth_file);
+        config_override!(
+            agent_config_builder,
+            agent_config,
+            simple_signing_sigstore_config
+        );
 
         // Populate the allowed endpoints hash set, if we got any from the config file.
         if let Some(endpoints) = agent_config_builder.endpoints {
@@ -234,7 +253,7 @@ impl AgentConfig {
         let config_position = args.iter().position(|a| a == "--config" || a == "-c");
         if let Some(config_position) = config_position {
             if let Some(config_file) = args.get(config_position + 1) {
-                return AgentConfig::from_config_file(config_file);
+                return AgentConfig::from_config_file(config_file).context("AgentConfig from args");
             } else {
                 panic!("The config argument wasn't formed properly: {:?}", args);
             }
@@ -254,7 +273,9 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                config = AgentConfig::from_config_file(&config_file)?;
+                config = AgentConfig::from_config_file(&config_file)
+                    .context("AgentConfig from kernel cmdline")
+                    .unwrap();
                 using_config_file = true;
                 break;
             }
@@ -337,6 +358,31 @@ impl AgentConfig {
                 config.enable_signature_verification,
                 get_bool_value
             );
+
+            // URI of the image security file
+            parse_cmdline_param!(
+                param,
+                IMAGE_POLICY_FILE,
+                config.image_policy_file,
+                get_string_value
+            );
+
+            // URI of the registry auth file
+            parse_cmdline_param!(
+                param,
+                IMAGE_REGISTRY_AUTH_FILE,
+                config.image_registry_auth_file,
+                get_string_value
+            );
+
+            // URI of the simple signing sigstore file
+            // used when simple signing verification is used
+            parse_cmdline_param!(
+                param,
+                SIMPLE_SIGNING_SIGSTORE_CONFIG,
+                config.simple_signing_sigstore_config,
+                get_string_value
+            );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
@@ -365,7 +411,8 @@ impl AgentConfig {
 
     #[instrument]
     pub fn from_config_file(file: &str) -> Result<AgentConfig> {
-        let config = fs::read_to_string(file)?;
+        let config = fs::read_to_string(file)
+            .with_context(|| format!("Failed to read config file {}", file))?;
         AgentConfig::from_str(&config)
     }
 
@@ -518,6 +565,9 @@ mod tests {
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
         assert_eq!(config.container_policy_path, "");
         assert!(config.enable_signature_verification);
+        assert_eq!(config.image_policy_file, "");
+        assert_eq!(config.image_registry_auth_file, "");
+        assert_eq!(config.simple_signing_sigstore_config, "");
     }
 
     #[test]
@@ -542,6 +592,9 @@ mod tests {
             no_proxy: &'a str,
             data_integrity: bool,
             enable_signature_verification: bool,
+            image_policy_file: &'a str,
+            image_registry_auth_file: &'a str,
+            simple_signing_sigstore_config: &'a str,
         }
 
         impl Default for TestData<'_> {
@@ -563,6 +616,9 @@ mod tests {
                     no_proxy: "",
                     data_integrity: false,
                     enable_signature_verification: true,
+                    image_policy_file: "",
+                    image_registry_auth_file: "",
+                    simple_signing_sigstore_config: "",
                 }
             }
         }
@@ -1007,6 +1063,51 @@ mod tests {
                 enable_signature_verification: false,
                 ..Default::default()
             },
+            TestData {
+                contents: "agent.image_policy=file:///etc/policy.json",
+                image_policy_file: "file:///etc/policy.json",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_policy=kbs:///default/security-policy/test",
+                image_policy_file: "kbs:///default/security-policy/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_policy=kbs://example.kbs.org/default/security-policy/test",
+                image_policy_file: "kbs://example.kbs.org/default/security-policy/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_registry_auth=file:///etc/auth.json",
+                image_registry_auth_file: "file:///etc/auth.json",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_registry_auth=kbs:///default/credential/test",
+                image_registry_auth_file: "kbs:///default/credential/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_registry_auth=kbs://example.kbs.org/default/credential/test",
+                image_registry_auth_file: "kbs://example.kbs.org/default/credential/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.simple_signing_sigstore_config=file:///etc/containers/signature/default.yml",
+                simple_signing_sigstore_config: "file:///etc/containers/signature/default.yml",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.simple_signing_sigstore_config=kbs:///default/sigstore-config/test",
+                simple_signing_sigstore_config: "kbs:///default/sigstore-config/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.simple_signing_sigstore_config=kbs://example.kbs.org/default/sigstore-config/test",
+                simple_signing_sigstore_config: "kbs://example.kbs.org/default/sigstore-config/test",
+                ..Default::default()
+            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -1068,6 +1169,17 @@ mod tests {
                 "{}",
                 msg
             );
+            assert_eq!(d.image_policy_file, config.image_policy_file, "{}", msg);
+            assert_eq!(
+                d.image_registry_auth_file, config.image_registry_auth_file,
+                "{}",
+                msg
+            );
+            assert_eq!(
+                d.simple_signing_sigstore_config, config.simple_signing_sigstore_config,
+                "{}",
+                msg
+            );
 
             for v in vars_to_unset {
                 env::remove_var(v);

src/agent/src/device.rs

@@ -26,15 +26,13 @@ use oci::{LinuxDeviceCgroup, LinuxResources, Spec};
 use protocols::agent::Device;
 use tracing::instrument;
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger().new(o!("subsystem" => "device"))
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "device"))
 }
 
 const VM_ROOTFS: &str = "/";
-
+const BLOCK: &str = "block";
 pub const DRIVER_9P_TYPE: &str = "9p";
 pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
 pub const DRIVER_BLK_TYPE: &str = "blk";
@@ -78,7 +76,7 @@ where
 {
     let syspci = Path::new(&syspci);
     let drv = drv.as_ref();
-    info!(sl!(), "rebind_pci_driver: {} => {:?}", dev, drv);
+    info!(sl(), "rebind_pci_driver: {} => {:?}", dev, drv);
 
     let devpath = syspci.join("devices").join(dev.to_string());
     let overridepath = &devpath.join("driver_override");
@@ -204,7 +202,7 @@ impl ScsiBlockMatcher {
 
 impl UeventMatcher for ScsiBlockMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == "block" && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
+        uev.subsystem == BLOCK && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
     }
 }
 
@@ -238,7 +236,7 @@ impl VirtioBlkPciMatcher {
 
 impl UeventMatcher for VirtioBlkPciMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == "block" && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
+        uev.subsystem == BLOCK && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
     }
 }
 
@@ -311,7 +309,7 @@ impl PmemBlockMatcher {
 
 impl UeventMatcher for PmemBlockMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == "block"
+        uev.subsystem == BLOCK
             && uev.devpath.starts_with(ACPI_DEV_PATH)
             && uev.devpath.ends_with(&self.suffix)
             && !uev.devname.is_empty()
@@ -441,6 +439,48 @@ async fn wait_for_ap_device(sandbox: &Arc<Mutex<Sandbox>>, address: ap::Address)
     Ok(())
 }
 
+#[derive(Debug)]
+struct MmioBlockMatcher {
+    suffix: String,
+}
+
+impl MmioBlockMatcher {
+    fn new(devname: &str) -> MmioBlockMatcher {
+        MmioBlockMatcher {
+            suffix: format!(r"/block/{}", devname),
+        }
+    }
+}
+
+impl UeventMatcher for MmioBlockMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == BLOCK && uev.devpath.ends_with(&self.suffix) && !uev.devname.is_empty()
+    }
+}
+
+#[instrument]
+pub async fn get_virtio_mmio_device_name(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    devpath: &str,
+) -> Result<()> {
+    let devname = devpath
+        .strip_prefix("/dev/")
+        .ok_or_else(|| anyhow!("Storage source '{}' must start with /dev/", devpath))?;
+
+    let matcher = MmioBlockMatcher::new(devname);
+    let uev = wait_for_uevent(sandbox, matcher)
+        .await
+        .context("failed to wait for uevent")?;
+    if uev.devname != devname {
+        return Err(anyhow!(
+            "Unexpected device name {} for mmio device (expected {})",
+            uev.devname,
+            devname
+        ));
+    }
+    Ok(())
+}
+
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 #[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
@@ -564,7 +604,7 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
             let host_minor = specdev.minor;
 
             info!(
-                sl!(),
+                sl(),
                 "update_spec_devices() updating device";
                 "container_path" => &specdev.path,
                 "type" => &specdev.r#type,
@@ -617,7 +657,7 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
                 if let Some(update) = res_updates.get(&(host_type.as_str(), host_major, host_minor))
                 {
                     info!(
-                        sl!(),
+                        sl(),
                         "update_spec_devices() updating resource";
                         "type" => &host_type,
                         "host_major" => host_major,
@@ -678,12 +718,18 @@ pub fn update_env_pci(
 #[instrument]
 async fn virtiommio_blk_device_handler(
     device: &Device,
-    _sandbox: &Arc<Mutex<Sandbox>>,
+    sandbox: &Arc<Mutex<Sandbox>>,
 ) -> Result<SpecUpdate> {
     if device.vm_path.is_empty() {
         return Err(anyhow!("Invalid path for virtio mmio blk device"));
     }
 
+    if !Path::new(&device.vm_path).exists() {
+        get_virtio_mmio_device_name(sandbox, &device.vm_path.to_string())
+            .await
+            .context("failed to get mmio device name")?;
+    }
+
     Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into())
 }
 
@@ -761,7 +807,7 @@ async fn vfio_pci_device_handler(
     device: &Device,
     sandbox: &Arc<Mutex<Sandbox>>,
 ) -> Result<SpecUpdate> {
-    let vfio_in_guest = device.field_type != DRIVER_VFIO_PCI_GK_TYPE;
+    let vfio_in_guest = device.type_ != DRIVER_VFIO_PCI_GK_TYPE;
     let mut pci_fixups = Vec::<(pci::Address, pci::Address)>::new();
     let mut group = None;
 
@@ -875,10 +921,10 @@ pub async fn add_devices(
 #[instrument]
 async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
     // log before validation to help with debugging gRPC protocol version differences.
-    info!(sl!(), "device-id: {}, device-type: {}, device-vm-path: {}, device-container-path: {}, device-options: {:?}",
-          device.id, device.field_type, device.vm_path, device.container_path, device.options);
+    info!(sl(), "device-id: {}, device-type: {}, device-vm-path: {}, device-container-path: {}, device-options: {:?}",
+          device.id, device.type_, device.vm_path, device.container_path, device.options);
 
-    if device.field_type.is_empty() {
+    if device.type_.is_empty() {
         return Err(anyhow!("invalid type for device {:?}", device));
     }
 
@@ -890,7 +936,7 @@ async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<Sp
         return Err(anyhow!("invalid container path for device {:?}", device));
     }
 
-    match device.field_type.as_str() {
+    match device.type_.as_str() {
         DRIVER_BLK_TYPE => virtio_blk_device_handler(device, sandbox).await,
         DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, sandbox).await,
         DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, sandbox).await,
@@ -900,7 +946,7 @@ async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<Sp
             vfio_pci_device_handler(device, sandbox).await
         }
         DRIVER_VFIO_AP_TYPE => vfio_ap_device_handler(device, sandbox).await,
-        _ => Err(anyhow!("Unknown device type {}", device.field_type)),
+        _ => Err(anyhow!("Unknown device type {}", device.type_)),
     }
 }
 
@@ -1396,7 +1442,7 @@ mod tests {
 
         let mut uev = crate::uevent::Uevent::default();
         uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev.subsystem = "block".to_string();
+        uev.subsystem = BLOCK.to_string();
         uev.devpath = devpath.clone();
         uev.devname = devname.to_string();
 
@@ -1430,7 +1476,7 @@ mod tests {
         let mut uev_a = crate::uevent::Uevent::default();
         let relpath_a = "/0000:00:0a.0";
         uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev_a.subsystem = "block".to_string();
+        uev_a.subsystem = BLOCK.to_string();
         uev_a.devname = devname.to_string();
         uev_a.devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath_a, devname);
         let matcher_a = VirtioBlkPciMatcher::new(relpath_a);
@@ -1514,7 +1560,7 @@ mod tests {
         let mut uev_a = crate::uevent::Uevent::default();
         let addr_a = "0:0";
         uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev_a.subsystem = "block".to_string();
+        uev_a.subsystem = BLOCK.to_string();
         uev_a.devname = devname.to_string();
         uev_a.devpath = format!(
             "{}/0000:00:00.0/virtio0/host0/target0:0:0/0:0:{}/block/sda",
@@ -1557,6 +1603,33 @@ mod tests {
         assert!(!matcher_a.is_match(&uev_b));
     }
 
+    #[tokio::test]
+    async fn test_mmio_block_matcher() {
+        let devname_a = "vda";
+        let devname_b = "vdb";
+        let mut uev_a = crate::uevent::Uevent::default();
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.subsystem = BLOCK.to_string();
+        uev_a.devname = devname_a.to_string();
+        uev_a.devpath = format!(
+            "/sys/devices/virtio-mmio-cmdline/virtio-mmio.0/virtio0/block/{}",
+            devname_a
+        );
+        let matcher_a = MmioBlockMatcher::new(devname_a);
+
+        let mut uev_b = uev_a.clone();
+        uev_b.devpath = format!(
+            "/sys/devices/virtio-mmio-cmdline/virtio-mmio.4/virtio4/block/{}",
+            devname_b
+        );
+        let matcher_b = MmioBlockMatcher::new(devname_b);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+
     #[test]
     fn test_split_vfio_pci_option() {
         assert_eq!(

src/agent/src/image_rpc.rs

@@ -38,11 +38,9 @@ const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/";
 const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle";
 const CONFIG_JSON: &str = "config.json";
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger()
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "cgroups"))
 }
 
 pub struct ImageService {
@@ -53,12 +51,29 @@ pub struct ImageService {
 }
 
 impl ImageService {
-    pub fn new(sandbox: Arc<Mutex<Sandbox>>) -> Self {
+    pub async fn new(sandbox: Arc<Mutex<Sandbox>>) -> Self {
         env::set_var("CC_IMAGE_WORK_DIR", KATA_CC_IMAGE_WORK_DIR);
+        let mut image_client = ImageClient::default();
+
+        let image_policy_file = &AGENT_CONFIG.image_policy_file;
+        if !image_policy_file.is_empty() {
+            image_client.config.file_paths.sigstore_config = image_policy_file.clone();
+        }
+
+        let simple_signing_sigstore_config = &AGENT_CONFIG.simple_signing_sigstore_config;
+        if !simple_signing_sigstore_config.is_empty() {
+            image_client.config.file_paths.sigstore_config = simple_signing_sigstore_config.clone();
+        }
+
+        let image_registry_auth_file = &AGENT_CONFIG.image_registry_auth_file;
+        if !image_registry_auth_file.is_empty() {
+            image_client.config.file_paths.auth_file = image_registry_auth_file.clone();
+        }
+
         Self {
             sandbox,
             attestation_agent_started: AtomicBool::new(false),
-            image_client: Arc::new(Mutex::new(ImageClient::default())),
+            image_client: Arc::new(Mutex::new(image_client)),
             container_count: Arc::new(AtomicU16::new(0)),
         }
     }
@@ -70,7 +85,7 @@ impl ImageService {
             return Err(anyhow!("Pause image not present in rootfs"));
         }
 
-        info!(sl!(), "use guest pause image cid {:?}", cid);
+        info!(sl(), "use guest pause image cid {:?}", cid);
         let pause_bundle = Path::new(CONTAINER_BASE).join(cid);
         let pause_rootfs = pause_bundle.join("rootfs");
         let pause_config = pause_bundle.join(CONFIG_JSON);
@@ -122,17 +137,17 @@ impl ImageService {
     /// If the request specifies a non-empty id, use it; otherwise derive it from the image path.
     /// In either case, verify that the chosen id is valid.
     fn cid_from_request(&self, req: &image::PullImageRequest) -> Result<String> {
-        let req_cid = req.get_container_id();
+        let req_cid = req.container_id();
         let cid = if !req_cid.is_empty() {
             req_cid.to_string()
-        } else if let Some(last) = req.get_image().rsplit('/').next() {
+        } else if let Some(last) = req.image().rsplit('/').next() {
             // Support multiple containers with same image
             let index = self.container_count.fetch_add(1, Ordering::Relaxed);
 
             // ':' not valid for container id
             format!("{}_{}", last.replace(':', "_"), index)
         } else {
-            return Err(anyhow!("Invalid image name. {}", req.get_image()));
+            return Err(anyhow!("Invalid image name. {}", req.image()));
         };
         verify_cid(&cid)?;
         Ok(cid)
@@ -141,18 +156,18 @@ impl ImageService {
     async fn pull_image(&self, req: &image::PullImageRequest) -> Result<String> {
         env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
 
-        let https_proxy = &AGENT_CONFIG.read().await.https_proxy;
+        let https_proxy = &AGENT_CONFIG.https_proxy;
         if !https_proxy.is_empty() {
             env::set_var("HTTPS_PROXY", https_proxy);
         }
 
-        let no_proxy = &AGENT_CONFIG.read().await.no_proxy;
+        let no_proxy = &AGENT_CONFIG.no_proxy;
         if !no_proxy.is_empty() {
             env::set_var("NO_PROXY", no_proxy);
         }
 
         let cid = self.cid_from_request(req)?;
-        let image = req.get_image();
+        let image = req.image();
         if cid.starts_with("pause") {
             Self::unpack_pause_image(&cid)?;
 
@@ -161,7 +176,7 @@ impl ImageService {
             return Ok(image.to_owned());
         }
 
-        let aa_kbc_params = &AGENT_CONFIG.read().await.aa_kbc_params;
+        let aa_kbc_params = &AGENT_CONFIG.aa_kbc_params;
         if !aa_kbc_params.is_empty() {
             match self.attestation_agent_started.compare_exchange_weak(
                 false,
@@ -170,34 +185,33 @@ impl ImageService {
                 Ordering::SeqCst,
             ) {
                 Ok(_) => Self::init_attestation_agent()?,
-                Err(_) => info!(sl!(), "Attestation Agent already running"),
+                Err(_) => info!(sl(), "Attestation Agent already running"),
             }
         }
         // If the attestation-agent is being used, then enable the authenticated credentials support
         info!(
-            sl!(),
+            sl(),
             "image_client.config.auth set to: {}",
             !aa_kbc_params.is_empty()
         );
         self.image_client.lock().await.config.auth = !aa_kbc_params.is_empty();
 
         // Read enable signature verification from the agent config and set it in the image_client
-        let enable_signature_verification =
-            &AGENT_CONFIG.read().await.enable_signature_verification;
+        let enable_signature_verification = &AGENT_CONFIG.enable_signature_verification;
         info!(
-            sl!(),
+            sl(),
             "enable_signature_verification set to: {}", enable_signature_verification
         );
         self.image_client.lock().await.config.security_validate = *enable_signature_verification;
 
-        let source_creds = (!req.get_source_creds().is_empty()).then(|| req.get_source_creds());
+        let source_creds = (!req.source_creds().is_empty()).then(|| req.source_creds());
 
         let bundle_path = Path::new(CONTAINER_BASE).join(&cid);
         fs::create_dir_all(&bundle_path)?;
 
         let decrypt_config = format!("provider:attestation-agent:{}", aa_kbc_params);
 
-        info!(sl!(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
+        info!(sl(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
         // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles
         // with rootfs and config.json will store under CONTAINER_BASE/cid.
         let res = self
@@ -210,13 +224,13 @@ impl ImageService {
         match res {
             Ok(image) => {
                 info!(
-                    sl!(),
+                    sl(),
                     "pull and unpack image {:?}, cid: {:?}, with image-rs succeed. ", image, cid
                 );
             }
             Err(e) => {
                 error!(
-                    sl!(),
+                    sl(),
                     "pull and unpack image {:?}, cid: {:?}, with image-rs failed with {:?}. ",
                     image,
                     cid,
@@ -333,7 +347,7 @@ mod tests {
 
         let logger = slog::Logger::root(slog::Discard, o!());
         let s = Sandbox::new(&logger).unwrap();
-        let image_service = ImageService::new(Arc::new(Mutex::new(s)));
+        let image_service = ImageService::new(Arc::new(Mutex::new(s))).await;
         for case in &cases {
             let mut req = image::PullImageRequest::new();
             req.set_image(case.image.to_string());

src/agent/src/linux_abi.rs

@@ -81,7 +81,8 @@ cfg_if! {
 // sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00
 pub const ACPI_DEV_PATH: &str = "/devices/LNXSYSTM";
 
-pub const SYSFS_CPU_ONLINE_PATH: &str = "/sys/devices/system/cpu";
+pub const SYSFS_CPU_PATH: &str = "/sys/devices/system/cpu";
+pub const SYSFS_CPU_ONLINE_PATH: &str = "/sys/devices/system/cpu/online";
 
 pub const SYSFS_MEMORY_BLOCK_SIZE_PATH: &str = "/sys/devices/system/memory/block_size_bytes";
 pub const SYSFS_MEMORY_HOTPLUG_PROBE_PATH: &str = "/sys/devices/system/memory/probe";

src/agent/src/main.rs

@@ -65,7 +65,7 @@ use tokio::{
     io::AsyncWrite,
     sync::{
         watch::{channel, Receiver},
-        Mutex, RwLock,
+        Mutex,
     },
     task::JoinHandle,
 };
@@ -84,12 +84,11 @@ cfg_if! {
 const NAME: &str = "kata-agent";
 
 lazy_static! {
-    static ref AGENT_CONFIG: Arc<RwLock<AgentConfig>> = Arc::new(RwLock::new(
+    static ref AGENT_CONFIG: AgentConfig =
         // Note: We can't do AgentOpts.parse() here to send through the processed arguments to AgentConfig
         // clap::Parser::parse() greedily process all command line input including cargo test parameters,
         // so should only be used inside main.
-        AgentConfig::from_cmdline("/proc/cmdline", env::args().collect()).unwrap()
-    ));
+        AgentConfig::from_cmdline("/proc/cmdline", env::args().collect()).unwrap();
 }
 
 #[derive(Parser)]
@@ -182,13 +181,13 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
 
         lazy_static::initialize(&AGENT_CONFIG);
 
-        init_agent_as_init(&logger, AGENT_CONFIG.read().await.unified_cgroup_hierarchy)?;
+        init_agent_as_init(&logger, AGENT_CONFIG.unified_cgroup_hierarchy)?;
         drop(logger_async_guard);
     } else {
         lazy_static::initialize(&AGENT_CONFIG);
     }
 
-    let config = AGENT_CONFIG.read().await;
+    let config = &AGENT_CONFIG;
     let log_vport = config.log_vport as u32;
 
     let log_handle = tokio::spawn(create_logger_task(rfd, log_vport, shutdown_rx.clone()));
@@ -201,7 +200,7 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     let (logger, logger_async_guard) =
         logging::create_logger(NAME, "agent", config.log_level, writer);
 
-    announce(&logger, &config);
+    announce(&logger, config);
 
     // This variable is required as it enables the global (and crucially static) logger,
     // which is required to satisfy the the lifetime constraints of the auto-generated gRPC code.
@@ -229,7 +228,7 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     let span_guard = root_span.enter();
 
     // Start the sandbox and wait for its ttRPC server to end
-    start_sandbox(&logger, &config, init_mode, &mut tasks, shutdown_rx.clone()).await?;
+    start_sandbox(&logger, config, init_mode, &mut tasks, shutdown_rx.clone()).await?;
 
     // Install a NOP logger for the remainder of the shutdown sequence
     // to ensure any log calls made by local crates using the scope logger
@@ -346,7 +345,7 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode)?;
+    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode).await?;
     server.start().await?;
 
     rx.await?;
@@ -443,9 +442,8 @@ mod tests {
             let msg = format!("test[{}]: {:?}", i, d);
             let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap();
             defer!({
-                // rfd is closed by the use of PipeStream in the crate_logger_task function,
-                // but we will attempt to close in case of a failure
-                let _ = unistd::close(rfd);
+                // XXX: Never try to close rfd, because it will be closed by PipeStream in
+                // create_logger_task() and it's not safe to close the same fd twice time.
                 unistd::close(wfd).unwrap();
             });
 

src/agent/src/metrics.rs

@@ -15,11 +15,9 @@ use tracing::instrument;
 const NAMESPACE_KATA_AGENT: &str = "kata_agent";
 const NAMESPACE_KATA_GUEST: &str = "kata_guest";
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger().new(o!("subsystem" => "metrics"))
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "metrics"))
 }
 
 lazy_static! {
@@ -139,7 +137,7 @@ fn update_agent_metrics() -> Result<()> {
         Ok(p) => p,
         Err(e) => {
             // FIXME: return Ok for all errors?
-            warn!(sl!(), "failed to create process instance: {:?}", e);
+            warn!(sl(), "failed to create process instance: {:?}", e);
 
             return Ok(());
         }
@@ -160,7 +158,7 @@ fn update_agent_metrics() -> Result<()> {
     // io
     match me.io() {
         Err(err) => {
-            info!(sl!(), "failed to get process io stat: {:?}", err);
+            info!(sl(), "failed to get process io stat: {:?}", err);
         }
         Ok(io) => {
             set_gauge_vec_proc_io(&AGENT_IO_STAT, &io);
@@ -169,7 +167,7 @@ fn update_agent_metrics() -> Result<()> {
 
     match me.stat() {
         Err(err) => {
-            info!(sl!(), "failed to get process stat: {:?}", err);
+            info!(sl(), "failed to get process stat: {:?}", err);
         }
         Ok(stat) => {
             set_gauge_vec_proc_stat(&AGENT_PROC_STAT, &stat);
@@ -177,7 +175,7 @@ fn update_agent_metrics() -> Result<()> {
     }
 
     match me.status() {
-        Err(err) => error!(sl!(), "failed to get process status: {:?}", err),
+        Err(err) => error!(sl(), "failed to get process status: {:?}", err),
         Ok(status) => set_gauge_vec_proc_status(&AGENT_PROC_STATUS, &status),
     }
 
@@ -189,7 +187,7 @@ fn update_guest_metrics() {
     // try get load and task info
     match procfs::LoadAverage::new() {
         Err(err) => {
-            info!(sl!(), "failed to get guest LoadAverage: {:?}", err);
+            info!(sl(), "failed to get guest LoadAverage: {:?}", err);
         }
         Ok(load) => {
             GUEST_LOAD
@@ -209,7 +207,7 @@ fn update_guest_metrics() {
     // try to get disk stats
     match procfs::diskstats() {
         Err(err) => {
-            info!(sl!(), "failed to get guest diskstats: {:?}", err);
+            info!(sl(), "failed to get guest diskstats: {:?}", err);
         }
         Ok(diskstats) => {
             for diskstat in diskstats {
@@ -221,7 +219,7 @@ fn update_guest_metrics() {
     // try to get vm stats
     match procfs::vmstat() {
         Err(err) => {
-            info!(sl!(), "failed to get guest vmstat: {:?}", err);
+            info!(sl(), "failed to get guest vmstat: {:?}", err);
         }
         Ok(vmstat) => {
             for (k, v) in vmstat {
@@ -233,7 +231,7 @@ fn update_guest_metrics() {
     // cpu stat
     match procfs::KernelStats::new() {
         Err(err) => {
-            info!(sl!(), "failed to get guest KernelStats: {:?}", err);
+            info!(sl(), "failed to get guest KernelStats: {:?}", err);
         }
         Ok(kernel_stats) => {
             set_gauge_vec_cpu_time(&GUEST_CPU_TIME, "total", &kernel_stats.total);
@@ -246,7 +244,7 @@ fn update_guest_metrics() {
     // try to get net device stats
     match procfs::net::dev_status() {
         Err(err) => {
-            info!(sl!(), "failed to get guest net::dev_status: {:?}", err);
+            info!(sl(), "failed to get guest net::dev_status: {:?}", err);
         }
         Ok(devs) => {
             // netdev: map[string]procfs::net::DeviceStatus
@@ -259,7 +257,7 @@ fn update_guest_metrics() {
     // get statistics about memory from /proc/meminfo
     match procfs::Meminfo::new() {
         Err(err) => {
-            info!(sl!(), "failed to get guest Meminfo: {:?}", err);
+            info!(sl(), "failed to get guest Meminfo: {:?}", err);
         }
         Ok(meminfo) => {
             set_gauge_vec_meminfo(&GUEST_MEMINFO, &meminfo);

src/agent/src/mount.rs

@@ -21,10 +21,11 @@ use nix::unistd::{Gid, Uid};
 use regex::Regex;
 
 use crate::device::{
-    get_scsi_device_name, get_virtio_blk_pci_device_name, online_device, wait_for_pmem_device,
-    DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE,
-    DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE,
-    DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE, FS_TYPE_HUGETLB,
+    get_scsi_device_name, get_virtio_blk_pci_device_name, get_virtio_mmio_device_name,
+    online_device, wait_for_pmem_device, DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE,
+    DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE,
+    DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE, DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE,
+    FS_TYPE_HUGETLB,
 };
 use crate::linux_abi::*;
 use crate::pci;
@@ -211,10 +212,10 @@ async fn ephemeral_storage_handler(
     // By now we only support one option field: "fsGroup" which
     // isn't an valid mount option, thus we should remove it when
     // do mount.
-    if storage.options.len() > 0 {
+    if !storage.options.is_empty() {
         // ephemeral_storage didn't support mount options except fsGroup.
         let mut new_storage = storage.clone();
-        new_storage.options = protobuf::RepeatedField::default();
+        new_storage.options = Default::default();
         common_storage_handler(logger, &new_storage)?;
 
         let opts_vec: Vec<String> = storage.options.to_vec();
@@ -473,8 +474,14 @@ async fn virtiommio_blk_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
+    let storage = storage.clone();
+    if !Path::new(&storage.source).exists() {
+        get_virtio_mmio_device_name(&sandbox, &storage.source)
+            .await
+            .context("failed to get mmio device name")?;
+    }
     //The source path is VmPath
-    common_storage_handler(logger, storage)
+    common_storage_handler(logger, &storage)
 }
 
 // virtiofs_storage_handler handles the storage for virtio-fs.
@@ -654,7 +661,7 @@ pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> {
     if storage.fs_group.is_none() {
         return Ok(());
     }
-    let fs_group = storage.get_fs_group();
+    let fs_group = storage.fs_group();
 
     let mut read_only = false;
     let opts_vec: Vec<String> = storage.options.to_vec();
@@ -671,7 +678,7 @@ pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> {
         err
     })?;
 
-    if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch
+    if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch.into()
         && metadata.gid() == fs_group.group_id
     {
         let mut mask = if read_only { RO_MASK } else { RW_MASK };
@@ -1094,7 +1101,6 @@ fn parse_options(option_list: Vec<String>) -> HashMap<String, String> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use protobuf::RepeatedField;
     use protocols::agent::FSGroup;
     use std::fs::File;
     use std::fs::OpenOptions;
@@ -2015,9 +2021,8 @@ mod tests {
                 mount_path: "rw_mount",
                 fs_group: Some(FSGroup {
                     group_id: 3000,
-                    group_change_policy: FSGroupChangePolicy::Always,
-                    unknown_fields: Default::default(),
-                    cached_size: Default::default(),
+                    group_change_policy: FSGroupChangePolicy::Always.into(),
+                    ..Default::default()
                 }),
                 read_only: false,
                 expected_group_id: 3000,
@@ -2027,9 +2032,8 @@ mod tests {
                 mount_path: "ro_mount",
                 fs_group: Some(FSGroup {
                     group_id: 3000,
-                    group_change_policy: FSGroupChangePolicy::OnRootMismatch,
-                    unknown_fields: Default::default(),
-                    cached_size: Default::default(),
+                    group_change_policy: FSGroupChangePolicy::OnRootMismatch.into(),
+                    ..Default::default()
                 }),
                 read_only: true,
                 expected_group_id: 3000,
@@ -2049,10 +2053,7 @@ mod tests {
             let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
             let mut storage_data = Storage::new();
             if d.read_only {
-                storage_data.set_options(RepeatedField::from_slice(&[
-                    "foo".to_string(),
-                    "ro".to_string(),
-                ]));
+                storage_data.set_options(vec!["foo".to_string(), "ro".to_string()]);
             }
             if let Some(fs_group) = d.fs_group.clone() {
                 storage_data.set_fs_group(fs_group);

src/agent/src/netlink.rs

@@ -7,7 +7,6 @@ use anyhow::{anyhow, Context, Result};
 use futures::{future, StreamExt, TryStreamExt};
 use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network};
 use nix::errno::Errno;
-use protobuf::RepeatedField;
 use protocols::types::{ARPNeighbor, IPAddress, IPFamily, Interface, Route};
 use rtnetlink::{new_connection, packet, IpVersion};
 use std::convert::{TryFrom, TryInto};
@@ -83,13 +82,34 @@ impl Handle {
 
         // Add new ip addresses from request
         for ip_address in &iface.IPAddresses {
-            let ip = IpAddr::from_str(ip_address.get_address())?;
-            let mask = ip_address.get_mask().parse::<u8>()?;
+            let ip = IpAddr::from_str(ip_address.address())?;
+            let mask = ip_address.mask().parse::<u8>()?;
 
             self.add_addresses(link.index(), std::iter::once(IpNetwork::new(ip, mask)?))
                 .await?;
         }
 
+        // we need to update the link's interface name, thus we should rename the existed link whose name
+        // is the same with the link's request name, otherwise, it would update the link failed with the
+        // name conflicted.
+        let mut new_link = None;
+        if link.name() != iface.name {
+            if let Ok(link) = self.find_link(LinkFilter::Name(iface.name.as_str())).await {
+                // update the existing interface name with a temporary name, otherwise
+                // it would failed to udpate this interface with an existing name.
+                let mut request = self.handle.link().set(link.index());
+                request.message_mut().header = link.header.clone();
+
+                request
+                    .name(format!("{}_temp", link.name()))
+                    .up()
+                    .execute()
+                    .await?;
+
+                new_link = Some(link);
+            }
+        }
+
         // Update link
         let mut request = self.handle.link().set(link.index());
         request.message_mut().header = link.header.clone();
@@ -102,6 +122,14 @@ impl Handle {
             .execute()
             .await?;
 
+        // swap the updated iface's name.
+        if let Some(nlink) = new_link {
+            let mut request = self.handle.link().set(nlink.index());
+            request.message_mut().header = nlink.header.clone();
+
+            request.name(link.name()).up().execute().await?;
+        }
+
         Ok(())
     }
 
@@ -152,7 +180,7 @@ impl Handle {
                 .map(|p| p.try_into())
                 .collect::<Result<Vec<IPAddress>>>()?;
 
-            iface.IPAddresses = RepeatedField::from_vec(ips);
+            iface.IPAddresses = ips;
 
             list.push(iface);
         }
@@ -334,7 +362,7 @@ impl Handle {
 
             // `rtnetlink` offers a separate request builders for different IP versions (IP v4 and v6).
             // This if branch is a bit clumsy because it does almost the same.
-            if route.get_family() == IPFamily::v6 {
+            if route.family() == IPFamily::v6 {
                 let dest_addr = if !route.dest.is_empty() {
                     Ipv6Network::from_str(&route.dest)?
                 } else {
@@ -368,9 +396,9 @@ impl Handle {
                     if Errno::from_i32(message.code.abs()) != Errno::EEXIST {
                         return Err(anyhow!(
                             "Failed to add IP v6 route (src: {}, dst: {}, gtw: {},Err: {})",
-                            route.get_source(),
-                            route.get_dest(),
-                            route.get_gateway(),
+                            route.source(),
+                            route.dest(),
+                            route.gateway(),
                             message
                         ));
                     }
@@ -409,9 +437,9 @@ impl Handle {
                     if Errno::from_i32(message.code.abs()) != Errno::EEXIST {
                         return Err(anyhow!(
                             "Failed to add IP v4 route (src: {}, dst: {}, gtw: {},Err: {})",
-                            route.get_source(),
-                            route.get_dest(),
-                            route.get_gateway(),
+                            route.source(),
+                            route.dest(),
+                            route.gateway(),
                             message
                         ));
                     }
@@ -506,7 +534,7 @@ impl Handle {
             self.add_arp_neighbor(&neigh).await.map_err(|err| {
                 anyhow!(
                     "Failed to add ARP neighbor {}: {:?}",
-                    neigh.get_toIPAddress().get_address(),
+                    neigh.toIPAddress().address(),
                     err
                 )
             })?;
@@ -725,7 +753,7 @@ impl TryFrom<Address> for IPAddress {
         let mask = format!("{}", value.0.header.prefix_len);
 
         Ok(IPAddress {
-            family,
+            family: family.into(),
             address,
             mask,
             ..Default::default()

src/agent/src/sandbox.rs

@@ -12,6 +12,7 @@ use crate::pci;
 use crate::uevent::{Uevent, UeventMatcher};
 use crate::watcher::BindWatcher;
 use anyhow::{anyhow, Context, Result};
+use kata_types::cpu::CpuSet;
 use libc::pid_t;
 use oci::{Hook, Hooks};
 use protocols::agent::OnlineCPUMemRequest;
@@ -25,6 +26,7 @@ use std::collections::HashMap;
 use std::fs;
 use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
+use std::str::FromStr;
 use std::sync::Arc;
 use std::{thread, time};
 use tokio::sync::mpsc::{channel, Receiver, Sender};
@@ -265,12 +267,12 @@ impl Sandbox {
     pub fn online_cpu_memory(&self, req: &OnlineCPUMemRequest) -> Result<()> {
         if req.nb_cpus > 0 {
             // online cpus
-            online_cpus(&self.logger, req.nb_cpus as i32)?;
+            online_cpus(&self.logger, req.nb_cpus as i32).context("online cpus")?;
         }
 
         if !req.cpu_only {
             // online memory
-            online_memory(&self.logger)?;
+            online_memory(&self.logger).context("online memory")?;
         }
 
         if req.nb_cpus == 0 {
@@ -434,23 +436,33 @@ fn online_resources(logger: &Logger, path: &str, pattern: &str, num: i32) -> Res
 
 // max wait for all CPUs to online will use 50 * 100 = 5 seconds.
 const ONLINE_CPUMEM_WATI_MILLIS: u64 = 50;
-const ONLINE_CPUMEM_MAX_RETRIES: u32 = 100;
+const ONLINE_CPUMEM_MAX_RETRIES: i32 = 100;
 
 #[instrument]
 fn online_cpus(logger: &Logger, num: i32) -> Result<i32> {
-    let mut onlined_count: i32 = 0;
+    let mut onlined_cpu_count = onlined_cpus().context("onlined cpu count")?;
+    // for some vmms, like dragonball, they will online cpus for us
+    // so check first whether agent need to do the online operation
+    if onlined_cpu_count >= num {
+        return Ok(num);
+    }
 
     for i in 0..ONLINE_CPUMEM_MAX_RETRIES {
-        let r = online_resources(
+        // online num resources
+        online_resources(
             logger,
-            SYSFS_CPU_ONLINE_PATH,
+            SYSFS_CPU_PATH,
             r"cpu[0-9]+",
-            num - onlined_count,
-        );
+            num - onlined_cpu_count,
+        )
+        .context("online cpu resource")?;
 
-        onlined_count += r?;
-        if onlined_count == num {
-            info!(logger, "online {} CPU(s) after {} retries", num, i);
+        onlined_cpu_count = onlined_cpus().context("onlined cpu count")?;
+        if onlined_cpu_count >= num {
+            info!(
+                logger,
+                "Currently {} onlined CPU(s) after {} retries", onlined_cpu_count, i
+            );
             return Ok(num);
         }
         thread::sleep(time::Duration::from_millis(ONLINE_CPUMEM_WATI_MILLIS));
@@ -465,10 +477,18 @@ fn online_cpus(logger: &Logger, num: i32) -> Result<i32> {
 
 #[instrument]
 fn online_memory(logger: &Logger) -> Result<()> {
-    online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1)?;
+    online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1)
+        .context("online memory resource")?;
     Ok(())
 }
 
+fn onlined_cpus() -> Result<i32> {
+    let content =
+        fs::read_to_string(SYSFS_CPU_ONLINE_PATH).context("read sysfs cpu online file")?;
+    let online_cpu_set = CpuSet::from_str(content.trim())?;
+    Ok(online_cpu_set.len() as i32)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/agent/src/signal.rs

@@ -24,7 +24,7 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
     loop {
         // Avoid reaping the undesirable child's signal, e.g., execute_hook's
         // The lock should be released immediately.
-        let _ = rustjail::container::WAIT_PID_LOCKER.lock().await;
+        let _locker = rustjail::container::WAIT_PID_LOCKER.lock().await;
         let result = wait::waitpid(
             Some(Pid::from_raw(-1)),
             Some(WaitPidFlag::WNOHANG | WaitPidFlag::__WALL),

src/agent/src/tracer.rs

@@ -69,7 +69,7 @@ macro_rules! trace_rpc_call {
             propagator.extract(&extract_carrier_from_ttrpc($ctx))
         });
 
-        info!(sl!(), "rpc call from shim to agent: {:?}", $name);
+        info!(sl(), "rpc call from shim to agent: {:?}", $name);
 
         // generate tracing span
         let rpc_span = span!(tracing::Level::INFO, $name, "mod"="rpc.rs", req=?$req);

src/agent/src/uevent.rs

@@ -19,11 +19,9 @@ use tokio::sync::watch::Receiver;
 use tokio::sync::Mutex;
 use tracing::instrument;
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger().new(o!("subsystem" => "uevent"))
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "uevent"))
 }
 
 #[derive(Debug, Default, Clone, PartialEq, Eq)]
@@ -120,11 +118,11 @@ pub async fn wait_for_uevent(
 ) -> Result<Uevent> {
     let logprefix = format!("Waiting for {:?}", &matcher);
 
-    info!(sl!(), "{}", logprefix);
+    info!(sl(), "{}", logprefix);
     let mut sb = sandbox.lock().await;
     for uev in sb.uevent_map.values() {
         if matcher.is_match(uev) {
-            info!(sl!(), "{}: found {:?} in uevent map", logprefix, &uev);
+            info!(sl(), "{}: found {:?} in uevent map", logprefix, &uev);
             return Ok(uev.clone());
         }
     }
@@ -139,9 +137,9 @@ pub async fn wait_for_uevent(
     sb.uevent_watchers.push(Some((Box::new(matcher), tx)));
     drop(sb); // unlock
 
-    info!(sl!(), "{}: waiting on channel", logprefix);
+    info!(sl(), "{}: waiting on channel", logprefix);
 
-    let hotplug_timeout = AGENT_CONFIG.read().await.hotplug_timeout;
+    let hotplug_timeout = AGENT_CONFIG.hotplug_timeout;
 
     let uev = match tokio::time::timeout(hotplug_timeout, rx).await {
         Ok(v) => v?,
@@ -157,7 +155,7 @@ pub async fn wait_for_uevent(
         }
     };
 
-    info!(sl!(), "{}: found {:?} on channel", logprefix, &uev);
+    info!(sl(), "{}: found {:?} on channel", logprefix, &uev);
     Ok(uev)
 }
 

src/agent/vsock-exporter/Cargo.toml

@@ -18,4 +18,4 @@ bincode = "1.3.3"
 byteorder = "1.4.3"
 slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
 async-trait = "0.1.50"
-tokio = "1.2.0"
+tokio = "1.28.1"

src/dragonball/Cargo.lock

@@ -180,9 +180,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.7"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c"
+checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
 dependencies = [
  "cfg-if",
  "crossbeam-utils",
@@ -209,11 +209,12 @@ dependencies = [
 
 [[package]]
 name = "dbs-address-space"
-version = "0.2.2"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bcc37dc0b8ffae1c5911d13ae630dc7a9020fa0de0edd178d6ab71daf56c8fc"
+checksum = "95e20d28a9cd13bf00d0ecd1bd073d242242b04f0acb663d7adfc659f8879322"
 dependencies = [
  "arc-swap",
+ "lazy_static",
  "libc",
  "nix 0.23.2",
  "thiserror",
@@ -247,9 +248,9 @@ dependencies = [
 
 [[package]]
 name = "dbs-boot"
-version = "0.3.1"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a74a8c05a1674d3032e610b4f201c7440c345559bad3dfe6b455ce195785108"
+checksum = "5466a92f75aa928a9103dcb2088f6d1638ef9da8945fad7389a73864dfa0182c"
 dependencies = [
  "dbs-arch",
  "kvm-bindings",
@@ -300,9 +301,9 @@ dependencies = [
 
 [[package]]
 name = "dbs-upcall"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "699e62afa444ae4b00d474fd91bc37785ba050acdfbe179731c81898e32efc3f"
+checksum = "ea3a78128fd0be8b8b10257675c262b378dc5d00b1e18157736a6c27e45ce4fb"
 dependencies = [
  "anyhow",
  "dbs-utils",
@@ -330,9 +331,9 @@ dependencies = [
 
 [[package]]
 name = "dbs-virtio-devices"
-version = "0.2.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88e5c6c48b766afb95851b04b6b193871a59d0b2a3ed19990d4f8f651ae5c668"
+checksum = "24d671cc3e5f98b84ef6b6bed007d28f72f16d3aea8eb38e2d42b00b2973c1d8"
 dependencies = [
  "byteorder",
  "caps",
@@ -346,7 +347,7 @@ dependencies = [
  "kvm-ioctls",
  "libc",
  "log",
- "nix 0.23.2",
+ "nix 0.24.3",
  "nydus-api",
  "nydus-blobfs",
  "nydus-rafs",
@@ -444,13 +445,13 @@ dependencies = [
 
 [[package]]
 name = "errno"
-version = "0.2.8"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
+checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
 dependencies = [
  "errno-dragonfly",
  "libc",
- "winapi",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -482,7 +483,7 @@ dependencies = [
  "cfg-if",
  "libc",
  "redox_syscall",
- "windows-sys",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -618,9 +619,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
+checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
 dependencies = [
  "cfg-if",
  "libc",
@@ -670,7 +671,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3"
 dependencies = [
  "libc",
- "windows-sys",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -841,7 +842,7 @@ dependencies = [
  "libc",
  "log",
  "wasi",
- "windows-sys",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -1044,7 +1045,7 @@ dependencies = [
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-sys",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -1120,16 +1121,16 @@ checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
 
 [[package]]
 name = "rustix"
-version = "0.36.8"
+version = "0.36.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644"
+checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62"
 dependencies = [
  "bitflags",
  "errno",
  "io-lifetimes",
  "libc",
  "linux-raw-sys",
- "windows-sys",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -1167,18 +1168,18 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.152"
+version = "1.0.156"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
+checksum = "314b5b092c0ade17c00142951e50ced110ec27cea304b1037c6969246c2469a4"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.152"
+version = "1.0.156"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
+checksum = "d7e29c4601e36bcec74a223228dce795f4cd3616341a4af93520ca1a837c087d"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1187,9 +1188,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.93"
+version = "1.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
+checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
 dependencies = [
  "itoa",
  "ryu",
@@ -1422,7 +1423,7 @@ dependencies = [
  "pin-project-lite",
  "socket2",
  "tokio-macros",
- "windows-sys",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -1610,7 +1611,16 @@ version = "0.45.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.42.1",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.0",
 ]
 
 [[package]]
@@ -1619,13 +1629,28 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
 dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
+ "windows_aarch64_gnullvm 0.42.1",
+ "windows_aarch64_msvc 0.42.1",
+ "windows_i686_gnu 0.42.1",
+ "windows_i686_msvc 0.42.1",
+ "windows_x86_64_gnu 0.42.1",
+ "windows_x86_64_gnullvm 0.42.1",
+ "windows_x86_64_msvc 0.42.1",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.0",
+ "windows_aarch64_msvc 0.48.0",
+ "windows_i686_gnu 0.48.0",
+ "windows_i686_msvc 0.48.0",
+ "windows_x86_64_gnu 0.48.0",
+ "windows_x86_64_gnullvm 0.48.0",
+ "windows_x86_64_msvc 0.48.0",
 ]
 
 [[package]]
@@ -1634,42 +1659,84 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+
 [[package]]
 name = "xattr"
 version = "0.2.3"

src/dragonball/Cargo.toml

@@ -12,16 +12,16 @@ edition = "2018"
 [dependencies]
 arc-swap = "1.5.0"
 bytes = "1.1.0"
-dbs-address-space = "0.2.0"
+dbs-address-space = "0.3.0"
 dbs-allocator = "0.1.0"
 dbs-arch = "0.2.0"
-dbs-boot = "0.3.0"
+dbs-boot = "0.4.0"
 dbs-device = "0.2.0"
 dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] }
 dbs-legacy-devices = "0.1.0"
-dbs-upcall = { version = "0.2.0", optional = true }
+dbs-upcall = { version = "0.3.0", optional = true }
 dbs-utils = "0.2.0"
-dbs-virtio-devices = { version = "0.2.0", optional = true, features = ["virtio-mmio"] }
+dbs-virtio-devices = { version = "0.3.1", optional = true, features = ["virtio-mmio"] }
 kvm-bindings = "0.6.0"
 kvm-ioctls = "0.12.0"
 lazy_static = "1.2"
@@ -55,3 +55,5 @@ virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]
 virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"]
 # virtio-fs only work on atomic-guest-memory
 virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"]
+virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"]
+virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"]

src/dragonball/src/api/v1/vmm_action.rs

@@ -19,6 +19,8 @@ use crate::vmm::Vmm;
 use self::VmConfigError::*;
 use self::VmmActionError::MachineConfig;
 
+#[cfg(feature = "virtio-balloon")]
+pub use crate::device_manager::balloon_dev_mgr::{BalloonDeviceConfigInfo, BalloonDeviceError};
 #[cfg(feature = "virtio-blk")]
 pub use crate::device_manager::blk_dev_mgr::{
     BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr,
@@ -27,6 +29,8 @@ pub use crate::device_manager::blk_dev_mgr::{
 pub use crate::device_manager::fs_dev_mgr::{
     FsDeviceConfigInfo, FsDeviceConfigUpdateInfo, FsDeviceError, FsDeviceMgr, FsMountConfigInfo,
 };
+#[cfg(feature = "virtio-mem")]
+pub use crate::device_manager::mem_dev_mgr::{MemDeviceConfigInfo, MemDeviceError};
 #[cfg(feature = "virtio-net")]
 pub use crate::device_manager::virtio_net_dev_mgr::{
     VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError,
@@ -34,7 +38,6 @@ pub use crate::device_manager::virtio_net_dev_mgr::{
 };
 #[cfg(feature = "virtio-vsock")]
 pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError};
-
 #[cfg(feature = "hotplug")]
 pub use crate::vcpu::{VcpuResizeError, VcpuResizeInfo};
 
@@ -97,6 +100,20 @@ pub enum VmmActionError {
     /// The action `ResizeVcpu` Failed
     #[error("vcpu resize error : {0}")]
     ResizeVcpu(#[source] VcpuResizeError),
+
+    /// Cannot access address space.
+    #[error("Cannot access address space.")]
+    AddressSpaceNotInitialized,
+
+    #[cfg(feature = "virtio-mem")]
+    /// Mem device related errors.
+    #[error("virtio-mem device error: {0}")]
+    Mem(#[source] MemDeviceError),
+
+    #[cfg(feature = "virtio-balloon")]
+    /// Balloon device related errors.
+    #[error("virtio-balloon device error: {0}")]
+    Balloon(#[source] BalloonDeviceError),
 }
 
 /// This enum represents the public interface of the VMM. Each action contains various
@@ -172,6 +189,15 @@ pub enum VmmAction {
     #[cfg(feature = "hotplug")]
     /// Resize Vcpu number in the guest.
     ResizeVcpu(VcpuResizeInfo),
+
+    #[cfg(feature = "virtio-mem")]
+    /// Add a new mem device or update one that already exists using the `MemDeviceConfig` as input.
+    InsertMemDevice(MemDeviceConfigInfo),
+
+    #[cfg(feature = "virtio-balloon")]
+    /// Add a new balloon device or update one that already exists using the `BalloonDeviceConfig`
+    /// as input.
+    InsertBalloonDevice(BalloonDeviceConfigInfo),
 }
 
 /// The enum represents the response sent by the VMM in case of success. The response is either
@@ -274,6 +300,12 @@ impl VmmService {
             }
             #[cfg(feature = "hotplug")]
             VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg),
+            #[cfg(feature = "virtio-mem")]
+            VmmAction::InsertMemDevice(mem_cfg) => self.add_mem_device(vmm, event_mgr, mem_cfg),
+            #[cfg(feature = "virtio-balloon")]
+            VmmAction::InsertBalloonDevice(balloon_cfg) => {
+                self.add_balloon_device(vmm, event_mgr, balloon_cfg)
+            }
         };
 
         debug!("send vmm response: {:?}", response);
@@ -486,7 +518,9 @@ impl VmmService {
                 VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot)
             })?;
 
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config)
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::Block)
     }
@@ -500,7 +534,9 @@ impl VmmService {
     ) -> VmmRequestResult {
         let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
 
-        BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config)
+        vm.device_manager_mut()
+            .block_manager
+            .update_device_ratelimiters(config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::Block)
     }
@@ -518,7 +554,9 @@ impl VmmService {
             .create_device_op_context(Some(event_mgr.epoll_manager()))
             .map_err(|_| VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot))?;
 
-        BlockDeviceMgr::remove_device(vm.device_manager_mut(), ctx, drive_id)
+        vm.device_manager_mut()
+            .block_manager
+            .remove_device(ctx, drive_id)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::Block)
     }
@@ -543,7 +581,9 @@ impl VmmService {
                 }
             })?;
 
-        VirtioNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config)
+        vm.device_manager_mut()
+            .virtio_net_manager
+            .insert_device(ctx, config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::VirtioNet)
     }
@@ -556,7 +596,9 @@ impl VmmService {
     ) -> VmmRequestResult {
         let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
 
-        VirtioNetDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config)
+        vm.device_manager_mut()
+            .virtio_net_manager
+            .update_device_ratelimiters(config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::VirtioNet)
     }
@@ -616,12 +658,6 @@ impl VmmService {
 
     #[cfg(feature = "hotplug")]
     fn resize_vcpu(&mut self, vmm: &mut Vmm, config: VcpuResizeInfo) -> VmmRequestResult {
-        if !cfg!(target_arch = "x86_64") {
-            // TODO: Arm need to support vcpu hotplug. issue: #6010
-            warn!("This arch do not support vm resize!");
-            return Ok(VmmData::Empty);
-        }
-
         if !cfg!(feature = "dbs-upcall") {
             warn!("We only support cpu resize through upcall server in the guest kernel now, please enable dbs-upcall feature.");
             return Ok(VmmData::Empty);
@@ -644,6 +680,62 @@ impl VmmService {
 
         Ok(VmmData::Empty)
     }
+
+    #[cfg(feature = "virtio-mem")]
+    fn add_mem_device(
+        &mut self,
+        vmm: &mut Vmm,
+        event_mgr: &mut EventManager,
+        config: MemDeviceConfigInfo,
+    ) -> VmmRequestResult {
+        let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
+
+        let ctx = vm
+            .create_device_op_context(Some(event_mgr.epoll_manager()))
+            .map_err(|e| {
+                if let StartMicroVmError::UpcallServerNotReady = e {
+                    VmmActionError::UpcallServerNotReady
+                } else {
+                    VmmActionError::StartMicroVm(e)
+                }
+            })?;
+
+        vm.device_manager_mut()
+            .mem_manager
+            .insert_or_update_device(ctx, config)
+            .map(|_| VmmData::Empty)
+            .map_err(VmmActionError::Mem)
+    }
+
+    #[cfg(feature = "virtio-balloon")]
+    fn add_balloon_device(
+        &mut self,
+        vmm: &mut Vmm,
+        event_mgr: &mut EventManager,
+        config: BalloonDeviceConfigInfo,
+    ) -> VmmRequestResult {
+        let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
+
+        if config.size_mib != 0 {
+            info!("add_balloon_device: wait prealloc");
+            vm.stop_prealloc().map_err(VmmActionError::StartMicroVm)?;
+        }
+        let ctx = vm
+            .create_device_op_context(Some(event_mgr.epoll_manager()))
+            .map_err(|e| {
+                if let StartMicroVmError::UpcallServerNotReady = e {
+                    VmmActionError::UpcallServerNotReady
+                } else {
+                    VmmActionError::StartMicroVm(e)
+                }
+            })?;
+
+        vm.device_manager_mut()
+            .balloon_manager
+            .insert_or_update_device(ctx, config)
+            .map(|_| VmmData::Empty)
+            .map_err(VmmActionError::Balloon)
+    }
 }
 
 fn handle_cpu_topology(
@@ -1452,4 +1544,84 @@ mod tests {
             t.check_request();
         }
     }
+
+    #[cfg(feature = "virtio-mem")]
+    #[test]
+    fn test_vmm_action_insert_mem_device() {
+        skip_if_not_root!();
+
+        let tests = &mut [
+            // hotplug unready
+            TestData::new(
+                VmmAction::InsertMemDevice(MemDeviceConfigInfo::default()),
+                InstanceState::Running,
+                &|result| {
+                    assert!(matches!(
+                        result,
+                        Err(VmmActionError::StartMicroVm(
+                            StartMicroVmError::UpcallMissVsock
+                        ))
+                    ));
+                    let err_string = format!("{}", result.unwrap_err());
+                    let expected_err = String::from(
+                        "failed to boot the VM: \
+                        the upcall client needs a virtio-vsock device for communication",
+                    );
+                    assert_eq!(err_string, expected_err);
+                },
+            ),
+            // success
+            TestData::new(
+                VmmAction::InsertMemDevice(MemDeviceConfigInfo::default()),
+                InstanceState::Uninitialized,
+                &|result| {
+                    assert!(result.is_ok());
+                },
+            ),
+        ];
+
+        for t in tests.iter_mut() {
+            t.check_request();
+        }
+    }
+
+    #[cfg(feature = "virtio-balloon")]
+    #[test]
+    fn test_vmm_action_insert_balloon_device() {
+        skip_if_not_root!();
+
+        let tests = &mut [
+            // hotplug unready
+            TestData::new(
+                VmmAction::InsertBalloonDevice(BalloonDeviceConfigInfo::default()),
+                InstanceState::Running,
+                &|result| {
+                    assert!(matches!(
+                        result,
+                        Err(VmmActionError::StartMicroVm(
+                            StartMicroVmError::UpcallMissVsock
+                        ))
+                    ));
+                    let err_string = format!("{}", result.unwrap_err());
+                    let expected_err = String::from(
+                        "failed to boot the VM: \
+                        the upcall client needs a virtio-vsock device for communication",
+                    );
+                    assert_eq!(err_string, expected_err);
+                },
+            ),
+            // success
+            TestData::new(
+                VmmAction::InsertBalloonDevice(BalloonDeviceConfigInfo::default()),
+                InstanceState::Uninitialized,
+                &|result| {
+                    assert!(result.is_ok());
+                },
+            ),
+        ];
+
+        for t in tests.iter_mut() {
+            t.check_request();
+        }
+    }
 }

src/dragonball/src/config_manager.rs

@@ -278,6 +278,11 @@ where
         self.info_list.iter_mut()
     }
 
+    /// Remove the last device config info from the `info_list`.
+    pub fn pop(&mut self) -> Option<DeviceConfigInfo<T>> {
+        self.info_list.pop()
+    }
+
     fn get_index_by_id(&self, config: &T) -> Option<usize> {
         self.info_list
             .iter()

src/dragonball/src/device_manager/balloon_dev_mgr.rs

@@ -0,0 +1,419 @@
+// Copyright 2020 Alibaba Cloud. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use dbs_virtio_devices as virtio;
+use serde_derive::{Deserialize, Serialize};
+use slog::{error, info};
+use virtio::balloon::{Balloon, BalloonConfig};
+use virtio::Error as VirtIoError;
+
+use crate::address_space_manager::GuestAddressSpaceImpl;
+use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos};
+use crate::device_manager::DbsMmioV2Device;
+use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext};
+
+// The flag of whether to use the shared irq.
+const USE_SHARED_IRQ: bool = true;
+// The flag of whether to use the generic irq.
+const USE_GENERIC_IRQ: bool = false;
+
+/// Errors associated with `BalloonDeviceConfig`.
+#[derive(Debug, thiserror::Error)]
+pub enum BalloonDeviceError {
+    /// The balloon device was already used.
+    #[error("the virtio-balloon ID was already added to a different device")]
+    BalloonDeviceAlreadyExists,
+
+    /// Cannot perform the requested operation after booting the microVM.
+    #[error("the update operation is not allowed after boot")]
+    UpdateNotAllowedPostBoot,
+
+    /// guest memory error
+    #[error("failed to access guest memory, {0}")]
+    GuestMemoryError(#[source] vm_memory::mmap::Error),
+
+    /// create balloon device error
+    #[error("failed to create virtio-balloon device, {0}")]
+    CreateBalloonDevice(#[source] virtio::Error),
+
+    /// hotplug balloon device error
+    #[error("cannot hotplug virtio-balloon device, {0}")]
+    HotplugDeviceFailed(#[source] DeviceMgrError),
+
+    /// create mmio device error
+    #[error("cannot create virtio-balloon mmio device, {0}")]
+    CreateMmioDevice(#[source] DeviceMgrError),
+
+    /// Cannot initialize a balloon device or add a device to the MMIO Bus.
+    #[error("failure while registering balloon device: {0}")]
+    RegisterBalloonDevice(#[source] DeviceMgrError),
+
+    /// resize balloon device error
+    #[error("failure while resizing virtio-balloon device, {0}")]
+    ResizeFailed(#[source] VirtIoError),
+
+    /// The balloon device id doesn't exist.
+    #[error("invalid balloon device id '{0}'")]
+    InvalidDeviceId(String),
+
+    /// balloon device does not exist
+    #[error("balloon device does not exist")]
+    NotExist,
+
+    /// The device manager errors.
+    #[error("DeviceManager error: {0}")]
+    DeviceManager(#[source] DeviceMgrError),
+}
+
+/// Configuration information for a virtio-balloon device.
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
+pub struct BalloonDeviceConfigInfo {
+    /// Unique identifier of the balloon device
+    pub balloon_id: String,
+    /// Resize balloon size in mib
+    pub size_mib: u64,
+    /// Use shared irq
+    pub use_shared_irq: Option<bool>,
+    /// Use generic irq
+    pub use_generic_irq: Option<bool>,
+    /// VIRTIO_BALLOON_F_DEFLATE_ON_OOM
+    pub f_deflate_on_oom: bool,
+    /// VIRTIO_BALLOON_F_REPORTING
+    pub f_reporting: bool,
+}
+
+impl ConfigItem for BalloonDeviceConfigInfo {
+    type Err = BalloonDeviceError;
+
+    fn id(&self) -> &str {
+        &self.balloon_id
+    }
+
+    fn check_conflicts(&self, other: &Self) -> Result<(), BalloonDeviceError> {
+        if self.balloon_id.as_str() == other.balloon_id.as_str() {
+            Err(BalloonDeviceError::BalloonDeviceAlreadyExists)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+/// Balloon Device Info
+pub type BalloonDeviceInfo = DeviceConfigInfo<BalloonDeviceConfigInfo>;
+
+impl ConfigItem for BalloonDeviceInfo {
+    type Err = BalloonDeviceError;
+
+    fn id(&self) -> &str {
+        &self.config.balloon_id
+    }
+
+    fn check_conflicts(&self, other: &Self) -> Result<(), BalloonDeviceError> {
+        if self.config.balloon_id.as_str() == other.config.balloon_id.as_str() {
+            Err(BalloonDeviceError::BalloonDeviceAlreadyExists)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+/// Wrapper for the collection that holds all the Balloon Devices Configs
+#[derive(Clone)]
+pub struct BalloonDeviceMgr {
+    /// A list of `BalloonDeviceConfig` objects.
+    info_list: DeviceConfigInfos<BalloonDeviceConfigInfo>,
+    pub(crate) use_shared_irq: bool,
+}
+
+impl BalloonDeviceMgr {
+    /// Inserts `balloon_cfg` in the virtio-balloon device configuration list.
+    /// If an entry with the same id already exists, it will attempt to update
+    /// the existing entry.
+    pub fn insert_or_update_device(
+        &mut self,
+        mut ctx: DeviceOpContext,
+        balloon_cfg: BalloonDeviceConfigInfo,
+    ) -> std::result::Result<(), BalloonDeviceError> {
+        if !cfg!(feature = "hotplug") && ctx.is_hotplug {
+            error!(ctx.logger(), "hotplug feature has been disabled.";
+            "subsystem" => "balloon_dev_mgr",);
+            return Err(BalloonDeviceError::UpdateNotAllowedPostBoot);
+        }
+
+        let epoll_mgr = ctx
+            .get_epoll_mgr()
+            .map_err(BalloonDeviceError::DeviceManager)?;
+
+        // If the id of the drive already exists in the list, the operation is update.
+        if let Some(index) = self.get_index_of_balloon_dev(&balloon_cfg.balloon_id) {
+            // Update an existing balloon device
+            if ctx.is_hotplug {
+                info!(ctx.logger(), "resize virtio balloon size to {:?}", balloon_cfg.size_mib; "subsystem" => "balloon_dev_mgr");
+                self.update_balloon_size(index, balloon_cfg.size_mib)?;
+            }
+            self.info_list.insert_or_update(&balloon_cfg)?;
+        } else {
+            // Create a new balloon device
+            if !self.info_list.is_empty() {
+                error!(ctx.logger(), "only support one balloon device!"; "subsystem" => "balloon_dev_mgr");
+                return Err(BalloonDeviceError::BalloonDeviceAlreadyExists);
+            }
+
+            if !ctx.is_hotplug {
+                self.info_list.insert_or_update(&balloon_cfg)?;
+                return Ok(());
+            }
+
+            info!(ctx.logger(), "hotplug balloon device: {}", balloon_cfg.balloon_id; "subsystem" => "balloon_dev_mgr");
+            let device = Box::new(
+                virtio::balloon::Balloon::new(
+                    epoll_mgr,
+                    BalloonConfig {
+                        f_deflate_on_oom: balloon_cfg.f_deflate_on_oom,
+                        f_reporting: balloon_cfg.f_reporting,
+                    },
+                )
+                .map_err(BalloonDeviceError::CreateBalloonDevice)?,
+            );
+
+            let mmio_dev =
+                DeviceManager::create_mmio_virtio_device_with_device_change_notification(
+                    device,
+                    &mut ctx,
+                    balloon_cfg.use_shared_irq.unwrap_or(self.use_shared_irq),
+                    balloon_cfg.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
+                )
+                .map_err(BalloonDeviceError::CreateMmioDevice)?;
+            ctx.insert_hotplug_mmio_device(&mmio_dev, None)
+                .map_err(|e| {
+                    error!(
+                        ctx.logger(),
+                        "hotplug balloon device {} error: {}",
+                        &balloon_cfg.balloon_id, e;
+                        "subsystem" => "balloon_dev_mgr"
+                    );
+                    BalloonDeviceError::HotplugDeviceFailed(e)
+                })?;
+            let index = self.info_list.insert_or_update(&balloon_cfg)?;
+            self.info_list[index].set_device(mmio_dev);
+        }
+        Ok(())
+    }
+
+    /// Attaches all virtio-balloon devices from the BalloonDevicesConfig.
+    pub fn attach_devices(
+        &mut self,
+        ctx: &mut DeviceOpContext,
+    ) -> std::result::Result<(), BalloonDeviceError> {
+        let epoll_mgr = ctx
+            .get_epoll_mgr()
+            .map_err(BalloonDeviceError::DeviceManager)?;
+
+        for info in self.info_list.iter_mut() {
+            info!(ctx.logger(), "attach balloon device: {}", info.config.balloon_id; "subsystem" => "balloon_dev_mgr");
+
+            let device = Balloon::new(
+                epoll_mgr.clone(),
+                BalloonConfig {
+                    f_deflate_on_oom: info.config.f_deflate_on_oom,
+                    f_reporting: info.config.f_reporting,
+                },
+            )
+            .map_err(BalloonDeviceError::CreateBalloonDevice)?;
+            let mmio_dev =
+                DeviceManager::create_mmio_virtio_device_with_device_change_notification(
+                    Box::new(device),
+                    ctx,
+                    info.config.use_shared_irq.unwrap_or(self.use_shared_irq),
+                    info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
+                )
+                .map_err(BalloonDeviceError::RegisterBalloonDevice)?;
+            info.set_device(mmio_dev);
+        }
+
+        Ok(())
+    }
+
+    fn update_balloon_size(
+        &self,
+        index: usize,
+        size_mib: u64,
+    ) -> std::result::Result<(), BalloonDeviceError> {
+        let device = self.info_list[index]
+            .device
+            .as_ref()
+            .ok_or_else(|| BalloonDeviceError::NotExist)?;
+        if let Some(mmio_dev) = device.as_any().downcast_ref::<DbsMmioV2Device>() {
+            let guard = mmio_dev.state();
+            let inner_dev = guard.get_inner_device();
+            if let Some(balloon_dev) = inner_dev
+                .as_any()
+                .downcast_ref::<Balloon<GuestAddressSpaceImpl>>()
+            {
+                return balloon_dev
+                    .set_size(size_mib)
+                    .map_err(BalloonDeviceError::ResizeFailed);
+            }
+        }
+        Ok(())
+    }
+
+    fn get_index_of_balloon_dev(&self, balloon_id: &str) -> Option<usize> {
+        self.info_list
+            .iter()
+            .position(|info| info.config.balloon_id.eq(balloon_id))
+    }
+}
+
+impl Default for BalloonDeviceMgr {
+    /// Create a new `BalloonDeviceMgr` object..
+    fn default() -> Self {
+        BalloonDeviceMgr {
+            info_list: DeviceConfigInfos::new(),
+            use_shared_irq: USE_SHARED_IRQ,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::tests::create_vm_for_test;
+
+    impl Default for BalloonDeviceConfigInfo {
+        fn default() -> Self {
+            BalloonDeviceConfigInfo {
+                balloon_id: "".to_string(),
+                size_mib: 0,
+                use_generic_irq: None,
+                use_shared_irq: None,
+                f_deflate_on_oom: false,
+                f_reporting: false,
+            }
+        }
+    }
+
+    #[test]
+    fn test_balloon_config_check_conflicts() {
+        let config = BalloonDeviceConfigInfo::default();
+        let mut config2 = BalloonDeviceConfigInfo::default();
+        assert!(config.check_conflicts(&config2).is_err());
+        config2.balloon_id = "dummy_balloon".to_string();
+        assert!(config.check_conflicts(&config2).is_ok());
+    }
+
+    #[test]
+    fn test_create_balloon_devices_configs() {
+        let mgr = BalloonDeviceMgr::default();
+        assert_eq!(mgr.info_list.len(), 0);
+        assert_eq!(mgr.get_index_of_balloon_dev(""), None);
+    }
+
+    #[test]
+    fn test_balloon_insert_or_update_device() {
+        //Init vm for test.
+        let mut vm = create_vm_for_test();
+
+        // Test for standard config
+        let device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            None,
+            false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
+        );
+
+        let dummy_balloon_device = BalloonDeviceConfigInfo::default();
+        vm.device_manager_mut()
+            .balloon_manager
+            .insert_or_update_device(device_op_ctx, dummy_balloon_device)
+            .unwrap();
+        assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1);
+    }
+
+    #[test]
+    fn test_balloon_attach_device() {
+        //Init vm and insert balloon config for test.
+        let mut vm = create_vm_for_test();
+        let device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            None,
+            false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
+        );
+
+        let dummy_balloon_device = BalloonDeviceConfigInfo::default();
+        vm.device_manager_mut()
+            .balloon_manager
+            .insert_or_update_device(device_op_ctx, dummy_balloon_device)
+            .unwrap();
+        assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1);
+
+        // Test for standard config
+        let mut device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            None,
+            false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
+        );
+        assert!(vm
+            .device_manager_mut()
+            .balloon_manager
+            .attach_devices(&mut device_op_ctx)
+            .is_ok());
+        assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1);
+    }
+
+    #[test]
+    fn test_balloon_update_device() {
+        //Init vm for test.
+        let mut vm = create_vm_for_test();
+        let device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            None,
+            false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
+        );
+
+        let dummy_balloon_device = BalloonDeviceConfigInfo::default();
+        vm.device_manager_mut()
+            .balloon_manager
+            .insert_or_update_device(device_op_ctx, dummy_balloon_device)
+            .unwrap();
+        assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1);
+
+        let mut device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            None,
+            false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
+        );
+
+        assert!(vm
+            .device_manager_mut()
+            .balloon_manager
+            .attach_devices(&mut device_op_ctx)
+            .is_ok());
+        assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1);
+
+        assert!(vm
+            .device_manager()
+            .balloon_manager
+            .update_balloon_size(0, 200)
+            .is_ok());
+    }
+}

src/dragonball/src/device_manager/blk_dev_mgr.rs

@@ -114,6 +114,8 @@ pub enum BlockDeviceType {
     /// SPOOL is a reliable NVMe virtualization system for the cloud environment.
     /// You could learn more SPOOL here: https://www.usenix.org/conference/atc20/presentation/xue
     Spool,
+    /// The standard vhost-user-blk based device such as Spdk device.
+    Spdk,
     /// Local disk/file based low level device.
     RawBlock,
 }
@@ -124,6 +126,8 @@ impl BlockDeviceType {
         // SPOOL path should be started with "spool", e.g. "spool:/device1"
         if path.starts_with("spool:/") {
             BlockDeviceType::Spool
+        } else if path.starts_with("spdk:/") {
+            BlockDeviceType::Spdk
         } else {
             BlockDeviceType::RawBlock
         }
@@ -340,7 +344,7 @@ impl BlockDeviceMgr {
     /// the existing entry.
     /// Inserting a secondary root block device will fail.
     pub fn insert_device(
-        device_mgr: &mut DeviceManager,
+        &mut self,
         mut ctx: DeviceOpContext,
         config: BlockDeviceConfigInfo,
     ) -> std::result::Result<(), BlockDeviceError> {
@@ -348,10 +352,8 @@ impl BlockDeviceMgr {
             return Err(BlockDeviceError::UpdateNotAllowedPostBoot);
         }
 
-        let mgr = &mut device_mgr.block_manager;
-
         // If the id of the drive already exists in the list, the operation is update.
-        match mgr.get_index_of_drive_id(config.id()) {
+        match self.get_index_of_drive_id(config.id()) {
             Some(index) => {
                 // No support for runtime update yet.
                 if ctx.is_hotplug {
@@ -359,19 +361,19 @@ impl BlockDeviceMgr {
                         config.path_on_host.clone(),
                     ))
                 } else {
-                    for (idx, info) in mgr.info_list.iter().enumerate() {
+                    for (idx, info) in self.info_list.iter().enumerate() {
                         if idx != index {
                             info.config.check_conflicts(&config)?;
                         }
                     }
-                    mgr.update(index, config)
+                    self.update(index, config)
                 }
             }
             None => {
-                for info in mgr.info_list.iter() {
+                for info in self.info_list.iter() {
                     info.config.check_conflicts(&config)?;
                 }
-                let index = mgr.create(config.clone())?;
+                let index = self.create(config.clone())?;
                 if !ctx.is_hotplug {
                     return Ok(());
                 }
@@ -383,17 +385,16 @@ impl BlockDeviceMgr {
                         let dev = DeviceManager::create_mmio_virtio_device(
                             device,
                             &mut ctx,
-                            config.use_shared_irq.unwrap_or(mgr.use_shared_irq),
+                            config.use_shared_irq.unwrap_or(self.use_shared_irq),
                             config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
                         )
                         .map_err(BlockDeviceError::DeviceManager)?;
-                        mgr.update_device_by_index(index, Arc::clone(&dev))?;
+                        self.update_device_by_index(index, Arc::clone(&dev))?;
                         // live-upgrade need save/restore device from info.device.
-                        mgr.info_list[index].set_device(dev.clone());
+                        self.info_list[index].set_device(dev.clone());
                         ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| {
                             let logger = ctx.logger().new(slog::o!());
-                            BlockDeviceMgr::remove_device(device_mgr, ctx, &config.drive_id)
-                                .unwrap();
+                            self.remove_device(ctx, &config.drive_id).unwrap();
                             error!(
                                 logger,
                                 "failed to hot-add virtio block device {}, {:?}",
@@ -403,6 +404,10 @@ impl BlockDeviceMgr {
                             BlockDeviceError::DeviceManager(e)
                         })
                     }
+                    BlockDeviceType::Spool | BlockDeviceType::Spdk => {
+                        // TBD
+                        todo!()
+                    }
                     _ => Err(BlockDeviceError::InvalidBlockDeviceType),
                 }
             }
@@ -466,7 +471,7 @@ impl BlockDeviceMgr {
 
     /// remove a block device, it basically is the inverse operation of `insert_device``
     pub fn remove_device(
-        dev_mgr: &mut DeviceManager,
+        &mut self,
         mut ctx: DeviceOpContext,
         drive_id: &str,
     ) -> std::result::Result<(), BlockDeviceError> {
@@ -474,8 +479,7 @@ impl BlockDeviceMgr {
             return Err(BlockDeviceError::UpdateNotAllowedPostBoot);
         }
 
-        let mgr = &mut dev_mgr.block_manager;
-        match mgr.remove(drive_id) {
+        match self.remove(drive_id) {
             Some(mut info) => {
                 info!(ctx.logger(), "remove drive {}", info.config.drive_id);
                 if let Some(device) = info.device.take() {
@@ -731,15 +735,14 @@ impl BlockDeviceMgr {
 
     /// Update the ratelimiter settings of a virtio blk device.
     pub fn update_device_ratelimiters(
-        device_mgr: &mut DeviceManager,
+        &mut self,
         new_cfg: BlockDeviceConfigUpdateInfo,
     ) -> std::result::Result<(), BlockDeviceError> {
-        let mgr = &mut device_mgr.block_manager;
-        match mgr.get_index_of_drive_id(&new_cfg.drive_id) {
+        match self.get_index_of_drive_id(&new_cfg.drive_id) {
             Some(index) => {
-                let config = &mut mgr.info_list[index].config;
+                let config = &mut self.info_list[index].config;
                 config.rate_limiter = new_cfg.rate_limiter.clone();
-                let device = mgr.info_list[index]
+                let device = self.info_list[index]
                     .device
                     .as_mut()
                     .ok_or_else(|| BlockDeviceError::InvalidDeviceId("".to_owned()))?;
@@ -827,12 +830,11 @@ mod tests {
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(BlockDeviceMgr::insert_device(
-            vm.device_manager_mut(),
-            ctx,
-            dummy_block_device.clone(),
-        )
-        .is_ok());
+        assert!(vm
+            .device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device.clone(),)
+            .is_ok());
 
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
         assert!(!vm.device_manager().block_manager.has_root_block_device());
@@ -877,6 +879,8 @@ mod tests {
             Some(vm.vm_as().unwrap().clone()),
             None,
             false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
         );
 
         let dummy_file = TempFile::new().unwrap();
@@ -897,7 +901,9 @@ mod tests {
             use_shared_irq: None,
             use_generic_irq: None,
         };
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), device_op_ctx, dummy_block_device)
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(device_op_ctx, dummy_block_device)
             .unwrap();
 
         let cfg = BlockDeviceConfigUpdateInfo {
@@ -911,6 +917,8 @@ mod tests {
             Some(vm.vm_as().unwrap().clone()),
             None,
             false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
         );
 
         vm.device_manager_mut()
@@ -923,7 +931,9 @@ mod tests {
         let expected_error = "could not send patch message to the block epoll handler".to_string();
 
         assert_eq!(
-            BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg)
+            vm.device_manager_mut()
+                .block_manager
+                .update_device_ratelimiters(cfg)
                 .unwrap_err()
                 .to_string(),
             expected_error
@@ -938,7 +948,9 @@ mod tests {
         let expected_error = format!("invalid block device id '{0}'", cfg2.drive_id);
 
         assert_eq!(
-            BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg2)
+            vm.device_manager_mut()
+                .block_manager
+                .update_device_ratelimiters(cfg2)
                 .unwrap_err()
                 .to_string(),
             expected_error
@@ -968,12 +980,11 @@ mod tests {
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(BlockDeviceMgr::insert_device(
-            vm.device_manager_mut(),
-            ctx,
-            dummy_block_device.clone(),
-        )
-        .is_ok());
+        assert!(vm
+            .device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device.clone(),)
+            .is_ok());
 
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
         assert!(vm.device_manager().block_manager.has_root_block);
@@ -1027,12 +1038,16 @@ mod tests {
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_1).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device_1)
+            .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(
-            BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_2)
-                .is_err()
-        );
+        assert!(vm
+            .device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device_2)
+            .is_err());
     }
 
     #[test]
@@ -1112,13 +1127,22 @@ mod tests {
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 3);
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device)
+            .unwrap();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2)
+            .unwrap();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_3)
+            .unwrap();
     }
 
     #[test]
@@ -1182,13 +1206,19 @@ mod tests {
         let mut vm = crate::vm::tests::create_vm_instance();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3.clone())
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_3.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device.clone())
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device.clone())
             .unwrap();
 
         assert!(vm.device_manager().block_manager.has_root_block_device(),);
@@ -1255,9 +1285,14 @@ mod tests {
 
         // Add 2 block devices.
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device)
+            .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2.clone())
             .unwrap();
 
         // Get index zero.
@@ -1286,7 +1321,9 @@ mod tests {
         // Update OK.
         dummy_block_device_2.is_read_only = true;
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2.clone())
             .unwrap();
 
         let index = vm
@@ -1306,21 +1343,21 @@ mod tests {
         let dummy_path_3 = PathBuf::from(dummy_filename_3);
         dummy_block_device_2.path_on_host = dummy_path_3;
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(BlockDeviceMgr::insert_device(
-            vm.device_manager_mut(),
-            ctx,
-            dummy_block_device_2.clone(),
-        )
-        .is_err());
+        assert!(vm
+            .device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2.clone(),)
+            .is_err());
 
         // Update with 2 root block devices.
         dummy_block_device_2.path_on_host = dummy_path_2.clone();
         dummy_block_device_2.is_root_device = true;
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(
-            BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2,)
-                .is_err(),
-        );
+        assert!(vm
+            .device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2,)
+            .is_err(),);
 
         // Switch roots and add a PARTUUID for the new one.
         let root_block_device_old = BlockDeviceConfigInfo {
@@ -1354,9 +1391,15 @@ mod tests {
             use_generic_irq: None,
         };
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_old).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device_old)
+            .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_new).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device_new)
+            .unwrap();
         assert!(vm.device_manager().block_manager.has_part_uuid_root);
     }
 }

src/dragonball/src/device_manager/console_manager.rs

@@ -79,16 +79,17 @@ impl ConsoleManager {
             .unwrap()
             .set_output_stream(Some(Box::new(std::io::stdout())));
         let stdin_handle = std::io::stdin();
-        stdin_handle
-            .lock()
-            .set_raw_mode()
-            .map_err(|e| DeviceMgrError::ConsoleManager(ConsoleManagerError::StdinHandle(e)))?;
-        stdin_handle
-            .lock()
-            .set_non_block(true)
-            .map_err(ConsoleManagerError::StdinHandle)
-            .map_err(DeviceMgrError::ConsoleManager)?;
-
+        {
+            let guard = stdin_handle.lock();
+            guard
+                .set_raw_mode()
+                .map_err(ConsoleManagerError::StdinHandle)
+                .map_err(DeviceMgrError::ConsoleManager)?;
+            guard
+                .set_non_block(true)
+                .map_err(ConsoleManagerError::StdinHandle)
+                .map_err(DeviceMgrError::ConsoleManager)?;
+        }
         let handler = ConsoleEpollHandler::new(device, Some(stdin_handle), None, &self.logger);
         self.subscriber_id = Some(self.epoll_mgr.add_subscriber(Box::new(handler)));
         self.backend = Some(Backend::StdinHandle(std::io::stdin()));

src/dragonball/src/device_manager/mem_dev_mgr.rs

@@ -0,0 +1,733 @@
+// Copyright 2020 Alibaba Cloud. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::io;
+use std::sync::{Arc, Mutex};
+
+use dbs_address_space::{
+    AddressSpace, AddressSpaceError, AddressSpaceRegion, MPOL_MF_MOVE, MPOL_PREFERRED, USABLE_END,
+};
+use dbs_utils::epoll_manager::EpollManager;
+use dbs_virtio_devices as virtio;
+use kvm_bindings::kvm_userspace_memory_region;
+use kvm_ioctls::VmFd;
+use nix::sys::mman;
+use serde_derive::{Deserialize, Serialize};
+use slog::{debug, error, info, warn};
+use virtio::mem::{Mem, MemRegionFactory};
+use virtio::Error as VirtIoError;
+use vm_memory::{
+    Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestRegionMmap, GuestUsize, MmapRegion,
+};
+
+use crate::address_space_manager::GuestAddressSpaceImpl;
+use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos};
+use crate::device_manager::DbsMmioV2Device;
+use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext};
+use crate::vm::VmConfigInfo;
+
+// The flag of whether to use the shared irq.
+const USE_SHARED_IRQ: bool = true;
+// The flag of whether to use the generic irq.
+const USE_GENERIC_IRQ: bool = false;
+
+const HUGE_PAGE_2M: usize = 0x200000;
+
+// max numa node ids on host
+const MAX_NODE: u32 = 64;
+
+/// Errors associated with `MemDeviceConfig`.
+#[derive(Debug, thiserror::Error)]
+pub enum MemDeviceError {
+    /// The mem device was already used.
+    #[error("the virtio-mem ID was already added to a different device")]
+    MemDeviceAlreadyExists,
+
+    /// Cannot perform the requested operation after booting the microVM.
+    #[error("the update operation is not allowed after boot")]
+    UpdateNotAllowedPostBoot,
+
+    /// insert mem device error
+    #[error("cannot add virtio-mem device, {0}")]
+    InsertDeviceFailed(#[source] DeviceMgrError),
+
+    /// create mem device error
+    #[error("cannot create virito-mem device, {0}")]
+    CreateMemDevice(#[source] DeviceMgrError),
+
+    /// create mmio device error
+    #[error("cannot create virito-mem mmio device, {0}")]
+    CreateMmioDevice(#[source] DeviceMgrError),
+
+    /// resize mem device error
+    #[error("failure while resizing virtio-mem device, {0}")]
+    ResizeFailed(#[source] VirtIoError),
+
+    /// mem device does not exist
+    #[error("mem device does not exist")]
+    DeviceNotExist,
+
+    /// address space region error
+    #[error("address space region error, {0}")]
+    AddressSpaceRegion(#[source] AddressSpaceError),
+
+    /// Cannot initialize a mem device or add a device to the MMIO Bus.
+    #[error("failure while registering mem device: {0}")]
+    RegisterMemDevice(#[source] DeviceMgrError),
+
+    /// The mem device id doesn't exist.
+    #[error("invalid mem device id '{0}'")]
+    InvalidDeviceId(String),
+
+    /// The device manager errors.
+    #[error("DeviceManager error: {0}")]
+    DeviceManager(#[source] DeviceMgrError),
+}
+
+/// Configuration information for a virtio-mem device.
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
+pub struct MemDeviceConfigInfo {
+    /// Unique identifier of the pmem device
+    pub mem_id: String,
+    /// Memory size mib
+    pub size_mib: u64,
+    /// Memory capacity mib
+    pub capacity_mib: u64,
+    /// Use multi_region or not
+    pub multi_region: bool,
+    /// host numa node id
+    pub host_numa_node_id: Option<u32>,
+    /// guest numa node id
+    pub guest_numa_node_id: Option<u16>,
+    /// Use shared irq
+    pub use_shared_irq: Option<bool>,
+    /// Use generic irq
+    pub use_generic_irq: Option<bool>,
+}
+
+impl ConfigItem for MemDeviceConfigInfo {
+    type Err = MemDeviceError;
+
+    fn id(&self) -> &str {
+        &self.mem_id
+    }
+
+    fn check_conflicts(&self, other: &Self) -> Result<(), MemDeviceError> {
+        if self.mem_id.as_str() == other.mem_id.as_str() {
+            Err(MemDeviceError::MemDeviceAlreadyExists)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+/// Mem Device Info
+pub type MemDeviceInfo = DeviceConfigInfo<MemDeviceConfigInfo>;
+
+impl ConfigItem for MemDeviceInfo {
+    type Err = MemDeviceError;
+
+    fn id(&self) -> &str {
+        &self.config.mem_id
+    }
+
+    fn check_conflicts(&self, other: &Self) -> Result<(), MemDeviceError> {
+        if self.config.mem_id.as_str() == other.config.mem_id.as_str() {
+            Err(MemDeviceError::MemDeviceAlreadyExists)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+/// Wrapper for the collection that holds all the Mem Devices Configs
+#[derive(Clone)]
+pub struct MemDeviceMgr {
+    /// A list of `MemDeviceConfig` objects.
+    info_list: DeviceConfigInfos<MemDeviceConfigInfo>,
+    pub(crate) use_shared_irq: bool,
+}
+
+impl MemDeviceMgr {
+    /// Inserts `mem_cfg` in the virtio-mem device configuration list.
+    /// If an entry with the same id already exists, it will attempt to update
+    /// the existing entry.
+    pub fn insert_or_update_device(
+        &mut self,
+        mut ctx: DeviceOpContext,
+        mem_cfg: MemDeviceConfigInfo,
+    ) -> std::result::Result<(), MemDeviceError> {
+        if !cfg!(feature = "hotplug") && ctx.is_hotplug {
+            error!(ctx.logger(), "hotplug feature has been disabled.";
+            "subsystem" => "virito-mem");
+            return Err(MemDeviceError::UpdateNotAllowedPostBoot);
+        }
+
+        let epoll_mgr = ctx.get_epoll_mgr().map_err(MemDeviceError::DeviceManager)?;
+
+        // If the id of the drive already exists in the list, the operation is update.
+        if let Some(index) = self.get_index_of_mem_dev(&mem_cfg.mem_id) {
+            // Update an existing memory device
+            if ctx.is_hotplug {
+                info!(
+                    ctx.logger(),
+                    "update memory device: {}, size: 0x{:x}MB.",
+                    mem_cfg.mem_id,
+                    mem_cfg.size_mib;
+                    "subsystem" => "virito-mem"
+                );
+                self.update_memory_size(index, mem_cfg.size_mib)?;
+            }
+            self.info_list.insert_or_update(&mem_cfg)?;
+        } else {
+            // Create a new memory device
+            if !ctx.is_hotplug {
+                self.info_list.insert_or_update(&mem_cfg)?;
+                return Ok(());
+            }
+
+            info!(
+                ctx.logger(),
+                "hot-add memory device: {}, size: 0x{:x}MB.", mem_cfg.mem_id, mem_cfg.size_mib;
+                "subsystem" => "virito-mem"
+            );
+
+            let device = Self::create_memory_device(&mem_cfg, &ctx, &epoll_mgr)
+                .map_err(MemDeviceError::CreateMemDevice)?;
+            let mmio_device =
+                DeviceManager::create_mmio_virtio_device_with_device_change_notification(
+                    Box::new(device),
+                    &mut ctx,
+                    mem_cfg.use_shared_irq.unwrap_or(self.use_shared_irq),
+                    mem_cfg.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
+                )
+                .map_err(MemDeviceError::CreateMmioDevice)?;
+
+            #[cfg(not(test))]
+            ctx.insert_hotplug_mmio_device(&mmio_device, None)
+                .map_err(|e| {
+                    error!(
+                        ctx.logger(),
+                        "failed to hot-add virtio-mem device {}, {}", &mem_cfg.mem_id, e;
+                        "subsystem" => "virito-mem"
+                    );
+                    MemDeviceError::InsertDeviceFailed(e)
+                })?;
+
+            let index = self.info_list.insert_or_update(&mem_cfg)?;
+            self.info_list[index].set_device(mmio_device);
+        }
+
+        Ok(())
+    }
+
+    /// Attaches all virtio-mem devices from the MemDevicesConfig.
+    pub fn attach_devices(
+        &mut self,
+        ctx: &mut DeviceOpContext,
+    ) -> std::result::Result<(), MemDeviceError> {
+        let epoll_mgr = ctx.get_epoll_mgr().map_err(MemDeviceError::DeviceManager)?;
+
+        for info in self.info_list.iter_mut() {
+            let config = &info.config;
+            info!(
+                ctx.logger(),
+                "attach virtio-mem device {}, size 0x{:x}.", config.mem_id, config.size_mib;
+                "subsystem" => "virito-mem"
+            );
+            // Ignore virtio-mem device with zero memory capacity.
+            if config.size_mib == 0 {
+                debug!(
+                    ctx.logger(),
+                    "ignore zero-sizing memory device {}.", config.mem_id;
+                    "subsystem" => "virito-mem"
+                );
+                continue;
+            }
+
+            let device = Self::create_memory_device(config, ctx, &epoll_mgr)
+                .map_err(MemDeviceError::CreateMemDevice)?;
+            let mmio_device =
+                DeviceManager::create_mmio_virtio_device_with_device_change_notification(
+                    Box::new(device),
+                    ctx,
+                    config.use_shared_irq.unwrap_or(self.use_shared_irq),
+                    config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
+                )
+                .map_err(MemDeviceError::RegisterMemDevice)?;
+
+            info.set_device(mmio_device);
+        }
+
+        Ok(())
+    }
+
+    fn get_index_of_mem_dev(&self, mem_id: &str) -> Option<usize> {
+        self.info_list
+            .iter()
+            .position(|info| info.config.mem_id.eq(mem_id))
+    }
+
+    fn create_memory_device(
+        config: &MemDeviceConfigInfo,
+        ctx: &DeviceOpContext,
+        epoll_mgr: &EpollManager,
+    ) -> std::result::Result<virtio::mem::Mem<GuestAddressSpaceImpl>, DeviceMgrError> {
+        let factory = Arc::new(Mutex::new(MemoryRegionFactory::new(
+            ctx,
+            config.mem_id.clone(),
+            config.host_numa_node_id,
+        )?));
+
+        let mut capacity_mib = config.capacity_mib;
+        if capacity_mib == 0 {
+            capacity_mib = *USABLE_END >> 20;
+        }
+        // get boot memory size for calculate alignment
+        let boot_mem_size = {
+            let boot_size = (ctx.get_vm_config()?.mem_size_mib << 20) as u64;
+            // increase 1G memory because of avoiding mmio hole
+            match boot_size {
+                x if x > dbs_boot::layout::MMIO_LOW_START => x + (1 << 30),
+                _ => boot_size,
+            }
+        };
+
+        virtio::mem::Mem::new(
+            config.mem_id.clone(),
+            capacity_mib,
+            config.size_mib,
+            config.multi_region,
+            config.guest_numa_node_id,
+            epoll_mgr.clone(),
+            factory,
+            boot_mem_size,
+        )
+        .map_err(DeviceMgrError::Virtio)
+    }
+
+    /// Removes all virtio-mem devices
+    pub fn remove_devices(&self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> {
+        for info in self.info_list.iter() {
+            if let Some(device) = &info.device {
+                DeviceManager::destroy_mmio_virtio_device(device.clone(), ctx)?;
+            }
+        }
+
+        Ok(())
+    }
+
+    fn update_memory_size(
+        &self,
+        index: usize,
+        size_mib: u64,
+    ) -> std::result::Result<(), MemDeviceError> {
+        let device = self.info_list[index]
+            .device
+            .as_ref()
+            .ok_or_else(|| MemDeviceError::DeviceNotExist)?;
+        if let Some(mmio_dev) = device.as_any().downcast_ref::<DbsMmioV2Device>() {
+            let guard = mmio_dev.state();
+            let inner_dev = guard.get_inner_device();
+            if let Some(mem_dev) = inner_dev
+                .as_any()
+                .downcast_ref::<Mem<GuestAddressSpaceImpl>>()
+            {
+                return mem_dev
+                    .set_requested_size(size_mib)
+                    .map_err(MemDeviceError::ResizeFailed);
+            }
+        }
+        Ok(())
+    }
+}
+
+impl Default for MemDeviceMgr {
+    /// Create a new `MemDeviceMgr` object..
+    fn default() -> Self {
+        MemDeviceMgr {
+            info_list: DeviceConfigInfos::new(),
+            use_shared_irq: USE_SHARED_IRQ,
+        }
+    }
+}
+
+struct MemoryRegionFactory {
+    mem_id: String,
+    vm_as: GuestAddressSpaceImpl,
+    address_space: AddressSpace,
+    vm_config: VmConfigInfo,
+    vm_fd: Arc<VmFd>,
+    logger: Arc<slog::Logger>,
+    host_numa_node_id: Option<u32>,
+    instance_id: String,
+}
+
+impl MemoryRegionFactory {
+    fn new(
+        ctx: &DeviceOpContext,
+        mem_id: String,
+        host_numa_node_id: Option<u32>,
+    ) -> Result<Self, DeviceMgrError> {
+        let vm_as = ctx.get_vm_as()?;
+        let address_space = ctx.get_address_space()?;
+        let vm_config = ctx.get_vm_config()?;
+        let logger = Arc::new(ctx.logger().new(slog::o!()));
+
+        let shared_info = ctx.shared_info.read().unwrap();
+        let instance_id = shared_info.id.clone();
+
+        Ok(MemoryRegionFactory {
+            mem_id,
+            vm_as,
+            address_space,
+            vm_config,
+            vm_fd: ctx.vm_fd.clone(),
+            logger,
+            host_numa_node_id,
+            instance_id,
+        })
+    }
+
+    fn configure_anon_mem(&self, mmap_reg: &MmapRegion) -> Result<(), VirtIoError> {
+        unsafe {
+            mman::madvise(
+                mmap_reg.as_ptr() as *mut libc::c_void,
+                mmap_reg.size(),
+                mman::MmapAdvise::MADV_DONTFORK,
+            )
+        }
+        .map_err(VirtIoError::Madvise)?;
+
+        Ok(())
+    }
+
+    fn configure_numa(&self, mmap_reg: &MmapRegion, node_id: u32) -> Result<(), VirtIoError> {
+        let nodemask = 1_u64
+            .checked_shl(node_id)
+            .ok_or(VirtIoError::InvalidInput)?;
+        let res = unsafe {
+            libc::syscall(
+                libc::SYS_mbind,
+                mmap_reg.as_ptr() as *mut libc::c_void,
+                mmap_reg.size(),
+                MPOL_PREFERRED,
+                &nodemask as *const u64,
+                MAX_NODE,
+                MPOL_MF_MOVE,
+            )
+        };
+        if res < 0 {
+            warn!(
+                self.logger,
+                "failed to mbind memory to host_numa_node_id {}: this may affect performance",
+                node_id;
+                "subsystem" => "virito-mem"
+            );
+        }
+        Ok(())
+    }
+
+    fn configure_thp(&mut self, mmap_reg: &MmapRegion) -> Result<(), VirtIoError> {
+        debug!(
+            self.logger,
+            "Setting MADV_HUGEPAGE on AddressSpaceRegion addr {:x?} len {:x?}",
+            mmap_reg.as_ptr(),
+            mmap_reg.size();
+            "subsystem" => "virito-mem"
+        );
+
+        // Safe because we just create the MmapRegion
+        unsafe {
+            mman::madvise(
+                mmap_reg.as_ptr() as *mut libc::c_void,
+                mmap_reg.size(),
+                mman::MmapAdvise::MADV_HUGEPAGE,
+            )
+        }
+        .map_err(VirtIoError::Madvise)?;
+
+        Ok(())
+    }
+
+    fn map_to_kvm(
+        &mut self,
+        slot: u32,
+        reg: &Arc<AddressSpaceRegion>,
+        mmap_reg: &MmapRegion,
+    ) -> Result<(), VirtIoError> {
+        let host_addr = mmap_reg.as_ptr() as u64;
+
+        let flags = 0u32;
+
+        let mem_region = kvm_userspace_memory_region {
+            slot,
+            guest_phys_addr: reg.start_addr().raw_value(),
+            memory_size: reg.len(),
+            userspace_addr: host_addr,
+            flags,
+        };
+
+        // Safe because the user mem region is just created, and kvm slot is allocated
+        // by resource allocator.
+        unsafe { self.vm_fd.set_user_memory_region(mem_region) }
+            .map_err(VirtIoError::SetUserMemoryRegion)?;
+
+        Ok(())
+    }
+}
+
+impl MemRegionFactory for MemoryRegionFactory {
+    fn create_region(
+        &mut self,
+        guest_addr: GuestAddress,
+        region_len: GuestUsize,
+        kvm_slot: u32,
+    ) -> std::result::Result<Arc<GuestRegionMmap>, VirtIoError> {
+        // create address space region
+        let mem_type = self.vm_config.mem_type.as_str();
+        let mut mem_file_path = self.vm_config.mem_file_path.clone();
+        let mem_file_name = format!(
+            "/virtiomem_{}_{}",
+            self.instance_id.as_str(),
+            self.mem_id.as_str()
+        );
+        mem_file_path.push_str(mem_file_name.as_str());
+        let region = Arc::new(
+            AddressSpaceRegion::create_default_memory_region(
+                guest_addr,
+                region_len,
+                self.host_numa_node_id,
+                mem_type,
+                mem_file_path.as_str(),
+                false,
+                true,
+            )
+            .map_err(|e| {
+                error!(self.logger, "failed to insert address space region: {}", e);
+                // dbs-virtio-devices should not depend on dbs-address-space.
+                // So here io::Error is used instead of AddressSpaceError directly.
+                VirtIoError::IOError(io::Error::new(
+                    io::ErrorKind::Other,
+                    format!(
+                        "invalid address space region ({0:#x}, {1:#x})",
+                        guest_addr.0, region_len
+                    ),
+                ))
+            })?,
+        );
+        info!(
+            self.logger,
+            "VM: mem_type: {} mem_file_path: {}, numa_node_id: {:?} file_offset: {:?}",
+            mem_type,
+            mem_file_path,
+            self.host_numa_node_id,
+            region.file_offset();
+            "subsystem" => "virito-mem"
+        );
+
+        let mmap_region = MmapRegion::build(
+            region.file_offset().cloned(),
+            region_len as usize,
+            region.prot_flags(),
+            region.perm_flags(),
+        )
+        .map_err(VirtIoError::NewMmapRegion)?;
+        let host_addr: u64 = mmap_region.as_ptr() as u64;
+
+        // thp
+        if mem_type == "hugeanon" || mem_type == "hugeshmem" {
+            self.configure_thp(&mmap_region)?;
+        }
+
+        // Handle numa
+        if let Some(numa_node_id) = self.host_numa_node_id {
+            self.configure_numa(&mmap_region, numa_node_id)?;
+        }
+
+        // add to guest memory mapping
+        self.map_to_kvm(kvm_slot, &region, &mmap_region)?;
+
+        info!(
+            self.logger,
+            "kvm set user memory region: slot: {}, flags: {}, guest_phys_addr: {:X}, memory_size: {}, userspace_addr: {:X}",
+            kvm_slot,
+            0,
+            guest_addr.raw_value(),
+            region_len,
+            host_addr;
+            "subsystem" => "virito-mem"
+        );
+
+        // All value should be valid.
+        let memory_region = Arc::new(
+            GuestRegionMmap::new(mmap_region, guest_addr).map_err(VirtIoError::InsertMmap)?,
+        );
+
+        let vm_as_new = self
+            .vm_as
+            .memory()
+            .insert_region(memory_region.clone())
+            .map_err(VirtIoError::InsertMmap)?;
+        self.vm_as.lock().unwrap().replace(vm_as_new);
+        self.address_space.insert_region(region).map_err(|e| {
+            error!(self.logger, "failed to insert address space region: {}", e);
+            // dbs-virtio-devices should not depend on dbs-address-space.
+            // So here io::Error is used instead of AddressSpaceError directly.
+            VirtIoError::IOError(io::Error::new(
+                io::ErrorKind::Other,
+                format!(
+                    "invalid address space region ({0:#x}, {1:#x})",
+                    guest_addr.0, region_len
+                ),
+            ))
+        })?;
+
+        Ok(memory_region)
+    }
+
+    fn restore_region_addr(
+        &self,
+        guest_addr: GuestAddress,
+    ) -> std::result::Result<*mut u8, VirtIoError> {
+        let memory = self.vm_as.memory();
+        // NOTE: We can't clone `GuestRegionMmap` reference directly!!!
+        //
+        // Since an important role of the member `mapping` (type is
+        // `MmapRegion`) in `GuestRegionMmap` is to mmap the memory during
+        // construction and munmap the memory during drop. However, when the
+        // life time of cloned data is over, the drop operation will be
+        // performed, which will munmap the origional mmap memory, which will
+        // cause some memory in dragonall to be inaccessable. And remember the
+        // data structure that was cloned is still alive now, when its life time
+        // is over, it will perform the munmap operation again, which will cause
+        // a memory exception!
+        memory
+            .get_host_address(guest_addr)
+            .map_err(VirtIoError::GuestMemory)
+    }
+
+    fn get_host_numa_node_id(&self) -> Option<u32> {
+        self.host_numa_node_id
+    }
+
+    fn set_host_numa_node_id(&mut self, host_numa_node_id: Option<u32>) {
+        self.host_numa_node_id = host_numa_node_id;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use vm_memory::GuestMemoryRegion;
+
+    use super::*;
+    use crate::test_utils::tests::create_vm_for_test;
+
+    impl Default for MemDeviceConfigInfo {
+        fn default() -> Self {
+            MemDeviceConfigInfo {
+                mem_id: "".to_string(),
+                size_mib: 0,
+                capacity_mib: 1024,
+                multi_region: true,
+                host_numa_node_id: None,
+                guest_numa_node_id: None,
+                use_generic_irq: None,
+                use_shared_irq: None,
+            }
+        }
+    }
+
+    #[test]
+    fn test_mem_config_check_conflicts() {
+        let config = MemDeviceConfigInfo::default();
+        let mut config2 = MemDeviceConfigInfo::default();
+        assert!(config.check_conflicts(&config2).is_err());
+        config2.mem_id = "dummy_mem".to_string();
+        assert!(config.check_conflicts(&config2).is_ok());
+    }
+
+    #[test]
+    fn test_create_mem_devices_configs() {
+        let mgr = MemDeviceMgr::default();
+        assert_eq!(mgr.info_list.len(), 0);
+        assert_eq!(mgr.get_index_of_mem_dev(""), None);
+    }
+
+    #[test]
+    fn test_mem_insert_or_update_device() {
+        // Init vm for test.
+        let mut vm = create_vm_for_test();
+
+        // We don't need to use virtio-mem before start vm
+        // Test for standard config with hotplug
+        let device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            vm.vm_address_space().cloned(),
+            true,
+            Some(VmConfigInfo::default()),
+            vm.shared_info().clone(),
+        );
+
+        let dummy_mem_device = MemDeviceConfigInfo::default();
+        vm.device_manager_mut()
+            .mem_manager
+            .insert_or_update_device(device_op_ctx, dummy_mem_device)
+            .unwrap();
+        assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1);
+    }
+
+    #[test]
+    fn test_mem_attach_device() {
+        // Init vm and insert mem config for test.
+        let mut vm = create_vm_for_test();
+        let dummy_mem_device = MemDeviceConfigInfo::default();
+        vm.device_manager_mut()
+            .mem_manager
+            .info_list
+            .insert_or_update(&dummy_mem_device)
+            .unwrap();
+        assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1);
+
+        // Test for standard config
+        let mut device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            vm.vm_address_space().cloned(),
+            false,
+            Some(VmConfigInfo::default()),
+            vm.shared_info().clone(),
+        );
+        vm.device_manager_mut()
+            .mem_manager
+            .attach_devices(&mut device_op_ctx)
+            .unwrap();
+        assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1);
+    }
+
+    #[test]
+    fn test_mem_create_region() {
+        let vm = create_vm_for_test();
+        let ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            vm.vm_address_space().cloned(),
+            true,
+            Some(VmConfigInfo::default()),
+            vm.shared_info().clone(),
+        );
+        let mem_id = String::from("mem0");
+        let guest_addr = GuestAddress(0x1_0000_0000);
+        let region_len = 0x1000_0000;
+        let kvm_slot = 2;
+
+        // no vfio manager, no numa node
+        let mut factory = MemoryRegionFactory::new(&ctx, mem_id, None).unwrap();
+        let region_opt = factory.create_region(guest_addr, region_len, kvm_slot);
+        assert_eq!(region_opt.unwrap().len(), region_len);
+    }
+}

src/dragonball/src/device_manager/memory_region_handler.rs

@@ -55,6 +55,7 @@ impl DeviceVirtioRegionHandler {
             None,
             file_offset,
             region.flags(),
+            region.prot(),
             false,
         ));
 

src/dragonball/src/device_manager/mod.rs

@@ -7,7 +7,7 @@
 use std::collections::HashMap;
 
 use std::io;
-use std::sync::{Arc, Mutex, MutexGuard};
+use std::sync::{Arc, Mutex, MutexGuard, RwLock};
 
 use arc_swap::ArcSwap;
 use dbs_address_space::AddressSpace;
@@ -45,9 +45,10 @@ use dbs_upcall::{
 use dbs_virtio_devices::vsock::backend::VsockInnerConnector;
 
 use crate::address_space_manager::GuestAddressSpaceImpl;
+use crate::api::v1::InstanceInfo;
 use crate::error::StartMicroVmError;
 use crate::resource_manager::ResourceManager;
-use crate::vm::{KernelConfigInfo, Vm};
+use crate::vm::{KernelConfigInfo, Vm, VmConfigInfo};
 use crate::IoManagerCached;
 
 /// Virtual machine console device manager.
@@ -89,6 +90,18 @@ mod memory_region_handler;
 #[cfg(feature = "virtio-fs")]
 pub use self::memory_region_handler::*;
 
+#[cfg(feature = "virtio-mem")]
+/// Device manager for virtio-mem devices.
+pub mod mem_dev_mgr;
+#[cfg(feature = "virtio-mem")]
+use self::mem_dev_mgr::MemDeviceMgr;
+
+#[cfg(feature = "virtio-balloon")]
+/// Device manager for virtio-balloon devices.
+pub mod balloon_dev_mgr;
+#[cfg(feature = "virtio-balloon")]
+use self::balloon_dev_mgr::BalloonDeviceMgr;
+
 macro_rules! info(
     ($l:expr, $($args:tt)+) => {
         slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager"))
@@ -248,6 +261,8 @@ pub struct DeviceOpContext {
     upcall_client: Option<Arc<UpcallClient<DevMgrService>>>,
     #[cfg(feature = "dbs-virtio-devices")]
     virtio_devices: Vec<Arc<DbsMmioV2Device>>,
+    vm_config: Option<VmConfigInfo>,
+    shared_info: Arc<RwLock<InstanceInfo>>,
 }
 
 impl DeviceOpContext {
@@ -257,6 +272,8 @@ impl DeviceOpContext {
         vm_as: Option<GuestAddressSpaceImpl>,
         address_space: Option<AddressSpace>,
         is_hotplug: bool,
+        vm_config: Option<VmConfigInfo>,
+        shared_info: Arc<RwLock<InstanceInfo>>,
     ) -> Self {
         let irq_manager = device_mgr.irq_manager.clone();
         let res_manager = device_mgr.res_manager.clone();
@@ -282,11 +299,21 @@ impl DeviceOpContext {
             upcall_client: None,
             #[cfg(feature = "dbs-virtio-devices")]
             virtio_devices: Vec::new(),
+            vm_config,
+            shared_info,
         }
     }
 
     pub(crate) fn create_boot_ctx(vm: &Vm, epoll_mgr: Option<EpollManager>) -> Self {
-        Self::new(epoll_mgr, vm.device_manager(), None, None, false)
+        Self::new(
+            epoll_mgr,
+            vm.device_manager(),
+            None,
+            None,
+            false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
+        )
     }
 
     pub(crate) fn get_vm_as(&self) -> Result<GuestAddressSpaceImpl> {
@@ -296,6 +323,27 @@ impl DeviceOpContext {
         }
     }
 
+    pub(crate) fn get_vm_config(&self) -> Result<VmConfigInfo> {
+        match self.vm_config.as_ref() {
+            Some(v) => Ok(v.clone()),
+            None => Err(DeviceMgrError::InvalidOperation),
+        }
+    }
+
+    pub(crate) fn get_address_space(&self) -> Result<AddressSpace> {
+        match self.address_space.as_ref() {
+            Some(v) => Ok(v.clone()),
+            None => Err(DeviceMgrError::InvalidOperation),
+        }
+    }
+
+    pub(crate) fn get_epoll_mgr(&self) -> Result<EpollManager> {
+        match self.epoll_mgr.as_ref() {
+            Some(v) => Ok(v.clone()),
+            None => Err(DeviceMgrError::InvalidOperation),
+        }
+    }
+
     pub(crate) fn logger(&self) -> &slog::Logger {
         &self.logger
     }
@@ -386,6 +434,8 @@ impl DeviceOpContext {
             Some(vm_as),
             vm.vm_address_space().cloned(),
             true,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
         );
         ctx.upcall_client = vm.upcall_client().clone();
         ctx
@@ -463,7 +513,7 @@ pub struct DeviceManager {
     res_manager: Arc<ResourceManager>,
     vm_fd: Arc<VmFd>,
     pub(crate) logger: slog::Logger,
-
+    pub(crate) shared_info: Arc<RwLock<InstanceInfo>>,
     pub(crate) con_manager: ConsoleManager,
     pub(crate) legacy_manager: Option<LegacyDeviceManager>,
     #[cfg(target_arch = "aarch64")]
@@ -481,6 +531,12 @@ pub struct DeviceManager {
 
     #[cfg(feature = "virtio-fs")]
     fs_manager: Arc<Mutex<FsDeviceMgr>>,
+
+    #[cfg(feature = "virtio-mem")]
+    pub(crate) mem_manager: MemDeviceMgr,
+
+    #[cfg(feature = "virtio-balloon")]
+    pub(crate) balloon_manager: BalloonDeviceMgr,
 }
 
 impl DeviceManager {
@@ -490,6 +546,7 @@ impl DeviceManager {
         res_manager: Arc<ResourceManager>,
         epoll_manager: EpollManager,
         logger: &slog::Logger,
+        shared_info: Arc<RwLock<InstanceInfo>>,
     ) -> Self {
         DeviceManager {
             io_manager: Arc::new(ArcSwap::new(Arc::new(IoManager::new()))),
@@ -498,6 +555,7 @@ impl DeviceManager {
             res_manager,
             vm_fd,
             logger: logger.new(slog::o!()),
+            shared_info,
 
             con_manager: ConsoleManager::new(epoll_manager, logger),
             legacy_manager: None,
@@ -511,6 +569,10 @@ impl DeviceManager {
             virtio_net_manager: VirtioNetDeviceMgr::default(),
             #[cfg(feature = "virtio-fs")]
             fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())),
+            #[cfg(feature = "virtio-mem")]
+            mem_manager: MemDeviceMgr::default(),
+            #[cfg(feature = "virtio-balloon")]
+            balloon_manager: BalloonDeviceMgr::default(),
         }
     }
 
@@ -636,9 +698,9 @@ impl DeviceManager {
         vm_as: GuestAddressSpaceImpl,
         epoll_mgr: EpollManager,
         kernel_config: &mut KernelConfigInfo,
-        com1_sock_path: Option<String>,
         dmesg_fifo: Option<Box<dyn io::Write + Send>>,
         address_space: Option<&AddressSpace>,
+        vm_config: &VmConfigInfo,
     ) -> std::result::Result<(), StartMicroVmError> {
         let mut ctx = DeviceOpContext::new(
             Some(epoll_mgr),
@@ -646,8 +708,12 @@ impl DeviceManager {
             Some(vm_as),
             address_space.cloned(),
             false,
+            Some(vm_config.clone()),
+            self.shared_info.clone(),
         );
 
+        let com1_sock_path = vm_config.serial_path.clone();
+
         self.create_legacy_devices(&mut ctx)?;
         self.init_legacy_devices(dmesg_fifo, com1_sock_path, &mut ctx)?;
 
@@ -710,10 +776,20 @@ impl DeviceManager {
             Some(vm_as),
             address_space.cloned(),
             true,
+            None,
+            self.shared_info.clone(),
         );
 
         #[cfg(feature = "virtio-blk")]
         self.block_manager.remove_devices(&mut ctx)?;
+        // FIXME: To acquire the full abilities for gracefully removing
+        // virtio-net and virtio-vsock devices, updating dragonball-sandbox
+        // is required.
+        #[cfg(feature = "virtio-net")]
+        self.virtio_net_manager.remove_devices(&mut ctx)?;
+        #[cfg(feature = "virtio-vsock")]
+        self.vsock_manager.remove_devices(&mut ctx)?;
+
         Ok(())
     }
 }
@@ -866,6 +942,24 @@ impl DeviceManager {
         )
     }
 
+    /// Create an Virtio MMIO transport layer device for the virtio backend device with configure
+    /// change notification enabled.
+    pub fn create_mmio_virtio_device_with_device_change_notification(
+        device: DbsVirtioDevice,
+        ctx: &mut DeviceOpContext,
+        use_shared_irq: bool,
+        use_generic_irq: bool,
+    ) -> std::result::Result<Arc<DbsMmioV2Device>, DeviceMgrError> {
+        let features = DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY;
+        DeviceManager::create_mmio_virtio_device_with_features(
+            device,
+            ctx,
+            Some(features),
+            use_shared_irq,
+            use_generic_irq,
+        )
+    }
+
     /// Create an Virtio MMIO transport layer device for the virtio backend device with specified
     /// features.
     pub fn create_mmio_virtio_device_with_features(
@@ -1011,6 +1105,7 @@ mod tests {
     use vm_memory::{GuestAddress, MmapRegion};
 
     use super::*;
+    #[cfg(target_arch = "x86_64")]
     use crate::vm::CpuTopology;
 
     impl DeviceManager {
@@ -1021,6 +1116,10 @@ mod tests {
             let epoll_manager = EpollManager::default();
             let res_manager = Arc::new(ResourceManager::new(None));
             let logger = slog_scope::logger().new(slog::o!());
+            let shared_info = Arc::new(RwLock::new(InstanceInfo::new(
+                String::from("dragonball"),
+                String::from("1"),
+            )));
 
             DeviceManager {
                 vm_fd: Arc::clone(&vm_fd),
@@ -1039,10 +1138,15 @@ mod tests {
                 virtio_net_manager: VirtioNetDeviceMgr::default(),
                 #[cfg(feature = "virtio-vsock")]
                 vsock_manager: VsockDeviceMgr::default(),
+                #[cfg(feature = "virtio-mem")]
+                mem_manager: MemDeviceMgr::default(),
+                #[cfg(feature = "virtio-balloon")]
+                balloon_manager: BalloonDeviceMgr::default(),
                 #[cfg(target_arch = "aarch64")]
                 mmio_device_info: HashMap::new(),
 
                 logger,
+                shared_info,
             }
         }
     }
@@ -1082,7 +1186,7 @@ mod tests {
             },
             vpmu_feature: 0,
         };
-        vm.set_vm_config(vm_config);
+        vm.set_vm_config(vm_config.clone());
         vm.init_guest_memory().unwrap();
         vm.setup_interrupt_controller().unwrap();
         let vm_as = vm.vm_as().cloned().unwrap();
@@ -1108,8 +1212,8 @@ mod tests {
             event_mgr.epoll_manager(),
             &mut cmdline,
             None,
-            None,
             address_space.as_ref(),
+            &vm_config,
         )
         .unwrap();
         let guard = mgr.io_manager.load();
@@ -1133,6 +1237,8 @@ mod tests {
             Some(vm.vm_as().unwrap().clone()),
             vm.vm_address_space().cloned(),
             true,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
         );
         let guest_addr = GuestAddress(0x200000000000);
 

src/dragonball/src/device_manager/virtio_net_dev_mgr.rs

@@ -223,7 +223,7 @@ impl VirtioNetDeviceMgr {
 
     /// Insert or update a virtio net device into the manager.
     pub fn insert_device(
-        device_mgr: &mut DeviceManager,
+        &mut self,
         mut ctx: DeviceOpContext,
         config: VirtioNetDeviceConfigInfo,
     ) -> std::result::Result<(), VirtioNetDeviceError> {
@@ -234,8 +234,6 @@ impl VirtioNetDeviceMgr {
             return Err(VirtioNetDeviceError::UpdateNotAllowedPostBoot);
         }
 
-        let mgr = &mut device_mgr.virtio_net_manager;
-
         slog::info!(
             ctx.logger(),
             "add virtio-net device configuration";
@@ -244,7 +242,7 @@ impl VirtioNetDeviceMgr {
             "host_dev_name" => &config.host_dev_name,
         );
 
-        let device_index = mgr.info_list.insert_or_update(&config)?;
+        let device_index = self.info_list.insert_or_update(&config)?;
 
         if ctx.is_hotplug {
             slog::info!(
@@ -260,17 +258,17 @@ impl VirtioNetDeviceMgr {
                     let dev = DeviceManager::create_mmio_virtio_device(
                         device,
                         &mut ctx,
-                        config.use_shared_irq.unwrap_or(mgr.use_shared_irq),
+                        config.use_shared_irq.unwrap_or(self.use_shared_irq),
                         config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
                     )
                     .map_err(VirtioNetDeviceError::DeviceManager)?;
                     ctx.insert_hotplug_mmio_device(&dev, None)
                         .map_err(VirtioNetDeviceError::DeviceManager)?;
                     // live-upgrade need save/restore device from info.device.
-                    mgr.info_list[device_index].set_device(dev);
+                    self.info_list[device_index].set_device(dev);
                 }
                 Err(e) => {
-                    mgr.info_list.remove(device_index);
+                    self.info_list.remove(device_index);
                     return Err(VirtioNetDeviceError::Virtio(e));
                 }
             }
@@ -281,16 +279,15 @@ impl VirtioNetDeviceMgr {
 
     /// Update the ratelimiter settings of a virtio net device.
     pub fn update_device_ratelimiters(
-        device_mgr: &mut DeviceManager,
+        &mut self,
         new_cfg: VirtioNetDeviceConfigUpdateInfo,
     ) -> std::result::Result<(), VirtioNetDeviceError> {
-        let mgr = &mut device_mgr.virtio_net_manager;
-        match mgr.get_index_of_iface_id(&new_cfg.iface_id) {
+        match self.get_index_of_iface_id(&new_cfg.iface_id) {
             Some(index) => {
-                let config = &mut mgr.info_list[index].config;
+                let config = &mut self.info_list[index].config;
                 config.rx_rate_limiter = new_cfg.rx_rate_limiter.clone();
                 config.tx_rate_limiter = new_cfg.tx_rate_limiter.clone();
-                let device = mgr.info_list[index].device.as_mut().ok_or_else(|| {
+                let device = self.info_list[index].device.as_mut().ok_or_else(|| {
                     VirtioNetDeviceError::InvalidIfaceId(new_cfg.iface_id.clone())
                 })?;
 
@@ -374,6 +371,21 @@ impl VirtioNetDeviceMgr {
 
         Ok(Box::new(net_device))
     }
+
+    /// Remove all virtio-net devices.
+    pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> {
+        while let Some(mut info) = self.info_list.pop() {
+            slog::info!(
+                ctx.logger(),
+                "remove virtio-net device: {}",
+                info.config.iface_id
+            );
+            if let Some(device) = info.device.take() {
+                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+            }
+        }
+        Ok(())
+    }
 }
 
 impl Default for VirtioNetDeviceMgr {

src/dragonball/src/device_manager/vsock_dev_mgr.rs

@@ -17,7 +17,7 @@ use dbs_virtio_devices::vsock::Vsock;
 use dbs_virtio_devices::Error as VirtioError;
 use serde_derive::{Deserialize, Serialize};
 
-use super::StartMicroVmError;
+use super::{DeviceMgrError, StartMicroVmError};
 use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos};
 use crate::device_manager::{DeviceManager, DeviceOpContext};
 
@@ -284,6 +284,21 @@ impl VsockDeviceMgr {
         // safe to unwrap, because we created the inner connector before
         Ok(self.default_inner_connector.clone().unwrap())
     }
+
+    /// Remove all virtio-vsock devices
+    pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> {
+        while let Some(mut info) = self.info_list.pop() {
+            slog::info!(
+                ctx.logger(),
+                "remove virtio-vsock device: {}",
+                info.config.id
+            );
+            if let Some(device) = info.device.take() {
+                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+            }
+        }
+        Ok(())
+    }
 }
 
 impl Default for VsockDeviceMgr {

src/dragonball/src/error.rs

@@ -192,6 +192,11 @@ pub enum StartMicroVmError {
     /// Virtio-fs errors.
     #[error("virtio-fs errors: {0}")]
     FsDeviceError(#[source] device_manager::fs_dev_mgr::FsDeviceError),
+
+    #[cfg(feature = "virtio-balloon")]
+    /// Virtio-balloon errors.
+    #[error("virtio-balloon errors: {0}")]
+    BalloonDeviceError(#[source] device_manager::balloon_dev_mgr::BalloonDeviceError),
 }
 
 /// Errors associated with starting the instance.

src/dragonball/src/vcpu/vcpu_manager.rs

@@ -374,11 +374,17 @@ impl VcpuManager {
         entry_addr: GuestAddress,
     ) -> Result<()> {
         info!("create boot vcpus");
-        self.create_vcpus(
-            self.vcpu_config.boot_vcpu_count,
-            Some(request_ts),
-            Some(entry_addr),
-        )?;
+        let boot_vcpu_count = if cfg!(target_arch = "aarch64") {
+            // On aarch64, kvm doesn't allow to call KVM_CREATE_VCPU ioctl after vm has been booted
+            // because of vgic check. To support vcpu hotplug/hotunplug feature, we should create
+            // all the vcpufd at booting procedure.
+            // SetVmConfiguration API will ensure max_vcpu_count >= boot_vcpu_count, so it is safe
+            // to directly use max_vcpu_count here.
+            self.vcpu_config.max_vcpu_count
+        } else {
+            self.vcpu_config.boot_vcpu_count
+        };
+        self.create_vcpus(boot_vcpu_count, Some(request_ts), Some(entry_addr))?;
 
         Ok(())
     }
@@ -1213,7 +1219,10 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         // test start boot vcpus
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
@@ -1267,8 +1276,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
+
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // invalid cpuid for pause
         let cpu_indexes = vec![2];
@@ -1304,9 +1319,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // invalid cpuid for exit
         let cpu_indexes = vec![2];
@@ -1330,9 +1350,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // exit all success
         assert!(vcpu_manager.exit_all_vcpus().is_ok());
@@ -1351,9 +1376,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // invalid cpuid for exit
         let cpu_indexes = vec![2];
@@ -1377,9 +1407,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // revalidate all success
         assert!(vcpu_manager.revalidate_all_vcpus_cache().is_ok());
@@ -1395,9 +1430,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // set vcpus in hotplug action
         let cpu_ids = vec![0];

src/dragonball/src/vm/aarch64.rs

@@ -6,70 +6,35 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the THIRD-PARTY file.
 
-use std::collections::HashMap;
-use std::fmt::Debug;
 use std::ops::Deref;
+use std::sync::MutexGuard;
 
 use dbs_arch::gic::GICDevice;
 use dbs_arch::pmu::initialize_pmu;
-use dbs_arch::{DeviceInfoForFDT, DeviceType, VpmuFeatureLevel};
+use dbs_arch::{MMIODeviceInfo, VpmuFeatureLevel};
+use dbs_boot::fdt_utils::*;
 use dbs_boot::InitrdConfig;
 use dbs_utils::epoll_manager::EpollManager;
 use dbs_utils::time::TimestampUs;
-use linux_loader::loader::Cmdline;
-use vm_memory::{GuestAddressSpace, GuestMemory};
+use linux_loader::cmdline::{Cmdline, Error as CmdlineError};
+use vm_memory::GuestAddressSpace;
 use vmm_sys_util::eventfd::EventFd;
 
 use super::{Vm, VmError};
 use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl};
 use crate::error::{Error, StartMicroVmError};
 use crate::event_manager::EventManager;
+use crate::vcpu::VcpuManager;
 
-/// Configures the system and should be called once per vm before starting vcpu threads.
-/// For aarch64, we only setup the FDT.
-///
-/// # Arguments
-///
-/// * `guest_mem` - The memory to be used by the guest.
-/// * `cmdline` - The kernel commandline.
-/// * `vcpu_mpidr` - Array of MPIDR register values per vcpu.
-/// * `device_info` - A hashmap containing the attached devices for building FDT device nodes.
-/// * `gic_device` - The GIC device.
-/// * `initrd` - Information about an optional initrd.
-#[allow(clippy::borrowed_box)]
-fn configure_system<T: DeviceInfoForFDT + Clone + Debug, M: GuestMemory>(
-    guest_mem: &M,
-    cmdline: &str,
-    vcpu_mpidr: Vec<u64>,
-    device_info: Option<&HashMap<(DeviceType, String), T>>,
-    gic_device: &Box<dyn GICDevice>,
-    initrd: &Option<super::InitrdConfig>,
-    vpmu_feature: &VpmuFeatureLevel,
-) -> super::Result<()> {
-    dbs_boot::fdt::create_fdt(
-        guest_mem,
-        vcpu_mpidr,
-        cmdline,
-        device_info,
-        gic_device,
-        initrd,
-        vpmu_feature,
-    )
-    .map_err(Error::BootSystem)?;
-    Ok(())
-}
-
-#[cfg(target_arch = "aarch64")]
 impl Vm {
     /// Gets a reference to the irqchip of the VM
-    #[allow(clippy::borrowed_box)]
-    pub fn get_irqchip(&self) -> &Box<dyn GICDevice> {
-        self.irqchip_handle.as_ref().unwrap()
+    pub fn get_irqchip(&self) -> &dyn GICDevice {
+        self.irqchip_handle.as_ref().unwrap().as_ref()
     }
 
     /// Creates the irq chip in-kernel device model.
     pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicroVmError> {
-        let vcpu_count = self.vm_config.vcpu_count;
+        let vcpu_count = self.vm_config.max_vcpu_count;
 
         self.irqchip_handle = Some(
             dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into())
@@ -99,12 +64,11 @@ impl Vm {
     /// Initialize the virtual machine instance.
     ///
     /// It initialize the virtual machine instance by:
-    /// 1) initialize virtual machine global state and configuration.
-    /// 2) create system devices, such as interrupt controller.
-    /// 3) create and start IO devices, such as serial, console, block, net, vsock etc.
-    /// 4) create and initialize vCPUs.
-    /// 5) configure CPU power management features.
-    /// 6) load guest kernel image.
+    /// 1) Initialize virtual machine reset event fd.
+    /// 2) Create and initialize vCPUs.
+    /// 3) Create and initialize interrupt controller.
+    /// 4) Create and initialize vPMU device.
+    /// 5) Create and initialize devices, such as virtio, block, net, vsock, vfio etc.
     pub fn init_microvm(
         &mut self,
         epoll_mgr: EpollManager,
@@ -139,6 +103,52 @@ impl Vm {
         Ok(())
     }
 
+    /// Generate fdt information about VM.
+    fn get_fdt_vm_info<'a>(
+        &'a self,
+        vm_memory: &'a GuestMemoryImpl,
+        cmdline: &'a str,
+        initrd_config: Option<&'a InitrdConfig>,
+        vcpu_manager: &'a MutexGuard<VcpuManager>,
+    ) -> FdtVmInfo {
+        let guest_memory = vm_memory.memory();
+        let vcpu_mpidr = vcpu_manager
+            .vcpus()
+            .into_iter()
+            .map(|cpu| cpu.get_mpidr())
+            .collect();
+        let vm_config = self.vm_config();
+        let mut vcpu_boot_onlined = vec![1; vm_config.vcpu_count as usize];
+        vcpu_boot_onlined.resize(vm_config.max_vcpu_count as usize, 0);
+        let vpmu_feature = vcpu_manager.vpmu_feature();
+        // This configuration is used for passing cache information into guest.
+        // TODO: dragonball-sandbox #274; kata-containers #6969
+        let cache_passthrough_enabled = false;
+        let fdt_vcpu_info = FdtVcpuInfo::new(
+            vcpu_mpidr,
+            vcpu_boot_onlined,
+            vpmu_feature,
+            cache_passthrough_enabled,
+        );
+
+        FdtVmInfo::new(guest_memory, cmdline, initrd_config, fdt_vcpu_info)
+    }
+
+    // This method is used for passing cache/numa information into guest
+    // TODO: dragonball-sandbox #274,#275; kata-containers #6969
+    /// Generate fdt information about cache/numa
+    fn get_fdt_numa_info(&self) -> FdtNumaInfo {
+        FdtNumaInfo::default()
+    }
+
+    /// Generate fdt information about devices
+    fn get_fdt_device_info(&self) -> FdtDeviceInfo<MMIODeviceInfo> {
+        FdtDeviceInfo::new(
+            self.device_manager().get_mmio_device_info(),
+            self.get_irqchip(),
+        )
+    }
+
     /// Execute system architecture specific configurations.
     ///
     /// 1) set guest kernel boot parameters
@@ -150,24 +160,23 @@ impl Vm {
         initrd: Option<InitrdConfig>,
     ) -> std::result::Result<(), StartMicroVmError> {
         let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?;
-        let vpmu_feature = vcpu_manager.vpmu_feature();
-        let vcpu_mpidr = vcpu_manager
-            .vcpus()
-            .into_iter()
-            .map(|cpu| cpu.get_mpidr())
-            .collect();
-        let guest_memory = vm_memory.memory();
+        let cmdline_cstring = cmdline
+            .as_cstring()
+            .map_err(StartMicroVmError::ProcessCommandlne)?;
+        let fdt_vm_info = self.get_fdt_vm_info(
+            vm_memory,
+            cmdline_cstring
+                .to_str()
+                .map_err(|_| StartMicroVmError::ProcessCommandlne(CmdlineError::InvalidAscii))?,
+            initrd.as_ref(),
+            &vcpu_manager,
+        );
+        let fdt_numa_info = self.get_fdt_numa_info();
+        let fdt_device_info = self.get_fdt_device_info();
 
-        configure_system(
-            guest_memory,
-            cmdline.as_cstring().unwrap().to_str().unwrap(),
-            vcpu_mpidr,
-            self.device_manager.get_mmio_device_info(),
-            self.get_irqchip(),
-            &initrd,
-            &vpmu_feature,
-        )
-        .map_err(StartMicroVmError::ConfigureSystem)
+        dbs_boot::fdt::create_fdt(fdt_vm_info, fdt_numa_info, fdt_device_info)
+            .map(|_| ())
+            .map_err(|e| StartMicroVmError::ConfigureSystem(Error::BootSystem(e)))
     }
 
     pub(crate) fn register_events(

src/dragonball/src/vm/mod.rs

@@ -241,6 +241,7 @@ impl Vm {
             resource_manager.clone(),
             epoll_manager.clone(),
             &logger,
+            api_shared_info.clone(),
         );
 
         Ok(Vm {
@@ -413,6 +414,19 @@ impl Vm {
 
         (dragonball_version, instance_id)
     }
+
+    pub(crate) fn stop_prealloc(&mut self) -> std::result::Result<(), StartMicroVmError> {
+        if self.address_space.is_initialized() {
+            return self
+                .address_space
+                .wait_prealloc(true)
+                .map_err(StartMicroVmError::AddressManagerError);
+        }
+
+        Err(StartMicroVmError::AddressManagerError(
+            AddressManagerError::GuestMemoryNotInitialized,
+        ))
+    }
 }
 
 impl Vm {
@@ -481,7 +495,6 @@ impl Vm {
     ) -> std::result::Result<(), StartMicroVmError> {
         info!(self.logger, "VM: initializing devices ...");
 
-        let com1_sock_path = self.vm_config.serial_path.clone();
         let kernel_config = self
             .kernel_config
             .as_mut()
@@ -503,9 +516,9 @@ impl Vm {
             vm_as.clone(),
             epoll_manager,
             kernel_config,
-            com1_sock_path,
             self.dmesg_fifo.take(),
             self.address_space.address_space(),
+            &self.vm_config,
         )?;
 
         info!(self.logger, "VM: start devices");
@@ -888,6 +901,7 @@ impl Vm {
 
 #[cfg(test)]
 pub mod tests {
+    #[cfg(target_arch = "x86_64")]
     use kvm_ioctls::VcpuExit;
     use linux_loader::cmdline::Cmdline;
     use test_utils::skip_if_not_root;

src/dragonball/src/vm/x86_64.rs

@@ -7,7 +7,6 @@
 // found in the THIRD-PARTY file.
 
 use std::convert::TryInto;
-use std::mem;
 use std::ops::Deref;
 
 use dbs_address_space::AddressSpace;
@@ -16,8 +15,9 @@ use dbs_utils::epoll_manager::EpollManager;
 use dbs_utils::time::TimestampUs;
 use kvm_bindings::{kvm_irqchip, kvm_pit_config, kvm_pit_state2, KVM_PIT_SPEAKER_DUMMY};
 use linux_loader::cmdline::Cmdline;
+use linux_loader::configurator::{linux::LinuxBootConfigurator, BootConfigurator, BootParams};
 use slog::info;
-use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory};
+use vm_memory::{Address, GuestAddress, GuestAddressSpace, GuestMemory};
 
 use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl};
 use crate::error::{Error, Result, StartMicroVmError};
@@ -110,15 +110,11 @@ fn configure_system<M: GuestMemory>(
         }
     }
 
-    let zero_page_addr = GuestAddress(layout::ZERO_PAGE_START);
-    guest_mem
-        .checked_offset(zero_page_addr, mem::size_of::<bootparam::boot_params>())
-        .ok_or(Error::ZeroPagePastRamEnd)?;
-    guest_mem
-        .write_obj(params, zero_page_addr)
-        .map_err(|_| Error::ZeroPageSetup)?;
-
-    Ok(())
+    LinuxBootConfigurator::write_bootparams(
+        &BootParams::new(&params, GuestAddress(layout::ZERO_PAGE_START)),
+        guest_mem,
+    )
+    .map_err(|_| Error::ZeroPageSetup)
 }
 
 impl Vm {

src/libs/Cargo.lock

@@ -186,7 +186,7 @@ checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011"
 dependencies = [
  "lazy_static",
  "log",
- "rand 0.8.5",
+ "rand",
 ]
 
 [[package]]
@@ -299,17 +299,6 @@ dependencies = [
  "slab",
 ]
 
-[[package]]
-name = "getrandom"
-version = "0.1.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi 0.9.0+wasi-snapshot-preview1",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.2.6"
@@ -495,7 +484,7 @@ dependencies = [
  "num_cpus",
  "oci",
  "once_cell",
- "rand 0.7.3",
+ "rand",
  "serde_json",
  "serial_test",
  "slog",
@@ -519,6 +508,7 @@ dependencies = [
  "num_cpus",
  "oci",
  "regex",
+ "safe-path",
  "serde",
  "serde_json",
  "slog",
@@ -703,9 +693,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.9.0"
+version = "1.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
+checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
 
 [[package]]
 name = "parking_lot"
@@ -845,9 +835,16 @@ name = "protobuf"
 version = "2.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96"
+
+[[package]]
+name = "protobuf"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b55bad9126f378a853655831eb7363b7b01b81d19f8cb1218861086ca4a1a61e"
 dependencies = [
- "serde",
- "serde_derive",
+ "once_cell",
+ "protobuf-support",
+ "thiserror",
 ]
 
 [[package]]
@@ -856,17 +853,47 @@ version = "2.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707"
 dependencies = [
- "protobuf",
+ "protobuf 2.27.1",
 ]
 
 [[package]]
-name = "protobuf-codegen-pure"
-version = "2.27.1"
+name = "protobuf-codegen"
+version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6"
+checksum = "0dd418ac3c91caa4032d37cb80ff0d44e2ebe637b2fb243b6234bf89cdac4901"
 dependencies = [
- "protobuf",
- "protobuf-codegen",
+ "anyhow",
+ "once_cell",
+ "protobuf 3.2.0",
+ "protobuf-parse",
+ "regex",
+ "tempfile",
+ "thiserror",
+]
+
+[[package]]
+name = "protobuf-parse"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d39b14605eaa1f6a340aec7f320b34064feb26c93aec35d6a9a2272a8ddfa49"
+dependencies = [
+ "anyhow",
+ "indexmap",
+ "log",
+ "protobuf 3.2.0",
+ "protobuf-support",
+ "tempfile",
+ "thiserror",
+ "which",
+]
+
+[[package]]
+name = "protobuf-support"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5d4d7b8601c814cfb36bcebb79f0e61e45e1e93640cf778837833bbed05c372"
+dependencies = [
+ "thiserror",
 ]
 
 [[package]]
@@ -875,7 +902,7 @@ version = "0.1.0"
 dependencies = [
  "async-trait",
  "oci",
- "protobuf",
+ "protobuf 3.2.0",
  "serde",
  "serde_json",
  "ttrpc",
@@ -891,19 +918,6 @@ dependencies = [
  "proc-macro2",
 ]
 
-[[package]]
-name = "rand"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
-dependencies = [
- "getrandom 0.1.16",
- "libc",
- "rand_chacha 0.2.2",
- "rand_core 0.5.1",
- "rand_hc",
-]
-
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -911,18 +925,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha 0.3.1",
- "rand_core 0.6.3",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
-dependencies = [
- "ppv-lite86",
- "rand_core 0.5.1",
+ "rand_chacha",
+ "rand_core",
 ]
 
 [[package]]
@@ -932,16 +936,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core 0.6.3",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
-dependencies = [
- "getrandom 0.1.16",
+ "rand_core",
 ]
 
 [[package]]
@@ -950,16 +945,7 @@ version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
 dependencies = [
- "getrandom 0.2.6",
-]
-
-[[package]]
-name = "rand_hc"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
-dependencies = [
- "rand_core 0.5.1",
+ "getrandom",
 ]
 
 [[package]]
@@ -1314,9 +1300,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
 [[package]]
 name = "ttrpc"
-version = "0.6.1"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ecfff459a859c6ba6668ff72b34c2f1d94d9d58f7088414c2674ad0f31cc7d8"
+checksum = "a35f22a2964bea14afee161665bb260b83cb48e665e0260ca06ec0e775c8b06c"
 dependencies = [
  "async-trait",
  "byteorder",
@@ -1324,8 +1310,8 @@ dependencies = [
  "libc",
  "log",
  "nix 0.23.1",
- "protobuf",
- "protobuf-codegen-pure",
+ "protobuf 3.2.0",
+ "protobuf-codegen 3.2.0",
  "thiserror",
  "tokio",
  "tokio-vsock",
@@ -1333,28 +1319,28 @@ dependencies = [
 
 [[package]]
 name = "ttrpc-codegen"
-version = "0.2.0"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "809eda4e459820237104e4b61d6b41bbe6c9e1ce6adf4057955e6e6722a90408"
+checksum = "94d7f7631d7a9ebed715a47cd4cb6072cbc7ae1d4ec01598971bbec0024340c2"
 dependencies = [
- "protobuf",
- "protobuf-codegen",
- "protobuf-codegen-pure",
+ "protobuf 2.27.1",
+ "protobuf-codegen 3.2.0",
+ "protobuf-support",
  "ttrpc-compiler",
 ]
 
 [[package]]
 name = "ttrpc-compiler"
-version = "0.4.1"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2978ed3fa047d8fd55cbeb4d4a61d461fb3021a90c9618519c73ce7e5bb66c15"
+checksum = "ec3cb5dbf1f0865a34fe3f722290fe776cacb16f50428610b779467b76ddf647"
 dependencies = [
  "derive-new",
  "prost",
  "prost-build",
  "prost-types",
- "protobuf",
- "protobuf-codegen",
+ "protobuf 2.27.1",
+ "protobuf-codegen 2.27.1",
  "tempfile",
 ]
 
@@ -1390,12 +1376,6 @@ dependencies = [
  "try-lock",
 ]
 
-[[package]]
-name = "wasi"
-version = "0.9.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
-
 [[package]]
 name = "wasi"
 version = "0.10.2+wasi-snapshot-preview1"

src/libs/kata-sys-util/Cargo.toml

@@ -24,7 +24,7 @@ serde_json = "1.0.73"
 slog = "2.5.2"
 slog-scope = "4.4.0"
 subprocess = "0.2.8"
-rand = "0.7.2"
+rand = "0.8.5"
 thiserror = "1.0.30"
 
 kata-types = { path = "../kata-types" }