release: Bump version to 3.21.0

Bump VERSION and helm-chart versions

Signed-off-by: stevenhorsman <steven@uk.ibm.com>

.github/dependabot.yml

@@ -66,6 +66,9 @@ updates:
       rustix:
         patterns:
           - rustix
+      slab:
+        patterns:
+          - slab
       time:
         patterns:
           - time

.github/workflows/PR-wip-checks.yaml

@@ -9,8 +9,7 @@ on:
       - labeled
       - unlabeled
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/actionlint.yaml

@@ -11,8 +11,8 @@ on:
     paths:
       - '.github/workflows/**'
 
-permissions:
-  contents: read
+permissions: {}
+
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/basic-ci-amd64.yaml

@@ -13,8 +13,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-containerd-sandboxapi:
@@ -282,10 +281,7 @@ jobs:
       fail-fast: false
       matrix:
         vmm:
-          - clh
           - qemu
-          - dragonball
-          - cloud-hypervisor
     runs-on: ubuntu-22.04
     env:
       KATA_HYPERVISOR: ${{ matrix.vmm }}

.github/workflows/basic-ci-s390x.yaml

@@ -13,8 +13,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-containerd-sandboxapi:

.github/workflows/build-checks-preview-riscv64.yaml

@@ -12,8 +12,7 @@ on:
         required: true
         type: string
 
-permissions:
-  contents: read
+permissions: {}
 
 name: Build checks preview riscv64
 jobs:

.github/workflows/build-checks.yaml

@@ -5,8 +5,8 @@ on:
         required: true
         type: string
 
-permissions:
-  contents: read
+permissions: {}
+
 
 name: Build checks
 jobs:
@@ -42,6 +42,10 @@ jobs:
             path: src/runtime-rs
             needs:
               - rust
+          - name: libs
+            path: src/libs
+            needs:
+              - rust
           - name: agent-ctl
             path: src/tools/agent-ctl
             needs:

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -26,8 +26,7 @@ on:
       KBUILD_SIGN_PIN:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-asset:
@@ -144,7 +143,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -153,7 +152,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}-headers${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}-headers.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}-headers.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -224,7 +223,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -316,7 +315,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-amd64-shim-v2${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-shim-v2.tar.xz
+          path: kata-build/kata-static-shim-v2.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -353,6 +352,6 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
+          path: kata-static.tar.zst
           retention-days: 15
           if-no-files-found: error

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -24,8 +24,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: false
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-asset:
@@ -121,7 +120,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -130,7 +129,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}-headers${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}-headers.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}-headers.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -195,7 +194,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -282,7 +281,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-arm64-shim-v2${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-shim-v2.tar.xz
+          path: kata-build/kata-static-shim-v2.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -319,6 +318,6 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
+          path: kata-static.tar.zst
           retention-days: 15
           if-no-files-found: error

.github/workflows/build-kata-static-tarball-ppc64le.yaml

@@ -24,8 +24,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-asset:
@@ -83,7 +82,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 1
           if-no-files-found: error
 
@@ -148,7 +147,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 1
           if-no-files-found: error
 
@@ -221,7 +220,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-ppc64le-shim-v2${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-shim-v2.tar.xz
+          path: kata-build/kata-static-shim-v2.tar.zst
           retention-days: 1
           if-no-files-found: error
 
@@ -262,6 +261,6 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-static-tarball-ppc64le${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
+          path: kata-static.tar.zst
           retention-days: 1
           if-no-files-found: error

.github/workflows/build-kata-static-tarball-riscv64.yaml

@@ -24,8 +24,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-asset:
@@ -81,6 +80,6 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-riscv64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -27,8 +27,7 @@ on:
         required: true
 
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-asset:
@@ -115,7 +114,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -182,7 +181,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -230,7 +229,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-boot-image-se.tar.xz
+          path: kata-build/kata-static-boot-image-se.tar.zst
           retention-days: 1
           if-no-files-found: error
 
@@ -307,7 +306,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-s390x-shim-v2${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-shim-v2.tar.xz
+          path: kata-build/kata-static-shim-v2.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -348,6 +347,6 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
+          path: kata-static.tar.zst
           retention-days: 15
           if-no-files-found: error

.github/workflows/cargo-deny-runner.yaml

@@ -11,8 +11,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   cargo-deny-runner:

.github/workflows/ci-coco-stability.yaml

@@ -9,8 +9,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   kata-containers-ci-on-push:

.github/workflows/ci-devel.yaml

@@ -2,8 +2,7 @@ name: Kata Containers CI (manually triggered)
 on:
   workflow_dispatch:
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   kata-containers-ci-on-push:

.github/workflows/ci-nightly-s390x.yaml

@@ -4,8 +4,7 @@ on:
 
 name: Nightly CI for s390x
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   check-internal-test-result:
@@ -16,7 +15,8 @@ jobs:
         test_title:
           - kata-vfio-ap-e2e-tests
           - cc-vfio-ap-e2e-tests
-          - cc-se-e2e-tests
+          - cc-se-e2e-tests-go
+          - cc-se-e2e-tests-rs
     steps:
     - name: Fetch a test result for {{ matrix.test_title }}
       run: |

.github/workflows/ci-nightly.yaml

@@ -7,8 +7,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   kata-containers-ci-on-push:

.github/workflows/ci-on-push.yaml

@@ -13,8 +13,7 @@ on:
       - reopened
       - labeled
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/ci-weekly.yaml

@@ -30,8 +30,7 @@ on:
       KBUILD_SIGN_PIN:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-kata-static-tarball-amd64:

.github/workflows/ci.yaml

@@ -40,9 +40,7 @@ on:
       KBUILD_SIGN_PIN:
         required: true
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   build-kata-static-tarball-amd64:
@@ -292,6 +290,10 @@ jobs:
     if: ${{ inputs.skip-test != 'yes' }}
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
+
+    permissions:
+      contents: read
+      id-token: write # Used for OIDC access to log into Azure
     with:
       tarball-suffix: -${{ inputs.tag }}
       registry: ghcr.io
@@ -351,6 +353,9 @@ jobs:
      - build-and-publish-tee-confidential-unencrypted-image
      - publish-csi-driver-amd64
     uses: ./.github/workflows/run-kata-coco-tests.yaml
+    permissions:
+      contents: read
+      id-token: write # Used for OIDC access to log into Azure
     with:
       tarball-suffix: -${{ inputs.tag }}
       registry: ghcr.io

.github/workflows/cleanup-resources.yaml

@@ -4,13 +4,13 @@ on:
     - cron: "0 0 * * *"
   workflow_dispatch:
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   cleanup-resources:
     runs-on: ubuntu-22.04
+    permissions:
+      id-token: write # Used for OIDC access to log into Azure
     environment: ci
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

.github/workflows/codeql.yml

@@ -19,8 +19,8 @@ on:
   schedule:
     - cron: '45 0 * * 1'
 
-permissions:
-  contents: read
+permissions: {}
+
 
 jobs:
   analyze:

.github/workflows/commit-message-check.yaml

@@ -6,8 +6,7 @@ on:
       - reopened
       - synchronize
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -42,7 +41,7 @@ jobs:
         filter_out_pattern: '^Revert "|^Reapply "'
 
     - name: DCO Check
-      uses: tim-actions/dco@2fd0504dc0d27b33f542867c300c60840c6dcb20 # master (2020-04-28)
+      uses: tim-actions/dco@f2279e6e62d5a7d9115b0cb8e837b777b1b02e21 # v1.1.0
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
 

.github/workflows/darwin-tests.yaml

@@ -6,8 +6,7 @@ on:
       - reopened
       - synchronize
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -18,13 +17,15 @@ jobs:
   test:
     runs-on: macos-latest
     steps:
-    - name: Install Go
-      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
-      with:
-        go-version: 1.23.10
     - name: Checkout code
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         persist-credentials: false
+
+    - name: Install golang
+      run: |
+        ./tests/install_go.sh -f -p
+        echo "/usr/local/go/bin" >> "${GITHUB_PATH}"
+
     - name: Build utils
       run: ./ci/darwin-test.sh

.github/workflows/docs-url-alive-check.yaml

@@ -2,8 +2,7 @@ on:
   schedule:
     - cron:  '0 23 * * 0'
 
-permissions:
-  contents: read
+permissions: {}
 
 name: Docs URL Alive Check
 jobs:
@@ -14,23 +13,21 @@ jobs:
     env:
       target_branch: ${{ github.base_ref }}
     steps:
-    - name: Install Go
-      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
-      with:
-        go-version: 1.23.10
-      env:
-        GOPATH: ${{ github.workspace }}/kata-containers
     - name: Set env
       run: |
         echo "GOPATH=${{ github.workspace }}" >> "$GITHUB_ENV"
-        echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Checkout code
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         fetch-depth: 0
         persist-credentials: false
         path: ./src/github.com/${{ github.repository }}
-    # docs url alive check
+
+    - name: Install golang
+      run: |
+        ./tests/install_go.sh -f -p
+        echo "/usr/local/go/bin" >> "${GITHUB_PATH}"
+
     - name: Docs URL Alive Check
       run: |
         cd "${GOPATH}/src/github.com/${{ github.repository }}" && make docs-url-alive-check

.github/workflows/gatekeeper-skipper.yaml

@@ -31,8 +31,7 @@ on:
       skip_static:
         value: ${{ jobs.skipper.outputs.skip_static }}
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   skipper:

.github/workflows/gatekeeper.yaml

@@ -12,8 +12,7 @@ on:
       - reopened
       - labeled
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/govulncheck.yaml

@@ -3,8 +3,7 @@ on:
 
 name: Govulncheck
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   govulncheck:
@@ -14,12 +13,12 @@ jobs:
         include:
           - binary: "kata-runtime"
             make_target: "runtime"
-          - binary: "containerd-shim-kata-v2" 
+          - binary: "containerd-shim-kata-v2"
             make_target: "containerd-shim-v2"
           - binary: "kata-monitor"
             make_target: "monitor"
       fail-fast: false
-    
+
     steps:
       - name: Checkout the code
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

.github/workflows/kata-runtime-classes-sync.yaml

@@ -6,8 +6,7 @@ on:
       - reopened
       - synchronize
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/osv-scanner.yaml

@@ -15,6 +15,8 @@ on:
   push:
     branches: [ "main" ]
 
+permissions: {}
+
 jobs:
   scan-scheduled:
     permissions:

.github/workflows/payload-after-push.yaml

@@ -5,8 +5,7 @@ on:
       - main
   workflow_dispatch:
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/publish-kata-deploy-payload.yaml

@@ -34,8 +34,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   kata-payload:
@@ -85,6 +84,6 @@ jobs:
           TAG: ${{ inputs.tag }}
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          "$(pwd)/kata-static.tar.xz" \
+          "$(pwd)/kata-static.tar.zst" \
           "${REGISTRY}/${REPO}" \
           "${TAG}"

.github/workflows/release-amd64.yaml

@@ -11,8 +11,7 @@ on:
       KBUILD_SIGN_PIN:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-kata-static-tarball-amd64:
@@ -74,9 +73,9 @@ jobs:
           fi
           for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "ghcr.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "ghcr.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
           done

.github/workflows/release-arm64.yaml

@@ -9,8 +9,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-kata-static-tarball-arm64:
@@ -71,9 +70,9 @@ jobs:
           fi
           for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "ghcr.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "ghcr.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
           done

.github/workflows/release-ppc64le.yaml

@@ -9,8 +9,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-kata-static-tarball-ppc64le:
@@ -71,9 +70,9 @@ jobs:
           fi
           for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "ghcr.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "ghcr.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
           done

.github/workflows/release-s390x.yaml

@@ -11,8 +11,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-kata-static-tarball-s390x:
@@ -75,9 +74,9 @@ jobs:
           fi
           for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "ghcr.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "ghcr.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
           done

.github/workflows/release.yaml

@@ -2,8 +2,7 @@ name: Release Kata Containers
 on:
   workflow_dispatch
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   release:
@@ -127,7 +126,7 @@ jobs:
 
       - name: Set KATA_STATIC_TARBALL env var
         run: |
-          tarball=$(pwd)/kata-static.tar.xz
+          tarball=$(pwd)/kata-static.tar.zst
           echo "KATA_STATIC_TARBALL=${tarball}" >> "$GITHUB_ENV"
 
       - name: Download amd64 artifacts

.github/workflows/run-cri-containerd-tests.yaml

@@ -1,7 +1,6 @@
 name: CI | Run cri-containerd tests
 
-permissions:
-  contents: read
+permissions: {}
 
 on:
   workflow_call:

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -34,9 +34,7 @@ on:
         required: true
 
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   run-k8s-tests:
@@ -71,6 +69,9 @@ jobs:
             instance-type: normal
             auto-generate-policy: yes
     runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      id-token: write # Used for OIDC access to log into Azure
     environment: ci
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
@@ -146,6 +147,13 @@ jobs:
         timeout-minutes: 60
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
+      - name: Refresh OIDC token in case access token expired
+        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
+        with:
+          client-id: ${{ secrets.AZ_APPID }}
+          tenant-id: ${{ secrets.AZ_TENANT_ID }}
+          subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
       - name: Delete AKS cluster
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-k8s-tests-on-amd64.yaml

@@ -22,8 +22,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-k8s-tests-amd64:
@@ -31,11 +30,7 @@ jobs:
       fail-fast: false
       matrix:
         vmm:
-          - clh #cloud-hypervisor
-          - dragonball
-          - fc #firecracker
           - qemu
-          - cloud-hypervisor
         container_runtime:
           - containerd
         snapshotter:
@@ -73,6 +68,25 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: Remove unnecessary directories to free up space
+        run: |
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf /usr/lib/jvm
+          sudo rm -rf /usr/share/swift
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /usr/local/julia*
+          sudo rm -rf /opt/az
+          sudo rm -rf /usr/local/share/chromium
+          sudo rm -rf /opt/microsoft
+          sudo rm -rf /opt/google
+          sudo rm -rf /usr/lib/firefox
+
       - name: Configure CRI-O
         if: matrix.container_runtime == 'crio'
         run: bash tests/integration/kubernetes/gha-run.sh setup-crio

.github/workflows/run-k8s-tests-on-arm64.yaml

@@ -22,8 +22,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-k8s-tests-on-arm64:

.github/workflows/run-k8s-tests-on-ppc64le.yaml

@@ -22,8 +22,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-k8s-tests:

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -25,8 +25,7 @@ on:
       AUTHENTICATED_IMAGE_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-k8s-tests:

.github/workflows/run-kata-coco-stability-tests.yaml

@@ -35,9 +35,7 @@ on:
       AUTHENTICATED_IMAGE_PASSWORD:
         required: true
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   # Generate jobs for testing CoCo on non-TEE environments
@@ -52,6 +50,9 @@ jobs:
         pull-type:
           - guest-pull
     runs-on: ubuntu-22.04
+    permissions:
+
+      id-token: write # Used for OIDC access to log into Azure
     environment: ci
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
@@ -138,6 +139,13 @@ jobs:
         timeout-minutes: 300
         run: bash tests/stability/gha-stability-run.sh run-tests
 
+      - name: Refresh OIDC token in case access token expired
+        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
+        with:
+          client-id: ${{ secrets.AZ_APPID }}
+          tenant-id: ${{ secrets.AZ_TENANT_ID }}
+          subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
       - name: Delete AKS cluster
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-kata-coco-tests.yaml

@@ -36,9 +36,7 @@ on:
       ITA_KEY:
         required: true
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   run-k8s-tests-on-tdx:
@@ -223,6 +221,8 @@ jobs:
         pull-type:
           - guest-pull
     runs-on: ubuntu-22.04
+    permissions:
+      id-token: write # Used for OIDC access to log into Azure
     environment: ci
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
@@ -323,6 +323,13 @@ jobs:
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh report-tests
 
+      - name: Refresh OIDC token in case access token expired
+        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
+        with:
+          client-id: ${{ secrets.AZ_APPID }}
+          tenant-id: ${{ secrets.AZ_TENANT_ID }}
+          subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
       - name: Delete AKS cluster
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-kata-deploy-tests-on-aks.yaml

@@ -29,9 +29,7 @@ on:
       AZ_SUBSCRIPTION_ID:
         required: true
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   run-kata-deploy-tests:
@@ -50,6 +48,8 @@ jobs:
             vmm: clh
     runs-on: ubuntu-22.04
     environment: ci
+    permissions:
+      id-token: write # Used for OIDC access to log into Azure
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
@@ -102,6 +102,13 @@ jobs:
       - name: Run tests
         run: bash tests/functional/kata-deploy/gha-run.sh run-tests
 
+      - name: Refresh OIDC token in case access token expired
+        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
+        with:
+          client-id: ${{ secrets.AZ_APPID }}
+          tenant-id: ${{ secrets.AZ_TENANT_ID }}
+          subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
       - name: Delete AKS cluster
         if: always()
         run: bash tests/functional/kata-deploy/gha-run.sh delete-cluster

.github/workflows/run-kata-deploy-tests.yaml

@@ -22,8 +22,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-kata-deploy-tests:
@@ -59,6 +58,25 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: Remove unnecessary directories to free up space
+        run: |
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf /usr/lib/jvm
+          sudo rm -rf /usr/share/swift
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /usr/local/julia*
+          sudo rm -rf /opt/az
+          sudo rm -rf /usr/local/share/chromium
+          sudo rm -rf /opt/microsoft
+          sudo rm -rf /opt/google
+          sudo rm -rf /usr/lib/firefox
+
       - name: Deploy ${{ matrix.k8s }}
         run:  bash tests/functional/kata-deploy/gha-run.sh deploy-k8s
 

.github/workflows/run-kata-monitor-tests.yaml

@@ -13,8 +13,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-monitor:

.github/workflows/run-metrics.yaml

@@ -22,8 +22,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-metrics:

.github/workflows/run-runk-tests.yaml

@@ -13,8 +13,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-runk:

.github/workflows/shellcheck.yaml

@@ -10,8 +10,7 @@ on:
       - reopened
       - synchronize
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -27,6 +26,6 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
       - name: Run ShellCheck
-        uses: ludeeus/action-shellcheck@00b27aa7cb85167568cb48a3838b75f4265f2bca # master (2024-06-20)
+        uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # v2.0.0
         with:
           ignore_paths: "**/vendor/**"

.github/workflows/shellcheck_required.yaml

@@ -11,8 +11,7 @@ on:
       - reopened
       - synchronize
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -29,7 +28,7 @@ jobs:
           persist-credentials: false
 
       - name: Run ShellCheck
-        uses: ludeeus/action-shellcheck@00b27aa7cb85167568cb48a3838b75f4265f2bca # master (2024-06-20)
+        uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # v2.0.0
         with:
           severity: error
           ignore_paths: "**/vendor/**"

.github/workflows/stale.yaml

@@ -4,8 +4,7 @@ on:
     - cron: '0 0 * * *'
   workflow_dispatch:
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   stale:

.github/workflows/static-checks-self-hosted.yaml

@@ -6,8 +6,7 @@ on:
       - reopened
       - labeled # a workflow runs only when the 'ok-to-test' label is added
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/static-checks.yaml

@@ -7,8 +7,7 @@ on:
       - synchronize
   workflow_dispatch:
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/zizmor.yaml

@@ -2,11 +2,9 @@ name: GHA security analysis
 
 on:
   push:
-    branches: ["main"]
   pull_request:
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -27,3 +25,5 @@ jobs:
 
       - name: Run zizmor
         uses: zizmorcore/zizmor-action@f52a838cfabf134edcbaa7c8b3677dde20045018 # v0.1.1
+        with:
+          persona: auditor

README.md

@@ -105,7 +105,7 @@ Please raise an issue
 [in this repository](https://github.com/kata-containers/kata-containers/issues).
 
 > **Note:**
-> If you are reporting a security issue, please follow the [vulnerability reporting process](SECURITY.md)
+> If you are reporting a security issue, please follow the [vulnerability reporting process](https://github.com/kata-containers/community#vulnerability-handling)
 
 ## Developers
 

SECURITY.md

@@ -1,79 +0,0 @@
-# Security Policy
-
-Kata Containers is a **rolling-release** project: every monthly release replaces the previous one, and only the _current_ release series receives security fixes. There are **no long-term-support branches**.
-
----
-
-## Reporting a Vulnerability
-### How to report
-
-- **Keep it private first.**
-  Please **do not** open a public GitHub issue or pull request for security problems.
-
-- **Use GitHub’s built-in security advisory workflow.**
-  See GitHub’s official guide:
-  [Creating a repository security advisory](https://docs.github.com/en/code-security/security-advisories/working-with-repository-security-advisories/creating-a-repository-security-advisory#creating-a-security-advisory)
-
-### What happens after you submit
-
-We follow the OpenSSF vulnerability-handling guidelines.
-The table below shows the target timelines we hold ourselves to once we receive your report.
-
-| Stage | Target time | Notes |
-|-------|-------------|-------|
-| **Initial acknowledgement** | ≤ 14 calendar days | Maintainers confirm receipt and start triage. |
-| **Triage & CVSS-v3.1 scoring** | ≤ 30 days | We assign severity and plan remediation. |
-| **Fix availability** | Next scheduled monthly release<br />(or an out-of-band patch for Critical/High issues) | We may cut a `vX.Y.Z` patch if waiting a month poses undue risk. |
-
----
-
-## Supported Versions
-
-| Release | First published | Security-fix window |
-|---------|-----------------|---------------------|
-| **Latest monthly release** | see `git tag --sort=-creatordate \| head -n 1` | Actively maintained |
-| Any prior release | — | **Unsupported** – please upgrade |
-
-> **Why no backports?**  
-> Kata’s architecture evolves quickly; back-porting patches would re-introduce the very maintenance burden we avoid by using a rolling model.
-
----
-
-## Disclosure Process & Fix Delivery
-
-1. We develop the fix on a private branch.  
-2. Once validated, we coordinate embargo dates with downstream consumers when appropriate.  
-3. The fix ships in **either**:
-   * Common: The next regular monthly release (e.g., `v3.19`) when impact is moderate and waiting does not materially increase risk, **or**
-   * Exception: A point release (e.g., `v3.18.1`) if the vulnerability affects only the current series.
-4. After the fix is public, we request a CVE ID (if not already issued) and publish details.
-
----
-
-## Security Advisories & Release Notes
-
-* Each patch or monthly release includes a **Security Bulletin** section in its GitHub *Release Notes* summarizing:
-  * affected components & versions,
-  * CVE identifiers (if assigned),
-  * severity / CVSS score,
-  * mitigation steps,
-  * upgrade instructions.
-
-* We do **not** publish separate “stable-branch” advisories because unsupported branches receive no fixes.
-
----
-
-## Frequently Asked Questions
-
-**Q: I run `v3.16` – will you patch it?**  
-A: No. Upgrade to the latest monthly release.
-
-**Q: Can I get early access to embargoed fixes?**  
-A: Only project members under the disclosure agreement (see [SECURITY_CONTACTS](SECURITY_CONTACTS)) receive advance patches.
-
-**Q: Where can I discuss the vulnerability once it is public?**  
-A: Open/continue a GitHub issue **after** the advisory is published, or use `#kata-containers` on Slack with a link to the advisory.
-
----
-
-*Last updated:* 2025-06-27

SECURITY_CONTACTS

@@ -1,13 +0,0 @@
-# Copyright (c) 2025 Kata Containers Authors
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# Defined below are the security contacts for this repo.
-#
-# They are the contact point for the Product Security Committee to reach out
-# to for triaging and handling of incoming issues.
-#
-# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
-# INSTRUCTIONS AT [SECURITY.md](SECURITY.md)
-
-@kata-containers/architecture-committee

VERSION

@@ -1 +1 @@
-3.19.1
+3.21.0

ci/README.md

@@ -306,7 +306,7 @@ tarball to the newly created VM that will be used for debugging purposes.
 > [!NOTE]
 > Those artifacts are only available (for 15 days) when all jobs are finished.
 
-Once you have the `kata-static.tar.xz` in your VM, you can login to the VM with
+Once you have the `kata-static.tar.zst` in your VM, you can login to the VM with
 `kcli ssh debug-nerdctl-pr8070`, go ahead and then clone your development branch
 
 ```bash
@@ -323,15 +323,15 @@ $ git config --global user.name "Your Name"
 $ git rebase upstream/main
 ```
 
-Now copy the `kata-static.tar.xz` into your `kata-containers/kata-artifacts` directory
+Now copy the `kata-static.tar.zst` into your `kata-containers/kata-artifacts` directory
 
 ```bash
 $ mkdir kata-artifacts
-$ cp ../kata-static.tar.xz kata-artifacts/
+$ cp ../kata-static.tar.zst kata-artifacts/
 ```
 
 > [!NOTE]
-> If you downloaded the .zip from GitHub you need to uncompress first to see `kata-static.tar.xz`
+> If you downloaded the .zip from GitHub you need to uncompress first to see `kata-static.tar.zst`
 
 And finally run the tests following what's in the yaml file for the test you're
 debugging.
@@ -363,11 +363,11 @@ and have fun debugging and hacking!
 
 Steps for debugging the Kubernetes tests are very similar to the ones for
 debugging non-Kubernetes tests, with the caveat that what you'll need, this
-time, is not the `kata-static.tar.xz` tarball, but rather a payload to be used
+time, is not the `kata-static.tar.zst` tarball, but rather a payload to be used
 with kata-deploy.
 
 In order to generate your own kata-deploy image you can generate your own
-`kata-static.tar.xz` and then take advantage of the following script.  Be aware
+`kata-static.tar.zst` and then take advantage of the following script.  Be aware
 that the image generated and uploaded must be accessible by the VM where you'll
 be performing your tests.
 

ci/openshift-ci/peer-pods-azure.sh

@@ -116,33 +116,44 @@ az network vnet subnet update \
 for NODE_NAME in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do [[ "${NODE_NAME}" =~ 'worker' ]] && kubectl label node "${NODE_NAME}" node.kubernetes.io/worker=; done
 
 # CAA artifacts
-CAA_IMAGE="quay.io/confidential-containers/cloud-api-adaptor"
-TAGS="$(curl https://quay.io/api/v1/repository/confidential-containers/cloud-api-adaptor/tag/?onlyActiveTags=true)"
-DIGEST=$(echo "${TAGS}" | jq -r '.tags[] | select(.name | contains("latest-amd64")) | .manifest_digest')
-CAA_TAG="$(echo "${TAGS}" | jq -r '.tags[] | select(.manifest_digest | contains("'"${DIGEST}"'")) | .name' | grep -v "latest")"
+if [[ -z "${CAA_TAG}" ]]; then
+	if [[ -n "${CAA_IMAGE}" ]]; then
+		echo "CAA_IMAGE (${CAA_IMAGE}) is set but CAA_TAG isn't, which is not supported. Please specify both or none"
+		exit 1
+	fi
+	TAGS="$(curl https://quay.io/api/v1/repository/confidential-containers/cloud-api-adaptor/tag/?onlyActiveTags=true)"
+	DIGEST=$(echo "${TAGS}" | jq -r '.tags[] | select(.name | contains("latest-amd64")) | .manifest_digest')
+	CAA_TAG="$(echo "${TAGS}" | jq -r '.tags[] | select(.manifest_digest | contains("'"${DIGEST}"'")) | .name' | grep -v "latest")"
+fi
+if [[ -z "${CAA_IMAGE}" ]]; then
+	CAA_IMAGE="quay.io/confidential-containers/cloud-api-adaptor"
+fi
 
 # Get latest PP image
-SUCCESS_TIME=$(curl -s \
-  -H "Accept: application/vnd.github+json" \
-  "https://api.github.com/repos/confidential-containers/cloud-api-adaptor/actions/workflows/azure-nightly-build.yml/runs?status=success" \
-  | jq -r '.workflow_runs[0].updated_at')
-PP_IMAGE_ID="/CommunityGalleries/cocopodvm-d0e4f35f-5530-4b9c-8596-112487cdea85/Images/podvm_image0/Versions/$(date -u -jf "%Y-%m-%dT%H:%M:%SZ" "${SUCCESS_TIME}" "+%Y.%m.%d" 2>/dev/null || date -d "${SUCCESS_TIME}" +%Y.%m.%d)"
+if [[ -z "${PP_IMAGE_ID}" ]]; then
+	SUCCESS_TIME=$(curl -s \
+	  -H "Accept: application/vnd.github+json" \
+	  "https://api.github.com/repos/confidential-containers/cloud-api-adaptor/actions/workflows/azure-nightly-build.yml/runs?status=success" \
+	  | jq -r '.workflow_runs[0].updated_at')
+	PP_IMAGE_ID="/CommunityGalleries/cocopodvm-d0e4f35f-5530-4b9c-8596-112487cdea85/Images/podvm_image0/Versions/$(date -u -jf "%Y-%m-%dT%H:%M:%SZ" "${SUCCESS_TIME}" "+%Y.%m.%d" 2>/dev/null || date -d "${SUCCESS_TIME}" +%Y.%m.%d)"
+fi
 
-echo "AZURE_REGION: \"${AZURE_REGION}\""
-echo "PP_REGION: \"${PP_REGION}\""
-echo "AZURE_RESOURCE_GROUP: \"${AZURE_RESOURCE_GROUP}\""
-echo "PP_RESOURCE_GROUP: \"${PP_RESOURCE_GROUP}\""
-echo "PP_SUBNET_ID: \"${PP_SUBNET_ID}\""
-echo "CAA_TAG: \"${CAA_TAG}\""
-echo "PP_IMAGE_ID: \"${PP_IMAGE_ID}\""
+echo "AZURE_REGION=\"${AZURE_REGION}\""
+echo "PP_REGION=\"${PP_REGION}\""
+echo "AZURE_RESOURCE_GROUP=\"${AZURE_RESOURCE_GROUP}\""
+echo "PP_RESOURCE_GROUP=\"${PP_RESOURCE_GROUP}\""
+echo "PP_SUBNET_ID=\"${PP_SUBNET_ID}\""
+echo "CAA_IMAGE=\"${CAA_IMAGE}\""
+echo "CAA_TAG=\"${CAA_TAG}\""
+echo "PP_IMAGE_ID=\"${PP_IMAGE_ID}\""
 
 # Clone and configure caa
-git clone --depth 1 --no-checkout https://github.com/confidential-containers/cloud-api-adaptor.git
+git clone --revision "${CAA_GIT_SHA:-main}" --depth 1 --no-checkout https://github.com/confidential-containers/cloud-api-adaptor.git
 pushd cloud-api-adaptor
 git sparse-checkout init --cone
 git sparse-checkout set src/cloud-api-adaptor/install/
 git checkout
-echo "CAA_GIT_SHA: \"$(git rev-parse HEAD)\""
+echo "CAA_GIT_SHA=\"$(git rev-parse HEAD)\""
 pushd src/cloud-api-adaptor
 cat <<EOF > install/overlays/azure/workload-identity.yaml
 apiVersion: apps/v1
@@ -208,12 +219,12 @@ echo "AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}" >> install/overlays/azure/serv
 echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" >> install/overlays/azure/service-principal.env
 
 # Deploy Operator
-git clone --depth 1 --no-checkout https://github.com/confidential-containers/operator
+git clone --revision "${OPERATOR_SHA:-main}" --depth 1 --no-checkout https://github.com/confidential-containers/operator
 pushd operator
 git sparse-checkout init --cone
 git sparse-checkout set "config/"
 git checkout
-echo "OPERATOR_SHA: \"$(git rev-parse HEAD)\""
+echo "OPERATOR_SHA=\"$(git rev-parse HEAD)\""
 oc apply -k "config/release"
 oc apply -k "config/samples/ccruntime/peer-pods"
 popd
@@ -227,7 +238,7 @@ popd
 SECONDS=0
 ( while [[ "${SECONDS}" -lt 360 ]]; do
     kubectl get runtimeclass | grep -q kata-remote && exit 0
-done; exit 1 ) || { echo "kata-remote runtimeclass not initialized in 60s"; kubectl -n confidential-containers-system get all; echo; echo CAA; kubectl -n confidential-containers-system logs daemonset.apps/cloud-api-adaptor-daemonset; echo pre-install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-pre-install-daemon; echo install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-daemon-install; exit 1; }
+done; exit 1 ) || { echo "kata-remote runtimeclass not initialized in 60s"; kubectl -n confidential-containers-system get all; echo; echo "kubectl -n confidential-containers-system describe all"; kubectl -n confidential-containers-system describe all; echo; echo CAA; kubectl -n confidential-containers-system logs daemonset.apps/cloud-api-adaptor-daemonset; echo pre-install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-pre-install-daemon; echo install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-daemon-install; exit 1; }
 
 
 ################

docs/how-to/how-to-run-kata-containers-with-SE-VMs.md

@@ -89,16 +89,16 @@ However, if any of these components are absent, they must be built from the
 $ # Assume that the project is cloned at $GOPATH/src/github.com/kata-containers
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers
 $ make rootfs-initrd-confidential-tarball
-$ tar -tf build/kata-static-kernel-confidential.tar.xz | grep vmlinuz
+$ tar --zstd -tf build/kata-static-kernel-confidential.tar.zst | grep vmlinuz
 ./opt/kata/share/kata-containers/vmlinuz-confidential.container
 ./opt/kata/share/kata-containers/vmlinuz-6.7-136-confidential
 $ kernel_version=6.7-136
-$ tar -tf build/kata-static-rootfs-initrd-confidential.tar.xz | grep initrd
+$ tar --zstd -tf build/kata-static-rootfs-initrd-confidential.tar.zst | grep initrd
 ./opt/kata/share/kata-containers/kata-containers-initrd-confidential.img
 ./opt/kata/share/kata-containers/kata-ubuntu-20.04-confidential.initrd
 $ mkdir artifacts
-$ tar -xvf build/kata-static-kernel-confidential.tar.xz -C artifacts ./opt/kata/share/kata-containers/vmlinuz-${kernel_version}-confidential
-$ tar -xvf build/kata-static-rootfs-initrd-confidential.tar.xz -C artifacts ./opt/kata/share/kata-containers/kata-ubuntu-20.04-confidential.initrd
+$ tar --zstd -xvf build/kata-static-kernel-confidential.tar.zst -C artifacts ./opt/kata/share/kata-containers/vmlinuz-${kernel_version}-confidential
+$ tar --zstd -xvf build/kata-static-rootfs-initrd-confidential.tar.zst -C artifacts ./opt/kata/share/kata-containers/kata-ubuntu-20.04-confidential.initrd
 $ ls artifacts/opt/kata/share/kata-containers/
 kata-ubuntu-20.04-confidential.initrd  vmlinuz-${kernel_version}-confidential
 ```
@@ -190,8 +190,8 @@ can be easily accomplished by issuing the following make target:
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers
 $ mkdir hkd_dir && cp $host_key_document hkd_dir
 $ HKD_PATH=hkd_dir SE_KERNEL_PARAMS="agent.log=debug" make boot-image-se-tarball
-$ ls build/kata-static-boot-image-se.tar.xz
-build/kata-static-boot-image-se.tar.xz
+$ ls build/kata-static-boot-image-se.tar.zst
+build/kata-static-boot-image-se.tar.zst
 ```
 
 `SE_KERNEL_PARAMS` could be used to add any extra kernel parameters. If no additional kernel configuration is required, this can be omitted.
@@ -344,18 +344,18 @@ $ make virtiofsd-tarball
 $ make shim-v2-tarball
 $ mkdir kata-artifacts
 $ build_dir=$(readlink -f build)
-$ cp -r $build_dir/*.tar.xz kata-artifacts
+$ cp -r $build_dir/*.tar.zst kata-artifacts
 $ ls -1 kata-artifacts
-kata-static-agent.tar.xz
-kata-static-boot-image-se.tar.xz
-kata-static-coco-guest-components.tar.xz
-kata-static-kernel-confidential-modules.tar.xz
-kata-static-kernel-confidential.tar.xz
-kata-static-pause-image.tar.xz
-kata-static-qemu.tar.xz
-kata-static-rootfs-initrd-confidential.tar.xz
-kata-static-shim-v2.tar.xz
-kata-static-virtiofsd.tar.xz
+kata-static-agent.tar.zst
+kata-static-boot-image-se.tar.zst
+kata-static-coco-guest-components.tar.zst
+kata-static-kernel-confidential-modules.tar.zst
+kata-static-kernel-confidential.tar.zst
+kata-static-pause-image.tar.zst
+kata-static-qemu.tar.zst
+kata-static-rootfs-initrd-confidential.tar.zst
+kata-static-shim-v2.tar.zst
+kata-static-virtiofsd.tar.zst
 $ ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
 ```
 
@@ -369,7 +369,7 @@ command before running `kata-deploy-merge-builds.sh`:
 $ make rootfs-image-tarball
 ```
 
-At this point, you should have an archive file named `kata-static.tar.xz` at the project root,
+At this point, you should have an archive file named `kata-static.tar.zst` at the project root,
 which will be used to build a payload image. If you are using a local container registry at
 `localhost:5000`, proceed with the following:
 
@@ -381,7 +381,7 @@ Build and push a payload image with the name `localhost:5000/build-kata-deploy`
 `latest` using the following:
 
 ```
-$ ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh kata-static.tar.xz localhost:5000/build-kata-deploy latest
+$ ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh kata-static.tar.zst localhost:5000/build-kata-deploy latest
 ... logs ...
 Pushing the image localhost:5000/build-kata-deploy:latest to the registry
 The push refers to repository [localhost:5000/build-kata-deploy]

docs/how-to/how-to-use-the-kata-agent-policy.md

@@ -32,11 +32,24 @@ Kubernetes users can encode in `base64` format their Policy documents, and add t
 
 ### Encode a Policy file
 
-For example, the [`allow-all-except-exec-process.rego`](../../src/kata-opa/allow-all-except-exec-process.rego) sample policy file is different from the [default Policy](../../src/kata-opa/allow-all.rego) because it rejects any `ExecProcess` requests. You can encode this policy file:
+For example, the [`allow-all-except-exec-process.rego`](../../src/kata-opa/allow-all-except-exec-process.rego) sample policy file is different from the [default Policy](../../src/kata-opa/allow-all.rego) because it rejects any `ExecProcess` requests. To encode this policy file, you need to:
+- Embed the policy inside an init data struct
+- Compress
+- Base64 encode 
+For example:
 
 ```bash
-$ base64 -w 0 allow-all-except-exec-process.rego
-cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEdldE1ldHJpY3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHZXRPT01FdmVudFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEd1ZXN0RGV0YWlsc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IExpc3RJbnRlcmZhY2VzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgTGlzdFJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE1lbUhvdHBsdWdCeVByb2JlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgT25saW5lQ1BVTWVtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUGF1c2VDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBQdWxsSW1hZ2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZWFkU3RyZWFtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlU3RhbGVWaXJ0aW9mc1NoYXJlTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVzZWVkUmFuZG9tRGV2UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVzdW1lQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2V0R3Vlc3REYXRlVGltZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNldFBvbGljeVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNpZ25hbFByb2Nlc3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdGFydENvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0VHJhY2luZ1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXRzQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RvcFRyYWNpbmdSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBUdHlXaW5SZXNpemVSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVFcGhlbWVyYWxNb3VudHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVJbnRlcmZhY2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXYWl0UHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdyaXRlU3RyZWFtUmVxdWVzdCA6PSB0cnVlCgpkZWZhdWx0IEV4ZWNQcm9jZXNzUmVxdWVzdCA6PSBmYWxzZQo=
+$ STRING="$(< allow-all-except-exec-process.rego)"
+$ cat <<EOF | gzip -c | base64 -w0
+version = "0.1.0"
+algorithm = "sha256"
+
+[data]
+"policy.rego" = '''
+$STRING
+'''
+EOF
+H4sIAAAAAAAAA42UTW/TQBCG7/4Vq/QQOCQKQXCo1ENIAkRqiGWnpBJCaGKP7RXrXTM7DnV/PRMiVUh07R582J3H8/XO7AnJa2fVjRrNpm+ms1EEpnSkuarPd76C+bv3oyj6lgPD92jUOKOzbkpYupEA4/E4ulJL13Sky4rVq+y1ms/mb9VWZ+S8K1iM1DgClijRlcBpvLqf3OoMrcfJJkfLutBI12rRQFbhZD6dCRfJ4SeUqOSz/OMSNopyLKA1rBZ5vkjiLyhBj458gr9a9KyubxRTi/9i6W9oQualcR5TzrUNElLZR20waCcExqWzDNoi9WMp2PzoHkLQSi7JdQPUJ+QtMuksWLQQu912fZK+BZHz7QolaRN0c6s9bywjFZBhL5W4lsPEFuvPjhvTlh+6mNwx2MudNdLDZXwnf4SYGFo/3O64NWZTy+SEgAQhT1lECQZKsHan4UgXLGUw+FWTzHjh0woIt661HGxJgh4xT0RoV6/w1IO19XAOKfJFTxmxva6DRQsX/12jIKBLC0Y0Er2DuUutxMM5nak9QaZt2cOwf4En1ww42nN3OK+w14/B4u+a/CWLesHWTYU1Eph+GS/w0470Y/1LcgDNA40/yKOMzw/tE7N+wOx/NwUYj9H5qf4DsX93tO4FAAA=
 ```
 
 ### Attach the Policy to a pod
@@ -49,7 +62,7 @@ kind: Pod
 metadata:
   name: policy-exec-rejected
   annotations:
-    io.katacontainers.config.agent.policy: cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEdldE1ldHJpY3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHZXRPT01FdmVudFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEd1ZXN0RGV0YWlsc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IExpc3RJbnRlcmZhY2VzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgTGlzdFJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE1lbUhvdHBsdWdCeVByb2JlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgT25saW5lQ1BVTWVtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUGF1c2VDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBQdWxsSW1hZ2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZWFkU3RyZWFtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlU3RhbGVWaXJ0aW9mc1NoYXJlTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVzZWVkUmFuZG9tRGV2UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVzdW1lQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2V0R3Vlc3REYXRlVGltZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNldFBvbGljeVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNpZ25hbFByb2Nlc3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdGFydENvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0VHJhY2luZ1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXRzQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RvcFRyYWNpbmdSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBUdHlXaW5SZXNpemVSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVFcGhlbWVyYWxNb3VudHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVJbnRlcmZhY2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXYWl0UHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdyaXRlU3RyZWFtUmVxdWVzdCA6PSB0cnVlCgpkZWZhdWx0IEV4ZWNQcm9jZXNzUmVxdWVzdCA6PSBmYWxzZQo=
+    io.katacontainers.config.hypervisor.cc_init_data: H4sIAAAAAAAAA42UTW/TQBCG7/4Vq/QQOCQKQXCo1ENIAkRqiGWnpBJCaGKP7RXrXTM7DnV/PRMiVUh07R582J3H8/XO7AnJa2fVjRrNpm+ms1EEpnSkuarPd76C+bv3oyj6lgPD92jUOKOzbkpYupEA4/E4ulJL13Sky4rVq+y1ms/mb9VWZ+S8K1iM1DgClijRlcBpvLqf3OoMrcfJJkfLutBI12rRQFbhZD6dCRfJ4SeUqOSz/OMSNopyLKA1rBZ5vkjiLyhBj458gr9a9KyubxRTi/9i6W9oQualcR5TzrUNElLZR20waCcExqWzDNoi9WMp2PzoHkLQSi7JdQPUJ+QtMuksWLQQu912fZK+BZHz7QolaRN0c6s9bywjFZBhL5W4lsPEFuvPjhvTlh+6mNwx2MudNdLDZXwnf4SYGFo/3O64NWZTy+SEgAQhT1lECQZKsHan4UgXLGUw+FWTzHjh0woIt661HGxJgh4xT0RoV6/w1IO19XAOKfJFTxmxva6DRQsX/12jIKBLC0Y0Er2DuUutxMM5nak9QaZt2cOwf4En1ww42nN3OK+w14/B4u+a/CWLesHWTYU1Eph+GS/w0470Y/1LcgDNA40/yKOMzw/tE7N+wOx/NwUYj9H5qf4DsX93tO4FAAA=
 spec:
   runtimeClassName: kata
   containers:
@@ -66,7 +79,7 @@ Create the pod:
 $ kubectl apply -f pod1.yaml
 ```
 
-While creating the Pod sandbox, the Kata Shim will notice the `io.katacontainers.config.agent.policy` annotation and will send the Policy document to the Kata Agent - by sending a `SetPolicy` request. Note that this request will fail if the default Policy, included in the Guest image, doesn't allow this `SetPolicy` request. If the `SetPolicy` request is rejected by the Guest, the Kata Shim will fail to start the Pod sandbox.
+While creating the Pod sandbox, the Kata Shim will notice the `io.katacontainers.config.hypervisor.cc_init_data` annotation and will create the init data device on the host and mount it on the guest as a block device. The agent then reads the init data struct from this device and sets the policy if present.
 
 # How is the Policy being enforced?
 

src/agent/Cargo.lock

@@ -4038,12 +4038,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
 
 [[package]]
 name = "slab"
-version = "0.4.9"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
-dependencies = [
- "autocfg",
-]
+checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
 
 [[package]]
 name = "slash-formatter"

src/agent/src/confidential_data_hub/mod.rs

@@ -22,6 +22,8 @@ use protocols::{
 };
 use safe_path::scoped_join;
 use std::fs;
+use std::fs::File;
+use std::io::{self, Read};
 use std::path::Path;
 use std::{os::unix::fs::symlink, path::PathBuf};
 use tokio::sync::OnceCell;
@@ -235,8 +237,8 @@ pub async fn unseal_file(path: &str) -> Result<()> {
         }
 
         let secret_name = entry.file_name();
-        let contents = fs::read_to_string(&target_path)?;
-        if contents.starts_with(SEALED_SECRET_PREFIX) {
+        if content_starts_with_prefix(&target_path, SEALED_SECRET_PREFIX).await? {
+            let contents = fs::read_to_string(&target_path)?;
             // Get the directory name of the sealed secret file
             let dir_name = target_path
                 .parent()
@@ -262,6 +264,17 @@ pub async fn unseal_file(path: &str) -> Result<()> {
     Ok(())
 }
 
+pub async fn content_starts_with_prefix(path: &Path, prefix: &str) -> io::Result<bool> {
+    let mut file = File::open(path)?;
+    let mut buffer = vec![0u8; prefix.len()];
+
+    match file.read_exact(&mut buffer) {
+        Ok(()) => Ok(buffer == prefix.as_bytes()),
+        Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => Ok(false),
+        Err(e) => Err(e),
+    }
+}
+
 pub async fn secure_mount(
     volume_type: &str,
     options: &std::collections::HashMap<String, String>,
@@ -294,7 +307,7 @@ mod tests {
     use std::fs::File;
     use std::io::{Read, Write};
     use std::sync::Arc;
-    use tempfile::tempdir;
+    use tempfile::{tempdir, NamedTempFile};
     use test_utils::skip_if_not_root;
     use tokio::signal::unix::{signal, SignalKind};
     struct TestService;
@@ -416,4 +429,34 @@ mod tests {
         rt.shutdown_background();
         std::thread::sleep(std::time::Duration::from_secs(2));
     }
+
+    #[tokio::test]
+    async fn test_content_starts_with_prefix() {
+        // Normal case: content matches the prefix
+        let mut f = NamedTempFile::new().unwrap();
+        write!(f, "sealed.hello_world").unwrap();
+        assert!(content_starts_with_prefix(f.path(), "sealed.")
+            .await
+            .unwrap());
+
+        // Does not match the prefix
+        let mut f2 = NamedTempFile::new().unwrap();
+        write!(f2, "notsealed.hello_world").unwrap();
+        assert!(!content_starts_with_prefix(f2.path(), "sealed.")
+            .await
+            .unwrap());
+
+        // File length < prefix.len()
+        let mut f3 = NamedTempFile::new().unwrap();
+        write!(f3, "seal").unwrap();
+        assert!(!content_starts_with_prefix(f3.path(), "sealed.")
+            .await
+            .unwrap());
+
+        // Empty file
+        let f4 = NamedTempFile::new().unwrap();
+        assert!(!content_starts_with_prefix(f4.path(), "sealed.")
+            .await
+            .unwrap());
+    }
 }

src/agent/src/config.rs

@@ -202,7 +202,7 @@ macro_rules! config_override {
         }
     };
 
-    ($builder:ident, $config:ident, $field:ident, $func: ident) => {
+    ($builder:ident, $config:ident, $field:ident, $func:ident) => {
         if let Some(v) = $builder.$field {
             $config.$field = $func(&v)?;
         }
@@ -661,8 +661,8 @@ impl AgentConfig {
             self.server_addr = addr;
         }
 
-        if let Ok(addr) = env::var(LOG_LEVEL_ENV_VAR) {
-            if let Ok(level) = logrus_to_slog_level(&addr) {
+        if let Ok(level) = env::var(LOG_LEVEL_ENV_VAR) {
+            if let Ok(level) = logrus_to_slog_level(&level) {
                 self.log_level = level;
             }
         }

src/agent/src/main.rs

@@ -30,6 +30,7 @@ use nix::unistd::{self, dup, sync, Pid};
 use std::env;
 use std::ffi::OsStr;
 use std::fs::{self, File};
+use std::io::ErrorKind;
 use std::os::unix::fs::{self as unixfs, FileTypeExt};
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
@@ -465,8 +466,17 @@ fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs)
         _ => vec![],
     };
     for binary in binaries.iter() {
-        if !Path::new(binary).exists() {
-            warn!(logger, "{} not found", binary);
+        let exists = Path::new(binary)
+            .try_exists()
+            .unwrap_or_else(|error| match error.kind() {
+                ErrorKind::NotFound => {
+                    warn!(logger, "{} not found", binary);
+                    false
+                }
+                _ => panic!("Path existence check failed for '{}': {}", binary, error),
+            });
+
+        if !exists {
             return false;
         }
     }

src/dragonball/Cargo.lock

@@ -1922,12 +1922,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "slab"
-version = "0.4.9"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
-dependencies = [
- "autocfg",
-]
+checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
 
 [[package]]
 name = "slog"

src/dragonball/dbs_device/src/device_manager.rs

@@ -102,7 +102,7 @@ use crate::resources::Resource;
 use crate::{DeviceIo, IoAddress, IoSize, PioAddress};
 
 /// Error types for `IoManager` related operations.
-#[derive(Error, Debug)]
+#[derive(Error, Debug, PartialEq)]
 pub enum Error {
     /// The inserting device overlaps with a current device.
     #[error("device address conflicts with existing devices")]

src/dragonball/dbs_interrupt/src/notifier.rs

@@ -5,7 +5,9 @@
 
 use std::any::Any;
 use std::io::Error;
+use std::sync::atomic::{AtomicU16, Ordering};
 use std::sync::Arc;
+use std::sync::Mutex;
 
 use vmm_sys_util::eventfd::EventFd;
 
@@ -121,6 +123,77 @@ mod msi {
     }
 }
 
+/// Vector value used to disable MSI for a queue.
+pub const VIRTQ_MSI_NO_VECTOR: u16 = 0xffff;
+
+#[derive(Clone, PartialEq, Debug, Copy)]
+pub enum VirtioInterruptType {
+    Config,
+    Queue(u16),
+}
+
+#[derive(Clone)]
+pub struct VirtioNotifierMsix {
+    pub(crate) config_vector: Arc<AtomicU16>,
+    pub(crate) queues_vectors: Arc<Mutex<Vec<u16>>>,
+    pub(crate) interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+    pub(crate) interrupt_type: VirtioInterruptType,
+}
+
+impl VirtioNotifierMsix {
+    pub fn new(
+        config_vector: Arc<AtomicU16>,
+        queues_vectors: Arc<Mutex<Vec<u16>>>,
+        interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+        interrupt_type: VirtioInterruptType,
+    ) -> Self {
+        VirtioNotifierMsix {
+            config_vector,
+            queues_vectors,
+            interrupt_source_group,
+            interrupt_type,
+        }
+    }
+}
+
+impl InterruptNotifier for VirtioNotifierMsix {
+    fn notify(&self) -> std::result::Result<(), std::io::Error> {
+        let vector = match self.interrupt_type {
+            VirtioInterruptType::Config => self.config_vector.load(Ordering::Acquire),
+            VirtioInterruptType::Queue(queue_index) => {
+                self.queues_vectors.lock().unwrap()[queue_index as usize]
+            }
+        };
+        if vector == VIRTQ_MSI_NO_VECTOR {
+            return Ok(());
+        }
+
+        self.interrupt_source_group
+            .trigger(vector as InterruptIndex)
+    }
+    fn notifier(&self) -> Option<&EventFd> {
+        let vector = match self.interrupt_type {
+            VirtioInterruptType::Config => self.config_vector.load(Ordering::Acquire),
+            VirtioInterruptType::Queue(queue_index) => {
+                self.queues_vectors.lock().unwrap()[queue_index as usize]
+            }
+        };
+        if vector == VIRTQ_MSI_NO_VECTOR {
+            return None;
+        }
+
+        self.interrupt_source_group
+            .notifier(vector as InterruptIndex)
+    }
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn clone_boxed(&self) -> Box<dyn InterruptNotifier> {
+        Box::new(self.clone())
+    }
+}
+
 /// Struct to discard interrupts.
 #[derive(Copy, Clone, Debug, Default)]
 pub struct NoopNotifier {}

src/dragonball/dbs_pci/Cargo.toml

@@ -16,6 +16,8 @@ thiserror = "1"
 dbs-allocator = { workspace = true }
 dbs-boot = { workspace = true }
 dbs-device = { workspace = true }
+dbs-address-space = { workspace = true }
+dbs-virtio-devices = { workspace = true }
 dbs-interrupt = { workspace = true, features = [
     "kvm-irq",
     "kvm-legacy-irq",
@@ -23,12 +25,18 @@ dbs-interrupt = { workspace = true, features = [
 ] }
 downcast-rs = "1.2.0"
 byteorder = "1.4.3"
+serde = "1.0.27"
+
 vm-memory = {workspace = true}
 kvm-ioctls = {workspace = true}
 kvm-bindings = {workspace = true}
 vfio-ioctls = {workspace = true}
 vfio-bindings = {workspace = true}
 libc = "0.2.39"
+vmm-sys-util = {workspace = true}
+virtio-queue = {workspace = true}
+dbs-utils = {workspace = true}
+
 
 [dev-dependencies]
 dbs-arch = { workspace = true }

src/dragonball/dbs_pci/src/lib.rs

@@ -21,7 +21,7 @@
 //! - PCI configuration: a common framework to emulator PCI configuration space header.
 //! - PCI MSI/MSIx: structs to emulate PCI MSI/MSIx capabilities.
 
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 
 use dbs_device::device_manager::IoManagerContext;
 use dbs_interrupt::KvmIrqManager;
@@ -58,8 +58,17 @@ pub use msix::{MsixCap, MsixState, MSIX_TABLE_ENTRY_SIZE};
 mod vfio;
 pub use vfio::{VfioPciDevice, VfioPciError, VENDOR_NVIDIA};
 
+mod virtio_pci;
+pub use virtio_pci::{VirtioPciDevice, VirtioPciDeviceError, CAPABILITY_BAR_SIZE};
+
+mod pci_address;
+use dbs_virtio_devices::VirtioDevice;
+pub use pci_address::PciAddress;
+
+mod pci_common_config;
+
 /// Error codes related to PCI root/bus/device operations.
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, thiserror::Error, PartialEq)]
 pub enum Error {
     /// Failed to activate the PCI root/bus/device.
     #[error("failed to activate PCI device, {0:?}")]
@@ -110,6 +119,9 @@ pub enum Error {
     /// PCI ROM BAR address is invalid.
     #[error("address {0} size {1} too big")]
     RomBarAddressInvalid(u64, u64),
+    /// Invalid parameter
+    #[error("invalid pci device address")]
+    InvalidParameter,
 }
 
 /// Specialized `Result` for PCI related operations.
@@ -130,3 +142,8 @@ pub fn fill_config_data(data: &mut [u8]) {
         *pos = 0xff;
     }
 }
+
+/// we only support one pci bus
+pub const PCI_BUS_DEFAULT: u8 = 0;
+
+type ArcMutexBoxDynVirtioDevice<AS, Q, R> = Arc<Mutex<Box<dyn VirtioDevice<AS, Q, R>>>>;

src/dragonball/dbs_pci/src/pci_address.rs

@@ -0,0 +1,100 @@
+// Copyright (C) 2024 Alibaba Cloud. All rights reserved.
+//
+// Copyright (C) 2025 Ant Group. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
+
+use std::cmp::Ordering;
+use std::fmt;
+
+use crate::{Error, Result};
+
+const PCI_MAX_DEV_ID: u8 = 0x1f;
+const PCI_MAX_FUNC_ID: u8 = 0x7;
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+pub struct PciAddress {
+    /// Bus number, in the range [0, 0xff].
+    bus: u8,
+    /// Device id, in the range [0x0, 0x1f].
+    dev: u8,
+    /// Function id, in the range [0x0, 0x7].
+    func: u8,
+}
+
+impl PartialOrd for PciAddress {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for PciAddress {
+    fn cmp(&self, other: &Self) -> Ordering {
+        // Compare in the order of bus -> dev -> func.
+        self.bus
+            .cmp(&other.bus)
+            .then_with(|| self.dev.cmp(&other.dev))
+            .then_with(|| self.func.cmp(&other.func))
+    }
+}
+
+impl PciAddress {
+    /// Create a new PCI address from bus and device/function id.
+    ///
+    /// * `bus`: PCI bus number, in the range \[0x0, 0xff\].
+    /// * `dev`: PCI device id, in the range \[0x0, 0x1f\].
+    /// * `func`: PCI function id, in the range \[0x0, 0x7\].
+    pub fn new(bus: u8, dev: u8, func: u8) -> Result<Self> {
+        if dev > PCI_MAX_DEV_ID || func > PCI_MAX_FUNC_ID {
+            return Err(Error::InvalidParameter);
+        }
+
+        Ok(PciAddress { bus, dev, func })
+    }
+
+    /// Get PCI device id on the PCI bus, which is in [0x0, 0x1f]
+    pub fn dev_id(&self) -> u8 {
+        self.dev
+    }
+
+    /// Get PCI device function id, which is in [0x0, 0x7].
+    pub fn func_id(&self) -> u8 {
+        self.func
+    }
+
+    /// Get PCI device bus number, which is in [0x0, 0xff].
+    pub fn bus_id(&self) -> u8 {
+        self.bus
+    }
+}
+
+impl fmt::Debug for PciAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "PciAddress: {:02x}:{:02x}.{:02x}",
+            self.bus, self.dev, self.func
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_pci_address() {
+        // test invlaid device id
+        assert_eq!(PciAddress::new(0, 32, 0), Err(Error::InvalidParameter));
+
+        // test invalid function id
+        assert_eq!(PciAddress::new(0, 0, 8), Err(Error::InvalidParameter));
+
+        // test pci address
+        let (bus, dev, func) = (3, 5, 4);
+        let address = PciAddress::new(bus, dev, func).unwrap();
+        assert_eq!(address.bus_id(), bus);
+        assert_eq!(address.dev_id(), dev);
+        assert_eq!(address.func_id(), func);
+    }
+}

src/dragonball/dbs_pci/src/pci_common_config.rs

@@ -0,0 +1,507 @@
+// Copyright 2018 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE-BSD-3-Clause file.
+//
+// Copyright © 2019 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+//
+// Copyright (C) 2024 Alibaba Cloud. All rights reserved.
+//
+// Copyright (C) 2025 Ant Group. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0 or BSD-3-Clause
+
+use std::sync::atomic::{AtomicU16, Ordering};
+use std::sync::{Arc, Mutex};
+
+use byteorder::{ByteOrder, LittleEndian};
+use log::{error, trace, warn};
+use serde::{Deserialize, Serialize};
+use virtio_queue::QueueT;
+use vm_memory::{GuestAddressSpace, GuestMemoryRegion};
+
+use crate::ArcMutexBoxDynVirtioDevice;
+use dbs_virtio_devices::VirtioQueueConfig;
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct VirtioPciCommonConfigState {
+    pub driver_status: u8,
+    pub config_generation: u8,
+    pub device_feature_select: u32,
+    pub driver_feature_select: u32,
+    pub queue_select: u16,
+    pub msix_config: u16,
+    pub msix_queues: Vec<u16>,
+}
+
+/* The standard layout for the ring is a continuous chunk of memory which looks
+ * like this.  We assume num is a power of 2.
+ *
+ * struct vring
+ * {
+ *	// The actual descriptors (16 bytes each)
+ *	struct vring_desc desc[num];
+ *
+ *	// A ring of available descriptor heads with free-running index.
+ *	__virtio16 avail_flags;
+ *	__virtio16 avail_idx;
+ *	__virtio16 available[num];
+ *	__virtio16 used_event_idx;
+ *
+ *	// Padding to the next align boundary.
+ *	char pad[];
+ *
+ *	// A ring of used descriptor heads with free-running index.
+ *	__virtio16 used_flags;
+ *	__virtio16 used_idx;
+ *	struct vring_used_elem used[num];
+ *	__virtio16 avail_event_idx;
+ * };
+ * struct vring_desc {
+ *	__virtio64 addr;
+ *	__virtio32 len;
+ *	__virtio16 flags;
+ *	__virtio16 next;
+ * };
+ *
+ * struct vring_avail {
+ *	__virtio16 flags;
+ *	__virtio16 idx;
+ *	__virtio16 ring[];
+ * };
+ *
+ * // u32 is used here for ids for padding reasons.
+ * struct vring_used_elem {
+ *	// Index of start of used descriptor chain.
+ *	__virtio32 id;
+ *	// Total length of the descriptor chain which was used (written to)
+ *	__virtio32 len;
+ * };
+*
+ * Kernel header used for this reference: include/uapi/linux/virtio_ring.h
+ * Virtio Spec: https://docs.oasis-open.org/virtio/virtio/v1.2/csd01/virtio-v1.2-csd01.html
+ *
+ */
+
+/// Contains the data for reading and writing the common configuration structure of a virtio PCI
+/// device.
+///
+/// * Registers:
+///
+/// ** About the whole device.
+///    le32 device_feature_select;     // 0x00 // read-write
+///    le32 device_feature;            // 0x04 // read-only for driver
+///    le32 driver_feature_select;     // 0x08 // read-write
+///    le32 driver_feature;            // 0x0C // read-write
+///    le16 msix_config;               // 0x10 // read-write
+///    le16 num_queues;                // 0x12 // read-only for driver
+///    u8 device_status;               // 0x14 // read-write (driver_status)
+///    u8 config_generation;           // 0x15 // read-only for driver
+///
+/// ** About a specific virtqueue.
+///    le16 queue_select;              // 0x16 // read-write
+///    le16 queue_size;                // 0x18 // read-write, power of 2, or 0.
+///    le16 queue_msix_vector;         // 0x1A // read-write
+///    le16 queue_enable;              // 0x1C // read-write (Ready)
+///    le16 queue_notify_off;          // 0x1E // read-only for driver
+///    le64 queue_desc;                // 0x20 // read-write
+///    le64 queue_avail;               // 0x28 // read-write
+///    le64 queue_used;                // 0x30 // read-write
+pub struct VirtioPciCommonConfig {
+    pub driver_status: u8,
+    pub config_generation: u8,
+    pub device_feature_select: u32,
+    pub driver_feature_select: u32,
+    pub queue_select: u16,
+    pub msix_config: Arc<AtomicU16>,
+    pub msix_queues: Arc<Mutex<Vec<u16>>>,
+}
+
+impl VirtioPciCommonConfig {
+    pub fn new(state: VirtioPciCommonConfigState) -> Self {
+        VirtioPciCommonConfig {
+            driver_status: state.driver_status,
+            config_generation: state.config_generation,
+            device_feature_select: state.device_feature_select,
+            driver_feature_select: state.driver_feature_select,
+            queue_select: state.queue_select,
+            msix_config: Arc::new(AtomicU16::new(state.msix_config)),
+            msix_queues: Arc::new(Mutex::new(state.msix_queues)),
+        }
+    }
+
+    // TODO(fupan): use for live upgrade later
+    #[allow(dead_code)]
+    fn state(&self) -> VirtioPciCommonConfigState {
+        VirtioPciCommonConfigState {
+            driver_status: self.driver_status,
+            config_generation: self.config_generation,
+            device_feature_select: self.device_feature_select,
+            driver_feature_select: self.driver_feature_select,
+            queue_select: self.queue_select,
+            msix_config: self.msix_config.load(Ordering::Acquire),
+            msix_queues: self.msix_queues.lock().unwrap().clone(),
+        }
+    }
+
+    fn read_common_config_byte(&self, offset: u64) -> u8 {
+        trace!("read_common_config_byte: offset 0x{:x}", offset);
+        // The driver is only allowed to do aligned, properly sized access.
+        match offset {
+            0x14 => self.driver_status,
+            0x15 => self.config_generation,
+            _ => {
+                warn!("invalid virtio config byte read: 0x{:x}", offset);
+                0
+            }
+        }
+    }
+
+    fn write_common_config_byte(&mut self, offset: u64, value: u8) {
+        trace!(
+            "write_common_config_byte: offset 0x{:x} value 0x{:x}",
+            offset,
+            value
+        );
+        match offset {
+            0x14 => self.driver_status = value,
+            _ => {
+                warn!("invalid virtio config byte write: 0x{:x}", offset);
+            }
+        }
+    }
+
+    fn read_common_config_word<Q: QueueT + 'static>(
+        &self,
+        offset: u64,
+        queues: &[VirtioQueueConfig<Q>],
+    ) -> u16 {
+        trace!("read_common_config_word: offset 0x{:x}", offset);
+        match offset {
+            0x10 => self.msix_config.load(Ordering::Acquire),
+            0x12 => queues.len() as u16, // num_queues
+            0x16 => self.queue_select,
+            0x18 => self.with_queue(queues, |q| q.max_size()).unwrap_or(0),
+            0x1a => self.msix_queues.lock().unwrap()[self.queue_select as usize],
+            0x1c => u16::from(self.with_queue(queues, |q| q.ready()).unwrap_or(false)),
+            0x1e => self.queue_select, // notify_off
+            _ => {
+                warn!("invalid virtio register word read: 0x{:x}", offset);
+                0
+            }
+        }
+    }
+
+    fn write_common_config_word<Q: QueueT + 'static>(
+        &mut self,
+        offset: u64,
+        value: u16,
+        queues: &mut [VirtioQueueConfig<Q>],
+    ) {
+        trace!(
+            "write_common_config_word: offset 0x{:x} value 0x{:x}",
+            offset,
+            value
+        );
+        match offset {
+            0x10 => self.msix_config.store(value, Ordering::Release),
+            0x16 => self.queue_select = value,
+            0x18 => self.with_queue_mut(queues, |q| q.set_size(value)),
+            0x1a => self.msix_queues.lock().unwrap()[self.queue_select as usize] = value,
+            0x1c => self.with_queue_mut(queues, |q| {
+                let ready = value == 1;
+                q.set_ready(ready);
+            }),
+            _ => {
+                warn!("invalid virtio register word write: 0x{:x}", offset);
+            }
+        }
+    }
+
+    fn read_common_config_dword<
+        AS: GuestAddressSpace + 'static,
+        Q: QueueT + 'static,
+        R: 'static + GuestMemoryRegion,
+    >(
+        &self,
+        offset: u64,
+        device: ArcMutexBoxDynVirtioDevice<AS, Q, R>,
+    ) -> u32 {
+        trace!("read_common_config_dword: offset 0x{:x}", offset);
+        match offset {
+            0x00 => self.device_feature_select,
+            0x04 => {
+                // Only 64 bits of features (2 pages) are defined for now, so limit
+                // device_feature_select to avoid shifting by 64 or more bits.
+                let locked_device = device.lock().unwrap();
+                if self.device_feature_select < 2 {
+                    locked_device.get_avail_features(self.device_feature_select)
+                } else {
+                    0
+                }
+            }
+            0x08 => self.driver_feature_select,
+            _ => {
+                warn!("invalid virtio register dword read: 0x{:x}", offset);
+                0
+            }
+        }
+    }
+
+    fn write_common_config_dword<
+        AS: GuestAddressSpace + 'static,
+        Q: QueueT + 'static,
+        R: 'static + GuestMemoryRegion,
+    >(
+        &mut self,
+        offset: u64,
+        value: u32,
+        queues: &mut [VirtioQueueConfig<Q>],
+        device: ArcMutexBoxDynVirtioDevice<AS, Q, R>,
+    ) {
+        trace!(
+            "write_common_config_dword: offset 0x{:x} value 0x{:x}",
+            offset,
+            value
+        );
+
+        match offset {
+            0x00 => self.device_feature_select = value,
+            0x08 => self.driver_feature_select = value,
+            0x0c => {
+                if self.driver_feature_select < 2 {
+                    let mut locked_device = device.lock().unwrap();
+                    locked_device.set_acked_features(self.driver_feature_select, value);
+                } else {
+                    warn!(
+                        "invalid ack_features (page {}, value 0x{:x})",
+                        self.driver_feature_select, value
+                    );
+                }
+            }
+            0x20 => self.with_queue_mut(queues, |q| q.set_desc_table_address(Some(value), None)),
+            0x24 => self.with_queue_mut(queues, |q| q.set_desc_table_address(None, Some(value))),
+            0x28 => self.with_queue_mut(queues, |q| q.set_avail_ring_address(Some(value), None)),
+            0x2c => self.with_queue_mut(queues, |q| q.set_avail_ring_address(None, Some(value))),
+            0x30 => self.with_queue_mut(queues, |q| q.set_used_ring_address(Some(value), None)),
+            0x34 => self.with_queue_mut(queues, |q| q.set_used_ring_address(None, Some(value))),
+            _ => {
+                warn!("invalid virtio register dword write: 0x{:x}", offset);
+            }
+        }
+    }
+
+    fn read_common_config_qword(&self, _offset: u64) -> u64 {
+        trace!("read_common_config_qword: offset 0x{:x}", _offset);
+        0 // Assume the guest has no reason to read write-only registers.
+    }
+
+    fn write_common_config_qword<Q: QueueT + 'static>(
+        &mut self,
+        offset: u64,
+        value: u64,
+        queues: &mut [VirtioQueueConfig<Q>],
+    ) {
+        trace!(
+            "write_common_config_qword: offset 0x{:x}, value 0x{:x}",
+            offset,
+            value
+        );
+
+        let low = Some((value & 0xffff_ffff) as u32);
+        let high = Some((value >> 32) as u32);
+
+        match offset {
+            0x20 => self.with_queue_mut(queues, |q| q.set_desc_table_address(low, high)),
+            0x28 => self.with_queue_mut(queues, |q| q.set_avail_ring_address(low, high)),
+            0x30 => self.with_queue_mut(queues, |q| q.set_used_ring_address(low, high)),
+            _ => {
+                warn!("invalid virtio register qword write: 0x{:x}", offset);
+            }
+        }
+    }
+
+    fn with_queue<U, F, Q>(&self, queues: &[VirtioQueueConfig<Q>], f: F) -> Option<U>
+    where
+        F: FnOnce(&Q) -> U,
+        Q: QueueT + 'static,
+    {
+        queues.get(self.queue_select as usize).map(|q| f(&q.queue))
+    }
+
+    fn with_queue_mut<F: FnOnce(&mut Q), Q: QueueT + 'static>(
+        &self,
+        queues: &mut [VirtioQueueConfig<Q>],
+        f: F,
+    ) {
+        if let Some(queue) = queues.get_mut(self.queue_select as usize) {
+            f(&mut queue.queue);
+        }
+    }
+
+    pub fn read<
+        AS: GuestAddressSpace + 'static,
+        Q: QueueT + 'static,
+        R: 'static + GuestMemoryRegion,
+    >(
+        &self,
+        offset: u64,
+        data: &mut [u8],
+        queues: &[VirtioQueueConfig<Q>],
+        device: ArcMutexBoxDynVirtioDevice<AS, Q, R>,
+    ) {
+        assert!(data.len() <= 8);
+
+        match data.len() {
+            1 => {
+                let v = self.read_common_config_byte(offset);
+                data[0] = v;
+            }
+            2 => {
+                let v = self.read_common_config_word(offset, queues);
+                LittleEndian::write_u16(data, v);
+            }
+            4 => {
+                let v = self.read_common_config_dword(offset, device);
+                LittleEndian::write_u32(data, v);
+            }
+            8 => {
+                let v = self.read_common_config_qword(offset);
+                LittleEndian::write_u64(data, v);
+            }
+            _ => error!("invalid data length for virtio read: len {}", data.len()),
+        }
+    }
+
+    pub fn write<
+        AS: GuestAddressSpace + 'static,
+        Q: QueueT + 'static,
+        R: 'static + GuestMemoryRegion,
+    >(
+        &mut self,
+        offset: u64,
+        data: &[u8],
+        queues: &mut [VirtioQueueConfig<Q>],
+        device: ArcMutexBoxDynVirtioDevice<AS, Q, R>,
+    ) {
+        assert!(data.len() <= 8);
+
+        match data.len() {
+            1 => self.write_common_config_byte(offset, data[0]),
+            2 => self.write_common_config_word(offset, LittleEndian::read_u16(data), queues),
+            4 => {
+                self.write_common_config_dword(offset, LittleEndian::read_u32(data), queues, device)
+            }
+            8 => self.write_common_config_qword(offset, LittleEndian::read_u64(data), queues),
+            _ => error!("invalid data length for virtio write: len {}", data.len()),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::virtio_pci::tests::{DummyDevice, DUMMY_FEATURES};
+    use super::*;
+    use dbs_virtio_devices::VirtioDevice;
+    use virtio_queue::QueueSync;
+    use vm_memory::{GuestMemoryMmap, GuestRegionMmap};
+
+    #[test]
+    fn write_base_regs() {
+        let regs_state = VirtioPciCommonConfigState {
+            driver_status: 0xaa,
+            config_generation: 0x55,
+            device_feature_select: 0x0,
+            driver_feature_select: 0x0,
+            queue_select: 0xff,
+            msix_config: 0,
+            msix_queues: vec![0; 3],
+        };
+        let mut regs = VirtioPciCommonConfig::new(regs_state);
+
+        let dev: Arc<
+            Mutex<Box<dyn VirtioDevice<Arc<GuestMemoryMmap>, QueueSync, GuestRegionMmap>>>,
+        > = Arc::new(Mutex::new(Box::new(DummyDevice::new())));
+        let mut queues = Vec::new();
+        queues.push(VirtioQueueConfig::create(2, 0).unwrap());
+        queues.push(VirtioQueueConfig::create(2, 1).unwrap());
+
+        // Can set all bits of driver_status.
+        regs.write(0x14, &[0x55], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0x00];
+        regs.read(0x14, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0x55);
+
+        // The config generation register is read only.
+        regs.write(0x15, &[0xaa], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0x00];
+        regs.read(0x15, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0x55);
+
+        // Device features is read-only and passed through from the device.
+        regs.write(0x04, &[0, 0, 0, 0], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0, 0, 0, 0];
+        regs.read(0x04, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(LittleEndian::read_u32(&read_back), DUMMY_FEATURES as u32);
+
+        // Read device features with device_feature_select as 0
+        regs.write(0x00, &[0, 0, 0, 0], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0, 0, 0, 0];
+        regs.read(0x04, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(LittleEndian::read_u32(&read_back), DUMMY_FEATURES as u32);
+
+        // Read device features with device_feature_select as 1
+        regs.write(0x00, &[1, 0, 0, 0], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0, 0, 0, 0];
+        regs.read(0x04, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(
+            LittleEndian::read_u32(&read_back),
+            (DUMMY_FEATURES >> 32) as u32
+        );
+
+        // Feature select registers are read/write.
+        regs.write(0x00, &[1, 2, 3, 4], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0, 0, 0, 0];
+        regs.read(0x00, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(LittleEndian::read_u32(&read_back), 0x0403_0201);
+        regs.write(0x08, &[1, 2, 3, 4], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0, 0, 0, 0];
+        regs.read(0x08, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(LittleEndian::read_u32(&read_back), 0x0403_0201);
+
+        // 'queue_select' can be read and written.
+        regs.write(0x16, &[0xaa, 0x55], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0x00, 0x00];
+        regs.read(0x16, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0xaa);
+        assert_eq!(read_back[1], 0x55);
+
+        // write msix_queues by queue_select 2
+        regs.write(0x16, &[0x02, 0x00], &mut queues, Arc::clone(&dev));
+        regs.write(0x1a, &[0xbb, 0xcc], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0x00, 0x00];
+        regs.read(0x1a, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0xbb);
+        assert_eq!(read_back[1], 0xcc);
+
+        // 'msix_config' can be read and written.
+        regs.write(0x10, &[0xdd, 0xee], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0x00, 0x00];
+        regs.read(0x10, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0xdd);
+        assert_eq!(read_back[1], 0xee);
+
+        // 'queue_size' can be read and set.
+        let mut read_back = vec![0x00, 0x00];
+        // queue_select is 2 and queues[2] is None, so queue_size is 0
+        regs.read(0x18, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0x00);
+        assert_eq!(read_back[1], 0x00);
+        // queue_select is 1, so queue_size is 2
+        regs.write(0x16, &[0x01, 0x00], &mut queues, Arc::clone(&dev));
+        regs.read(0x18, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0x02);
+        assert_eq!(read_back[1], 0x00);
+    }
+}

src/dragonball/dbs_pci/src/vfio.rs

@@ -888,7 +888,7 @@ pub struct VfioPciDeviceState<C: PciSystemContext> {
     vfio_path: String,
     interrupt: Interrupt,
     vfio_dev: Arc<VfioDevice>,
-    context: Weak<C>,
+    context: Arc<Mutex<C>>,
     configuration: PciConfiguration,
     device: Option<Weak<dyn DeviceIo>>,
     regions: Vec<Region>,
@@ -904,7 +904,7 @@ impl<C: PciSystemContext> VfioPciDeviceState<C> {
         vfio_path: String,
         vfio_dev: Arc<VfioDevice>,
         bus: Weak<PciBus>,
-        context: Weak<C>,
+        context: Arc<Mutex<C>>,
         vendor_device_id: u32,
         clique_id: Option<u8>,
         vfio_container: Arc<VfioContainer>,
@@ -1277,11 +1277,7 @@ impl<C: PciSystemContext> VfioPciDeviceState<C> {
     }
 
     fn register_regions(&mut self, vm: &Arc<VmFd>) -> Result<()> {
-        let ctx = self
-            .context
-            .upgrade()
-            .ok_or(VfioPciError::BusIsDropped)?
-            .get_device_manager_context();
+        let ctx = self.context.lock().unwrap().get_device_manager_context();
         let mut tx = ctx.begin_tx();
 
         for region in self.regions.iter_mut() {
@@ -1336,22 +1332,7 @@ impl<C: PciSystemContext> VfioPciDeviceState<C> {
     }
 
     fn unregister_regions(&mut self, vm: &Arc<VmFd>) -> Result<()> {
-        // This routine handle VfioPciDevice dropped but not unmap memory
-        if self.context.upgrade().is_none() {
-            for region in self.regions.iter_mut() {
-                if region.mappable() {
-                    region.unmap(vm, &self.vfio_container)?;
-                }
-            }
-
-            return Ok(());
-        }
-
-        let ctx = self
-            .context
-            .upgrade()
-            .ok_or(VfioPciError::BusIsDropped)?
-            .get_device_manager_context();
+        let ctx = self.context.lock().unwrap().get_device_manager_context();
         let mut tx = ctx.begin_tx();
 
         for region in self.regions.iter_mut() {
@@ -1380,11 +1361,8 @@ impl<C: PciSystemContext> VfioPciDeviceState<C> {
                 } else {
                     // Safe to unwrap because activate() has set self.device to a valid value.
                     let device = self.device.as_ref().unwrap().clone();
-                    let ctx: <C as PciSystemContext>::D = self
-                        .context
-                        .upgrade()
-                        .ok_or(VfioPciError::BusIsDropped)?
-                        .get_device_manager_context();
+                    let ctx: <C as PciSystemContext>::D =
+                        self.context.lock().unwrap().get_device_manager_context();
                     let mut tx = ctx.begin_tx();
 
                     if let Err(e) = region.retrap(
@@ -1561,7 +1539,7 @@ impl<C: PciSystemContext> VfioPciDevice<C> {
         path: String,
         bus: Weak<PciBus>,
         device: VfioDevice,
-        context: Weak<C>,
+        context: Arc<Mutex<C>>,
         vm_fd: Arc<VmFd>,
         vendor_device_id: u32,
         clique_id: Option<u8>,
@@ -1649,11 +1627,7 @@ impl<C: PciSystemContext> VfioPciDevice<C> {
             state.interrupt.add_msi_irq_resource(base, size);
         }
 
-        let irq_manager = state
-            .context
-            .upgrade()
-            .ok_or(VfioPciError::BusIsDropped)?
-            .get_interrupt_manager();
+        let irq_manager = state.context.lock().unwrap().get_interrupt_manager();
         state.interrupt.initialize(irq_manager)?;
         #[cfg(target_arch = "aarch64")]
         self.set_device_id(&mut state);

src/dragonball/dbs_virtio_devices/src/lib.rs

@@ -145,6 +145,8 @@ pub enum ActivateError {
     #[cfg(feature = "vhost")]
     #[error("Vhost activate error")]
     VhostActivate(vhost_rs::Error),
+    #[error("VirtioPci error")]
+    VirtioPci,
 }
 
 impl std::convert::From<Error> for ActivateError {

src/dragonball/dbs_virtio_devices/src/mmio/mmio_state.rs

@@ -371,6 +371,7 @@ where
             }
 
             let _ = self.intr_mgr.reset();
+            self.unregister_ioevent_doorbell();
             self.unregister_ioevent();
             self.features_select = 0;
             self.acked_features_select = 0;

src/dragonball/src/api/v1/vmm_action.rs

@@ -205,6 +205,10 @@ pub enum VmmAction {
     /// input. This action can only be called before the microVM has booted.
     InsertBlockDevice(BlockDeviceConfigInfo),
 
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
+    /// Prepare to remove a block device that already exists
+    PrepareRemoveBlockDevice(String),
+
     #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     /// Remove a new block device for according to given drive_id
     RemoveBlockDevice(String),
@@ -356,6 +360,10 @@ impl VmmService {
                 self.add_block_device(vmm, event_mgr, block_device_config)
             }
             #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
+            VmmAction::PrepareRemoveBlockDevice(blkdev_id) => {
+                self.prepare_remove_block_device(vmm, &blkdev_id)
+            }
+            #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
             VmmAction::UpdateBlockDevice(blk_update) => {
                 self.update_blk_rate_limiters(vmm, blk_update)
             }
@@ -664,10 +672,17 @@ impl VmmService {
                 VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot)
             })?;
 
+        let (sender, receiver) = mpsc::channel();
+        let vmm_data = if ctx.is_hotplug() {
+            VmmData::SyncHotplug((sender.clone(), receiver))
+        } else {
+            VmmData::Empty
+        };
+
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, config)
-            .map(|_| VmmData::Empty)
+            .insert_device(ctx, config, sender.clone())
+            .map(|_| vmm_data)
             .map_err(VmmActionError::Block)
     }
 
@@ -688,6 +703,38 @@ impl VmmService {
             .map_err(VmmActionError::Block)
     }
 
+    // using upcall to unplug the block device in the guest
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
+    fn prepare_remove_block_device(
+        &mut self,
+        vmm: &mut Vmm,
+        blockdev_id: &str,
+    ) -> VmmRequestResult {
+        let vm = vmm.get_vm_mut().ok_or(VmmActionError::HostDeviceConfig(
+            VfioDeviceError::InvalidVMID,
+        ))?;
+
+        info!("prepare_remove_block_device: {:?}", blockdev_id);
+        let ctx = vm.create_device_op_context(None).map_err(|e| {
+            info!("create device op context error: {:?}", e);
+            if let StartMicroVmError::MicroVMAlreadyRunning = e {
+                VmmActionError::HostDeviceConfig(VfioDeviceError::UpdateNotAllowedPostBoot)
+            } else if let StartMicroVmError::UpcallServerNotReady = e {
+                VmmActionError::UpcallServerNotReady
+            } else {
+                VmmActionError::StartMicroVm(e)
+            }
+        })?;
+
+        let (sender, receiver) = mpsc::channel();
+
+        vm.device_manager_mut()
+            .block_manager
+            .prepare_remove_device(&ctx, blockdev_id, sender.clone())
+            .map(|_| VmmData::SyncHotplug((sender, receiver)))
+            .map_err(VmmActionError::Block)
+    }
+
     #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     // Remove the device
     #[instrument(skip(self, event_mgr))]
@@ -1526,6 +1573,7 @@ mod tests {
                     queue_size: 256,
                     use_shared_irq: None,
                     use_generic_irq: None,
+                    use_pci_bus: Some(true),
                 }),
                 InstanceState::Uninitialized,
                 &|result| {

src/dragonball/src/device_manager/blk_dev_mgr.rs

@@ -7,19 +7,28 @@
 // found in the THIRD-PARTY file.
 
 //! Device manager for virtio-blk and vhost-user-blk devices.
-use std::collections::{vec_deque, VecDeque};
 use std::convert::TryInto;
 use std::fs::OpenOptions;
 use std::os::unix::fs::OpenOptionsExt;
 use std::os::unix::io::AsRawFd;
 use std::path::{Path, PathBuf};
+use std::sync::mpsc::Sender;
 use std::sync::Arc;
+use std::{
+    collections::{vec_deque, VecDeque},
+    sync::mpsc,
+};
 
+use dbs_device::DeviceIo;
+use dbs_pci::VirtioPciDevice;
+use dbs_upcall::{DevMgrResponse, UpcallClientResponse};
 use dbs_virtio_devices as virtio;
 use dbs_virtio_devices::block::{aio::Aio, io_uring::IoUring, Block, LocalFile, Ufile};
 #[cfg(feature = "vhost-user-blk")]
 use dbs_virtio_devices::vhost::vhost_user::block::VhostUserBlock;
 use serde_derive::{Deserialize, Serialize};
+use virtio_queue::QueueSync;
+use vm_memory::GuestRegionMmap;
 
 use crate::address_space_manager::GuestAddressSpaceImpl;
 use crate::config_manager::{ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo};
@@ -190,6 +199,8 @@ pub struct BlockDeviceConfigInfo {
     pub use_shared_irq: Option<bool>,
     /// Use generic irq
     pub use_generic_irq: Option<bool>,
+    /// Use pci bus
+    pub use_pci_bus: Option<bool>,
 }
 
 impl std::default::Default for BlockDeviceConfigInfo {
@@ -208,6 +219,7 @@ impl std::default::Default for BlockDeviceConfigInfo {
             rate_limiter: None,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: None,
         }
     }
 }
@@ -349,6 +361,7 @@ impl BlockDeviceMgr {
         &mut self,
         mut ctx: DeviceOpContext,
         config: BlockDeviceConfigInfo,
+        sender: mpsc::Sender<Option<i32>>,
     ) -> std::result::Result<(), BlockDeviceError> {
         if !cfg!(feature = "hotplug") && ctx.is_hotplug {
             return Err(BlockDeviceError::UpdateNotAllowedPostBoot);
@@ -380,31 +393,68 @@ impl BlockDeviceMgr {
                     return Ok(());
                 }
 
+                let mut slot = 0;
+
+                let use_generic_irq = config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ);
+
                 match config.device_type {
                     BlockDeviceType::RawBlock => {
                         let device = Self::create_blk_device(&config, &mut ctx)
                             .map_err(BlockDeviceError::Virtio)?;
-                        let dev = DeviceManager::create_mmio_virtio_device(
-                            device,
-                            &mut ctx,
-                            config.use_shared_irq.unwrap_or(self.use_shared_irq),
-                            config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
-                        )
-                        .map_err(BlockDeviceError::DeviceManager)?;
-                        self.update_device_by_index(index, Arc::clone(&dev))?;
+
+                        let dev = if let Some(true) = config.use_pci_bus {
+                            let pci_dev = DeviceManager::create_virtio_pci_device(
+                                device,
+                                &mut ctx,
+                                use_generic_irq,
+                            )
+                            .map_err(BlockDeviceError::DeviceManager)?;
+
+                            let (_, devfn) = DeviceManager::get_pci_device_info(&pci_dev)?;
+                            slot = devfn >> 3;
+
+                            pci_dev
+                        } else {
+                            DeviceManager::create_mmio_virtio_device(
+                                device,
+                                &mut ctx,
+                                config.use_shared_irq.unwrap_or(self.use_shared_irq),
+                                use_generic_irq,
+                            )
+                            .map_err(BlockDeviceError::DeviceManager)?
+                        };
+
+                        let callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>> =
+                            Some(Box::new(move |_| {
+                                // send the pci device slot to caller.
+                                let _ = sender.send(Some(slot as i32));
+                            }));
+
+                        self.update_device_by_index(index, dev.clone())?;
                         // live-upgrade need save/restore device from info.device.
                         self.info_list[index].set_device(dev.clone());
-                        ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| {
+
+                        let mut cleanup = |e, ctx: DeviceOpContext| -> BlockDeviceError {
                             let logger = ctx.logger().new(slog::o!());
                             self.remove_device(ctx, &config.drive_id).unwrap();
                             error!(
                                 logger,
-                                "failed to hot-add virtio block device {}, {:?}",
+                                "failed to hot-add pci virtio block device {}, {:?}",
                                 &config.drive_id,
                                 e
                             );
                             BlockDeviceError::DeviceManager(e)
-                        })
+                        };
+
+                        if let Some(true) = config.use_pci_bus {
+                            let _ = ctx
+                                .insert_hotplug_pci_device(&dev, callback)
+                                .map_err(|e| cleanup(e, ctx))?;
+                            Ok(())
+                        } else {
+                            ctx.insert_hotplug_mmio_device(&dev, callback)
+                                .map_err(|e| cleanup(e, ctx))
+                        }
                     }
                     #[cfg(feature = "vhost-user-blk")]
                     BlockDeviceType::Spool | BlockDeviceType::Spdk => {
@@ -417,8 +467,13 @@ impl BlockDeviceMgr {
                             config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
                         )
                         .map_err(BlockDeviceError::DeviceManager)?;
+                        let callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>> =
+                            Some(Box::new(move |_| {
+                                let _ = sender.send(None);
+                            }));
+
                         self.update_device_by_index(index, Arc::clone(&dev))?;
-                        ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| {
+                        ctx.insert_hotplug_mmio_device(&dev, callback).map_err(|e| {
                             let logger = ctx.logger().new(slog::o!());
                             self.remove_device(ctx, &config.drive_id).unwrap();
                             error!(
@@ -450,15 +505,25 @@ impl BlockDeviceMgr {
                         info.config.drive_id,
                         info.config.path_on_host.to_str().unwrap_or("<unknown>")
                     );
+
+                    let use_shared_irq = info.config.use_shared_irq.unwrap_or(self.use_shared_irq);
+                    let use_generic_irq = info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ);
                     let device = Self::create_blk_device(&info.config, ctx)
                         .map_err(BlockDeviceError::Virtio)?;
-                    let device = DeviceManager::create_mmio_virtio_device(
-                        device,
-                        ctx,
-                        info.config.use_shared_irq.unwrap_or(self.use_shared_irq),
-                        info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
-                    )
-                    .map_err(BlockDeviceError::RegisterBlockDevice)?;
+
+                    let device = if let Some(true) = info.config.use_pci_bus {
+                        DeviceManager::create_virtio_pci_device(device, ctx, use_generic_irq)
+                            .map_err(BlockDeviceError::RegisterBlockDevice)?
+                    } else {
+                        DeviceManager::create_mmio_virtio_device(
+                            device,
+                            ctx,
+                            use_shared_irq,
+                            use_generic_irq,
+                        )
+                        .map_err(BlockDeviceError::RegisterBlockDevice)?
+                    };
+
                     info.device = Some(device);
                 }
                 #[cfg(feature = "vhost-user-blk")]
@@ -496,7 +561,7 @@ impl BlockDeviceMgr {
         while let Some(mut info) = self.info_list.pop_back() {
             info!(ctx.logger(), "remove drive {}", info.config.drive_id);
             if let Some(device) = info.device.take() {
-                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+                DeviceManager::destroy_virtio_device(device, ctx)?;
             }
         }
 
@@ -510,6 +575,62 @@ impl BlockDeviceMgr {
         }
     }
 
+    /// prepare to remove device
+    pub fn prepare_remove_device(
+        &self,
+        ctx: &DeviceOpContext,
+        blockdev_id: &str,
+        result_sender: Sender<Option<i32>>,
+    ) -> Result<(), BlockDeviceError> {
+        if !cfg!(feature = "hotplug") {
+            return Err(BlockDeviceError::UpdateNotAllowedPostBoot);
+        }
+
+        info!(ctx.logger(), "prepare remove block device");
+
+        let callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>> =
+            Some(Box::new(move |result| match result {
+                UpcallClientResponse::DevMgr(response) => {
+                    if let DevMgrResponse::Other(resp) = response {
+                        if let Err(e) = result_sender.send(Some(resp.result)) {
+                            log::error!("send upcall result failed, due to {:?}!", e);
+                        }
+                    }
+                }
+                UpcallClientResponse::UpcallReset => {
+                    if let Err(e) = result_sender.send(None) {
+                        log::error!("send upcall result failed, due to {:?}!", e);
+                    }
+                }
+                #[allow(unreachable_patterns)]
+                _ => {
+                    log::debug!("this arm should only be triggered under test");
+                }
+            }));
+
+        let device_index = self
+            .get_index_of_drive_id(blockdev_id)
+            .ok_or(BlockDeviceError::InvalidDeviceId(blockdev_id.to_string()))?;
+
+        let info = &self.info_list[device_index];
+        if let Some(device) = info.device.as_ref() {
+            if let Some(_mmio_dev) = device.as_any().downcast_ref::<DbsMmioV2Device>() {
+                if callback.is_some() {
+                    ctx.remove_hotplug_mmio_device(device, callback)?;
+                }
+            } else if let Some(_pci_dev) = device.as_any().downcast_ref::<VirtioPciDevice<
+                GuestAddressSpaceImpl,
+                QueueSync,
+                GuestRegionMmap,
+            >>() {
+                if callback.is_some() {
+                    ctx.remove_hotplug_pci_device(device, callback)?;
+                }
+            }
+        }
+        Ok(())
+    }
+
     /// remove a block device, it basically is the inverse operation of `insert_device``
     pub fn remove_device(
         &mut self,
@@ -524,7 +645,7 @@ impl BlockDeviceMgr {
             Some(mut info) => {
                 info!(ctx.logger(), "remove drive {}", info.config.drive_id);
                 if let Some(device) = info.device.take() {
-                    DeviceManager::destroy_mmio_virtio_device(device, &mut ctx)
+                    DeviceManager::destroy_virtio_device(device, &mut ctx)
                         .map_err(BlockDeviceError::DeviceManager)?;
                 }
             }
@@ -783,7 +904,7 @@ impl BlockDeviceMgr {
     pub fn update_device_by_index(
         &mut self,
         index: usize,
-        device: Arc<DbsMmioV2Device>,
+        device: Arc<dyn DeviceIo>,
     ) -> Result<(), BlockDeviceError> {
         if let Some(info) = self.info_list.get_mut(index) {
             info.device = Some(device);
@@ -818,6 +939,21 @@ impl BlockDeviceMgr {
                             .map(|_p| ())
                             .map_err(|_e| BlockDeviceError::BlockEpollHanderSendFail);
                     }
+                } else if let Some(pci_dev) = device.as_any().downcast_ref::<VirtioPciDevice<
+                    GuestAddressSpaceImpl,
+                    QueueSync,
+                    GuestRegionMmap,
+                >>() {
+                    let inner_dev = pci_dev.device();
+                    if let Some(blk_dev) = inner_dev
+                        .as_any()
+                        .downcast_ref::<virtio::block::Block<GuestAddressSpaceImpl>>()
+                    {
+                        return blk_dev
+                            .set_patch_rate_limiters(new_cfg.bytes(), new_cfg.ops())
+                            .map(|_p| ())
+                            .map_err(|_e| BlockDeviceError::BlockEpollHanderSendFail);
+                    }
                 }
                 Ok(())
             }
@@ -848,6 +984,7 @@ mod tests {
     use super::*;
     use crate::device_manager::tests::create_address_space;
     use crate::test_utils::tests::create_vm_for_test;
+    use std::sync::mpsc::channel;
 
     #[test]
     fn test_block_device_type() {
@@ -887,14 +1024,16 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        let (sender, _receiver) = channel();
         assert!(vm
             .device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device.clone(),)
+            .insert_device(ctx, dummy_block_device.clone(), sender)
             .is_ok());
 
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
@@ -961,10 +1100,12 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
+        let (sender, _receiver) = channel();
         vm.device_manager_mut()
             .block_manager
-            .insert_device(device_op_ctx, dummy_block_device)
+            .insert_device(device_op_ctx, dummy_block_device, sender)
             .unwrap();
 
         let cfg = BlockDeviceConfigUpdateInfo {
@@ -1037,14 +1178,16 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        let (sender, _receiver) = channel();
         assert!(vm
             .device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device.clone(),)
+            .insert_device(ctx, dummy_block_device.clone(), sender)
             .is_ok());
 
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
@@ -1077,6 +1220,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_2 = TempFile::new().unwrap();
@@ -1095,19 +1239,21 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        let (sender, _receiver) = channel();
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device_1)
+            .insert_device(ctx, root_block_device_1, sender.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         assert!(vm
             .device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device_2)
+            .insert_device(ctx, root_block_device_2, sender)
             .is_err());
     }
 
@@ -1131,6 +1277,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_2 = TempFile::new().unwrap();
@@ -1149,6 +1296,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_3 = TempFile::new().unwrap();
@@ -1167,6 +1315,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
@@ -1186,23 +1335,24 @@ mod tests {
         assert!(vm.device_manager().block_manager.has_root_block_device(),);
         assert!(!vm.device_manager().block_manager.has_part_uuid_root());
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 3);
+        let (sender, _receiver) = channel();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device)
+            .insert_device(ctx, root_block_device, sender.clone())
             .unwrap();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2)
+            .insert_device(ctx, dummy_block_device_2, sender.clone())
             .unwrap();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_3)
+            .insert_device(ctx, dummy_block_device_3, sender.clone())
             .unwrap();
     }
 
@@ -1226,6 +1376,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_2 = TempFile::new().unwrap();
@@ -1244,6 +1395,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_3 = TempFile::new().unwrap();
@@ -1262,24 +1414,26 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        let (sender, _receiver) = channel();
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone())
+            .insert_device(ctx, dummy_block_device_2.clone(), sender.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_3.clone())
+            .insert_device(ctx, dummy_block_device_3.clone(), sender.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device.clone())
+            .insert_device(ctx, root_block_device.clone(), sender.clone())
             .unwrap();
 
         assert!(vm.device_manager().block_manager.has_root_block_device(),);
@@ -1322,6 +1476,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_2 = TempFile::new().unwrap();
@@ -1340,20 +1495,22 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
+        let (sender, _receiver) = channel();
 
         // Add 2 block devices.
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device)
+            .insert_device(ctx, root_block_device, sender.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone())
+            .insert_device(ctx, dummy_block_device_2.clone(), sender.clone())
             .unwrap();
 
         // Get index zero.
@@ -1384,7 +1541,7 @@ mod tests {
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone())
+            .insert_device(ctx, dummy_block_device_2.clone(), sender.clone())
             .unwrap();
 
         let index = vm
@@ -1407,7 +1564,7 @@ mod tests {
         assert!(vm
             .device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone(),)
+            .insert_device(ctx, dummy_block_device_2.clone(), sender.clone())
             .is_err());
 
         // Update with 2 root block devices.
@@ -1417,7 +1574,7 @@ mod tests {
         assert!(vm
             .device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2,)
+            .insert_device(ctx, dummy_block_device_2, sender.clone())
             .is_err(),);
 
         // Switch roots and add a PARTUUID for the new one.
@@ -1435,6 +1592,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
         let root_block_device_new = BlockDeviceConfigInfo {
             path_on_host: dummy_path_2,
@@ -1450,16 +1608,17 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device_old)
+            .insert_device(ctx, root_block_device_old, sender.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device_new)
+            .insert_device(ctx, root_block_device_new, sender.clone())
             .unwrap();
         assert!(vm.device_manager().block_manager.has_part_uuid_root);
     }

src/dragonball/src/device_manager/mem_dev_mgr.rs

@@ -310,7 +310,7 @@ impl MemDeviceMgr {
     pub fn remove_devices(&self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> {
         for info in self.info_list.iter() {
             if let Some(device) = &info.device {
-                DeviceManager::destroy_mmio_virtio_device(device.clone(), ctx)?;
+                DeviceManager::destroy_mmio_device(device.clone(), ctx)?;
             }
         }
 

src/dragonball/src/device_manager/mod.rs

@@ -13,17 +13,19 @@ use arc_swap::ArcSwap;
 use dbs_address_space::AddressSpace;
 #[cfg(target_arch = "aarch64")]
 use dbs_arch::{DeviceType, MMIODeviceInfo};
+use dbs_boot::layout::MMIO_LOW_END;
 use dbs_device::device_manager::{Error as IoManagerError, IoManager, IoManagerContext};
-#[cfg(target_arch = "aarch64")]
 use dbs_device::resources::DeviceResources;
 use dbs_device::resources::Resource;
 use dbs_device::DeviceIo;
 use dbs_interrupt::KvmIrqManager;
 use dbs_legacy_devices::ConsoleHandler;
-#[cfg(all(feature = "host-device", target_arch = "aarch64"))]
-use dbs_pci::PciBusResources;
+#[cfg(feature = "dbs-virtio-devices")]
+use dbs_pci::CAPABILITY_BAR_SIZE;
 use dbs_utils::epoll_manager::EpollManager;
 use kvm_ioctls::VmFd;
+use log::error;
+use virtio_queue::QueueSync;
 
 #[cfg(feature = "dbs-virtio-devices")]
 use dbs_device::resources::ResourceConstraint;
@@ -40,6 +42,7 @@ use dbs_virtio_devices::{
 
 #[cfg(feature = "host-device")]
 use dbs_pci::VfioPciDevice;
+use dbs_pci::VirtioPciDevice;
 #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
 use dbs_upcall::{
     DevMgrRequest, DevMgrService, MmioDevRequest, PciDevRequest, UpcallClient, UpcallClientError,
@@ -57,6 +60,8 @@ use crate::resource_manager::ResourceManager;
 use crate::vm::{KernelConfigInfo, Vm, VmConfigInfo};
 use crate::IoManagerCached;
 
+use vm_memory::GuestRegionMmap;
+
 /// Virtual machine console device manager.
 pub mod console_manager;
 /// Console Manager for virtual machines console device.
@@ -131,6 +136,9 @@ macro_rules! info(
     };
 );
 
+// The flag of whether to use the shared irq.
+const USE_SHARED_IRQ: bool = true;
+
 /// Errors related to device manager operations.
 #[derive(Debug, thiserror::Error)]
 pub enum DeviceMgrError {
@@ -173,13 +181,28 @@ pub enum DeviceMgrError {
     HotplugDevice(#[source] UpcallClientError),
 
     /// Failed to free device resource.
-    #[error("failed to free device resources: {0}")]
+    #[error("failed to allocate/free device resources: {0}")]
     ResourceError(#[source] crate::resource_manager::ResourceError),
 
     #[cfg(feature = "host-device")]
     /// Error from Vfio Pci
     #[error("failed to do vfio pci operation: {0:?}")]
     VfioPci(#[source] dbs_pci::VfioPciError),
+    /// Error from Virtio Pci
+    #[error("failed to do virtio pci operation")]
+    VirtioPci,
+    /// PCI system manager error
+    #[error("Pci system manager error")]
+    PciSystemManager,
+    /// Dragonball pci system error
+    #[error("pci error: {0:?}")]
+    PciError(#[source] dbs_pci::Error),
+    /// Virtio Pci system error
+    #[error("virtio pci error: {0:?}")]
+    VirtioPciError(#[source] dbs_pci::VirtioPciDeviceError),
+    /// Unsupported pci device type
+    #[error("unsupported pci device type")]
+    InvalidPciDeviceType,
 }
 
 /// Specialized version of `std::result::Result` for device manager operations.
@@ -290,9 +313,10 @@ pub struct DeviceOpContext {
     #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
     upcall_client: Option<Arc<UpcallClient<DevMgrService>>>,
     #[cfg(feature = "dbs-virtio-devices")]
-    virtio_devices: Vec<Arc<DbsMmioV2Device>>,
+    virtio_devices: Vec<Arc<dyn DeviceIo>>,
     #[cfg(feature = "host-device")]
     vfio_manager: Option<Arc<Mutex<VfioDeviceMgr>>>,
+    pci_system_manager: Arc<Mutex<PciSystemManager>>,
     vm_config: Option<VmConfigInfo>,
     shared_info: Arc<RwLock<InstanceInfo>>,
 }
@@ -343,6 +367,7 @@ impl DeviceOpContext {
             shared_info,
             #[cfg(feature = "host-device")]
             vfio_manager: None,
+            pci_system_manager: device_mgr.pci_system_manager.clone(),
         }
     }
 
@@ -444,6 +469,11 @@ impl DeviceOpContext {
         }
         Ok(dev_info)
     }
+
+    /// check the hotplug context
+    pub fn is_hotplug(&self) -> bool {
+        self.is_hotplug
+    }
 }
 
 #[cfg(all(feature = "hotplug", not(feature = "dbs-upcall")))]
@@ -513,7 +543,7 @@ impl DeviceOpContext {
 
     pub(crate) fn insert_hotplug_mmio_device(
         &self,
-        dev: &Arc<DbsMmioV2Device>,
+        dev: &Arc<dyn DeviceIo>,
         callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>>,
     ) -> Result<()> {
         if !self.is_hotplug {
@@ -532,7 +562,7 @@ impl DeviceOpContext {
 
     pub(crate) fn remove_hotplug_mmio_device(
         &self,
-        dev: &Arc<DbsMmioV2Device>,
+        dev: &Arc<dyn DeviceIo>,
         callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>>,
     ) -> Result<()> {
         if !self.is_hotplug {
@@ -630,6 +660,7 @@ pub struct DeviceManager {
     vhost_user_net_manager: VhostUserNetDeviceMgr,
     #[cfg(feature = "host-device")]
     pub(crate) vfio_manager: Arc<Mutex<VfioDeviceMgr>>,
+    pub(crate) pci_system_manager: Arc<Mutex<PciSystemManager>>,
 }
 
 impl DeviceManager {
@@ -640,11 +671,25 @@ impl DeviceManager {
         epoll_manager: EpollManager,
         logger: &slog::Logger,
         shared_info: Arc<RwLock<InstanceInfo>>,
-    ) -> Self {
-        DeviceManager {
-            io_manager: Arc::new(ArcSwap::new(Arc::new(IoManager::new()))),
-            io_lock: Arc::new(Mutex::new(())),
-            irq_manager: Arc::new(KvmIrqManager::new(vm_fd.clone())),
+    ) -> Result<Self> {
+        let irq_manager = Arc::new(KvmIrqManager::new(vm_fd.clone()));
+        let io_manager = Arc::new(ArcSwap::new(Arc::new(IoManager::new())));
+        let io_lock = Arc::new(Mutex::new(()));
+        let io_context = DeviceManagerContext::new(io_manager.clone(), io_lock.clone());
+        let mut mgr = PciSystemManager::new(irq_manager.clone(), io_context, res_manager.clone())?;
+
+        let requirements = mgr.resource_requirements();
+        let resources = res_manager
+            .allocate_device_resources(&requirements, USE_SHARED_IRQ)
+            .map_err(DeviceMgrError::ResourceError)?;
+        mgr.activate(resources)?;
+
+        let pci_system_manager = Arc::new(Mutex::new(mgr));
+
+        Ok(DeviceManager {
+            io_manager,
+            io_lock,
+            irq_manager,
             res_manager,
             vm_fd: vm_fd.clone(),
             logger: logger.new(slog::o!()),
@@ -671,8 +716,13 @@ impl DeviceManager {
             #[cfg(feature = "vhost-user-net")]
             vhost_user_net_manager: VhostUserNetDeviceMgr::default(),
             #[cfg(feature = "host-device")]
-            vfio_manager: Arc::new(Mutex::new(VfioDeviceMgr::new(vm_fd, logger))),
-        }
+            vfio_manager: Arc::new(Mutex::new(VfioDeviceMgr::new(
+                vm_fd,
+                pci_system_manager.clone(),
+                logger,
+            ))),
+            pci_system_manager,
+        })
     }
 
     /// Get the underlying IoManager to dispatch IO read/write requests.
@@ -1024,7 +1074,7 @@ impl DeviceManager {
     }
 
     #[cfg(feature = "dbs-virtio-devices")]
-    fn get_virtio_mmio_device_info(device: &Arc<DbsMmioV2Device>) -> Result<(u64, u64, u32)> {
+    fn get_virtio_mmio_device_info(device: &Arc<dyn DeviceIo>) -> Result<(u64, u64, u32)> {
         let resources = device.get_assigned_resources();
         let irq = resources
             .get_legacy_irq()
@@ -1038,26 +1088,11 @@ impl DeviceManager {
 
         Err(DeviceMgrError::GetDeviceResource)
     }
-
-    /// Get pci bus resources for creating fdt.
-    #[cfg(feature = "host-device")]
-    pub fn get_pci_bus_resources(&self) -> Option<PciBusResources> {
-        let mut vfio_dev_mgr = self.vfio_manager.lock().unwrap();
-        let vfio_pci_mgr = vfio_dev_mgr.get_pci_manager();
-        vfio_pci_mgr.as_ref()?;
-        let pci_manager = vfio_pci_mgr.unwrap();
-        let ecam_space = pci_manager.get_ecam_space();
-        let bar_space = pci_manager.get_bar_space();
-        Some(PciBusResources {
-            ecam_space,
-            bar_space,
-        })
-    }
 }
 
 #[cfg(feature = "dbs-virtio-devices")]
 impl DeviceManager {
-    fn get_virtio_device_info(device: &Arc<DbsMmioV2Device>) -> Result<(u64, u64, u32)> {
+    fn get_virtio_device_info(device: &Arc<dyn DeviceIo>) -> Result<(u64, u64, u32)> {
         let resources = device.get_assigned_resources();
         let irq = resources
             .get_legacy_irq()
@@ -1079,7 +1114,7 @@ impl DeviceManager {
         ctx: &mut DeviceOpContext,
         use_shared_irq: bool,
         use_generic_irq: bool,
-    ) -> std::result::Result<Arc<DbsMmioV2Device>, DeviceMgrError> {
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
         let features = DRAGONBALL_FEATURE_INTR_USED | DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY;
         DeviceManager::create_mmio_virtio_device_with_features(
             device,
@@ -1097,7 +1132,7 @@ impl DeviceManager {
         ctx: &mut DeviceOpContext,
         use_shared_irq: bool,
         use_generic_irq: bool,
-    ) -> std::result::Result<Arc<DbsMmioV2Device>, DeviceMgrError> {
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
         let features = DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY;
         DeviceManager::create_mmio_virtio_device_with_features(
             device,
@@ -1116,7 +1151,7 @@ impl DeviceManager {
         features: Option<u32>,
         use_shared_irq: bool,
         use_generic_irq: bool,
-    ) -> std::result::Result<Arc<DbsMmioV2Device>, DeviceMgrError> {
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
         // Every emulated Virtio MMIO device needs a 4K configuration space,
         // and another 4K space for per queue notification.
         const MMIO_ADDRESS_DEFAULT: ResourceConstraint = ResourceConstraint::MmioAddress {
@@ -1147,12 +1182,18 @@ impl DeviceManager {
         Self::register_mmio_virtio_device(Arc::new(virtio_dev), ctx)
     }
 
-    /// Teardown the Virtio MMIO transport layer device associated with the virtio backend device.
-    pub fn destroy_mmio_virtio_device(
+    fn destroy_mmio_device(
         device: Arc<dyn DeviceIo>,
         ctx: &mut DeviceOpContext,
     ) -> std::result::Result<(), DeviceMgrError> {
-        Self::destroy_mmio_device(device.clone(), ctx)?;
+        // unregister IoManager
+        Self::deregister_virtio_device(&device, ctx)?;
+
+        // unregister Resource manager
+        let resources = device.get_assigned_resources();
+        ctx.res_manager
+            .free_device_resources(&resources)
+            .map_err(DeviceMgrError::ResourceError)?;
 
         let mmio_dev = device
             .as_any()
@@ -1164,27 +1205,11 @@ impl DeviceManager {
         Ok(())
     }
 
-    fn destroy_mmio_device(
-        device: Arc<dyn DeviceIo>,
-        ctx: &mut DeviceOpContext,
-    ) -> std::result::Result<(), DeviceMgrError> {
-        // unregister IoManager
-        Self::deregister_mmio_virtio_device(&device, ctx)?;
-
-        // unregister Resource manager
-        let resources = device.get_assigned_resources();
-        ctx.res_manager
-            .free_device_resources(&resources)
-            .map_err(DeviceMgrError::ResourceError)?;
-
-        Ok(())
-    }
-
     /// Create an Virtio MMIO transport layer device for the virtio backend device.
     pub fn register_mmio_virtio_device(
-        device: Arc<DbsMmioV2Device>,
+        device: Arc<dyn DeviceIo>,
         ctx: &mut DeviceOpContext,
-    ) -> std::result::Result<Arc<DbsMmioV2Device>, DeviceMgrError> {
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
         let (mmio_base, mmio_size, irq) = Self::get_virtio_device_info(&device)?;
         info!(
             ctx.logger(),
@@ -1226,8 +1251,210 @@ impl DeviceManager {
         }
     }
 
+    /// Create an Virtio PCI transport layer device for the virtio backend device.
+    pub fn create_virtio_pci_device(
+        mut device: DbsVirtioDevice,
+        ctx: &mut DeviceOpContext,
+        use_generic_irq: bool,
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
+        let pci_system_manager = ctx.pci_system_manager.lock().unwrap();
+
+        // We always use 64bit bars, we don't support 32bit bar now
+        // We aligned to the size of the bar itself, refers to cloud-hypervisor
+        // https://github.com/cloud-hypervisor/cloud-hypervisor/commit/bfc65bff2a5bdb9aca7dcd3284a0ced0e5cc7db8
+        //
+        // Allocate virtio-pci config bar below MMIO_LOW_END.
+        // Each bridge PCI bridge only has two bridge windows:
+        // - One is non-prefetchable and located below `MMIO_LOW_END`.
+        // - The other is prefetchable and located above `MMIO_LOW_END`.
+        // In reference to `clh`, the config BAR is set as non-prefetchable.
+        // Therefore, it must be allocated below `MMIO_LOW_END`.
+        const DEFAULE_VIRTIO_PCI_CONFIG_BAR: ResourceConstraint = ResourceConstraint::MmioAddress {
+            range: Some((0, MMIO_LOW_END)),
+            align: CAPABILITY_BAR_SIZE,
+            size: CAPABILITY_BAR_SIZE,
+        };
+
+        // Virtio pci device always use msi-x, extend irq resources to other_requests
+        let mut other_requests = vec![];
+        VirtioPciDevice::get_interrupt_requirements(device.as_ref(), &mut other_requests);
+
+        // allocate device resources by pci_bus, MmioAddress + KvmSlot?
+        let mut device_requests = vec![];
+        device.get_resource_requirements(&mut device_requests, use_generic_irq);
+
+        // Extend KvmSlot resources to other_requests
+        for req in device_requests.iter() {
+            if !matches!(
+                req,
+                ResourceConstraint::PioAddress { .. } | ResourceConstraint::MmioAddress { .. }
+            ) {
+                other_requests.push(*req);
+            }
+        }
+
+        // allocate PciMsixIrq and KvmSlot by res_manager
+        let other_resources = ctx
+            .res_manager
+            .allocate_device_resources(&other_requests, false)
+            .map_err(DeviceMgrError::ResourceError)?;
+
+        let pci_bus = pci_system_manager.pci_root_bus();
+        let dev_id = pci_system_manager
+            .new_device_id(None)
+            .ok_or(DeviceMgrError::VirtioPci)?;
+
+        // Allocate config bar resources by pci_bus
+        let default_config_req = vec![DEFAULE_VIRTIO_PCI_CONFIG_BAR];
+        let default_config_res = pci_bus
+            .allocate_resources(&default_config_req)
+            .map_err(DeviceMgrError::PciError)?;
+        assert!(default_config_res.get_all_resources().len() == 1);
+
+        // Allocate MmioAddress and PioAddress resource by pci bus, other resourece type will skip
+        let mut device_resource = pci_bus
+            .allocate_resources(&device_requests)
+            .map_err(DeviceMgrError::PciError)?;
+
+        // Extend PciMsixIrq and KvmSlot resources to device_resource
+        other_resources.get_all_resources().iter().for_each(|res| {
+            device_resource.append(res.clone());
+        });
+
+        // Do map for virtio share memory region by set_resource, this will use KvmSlot + MmioAddress resources, which should be allocated before
+        let _virito_shared_mem_list = device
+            .as_mut()
+            .set_resource(ctx.vm_fd.clone(), device_resource.clone())
+            .map_err(DeviceMgrError::Virtio)?;
+
+        // Extend config bar resources to device_resource
+        // Now device_resource contains all resources
+        default_config_res
+            .get_all_resources()
+            .iter()
+            .for_each(|res| {
+                device_resource.append(res.clone());
+            });
+
+        drop(pci_system_manager);
+
+        // new a virtio pci device
+        let mut virtio_dev = VirtioPciDevice::new(
+            ctx.vm_fd.clone(),
+            ctx.get_vm_as()?,
+            ctx.get_address_space()?,
+            ctx.irq_manager.clone(),
+            device_resource,
+            dev_id,
+            device,
+            true,
+            Arc::downgrade(&pci_bus),
+            default_config_res.get_all_resources()[0].clone(),
+        )
+        .map_err(DeviceMgrError::VirtioPciError)?;
+
+        virtio_dev
+            .alloc_bars()
+            .map_err(DeviceMgrError::VirtioPciError)?;
+
+        let arc_dev = Arc::new(virtio_dev);
+
+        pci_bus
+            .register_device(arc_dev.clone())
+            .map_err(DeviceMgrError::PciError)?;
+
+        Self::register_virtio_pci_device(arc_dev, ctx)
+    }
+
+    /// Create an Virtio PCI transport layer device for the virtio backend device.
+    pub fn register_virtio_pci_device(
+        device: Arc<dyn DeviceIo>,
+        ctx: &DeviceOpContext,
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
+        let resources = device.get_trapped_io_resources();
+        let mut tx = ctx.io_context.begin_tx();
+        if let Err(e) = ctx
+            .io_context
+            .register_device_io(&mut tx, device.clone(), &resources)
+        {
+            ctx.io_context.cancel_tx(tx);
+            Err(DeviceMgrError::IoManager(e))
+        } else {
+            ctx.io_context.commit_tx(tx);
+            Ok(device)
+        }
+    }
+
+    /// Deregister Virtio device from IoManager
+    pub fn deregister_virtio_device(
+        device: &Arc<dyn DeviceIo>,
+        ctx: &mut DeviceOpContext,
+    ) -> std::result::Result<(), DeviceMgrError> {
+        let resources = device.get_trapped_io_resources();
+        info!(
+            ctx.logger(),
+            "unregister pci virtio device: {:?}", resources
+        );
+        let mut tx = ctx.io_context.begin_tx();
+        if let Err(e) = ctx.io_context.unregister_device_io(&mut tx, &resources) {
+            ctx.io_context.cancel_tx(tx);
+            Err(DeviceMgrError::IoManager(e))
+        } else {
+            ctx.io_context.commit_tx(tx);
+            Ok(())
+        }
+    }
+
+    /// Destroy/Deregister resources for a Virtio PCI
+    fn destroy_pci_device(
+        device: Arc<dyn DeviceIo>,
+        ctx: &mut DeviceOpContext,
+        dev_id: u8,
+    ) -> std::result::Result<(), DeviceMgrError> {
+        // unregister IoManager
+        Self::deregister_virtio_device(&device, ctx)?;
+        // unregister Resource manager
+        let resources = device.get_assigned_resources();
+        let mut system_resources = DeviceResources::new();
+        resources.iter().for_each(|res| {
+            if !matches!(
+                res,
+                Resource::PioAddressRange { .. } | Resource::MmioAddressRange { .. }
+            ) {
+                system_resources.append(res.clone());
+            }
+        });
+        info!(
+            ctx.logger(),
+            "unregister resource {:?} from system resource manager for pci device",
+            system_resources
+        );
+        ctx.res_manager
+            .free_device_resources(&system_resources)
+            .map_err(DeviceMgrError::ResourceError)?;
+        let pci_system_manager = ctx.pci_system_manager.lock().unwrap();
+        let pci_bus = pci_system_manager.pci_root_bus();
+        info!(
+            ctx.logger(),
+            "unregister resource {:?} from pci bus resource manager for pci device", resources
+        );
+        pci_bus.free_resources(resources);
+        let _ = pci_system_manager.free_device_id(dev_id as u32);
+
+        let pci_dev = device
+            .as_any()
+            .downcast_ref::<VirtioPciDevice<GuestAddressSpaceImpl, QueueSync, GuestRegionMmap>>()
+            .ok_or(DeviceMgrError::InvalidOperation)?;
+
+        pci_dev.remove();
+
+        Ok(())
+    }
+
     #[cfg(feature = "host-device")]
     fn get_pci_device_info(device: &Arc<dyn DeviceIo>) -> Result<(u8, u8)> {
+        use virtio_queue::QueueSync;
+
         if let Some(pci_dev) = device
             .as_any()
             .downcast_ref::<VfioPciDevice<PciSystemManager>>()
@@ -1244,10 +1471,41 @@ impl DeviceManager {
             // together those 8 bits combined as devfn value
             let devfn = (((slot) & 0x1f) << 3) | ((func) & 0x07);
 
+            return Ok((busno, devfn));
+        } else if let Some(pci_dev) = device.as_any().downcast_ref::<VirtioPciDevice<
+            GuestAddressSpaceImpl,
+            QueueSync,
+            GuestRegionMmap,
+        >>() {
+            // reference from kernel: include/uapi/linux/pci.h
+            let busno = pci_dev.bus_id().map_err(DeviceMgrError::VirtioPciError)?;
+            let slot = pci_dev.device_id();
+            let func = 0;
+            let devfn = (((slot) & 0x1f) << 3) | ((func) & 0x07);
+
             return Ok((busno, devfn));
         }
 
-        Err(DeviceMgrError::GetDeviceResource)
+        Err(DeviceMgrError::InvalidPciDeviceType)
+    }
+
+    /// Teardown the Virtio PCI or MMIO transport layer device associated with the virtio backend device.
+    pub fn destroy_virtio_device(
+        device: Arc<dyn DeviceIo>,
+        ctx: &mut DeviceOpContext,
+    ) -> std::result::Result<(), DeviceMgrError> {
+        if let Some(mmio_dev) = device.as_any().downcast_ref::<DbsMmioV2Device>() {
+            Self::destroy_mmio_device(device.clone(), ctx)?;
+            mmio_dev.remove();
+        } else if let Some(pci_dev) = device.as_any().downcast_ref::<VirtioPciDevice<
+            GuestAddressSpaceImpl,
+            QueueSync,
+            GuestRegionMmap,
+        >>() {
+            Self::destroy_pci_device(device.clone(), ctx, pci_dev.device_id())?;
+        }
+
+        Ok(())
     }
 }
 
@@ -1312,12 +1570,29 @@ mod tests {
                 String::from("1"),
             )));
 
+            let irq_manager = Arc::new(KvmIrqManager::new(vm_fd.clone()));
+            let io_manager = Arc::new(ArcSwap::new(Arc::new(IoManager::new())));
+            let io_lock = Arc::new(Mutex::new(()));
+            let io_context = DeviceManagerContext::new(io_manager.clone(), io_lock.clone());
+            let mut mgr =
+                PciSystemManager::new(irq_manager.clone(), io_context, res_manager.clone())
+                    .unwrap();
+
+            let requirements = mgr.resource_requirements();
+            let resources = res_manager
+                .allocate_device_resources(&requirements, USE_SHARED_IRQ)
+                .map_err(DeviceMgrError::ResourceError)
+                .unwrap();
+            mgr.activate(resources).unwrap();
+
+            let pci_system_manager = Arc::new(Mutex::new(mgr));
+
             DeviceManager {
                 vm_fd: Arc::clone(&vm_fd),
                 con_manager: ConsoleManager::new(epoll_manager, &logger),
-                io_manager: Arc::new(ArcSwap::new(Arc::new(IoManager::new()))),
-                io_lock: Arc::new(Mutex::new(())),
-                irq_manager: Arc::new(KvmIrqManager::new(vm_fd.clone())),
+                io_manager,
+                io_lock,
+                irq_manager,
                 res_manager,
 
                 legacy_manager: None,
@@ -1340,7 +1615,12 @@ mod tests {
                 #[cfg(feature = "vhost-user-net")]
                 vhost_user_net_manager: VhostUserNetDeviceMgr::default(),
                 #[cfg(feature = "host-device")]
-                vfio_manager: Arc::new(Mutex::new(VfioDeviceMgr::new(vm_fd, &logger))),
+                vfio_manager: Arc::new(Mutex::new(VfioDeviceMgr::new(
+                    vm_fd,
+                    pci_system_manager.clone(),
+                    &logger,
+                ))),
+                pci_system_manager,
 
                 logger,
                 shared_info,

src/dragonball/src/device_manager/vfio_dev_mgr/mod.rs

@@ -17,12 +17,11 @@ use std::ops::Deref;
 use std::os::fd::RawFd;
 use std::path::Path;
 use std::sync::mpsc::Sender;
-use std::sync::{Arc, Weak};
+use std::sync::{Arc, Mutex, Weak};
 
 use dbs_device::resources::Resource::LegacyIrq;
 use dbs_device::resources::{DeviceResources, Resource, ResourceConstraint};
 use dbs_device::DeviceIo;
-use dbs_interrupt::KvmIrqManager;
 use dbs_pci::{VfioPciDevice, VENDOR_NVIDIA};
 use dbs_upcall::{DevMgrResponse, UpcallClientResponse};
 use kvm_ioctls::{DeviceFd, VmFd};
@@ -37,8 +36,8 @@ use vm_memory::{
 use super::StartMicroVmError;
 use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl};
 use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos};
-use crate::device_manager::{DeviceManagerContext, DeviceMgrError, DeviceOpContext};
-use crate::resource_manager::{ResourceError, ResourceManager};
+use crate::device_manager::{DeviceMgrError, DeviceOpContext};
+use crate::resource_manager::ResourceError;
 
 // The flag of whether to use the shared irq.
 const USE_SHARED_IRQ: bool = true;
@@ -230,7 +229,7 @@ pub struct VfioDeviceMgr {
     info_list: DeviceConfigInfos<HostDeviceConfig>,
     locked_vm_size: u64,
     vfio_container: Option<Arc<VfioContainer>>,
-    pci_vfio_manager: Option<Arc<PciSystemManager>>,
+    pci_system_manager: Arc<Mutex<PciSystemManager>>,
     pci_legacy_irqs: Option<HashMap<u8, u8>>,
     nvidia_shared_irq: Option<u32>,
     logger: slog::Logger,
@@ -238,13 +237,17 @@ pub struct VfioDeviceMgr {
 
 impl VfioDeviceMgr {
     /// Create a new VFIO device manager.
-    pub fn new(vm_fd: Arc<VmFd>, logger: &slog::Logger) -> Self {
+    pub fn new(
+        vm_fd: Arc<VmFd>,
+        pci_system_manager: Arc<Mutex<PciSystemManager>>,
+        logger: &slog::Logger,
+    ) -> Self {
         VfioDeviceMgr {
             vm_fd,
             info_list: DeviceConfigInfos::new(),
             locked_vm_size: 0,
             vfio_container: None,
-            pci_vfio_manager: None,
+            pci_system_manager,
             pci_legacy_irqs: Some(HashMap::new()),
             nvidia_shared_irq: None,
             logger: logger.new(slog::o!()),
@@ -288,17 +291,6 @@ impl VfioDeviceMgr {
         &mut self,
         ctx: &mut DeviceOpContext,
     ) -> std::result::Result<(), StartMicroVmError> {
-        // create and attach pci root bus
-        #[cfg(all(feature = "hotplug", feature = "host-device"))]
-        if ctx.pci_hotplug_enabled {
-            let _ = self
-                .create_pci_manager(
-                    ctx.irq_manager.clone(),
-                    ctx.io_context.clone(),
-                    ctx.res_manager.clone(),
-                )
-                .map_err(StartMicroVmError::CreateVfioDevice)?;
-        }
         for (idx, info) in self.info_list.clone().iter().enumerate() {
             self.create_device(&info.config, ctx, idx)
                 .map_err(StartMicroVmError::CreateVfioDevice)?;
@@ -574,12 +566,9 @@ impl VfioDeviceMgr {
             "subsystem" => "vfio_dev_mgr",
              "host_bdf" => &cfg.bus_slot_func,
         );
-        // safe to get pci_manager
-        let pci_manager = self.create_pci_manager(
-            ctx.irq_manager.clone(),
-            ctx.io_context.clone(),
-            ctx.res_manager.clone(),
-        )?;
+
+        let pci_manager = self.get_pci_manager();
+        let pci_manager = pci_manager.lock().unwrap();
         let pci_bus = pci_manager.pci_root_bus();
         let id = pci_manager
             .new_device_id(cfg.guest_dev_id)
@@ -607,7 +596,7 @@ impl VfioDeviceMgr {
                 sysfs_path,
                 Arc::downgrade(&pci_bus),
                 vfio_dev,
-                Arc::downgrade(self.get_pci_manager().unwrap()),
+                self.get_pci_manager(),
                 ctx.vm_fd.clone(),
                 cfg.vendor_device_id,
                 cfg.clique_id,
@@ -665,8 +654,8 @@ impl VfioDeviceMgr {
 
         // safe to unwrap because pci vfio manager is already created
         let _ = self
-            .pci_vfio_manager
-            .as_mut()
+            .pci_system_manager
+            .lock()
             .unwrap()
             .free_device_id(device_id)
             .ok_or(VfioDeviceError::InvalidDeviceID(device_id))?;
@@ -698,27 +687,9 @@ impl VfioDeviceMgr {
         Ok(())
     }
 
-    pub(crate) fn create_pci_manager(
-        &mut self,
-        irq_manager: Arc<KvmIrqManager>,
-        io_context: DeviceManagerContext,
-        res_manager: Arc<ResourceManager>,
-    ) -> Result<&mut Arc<PciSystemManager>> {
-        if self.pci_vfio_manager.is_none() {
-            let mut mgr = PciSystemManager::new(irq_manager, io_context, res_manager.clone())?;
-            let requirements = mgr.resource_requirements();
-            let resources = res_manager
-                .allocate_device_resources(&requirements, USE_SHARED_IRQ)
-                .or(Err(VfioDeviceError::NoResource))?;
-            mgr.activate(resources)?;
-            self.pci_vfio_manager = Some(Arc::new(mgr));
-        }
-        Ok(self.pci_vfio_manager.as_mut().unwrap())
-    }
-
     /// Get the PCI manager to support PCI device passthrough
-    pub fn get_pci_manager(&mut self) -> Option<&mut Arc<PciSystemManager>> {
-        self.pci_vfio_manager.as_mut()
+    pub fn get_pci_manager(&mut self) -> Arc<Mutex<PciSystemManager>> {
+        self.pci_system_manager.clone()
     }
 }
 

src/dragonball/src/device_manager/vfio_dev_mgr/pci_vfio.rs

@@ -12,14 +12,13 @@ use dbs_interrupt::KvmIrqManager;
 use dbs_pci::ECAM_SPACE_LENGTH;
 use dbs_pci::{create_pci_root_bus, PciBus, PciDevice, PciRootDevice, PciSystemContext};
 
-use super::{Result, VfioDeviceError};
+use super::DeviceMgrError;
 #[cfg(target_arch = "aarch64")]
 use crate::device_manager::vfio_dev_mgr::USE_SHARED_IRQ;
 use crate::device_manager::DeviceManagerContext;
 use crate::resource_manager::ResourceManager;
+use dbs_pci::PCI_BUS_DEFAULT;
 
-/// we only support one pci bus
-pub const PCI_BUS_DEFAULT: u8 = 0;
 /// The default mmio size for pci root bus.
 const PCI_MMIO_DEFAULT_SIZE: u64 = 2048u64 << 30;
 
@@ -38,13 +37,13 @@ impl PciSystemManager {
         irq_manager: Arc<KvmIrqManager>,
         io_context: DeviceManagerContext,
         res_manager: Arc<ResourceManager>,
-    ) -> std::result::Result<Self, VfioDeviceError> {
+    ) -> std::result::Result<Self, DeviceMgrError> {
         let resources = PciSystemManager::allocate_root_device_resources(res_manager)?;
         let pci_root = Arc::new(
-            PciRootDevice::create(PCI_BUS_DEFAULT, resources).map_err(VfioDeviceError::PciError)?,
+            PciRootDevice::create(PCI_BUS_DEFAULT, resources).map_err(DeviceMgrError::PciError)?,
         );
         let pci_root_bus =
-            create_pci_root_bus(PCI_BUS_DEFAULT).map_err(VfioDeviceError::PciError)?;
+            create_pci_root_bus(PCI_BUS_DEFAULT).map_err(DeviceMgrError::PciError)?;
 
         Ok(PciSystemManager {
             irq_manager,
@@ -58,7 +57,7 @@ impl PciSystemManager {
     #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
     fn allocate_root_device_resources(
         _res_manager: Arc<ResourceManager>,
-    ) -> Result<DeviceResources> {
+    ) -> std::result::Result<DeviceResources, DeviceMgrError> {
         let mut resources = DeviceResources::new();
         resources.append(Resource::PioAddressRange {
             // PCI CONFIG_ADDRESS port address 0xcf8 and uses 32 bits
@@ -76,7 +75,7 @@ impl PciSystemManager {
     #[cfg(target_arch = "aarch64")]
     fn allocate_root_device_resources(
         res_manager: Arc<ResourceManager>,
-    ) -> Result<DeviceResources> {
+    ) -> std::result::Result<DeviceResources, DeviceMgrError> {
         let requests = vec![ResourceConstraint::MmioAddress {
             range: Some((0x0, 0xffff_ffff)),
             align: 4096,
@@ -84,23 +83,26 @@ impl PciSystemManager {
         }];
         let resources = res_manager
             .allocate_device_resources(&requests, USE_SHARED_IRQ)
-            .map_err(VfioDeviceError::AllocateDeviceResource)?;
+            .map_err(DeviceMgrError::ResourceError)?;
         Ok(resources)
     }
 
     /// Activate the PCI subsystem.
-    pub fn activate(&mut self, resources: DeviceResources) -> Result<()> {
+    pub fn activate(
+        &mut self,
+        resources: DeviceResources,
+    ) -> std::result::Result<(), DeviceMgrError> {
         let bus_id = self.pci_root_bus.bus_id();
 
         self.pci_root
             .add_bus(self.pci_root_bus.clone(), bus_id)
-            .map_err(VfioDeviceError::PciError)?;
+            .map_err(DeviceMgrError::PciError)?;
         PciRootDevice::activate(self.pci_root.clone(), &mut self.io_context)
-            .map_err(VfioDeviceError::PciError)?;
+            .map_err(DeviceMgrError::PciError)?;
 
         self.pci_root_bus
             .assign_resources(resources)
-            .map_err(VfioDeviceError::PciError)?;
+            .map_err(DeviceMgrError::PciError)?;
 
         Ok(())
     }

src/dragonball/src/device_manager/vhost_net_dev_mgr.rs

@@ -256,7 +256,7 @@ impl VhostNetDeviceMgr {
                 info.config.iface_id
             );
             if let Some(device) = info.device.take() {
-                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+                DeviceManager::destroy_mmio_device(device, ctx)?;
             }
         }
 

src/dragonball/src/device_manager/virtio_net_dev_mgr.rs

@@ -382,7 +382,7 @@ impl VirtioNetDeviceMgr {
                 info.config.iface_id
             );
             if let Some(device) = info.device.take() {
-                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+                DeviceManager::destroy_mmio_device(device, ctx)?;
             }
         }
         Ok(())

src/dragonball/src/device_manager/vsock_dev_mgr.rs

@@ -294,7 +294,7 @@ impl VsockDeviceMgr {
                 info.config.id
             );
             if let Some(device) = info.device.take() {
-                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+                DeviceManager::destroy_mmio_device(device, ctx)?;
             }
         }
         Ok(())

src/dragonball/src/error.rs

@@ -77,6 +77,10 @@ pub enum Error {
     /// Cannot open the VM file descriptor.
     #[error(transparent)]
     Vm(vm::VmError),
+
+    /// Fail to create device manager system
+    #[error("failed to create device manager system: {0}")]
+    DeviceMgrError(#[source] device_manager::DeviceMgrError),
 }
 
 /// Errors associated with starting the instance.

src/dragonball/src/vm/mod.rs

@@ -35,7 +35,7 @@ use crate::address_space_manager::{
 use crate::api::v1::{InstanceInfo, InstanceState};
 use crate::device_manager::console_manager::DmesgWriter;
 use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext};
-use crate::error::{LoadInitrdError, Result, StartMicroVmError, StopMicrovmError};
+use crate::error::{Error, LoadInitrdError, Result, StartMicroVmError, StopMicrovmError};
 use crate::event_manager::EventManager;
 use crate::kvm_context::KvmContext;
 use crate::resource_manager::ResourceManager;
@@ -228,7 +228,8 @@ impl Vm {
             epoll_manager.clone(),
             &logger,
             api_shared_info.clone(),
-        );
+        )
+        .map_err(Error::DeviceMgrError)?;
 
         Ok(Vm {
             epoll_manager,

src/libs/kata-sys-util/src/hooks.rs

@@ -375,7 +375,11 @@ mod tests {
         fn build_oci_hook(self) -> oci::Hook {
             let mut hook = oci::Hook::default();
             hook.set_path(PathBuf::from(self.path));
-            hook.set_args(Some(self.args));
+            if self.args.is_empty() {
+                hook.set_args(None);
+            } else {
+                hook.set_args(Some(self.args));
+            }
             hook.set_env(Some(self.env));
             hook.set_timeout(self.timeout);
 

src/libs/kata-sys-util/src/k8s.rs

@@ -11,7 +11,6 @@
 
 use kata_types::mount;
 use oci_spec::runtime::{Mount, Spec};
-use std::path::Path;
 
 use crate::mount::get_linux_mount_info;
 
@@ -34,10 +33,8 @@ pub fn is_ephemeral_volume(mount: &Mount) -> bool {
             mount.destination(),
 
         ),
-        (Some("bind"), Some(source), dest) if get_linux_mount_info(source)
-            .map_or(false, |info| info.fs_type == "tmpfs") &&
-            (is_empty_dir(source) || dest.as_path() == Path::new("/dev/shm"))
-    )
+        (Some("bind"), Some(source), _dest) if get_linux_mount_info(source).is_ok_and(|info| info.fs_type == "tmpfs") &&
+            is_empty_dir(source))
 }
 
 /// Check whether the given path is a kubernetes empty-dir volume of medium "default".

src/libs/kata-sys-util/src/mount.rs

@@ -823,11 +823,11 @@ mod tests {
 
     #[test]
     fn test_get_linux_mount_info() {
-        let info = get_linux_mount_info("/sys/fs/cgroup").unwrap();
+        let info = get_linux_mount_info("/dev/shm").unwrap();
 
         assert_eq!(&info.device, "tmpfs");
         assert_eq!(&info.fs_type, "tmpfs");
-        assert_eq!(&info.path, "/sys/fs/cgroup");
+        assert_eq!(&info.path, "/dev/shm");
 
         assert!(matches!(
             get_linux_mount_info(""),

src/libs/kata-sys-util/src/numa.rs

@@ -168,7 +168,7 @@ pub fn is_valid_numa_cpu(cpus: &[u32]) -> Result<bool> {
     let numa_nodes = get_numa_nodes()?;
 
     for cpu in cpus {
-        if numa_nodes.get(cpu).is_none() {
+        if !numa_nodes.contains_key(cpu) {
             return Ok(false);
         }
     }

src/libs/kata-sys-util/src/pcilibs/devices.rs

@@ -66,7 +66,7 @@ impl PCIDevices for NvidiaPCIDevice {
             }
         }
 
-        return nvidia_devices;
+        nvidia_devices
     }
 }
 

src/libs/kata-sys-util/src/pcilibs/pci_manager.rs

@@ -7,7 +7,7 @@
 use std::collections::HashMap;
 use std::fs;
 use std::io;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use mockall::automock;
 use pci_ids::{Classes, Vendors};
@@ -61,24 +61,22 @@ pub(crate) trait MemoryResourceTrait {
 
 impl MemoryResourceTrait for MemoryResources {
     fn get_total_addressable_memory(&self, round_up: bool) -> (u64, u64) {
-        let mut num_bar = 0;
         let mut mem_size_32bit = 0u64;
         let mut mem_size_64bit = 0u64;
 
         let mut keys: Vec<_> = self.keys().cloned().collect();
         keys.sort();
 
-        for key in keys {
-            if key as usize >= PCI_IOV_NUM_BAR || num_bar == PCI_IOV_NUM_BAR {
+        for (num_bar, key) in keys.into_iter().enumerate() {
+            if key >= PCI_IOV_NUM_BAR || num_bar == PCI_IOV_NUM_BAR {
                 break;
             }
-            num_bar += 1;
 
             if let Some(region) = self.get(&key) {
                 let flags = region.flags & PCI_BASE_ADDRESS_MEM_TYPE_MASK;
                 let mem_type_32bit = flags == PCI_BASE_ADDRESS_MEM_TYPE32;
                 let mem_type_64bit = flags == PCI_BASE_ADDRESS_MEM_TYPE64;
-                let mem_size = (region.end - region.start + 1) as u64;
+                let mem_size = region.end - region.start + 1;
 
                 if mem_type_32bit {
                     mem_size_32bit += mem_size;
@@ -138,10 +136,10 @@ impl PCIDeviceManager {
         for entry in device_dirs {
             let device_dir = entry?;
             let device_address = device_dir.file_name().to_string_lossy().to_string();
-            if let Ok(device) = self.get_device_by_pci_bus_id(&device_address, vendor, &mut cache) {
-                if let Some(dev) = device {
-                    pci_devices.push(dev);
-                }
+            if let Ok(Some(dev)) =
+                self.get_device_by_pci_bus_id(&device_address, vendor, &mut cache)
+            {
+                pci_devices.push(dev);
             }
         }
 
@@ -238,7 +236,7 @@ impl PCIDeviceManager {
         Ok(Some(pci_device))
     }
 
-    fn parse_resources(&self, device_path: &PathBuf) -> io::Result<MemoryResources> {
+    fn parse_resources(&self, device_path: &Path) -> io::Result<MemoryResources> {
         let content = fs::read_to_string(device_path.join("resource"))?;
         let mut resources: MemoryResources = MemoryResources::new();
         for (i, line) in content.lines().enumerate() {
@@ -405,6 +403,8 @@ mod tests {
 
     #[test]
     fn test_parse_resources() {
+        setup_mock_device_files();
+
         let manager = PCIDeviceManager::new(MOCK_PCI_DEVICES_ROOT);
         let device_path = PathBuf::from(MOCK_PCI_DEVICES_ROOT).join("0000:ff:1f.0");
 
@@ -418,6 +418,8 @@ mod tests {
         assert_eq!(resource.start, 0x00000000);
         assert_eq!(resource.end, 0x0000ffff);
         assert_eq!(resource.flags, 0x00000404);
+
+        cleanup_mock_device_files();
     }
 
     #[test]
@@ -435,10 +437,7 @@ mod tests {
         file.write_all(&vec![0; 512]).unwrap();
 
         // It should be true
-        assert!(is_pcie_device(
-            &format!("ff:00.0"),
-            MOCK_SYS_BUS_PCI_DEVICES
-        ));
+        assert!(is_pcie_device("ff:00.0", MOCK_SYS_BUS_PCI_DEVICES));
 
         // Clean up
         let _ = fs::remove_file(config_path);

src/libs/kata-sys-util/src/protection.rs

@@ -142,14 +142,11 @@ pub fn arch_guest_protection(
 #[allow(dead_code)]
 pub fn available_guest_protection() -> Result<GuestProtection, ProtectionError> {
     if !Uid::effective().is_root() {
-        return Err(ProtectionError::NoPerms)?;
+        Err(ProtectionError::NoPerms)?;
     }
 
     let facilities = crate::cpu::retrieve_cpu_facilities().map_err(|err| {
-        ProtectionError::CheckFailed(format!(
-            "Error retrieving cpu facilities file : {}",
-            err.to_string()
-        ))
+        ProtectionError::CheckFailed(format!("Error retrieving cpu facilities file : {}", err))
     })?;
 
     // Secure Execution
@@ -219,6 +216,11 @@ mod tests {
         // Test snp
         let dir = tempdir().unwrap();
         let snp_file_path = dir.path().join("sev_snp");
+        if !snp_file_path.exists() {
+            println!("INFO: skipping {} which needs sev_snp", module_path!());
+            return;
+        }
+
         let path = snp_file_path.clone();
         let mut snp_file = fs::File::create(snp_file_path).unwrap();
         writeln!(snp_file, "Y").unwrap();
@@ -238,6 +240,11 @@ mod tests {
         // Test sev
         let dir = tempdir().unwrap();
         let sev_file_path = dir.path().join("sev");
+        if !sev_file_path.exists() {
+            println!("INFO: skipping {} which needs sev", module_path!());
+            return;
+        }
+
         let sev_path = sev_file_path.clone();
         let mut sev_file = fs::File::create(sev_file_path).unwrap();
         writeln!(sev_file, "Y").unwrap();
@@ -260,6 +267,11 @@ mod tests {
         let invalid_dir = invalid_dir.to_str().unwrap();
 
         let tdx_file_path = dir.path().join("tdx");
+        if !tdx_file_path.exists() {
+            println!("INFO: skipping {} which needs tdx", module_path!());
+            return;
+        }
+
         let tdx_path = tdx_file_path;
 
         std::fs::create_dir_all(tdx_path.clone()).unwrap();

src/libs/kata-types/src/annotations/mod.rs

@@ -8,7 +8,6 @@ use std::collections::HashMap;
 use std::fs::File;
 use std::io::{self, BufReader, Result};
 use std::result::{self};
-use std::u32;
 
 use serde::Deserialize;
 
@@ -273,7 +272,8 @@ pub const KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS: &str =
 /// A sandbox annotation to specify as the msize for 9p shares.
 pub const KATA_ANNO_CFG_HYPERVISOR_MSIZE_9P: &str = "io.katacontainers.config.hypervisor.msize_9p";
 /// The initdata annotation passed in when CVM launchs
-pub const KATA_ANNO_CFG_RUNTIME_INIT_DATA: &str = "io.katacontainers.config.runtime.cc_init_data";
+pub const KATA_ANNO_CFG_HYPERVISOR_INIT_DATA: &str =
+    "io.katacontainers.config.hypervisor.cc_init_data";
 
 /// GPU specific annotations for remote hypervisor to help with instance selection
 /// It's for minimum number of GPUs required for the VM.
@@ -462,12 +462,12 @@ impl Annotation {
     /// update config info by annotation
     pub fn update_config_by_annotation(&self, config: &mut TomlConfig) -> Result<()> {
         if let Some(hv) = self.annotations.get(KATA_ANNO_CFG_RUNTIME_HYPERVISOR) {
-            if config.hypervisor.get(hv).is_some() {
+            if config.hypervisor.contains_key(hv) {
                 config.runtime.hypervisor_name = hv.to_string();
             }
         }
         if let Some(ag) = self.annotations.get(KATA_ANNO_CFG_RUNTIME_AGENT) {
-            if config.agent.get(ag).is_some() {
+            if config.agent.contains_key(ag) {
                 config.runtime.agent_name = ag.to_string();
             }
         }
@@ -894,7 +894,7 @@ impl Annotation {
                         hv.security_info.validate_path(value)?;
                         hv.security_info.guest_hook_path = value.to_string();
                     }
-                    KATA_ANNO_CFG_RUNTIME_INIT_DATA => {
+                    KATA_ANNO_CFG_HYPERVISOR_INIT_DATA => {
                         hv.security_info.initdata =
                             add_hypervisor_initdata_overrides(value).unwrap();
                     }
@@ -943,8 +943,7 @@ impl Annotation {
                         }
                     }
                     KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS => {
-                        let args: Vec<String> =
-                            value.to_string().split(',').map(str::to_string).collect();
+                        let args: Vec<String> = value.split(',').map(str::to_string).collect();
                         for arg in args {
                             hv.shared_fs.virtio_fs_extra_args.push(arg.to_string());
                         }
@@ -970,7 +969,7 @@ impl Annotation {
                     // update agent config
                     KATA_ANNO_CFG_KERNEL_MODULES => {
                         let kernel_mod: Vec<String> =
-                            value.to_string().split(';').map(str::to_string).collect();
+                            value.split(';').map(str::to_string).collect();
                         for modules in kernel_mod {
                             ag.kernel_modules.push(modules.to_string());
                         }
@@ -991,14 +990,16 @@ impl Annotation {
                             return Err(u32_err);
                         }
                     },
-                    KATA_ANNO_CFG_RUNTIME_CREATE_CONTAINTER_TIMEOUT => match self.get_value::<u32>(key) {
-                        Ok(v) => {
-                            ag.request_timeout_ms = v.unwrap_or_default() * 1000;
+                    KATA_ANNO_CFG_RUNTIME_CREATE_CONTAINTER_TIMEOUT => {
+                        match self.get_value::<u32>(key) {
+                            Ok(v) => {
+                                ag.request_timeout_ms = v.unwrap_or_default() * 1000;
+                            }
+                            Err(_e) => {
+                                return Err(u32_err);
+                            }
                         }
-                        Err(_e) => {
-                            return Err(u32_err);
-                        }
-                    },
+                    }
                     // update runtime config
                     KATA_ANNO_CFG_RUNTIME_NAME => {
                         let runtime = vec!["virt-container", "linux-container", "wasm-container"];
@@ -1031,8 +1032,7 @@ impl Annotation {
                         }
                     },
                     KATA_ANNO_CFG_EXPERIMENTAL => {
-                        let args: Vec<String> =
-                            value.to_string().split(',').map(str::to_string).collect();
+                        let args: Vec<String> = value.split(',').map(str::to_string).collect();
                         for arg in args {
                             config.runtime.experimental.push(arg.to_string());
                         }

src/libs/kata-types/src/config/agent.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use serde::{Deserialize, Deserializer};
 use std::io::Result;
 
 use crate::config::{ConfigOps, TomlConfig};
@@ -115,7 +116,11 @@ pub struct Agent {
     /// This timeout value is used to set the maximum duration for the agent to process a CreateContainerRequest.
     /// It's also used to ensure that workloads, especially those involving large image pulls within the guest,
     /// have sufficient time to complete.
-    #[serde(default = "default_request_timeout", rename = "create_container_timeout")]
+    #[serde(
+        default = "default_request_timeout",
+        rename = "create_container_timeout",
+        deserialize_with = "deserialize_secs_to_millis"
+    )]
     pub request_timeout_ms: u32,
 
     /// Agent health check request timeout value in millisecond
@@ -127,12 +132,12 @@ pub struct Agent {
     /// These modules will be loaded in the guest kernel using modprobe(8).
     /// The following example can be used to load two kernel modules with parameters:
     ///  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
-    /// The first word is considered as the module name and the rest as its parameters.
-    /// Container will not be started when:
+    ///    The first word is considered as the module name and the rest as its parameters.
+    ///    Container will not be started when:
     /// - A kernel module is specified and the modprobe command is not installed in the guest
     ///   or it fails loading the module.
     /// - The module is not available in the guest or it doesn't met the guest kernel
-    ///    requirements, like architecture and version.
+    ///   requirements, like architecture and version.
     #[serde(default)]
     pub kernel_modules: Vec<String>,
 
@@ -145,6 +150,15 @@ pub struct Agent {
     pub mem_agent: MemAgent,
 }
 
+fn deserialize_secs_to_millis<'de, D>(deserializer: D) -> std::result::Result<u32, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let secs = u32::deserialize(deserializer)?;
+
+    Ok(secs.saturating_mul(1000))
+}
+
 impl std::default::Default for Agent {
     fn default() -> Self {
         Self {

src/libs/kata-types/src/config/hypervisor/dragonball.rs

@@ -6,7 +6,6 @@
 use std::io::Result;
 use std::path::Path;
 use std::sync::Arc;
-use std::u32;
 
 use super::{default, register_hypervisor_plugin};
 use crate::config::default::MAX_DRAGONBALL_VCPUS;