release: Bump version to 3.21.0

Bump VERSION and helm-chart versions

Signed-off-by: stevenhorsman <steven@uk.ibm.com>

.github/actionlint.yaml

@@ -23,3 +23,4 @@ self-hosted-runner:
     - s390x
     - s390x-large
     - tdx
+    - amd64-nvidia-a100

.github/dependabot.yml

@@ -66,6 +66,9 @@ updates:
       rustix:
         patterns:
           - rustix
+      slab:
+        patterns:
+          - slab
       time:
         patterns:
           - time

.github/workflows/PR-wip-checks.yaml

@@ -9,8 +9,7 @@ on:
       - labeled
       - unlabeled
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/actionlint.yaml

@@ -11,8 +11,8 @@ on:
     paths:
       - '.github/workflows/**'
 
-permissions:
-  contents: read
+permissions: {}
+
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/basic-ci-amd64.yaml

@@ -13,8 +13,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-containerd-sandboxapi:
@@ -49,6 +48,8 @@ jobs:
 
       - name: Install dependencies
         run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -89,6 +90,8 @@ jobs:
 
       - name: Install dependencies
         run: bash tests/stability/gha-run.sh install-dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -132,6 +135,8 @@ jobs:
 
       - name: Install dependencies
         run: bash tests/integration/nydus/gha-run.sh install-dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -209,6 +214,8 @@ jobs:
 
       - name: Install dependencies
         run: bash tests/functional/tracing/gha-run.sh install-dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -253,6 +260,8 @@ jobs:
 
       - name: Install dependencies
         run: bash tests/functional/vfio/gha-run.sh install-dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -272,10 +281,7 @@ jobs:
       fail-fast: false
       matrix:
         vmm:
-          - clh
           - qemu
-          - dragonball
-          - cloud-hypervisor
     runs-on: ubuntu-22.04
     env:
       KATA_HYPERVISOR: ${{ matrix.vmm }}
@@ -294,6 +300,8 @@ jobs:
 
       - name: Install dependencies
         run: bash tests/integration/docker/gha-run.sh install-dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -339,6 +347,7 @@ jobs:
       - name: Install dependencies
         env:
           GITHUB_API_TOKEN: ${{ github.token }}
+          GH_TOKEN: ${{ github.token }}
         run: bash tests/integration/nerdctl/gha-run.sh install-dependencies
 
       - name: get-kata-tarball
@@ -383,6 +392,8 @@ jobs:
 
       - name: Install dependencies
         run: bash tests/functional/kata-agent-apis/gha-run.sh install-dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0

.github/workflows/basic-ci-s390x.yaml

@@ -13,8 +13,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-containerd-sandboxapi:
@@ -48,7 +47,9 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Install dependencies
-        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+        run: bash tests/integration/cri-containerd/gha-run.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0

.github/workflows/build-checks-preview-riscv64.yaml

@@ -12,8 +12,7 @@ on:
         required: true
         type: string
 
-permissions:
-  contents: read
+permissions: {}
 
 name: Build checks preview riscv64
 jobs:

.github/workflows/build-checks.yaml

@@ -5,8 +5,8 @@ on:
         required: true
         type: string
 
-permissions:
-  contents: read
+permissions: {}
+
 
 name: Build checks
 jobs:
@@ -42,6 +42,10 @@ jobs:
             path: src/runtime-rs
             needs:
               - rust
+          - name: libs
+            path: src/libs
+            needs:
+              - rust
           - name: agent-ctl
             path: src/tools/agent-ctl
             needs:

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -23,9 +23,10 @@ on:
     secrets:
       QUAY_DEPLOYER_PASSWORD:
         required: false
+      KBUILD_SIGN_PIN:
+        required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-asset:
@@ -95,6 +96,7 @@ jobs:
       - name: Build ${{ matrix.asset }}
         id: build
         run: |
+          [[ "${KATA_ASSET}" == *"nvidia"* ]] && echo "KBUILD_SIGN_PIN=${{ secrets.KBUILD_SIGN_PIN }}" >> "${GITHUB_ENV}"
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
@@ -141,7 +143,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -150,7 +152,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}-headers${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}-headers.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}-headers.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -201,6 +203,7 @@ jobs:
       - name: Build ${{ matrix.asset }}
         id: build
         run: |
+          [[ "${KATA_ASSET}" == *"nvidia"* ]] && echo "KBUILD_SIGN_PIN=${{ secrets.KBUILD_SIGN_PIN }}" >> "${GITHUB_ENV}"
           ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
@@ -220,7 +223,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -312,7 +315,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-amd64-shim-v2${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-shim-v2.tar.xz
+          path: kata-build/kata-static-shim-v2.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -343,10 +346,12 @@ jobs:
       - name: merge-artifacts
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+        env:
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
       - name: store-artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
+          path: kata-static.tar.zst
           retention-days: 15
           if-no-files-found: error

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -24,8 +24,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: false
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-asset:
@@ -121,7 +120,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -130,7 +129,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}-headers${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}-headers.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}-headers.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -195,7 +194,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -282,7 +281,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-arm64-shim-v2${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-shim-v2.tar.xz
+          path: kata-build/kata-static-shim-v2.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -313,10 +312,12 @@ jobs:
       - name: merge-artifacts
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+        env:
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
       - name: store-artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
+          path: kata-static.tar.zst
           retention-days: 15
           if-no-files-found: error

.github/workflows/build-kata-static-tarball-ppc64le.yaml

@@ -24,8 +24,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-asset:
@@ -83,7 +82,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 1
           if-no-files-found: error
 
@@ -148,7 +147,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 1
           if-no-files-found: error
 
@@ -221,7 +220,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-ppc64le-shim-v2${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-shim-v2.tar.xz
+          path: kata-build/kata-static-shim-v2.tar.zst
           retention-days: 1
           if-no-files-found: error
 
@@ -256,10 +255,12 @@ jobs:
       - name: merge-artifacts
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+        env:
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
       - name: store-artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-static-tarball-ppc64le${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
+          path: kata-static.tar.zst
           retention-days: 1
           if-no-files-found: error

.github/workflows/build-kata-static-tarball-riscv64.yaml

@@ -24,8 +24,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-asset:
@@ -81,6 +80,6 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-riscv64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -27,8 +27,7 @@ on:
         required: true
 
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-asset:
@@ -115,7 +114,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -182,7 +181,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -230,7 +229,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-boot-image-se.tar.xz
+          path: kata-build/kata-static-boot-image-se.tar.zst
           retention-days: 1
           if-no-files-found: error
 
@@ -307,7 +306,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-s390x-shim-v2${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-shim-v2.tar.xz
+          path: kata-build/kata-static-shim-v2.tar.zst
           retention-days: 15
           if-no-files-found: error
 
@@ -342,10 +341,12 @@ jobs:
       - name: merge-artifacts
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+        env:
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
       - name: store-artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
+          path: kata-static.tar.zst
           retention-days: 15
           if-no-files-found: error

.github/workflows/cargo-deny-runner.yaml

@@ -11,8 +11,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   cargo-deny-runner:

.github/workflows/ci-coco-stability.yaml

@@ -9,8 +9,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   kata-containers-ci-on-push:
@@ -31,3 +30,4 @@ jobs:
       AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
       AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+      KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }}

.github/workflows/ci-devel.yaml

@@ -2,17 +2,15 @@ name: Kata Containers CI (manually triggered)
 on:
   workflow_dispatch:
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   kata-containers-ci-on-push:
     permissions:
-      actions: write
-      attestations: write
       contents: read
-      id-token: write
       packages: write
+      id-token: write
+      attestations: write
     uses: ./.github/workflows/ci.yaml
     with:
       commit-hash: ${{ github.sha }}
@@ -28,6 +26,8 @@ jobs:
       CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
       ITA_KEY: ${{ secrets.ITA_KEY }}
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+      NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
+      KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }}
 
   build-checks:
     uses: ./.github/workflows/build-checks.yaml

.github/workflows/ci-nightly-s390x.yaml

@@ -4,8 +4,7 @@ on:
 
 name: Nightly CI for s390x
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   check-internal-test-result:
@@ -16,7 +15,8 @@ jobs:
         test_title:
           - kata-vfio-ap-e2e-tests
           - cc-vfio-ap-e2e-tests
-          - cc-se-e2e-tests
+          - cc-se-e2e-tests-go
+          - cc-se-e2e-tests-rs
     steps:
     - name: Fetch a test result for {{ matrix.test_title }}
       run: |

.github/workflows/ci-nightly.yaml

@@ -7,17 +7,15 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   kata-containers-ci-on-push:
     permissions:
-      actions: write
-      attestations: write
       contents: read
-      id-token: write
       packages: write
+      id-token: write
+      attestations: write
     uses: ./.github/workflows/ci.yaml
     with:
       commit-hash: ${{ github.sha }}
@@ -32,3 +30,5 @@ jobs:
       CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
       ITA_KEY: ${{ secrets.ITA_KEY }}
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+      NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
+      KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }}

.github/workflows/ci-on-push.yaml

@@ -3,7 +3,6 @@ on:
   pull_request_target:
     branches:
       - 'main'
-      - 'stable-*'
     types:
       # Adding 'labeled' to the list of activity types that trigger this event
       # (default: opened, synchronize, reopened) so that we can run this
@@ -14,8 +13,7 @@ on:
       - reopened
       - labeled
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -33,11 +31,10 @@ jobs:
     needs: skipper
     if: ${{ needs.skipper.outputs.skip_build != 'yes' }}
     permissions:
-      actions: write
-      attestations: write
       contents: read
-      id-token: write
       packages: write
+      id-token: write
+      attestations: write
     uses: ./.github/workflows/ci.yaml
     with:
       commit-hash: ${{ github.event.pull_request.head.sha }}
@@ -53,3 +50,5 @@ jobs:
       CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
       ITA_KEY: ${{ secrets.ITA_KEY }}
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+      NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
+      KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }}

.github/workflows/ci-weekly.yaml

@@ -27,9 +27,10 @@ on:
         required: true
       QUAY_DEPLOYER_PASSWORD:
         required: true
+      KBUILD_SIGN_PIN:
+        required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-kata-static-tarball-amd64:
@@ -43,6 +44,8 @@ jobs:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
       target-branch: ${{ inputs.target-branch }}
+    secrets:
+      KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }}
 
   publish-kata-deploy-payload-amd64:
     needs: build-kata-static-tarball-amd64

.github/workflows/ci.yaml

@@ -35,10 +35,12 @@ on:
         required: true
       QUAY_DEPLOYER_PASSWORD:
         required: true
+      NGC_API_KEY:
+        required: true
+      KBUILD_SIGN_PIN:
+        required: true
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   build-kata-static-tarball-amd64:
@@ -52,6 +54,8 @@ jobs:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
       target-branch: ${{ inputs.target-branch }}
+    secrets:
+      KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }}
 
   publish-kata-deploy-payload-amd64:
     needs: build-kata-static-tarball-amd64
@@ -71,112 +75,112 @@ jobs:
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-  # build-kata-static-tarball-arm64:
-  #   permissions:
-  #     contents: read
-  #     packages: write
-  #     id-token: write
-  #     attestations: write
-  #   uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
+  build-kata-static-tarball-arm64:
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
 
-  # publish-kata-deploy-payload-arm64:
-  #   needs: build-kata-static-tarball-arm64
-  #   permissions:
-  #     contents: read
-  #     packages: write
-  #   uses: ./.github/workflows/publish-kata-deploy-payload.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     registry: ghcr.io
-  #     repo: ${{ github.repository_owner }}/kata-deploy-ci
-  #     tag: ${{ inputs.tag }}-arm64
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #     runner: ubuntu-22.04-arm
-  #     arch: arm64
-  #   secrets:
-  #     QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+  publish-kata-deploy-payload-arm64:
+    needs: build-kata-static-tarball-arm64
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/publish-kata-deploy-payload.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-arm64
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+      runner: ubuntu-22.04-arm
+      arch: arm64
+    secrets:
+      QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-  # build-kata-static-tarball-s390x:
-  #   permissions:
-  #     contents: read
-  #     packages: write
-  #     id-token: write
-  #     attestations: write
-  #   uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #   secrets:
-  #     CI_HKD_PATH: ${{ secrets.ci_hkd_path }}
-  #     QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+  build-kata-static-tarball-s390x:
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets:
+      CI_HKD_PATH: ${{ secrets.ci_hkd_path }}
+      QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-  # build-kata-static-tarball-ppc64le:
-  #   permissions:
-  #     contents: read
-  #     packages: write
-  #   uses: ./.github/workflows/build-kata-static-tarball-ppc64le.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #   secrets:
-  #     QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+  build-kata-static-tarball-ppc64le:
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/build-kata-static-tarball-ppc64le.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets:
+      QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-  # build-kata-static-tarball-riscv64:
-  #   permissions:
-  #     contents: read
-  #     packages: write
-  #     id-token: write
-  #     attestations: write
-  #   uses: ./.github/workflows/build-kata-static-tarball-riscv64.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #   secrets:
-  #     QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+  build-kata-static-tarball-riscv64:
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
+    uses: ./.github/workflows/build-kata-static-tarball-riscv64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets:
+      QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-  # publish-kata-deploy-payload-s390x:
-  #   needs: build-kata-static-tarball-s390x
-  #   permissions:
-  #     contents: read
-  #     packages: write
-  #   uses: ./.github/workflows/publish-kata-deploy-payload.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     registry: ghcr.io
-  #     repo: ${{ github.repository_owner }}/kata-deploy-ci
-  #     tag: ${{ inputs.tag }}-s390x
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #     runner: s390x
-  #     arch: s390x
-  #   secrets:
-  #     QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+  publish-kata-deploy-payload-s390x:
+    needs: build-kata-static-tarball-s390x
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/publish-kata-deploy-payload.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-s390x
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+      runner: s390x
+      arch: s390x
+    secrets:
+      QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-  # publish-kata-deploy-payload-ppc64le:
-  #   needs: build-kata-static-tarball-ppc64le
-  #   permissions:
-  #     contents: read
-  #     packages: write
-  #   uses: ./.github/workflows/publish-kata-deploy-payload.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     registry: ghcr.io
-  #     repo: ${{ github.repository_owner }}/kata-deploy-ci
-  #     tag: ${{ inputs.tag }}-ppc64le
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #     runner: ppc64le
-  #     arch: ppc64le
-  #   secrets:
-  #     QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+  publish-kata-deploy-payload-ppc64le:
+    needs: build-kata-static-tarball-ppc64le
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/publish-kata-deploy-payload.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-ppc64le
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+      runner: ppc64le
+      arch: ppc64le
+    secrets:
+      QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
   build-and-publish-tee-confidential-unencrypted-image:
     permissions:
@@ -275,61 +279,33 @@ jobs:
 
   run-kata-monitor-tests:
     if: ${{ inputs.skip-test != 'yes' }}
-    runs-on: ubuntu-22.04
     needs: build-kata-static-tarball-amd64
-    env:
-      GH_TOKEN: ${{ github.token }}
-    permissions:
-      actions: write # Permission to trigger the gh workflows
-    steps:
-      - name: Checkout the code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-      - name: Trigger kata monitor tests
-        run: |
-          ./tests/gha-helper.sh trigger-and-check-workflow \
-            run-kata-monitor-tests.yaml \
-            ${{ github.ref }} \
-            ${{ github.sha }} \
-            '{
-              "artifact-run-id":"${{ github.run_id }}",
-              "tarball-suffix":"-${{ inputs.tag }}",
-              "commit-hash":"${{ inputs.commit-hash }}",
-              "target-branch":"${{ inputs.target-branch }}"
-            }'
+    uses: ./.github/workflows/run-kata-monitor-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
 
   run-k8s-tests-on-aks:
     if: ${{ inputs.skip-test != 'yes' }}
-    runs-on: ubuntu-22.04
     needs: publish-kata-deploy-payload-amd64
-    env:
-      GH_TOKEN: ${{ github.token }}
+    uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
+
     permissions:
-      actions: write # Permission to trigger the gh workflows
-    steps:
-      - name: Checkout the code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-      - name: Trigger run-k8s-tests-on-aks workflow
-        run: |
-          ./tests/gha-helper.sh trigger-and-check-workflow \
-            run-k8s-tests-on-aks.yaml \
-            ${{ github.ref }} \
-            ${{ github.sha }} \
-            '{
-              "artifact-run-id":"${{ github.run_id }}",
-              "tarball-suffix":"-${{ inputs.tag }}",
-              "registry":"ghcr.io",
-              "repo":"${{ github.repository_owner }}/kata-deploy-ci",
-              "tag":"${{ inputs.tag }}-amd64",
-              "commit-hash":"${{ inputs.commit-hash }}",
-              "pr-number":"${{ inputs.pr-number }}",
-              "target-branch":"${{ inputs.target-branch }}"
-            }'
+      contents: read
+      id-token: write # Used for OIDC access to log into Azure
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets:
+      AZ_APPID: ${{ secrets.AZ_APPID }}
+      AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+      AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
 
   run-k8s-tests-on-amd64:
     if: ${{ inputs.skip-test != 'yes' }}
@@ -343,206 +319,198 @@ jobs:
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
 
-  # run-k8s-tests-on-arm64:
-  #   if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: publish-kata-deploy-payload-arm64
-  #   uses: ./.github/workflows/run-k8s-tests-on-arm64.yaml
-  #   with:
-  #     registry: ghcr.io
-  #     repo: ${{ github.repository_owner }}/kata-deploy-ci
-  #     tag: ${{ inputs.tag }}-arm64
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     pr-number: ${{ inputs.pr-number }}
-  #     target-branch: ${{ inputs.target-branch }}
+  run-k8s-tests-on-arm64:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: publish-kata-deploy-payload-arm64
+    uses: ./.github/workflows/run-k8s-tests-on-arm64.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-arm64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-k8s-tests-on-nvidia-gpu:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets:
+      NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
+
 
   run-kata-coco-tests:
     if: ${{ inputs.skip-test != 'yes' }}
-    runs-on: ubuntu-22.04
     needs:
-    - publish-kata-deploy-payload-amd64
-    - build-and-publish-tee-confidential-unencrypted-image
-    - publish-csi-driver-amd64
-    env:
-      GH_TOKEN: ${{ github.token }}
+     - publish-kata-deploy-payload-amd64
+     - build-and-publish-tee-confidential-unencrypted-image
+     - publish-csi-driver-amd64
+    uses: ./.github/workflows/run-kata-coco-tests.yaml
     permissions:
-      actions: write # Permission to trigger the gh workflows
-    steps:
-      - name: Checkout the code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-      - name: Trigger kata-coco-tests for amd64 arch
-        run: |
-          ./tests/gha-helper.sh trigger-and-check-workflow \
-            run-kata-coco-tests.yaml \
-            ${{ github.ref }} \
-            ${{ github.sha }} \
-            '{
-              "artifact-run-id":"${{ github.run_id }}",
-              "tarball-suffix":"-${{ inputs.tag }}",
-              "registry":"ghcr.io",
-              "repo":"${{ github.repository_owner }}/kata-deploy-ci",
-              "tag":"${{ inputs.tag }}-amd64",
-              "commit-hash":"${{ inputs.commit-hash }}",
-              "pr-number":"${{ inputs.pr-number }}",
-              "target-branch":"${{ inputs.target-branch }}"
-            }'
+      contents: read
+      id-token: write # Used for OIDC access to log into Azure
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets:
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      AZ_APPID: ${{ secrets.AZ_APPID }}
+      AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+      AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+      ITA_KEY: ${{ secrets.ITA_KEY }}
 
-  # run-k8s-tests-on-zvsi:
-  #   if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: [publish-kata-deploy-payload-s390x, build-and-publish-tee-confidential-unencrypted-image]
-  #   uses: ./.github/workflows/run-k8s-tests-on-zvsi.yaml
-  #   with:
-  #     registry: ghcr.io
-  #     repo: ${{ github.repository_owner }}/kata-deploy-ci
-  #     tag: ${{ inputs.tag }}-s390x
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     pr-number: ${{ inputs.pr-number }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #   secrets:
-  #     AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+  run-k8s-tests-on-zvsi:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: [publish-kata-deploy-payload-s390x, build-and-publish-tee-confidential-unencrypted-image]
+    uses: ./.github/workflows/run-k8s-tests-on-zvsi.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-s390x
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets:
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
 
-  # run-k8s-tests-on-ppc64le:
-  #   if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: publish-kata-deploy-payload-ppc64le
-  #   uses: ./.github/workflows/run-k8s-tests-on-ppc64le.yaml
-  #   with:
-  #     registry: ghcr.io
-  #     repo: ${{ github.repository_owner }}/kata-deploy-ci
-  #     tag: ${{ inputs.tag }}-ppc64le
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     pr-number: ${{ inputs.pr-number }}
-  #     target-branch: ${{ inputs.target-branch }}
+  run-k8s-tests-on-ppc64le:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: publish-kata-deploy-payload-ppc64le
+    uses: ./.github/workflows/run-k8s-tests-on-ppc64le.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-ppc64le
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
 
-  # run-kata-deploy-tests:
-  #   if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: [publish-kata-deploy-payload-amd64]
-  #   uses: ./.github/workflows/run-kata-deploy-tests.yaml
-  #   with:
-  #     registry: ghcr.io
-  #     repo: ${{ github.repository_owner }}/kata-deploy-ci
-  #     tag: ${{ inputs.tag }}-amd64
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     pr-number: ${{ inputs.pr-number }}
-  #     target-branch: ${{ inputs.target-branch }}
+  run-kata-deploy-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: [publish-kata-deploy-payload-amd64]
+    uses: ./.github/workflows/run-kata-deploy-tests.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
 
-  # run-metrics-tests:
-  #   # Skip metrics tests whilst runner is broken
-  #   if: false
-  #   # if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: build-kata-static-tarball-amd64
-  #   uses: ./.github/workflows/run-metrics.yaml
-  #   with:
-  #     registry: ghcr.io
-  #     repo: ${{ github.repository_owner }}/kata-deploy-ci
-  #     tag: ${{ inputs.tag }}-amd64
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     pr-number: ${{ inputs.pr-number }}
-  #     target-branch: ${{ inputs.target-branch }}
+  run-basic-amd64-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/basic-ci-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
 
-  # run-basic-amd64-tests:
-  #   if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: build-kata-static-tarball-amd64
-  #   uses: ./.github/workflows/basic-ci-amd64.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
+  run-basic-s390x-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-s390x
+    uses: ./.github/workflows/basic-ci-s390x.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
 
-  # run-basic-s390x-tests:
-  #   if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: build-kata-static-tarball-s390x
-  #   uses: ./.github/workflows/basic-ci-s390x.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
+  run-cri-containerd-amd64:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-amd64
+    strategy:
+      fail-fast: false
+      matrix:
+        params: [
+          { containerd_version: lts,    vmm: clh              },
+          { containerd_version: lts,    vmm: dragonball       },
+          { containerd_version: lts,    vmm: qemu             },
+          { containerd_version: lts,    vmm: stratovirt       },
+          { containerd_version: lts,    vmm: cloud-hypervisor },
+          { containerd_version: lts,    vmm: qemu-runtime-rs  },
+          { containerd_version: active, vmm: clh              },
+          { containerd_version: active, vmm: dragonball       },
+          { containerd_version: active, vmm: qemu             },
+          { containerd_version: active, vmm: stratovirt       },
+          { containerd_version: active, vmm: cloud-hypervisor },
+          { containerd_version: active, vmm: qemu-runtime-rs  },
+         ]
+    uses: ./.github/workflows/run-cri-containerd-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+      runner: ubuntu-22.04
+      arch: amd64
+      containerd_version: ${{ matrix.params.containerd_version }}
+      vmm: ${{ matrix.params.vmm }}
 
-  # run-cri-containerd-amd64:
-  #   if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: build-kata-static-tarball-amd64
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       params: [
-  #         { containerd_version: lts,    vmm: clh              },
-  #         { containerd_version: lts,    vmm: dragonball       },
-  #         { containerd_version: lts,    vmm: qemu             },
-  #         { containerd_version: lts,    vmm: stratovirt       },
-  #         { containerd_version: lts,    vmm: cloud-hypervisor },
-  #         { containerd_version: lts,    vmm: qemu-runtime-rs  },
-  #         { containerd_version: active, vmm: clh              },
-  #         { containerd_version: active, vmm: dragonball       },
-  #         { containerd_version: active, vmm: qemu             },
-  #         { containerd_version: active, vmm: stratovirt       },
-  #         { containerd_version: active, vmm: cloud-hypervisor },
-  #         { containerd_version: active, vmm: qemu-runtime-rs  },
-  #        ]
-  #   uses: ./.github/workflows/run-cri-containerd-tests.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #     runner: ubuntu-22.04
-  #     arch: amd64
-  #     containerd_version: ${{ matrix.params.containerd_version }}
-  #     vmm: ${{ matrix.params.vmm }}
+  run-cri-containerd-s390x:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-s390x
+    strategy:
+      fail-fast: false
+      matrix:
+        params: [
+          { containerd_version: active, vmm: qemu            },
+          { containerd_version: active, vmm: qemu-runtime-rs },
+         ]
+    uses: ./.github/workflows/run-cri-containerd-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+      runner: s390x-large
+      arch: s390x
+      containerd_version: ${{ matrix.params.containerd_version }}
+      vmm: ${{ matrix.params.vmm }}
 
-  # run-cri-containerd-s390x:
-  #   if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: build-kata-static-tarball-s390x
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       params: [
-  #         { containerd_version: active, vmm: qemu            },
-  #         { containerd_version: active, vmm: qemu-runtime-rs },
-  #        ]
-  #   uses: ./.github/workflows/run-cri-containerd-tests.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #     runner: s390x-large
-  #     arch: s390x
-  #     containerd_version: ${{ matrix.params.containerd_version }}
-  #     vmm: ${{ matrix.params.vmm }}
+  run-cri-containerd-tests-ppc64le:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-ppc64le
+    strategy:
+      fail-fast: false
+      matrix:
+        params: [
+          { containerd_version: active, vmm: qemu },
+         ]
+    uses: ./.github/workflows/run-cri-containerd-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+      runner: ppc64le
+      arch: ppc64le
+      containerd_version: ${{ matrix.params.containerd_version }}
+      vmm: ${{ matrix.params.vmm }}
 
-  # run-cri-containerd-tests-ppc64le:
-  #   if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: build-kata-static-tarball-ppc64le
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       params: [
-  #         { containerd_version: active, vmm: qemu },
-  #        ]
-  #   uses: ./.github/workflows/run-cri-containerd-tests.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #     runner: ppc64le
-  #     arch: ppc64le
-  #     containerd_version: ${{ matrix.params.containerd_version }}
-  #     vmm: ${{ matrix.params.vmm }}
-
-  # run-cri-containerd-tests-arm64:
-  #   if: ${{ inputs.skip-test != 'yes' }}
-  #   needs: build-kata-static-tarball-arm64
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       params: [
-  #         { containerd_version: active, vmm: qemu },
-  #        ]
-  #   uses: ./.github/workflows/run-cri-containerd-tests.yaml
-  #   with:
-  #     tarball-suffix: -${{ inputs.tag }}
-  #     commit-hash: ${{ inputs.commit-hash }}
-  #     target-branch: ${{ inputs.target-branch }}
-  #     runner: arm64-non-k8s
-  #     arch: arm64
-  #     containerd_version: ${{ matrix.params.containerd_version }}
-  #     vmm: ${{ matrix.params.vmm }}
+  run-cri-containerd-tests-arm64:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-arm64
+    strategy:
+      fail-fast: false
+      matrix:
+        params: [
+          { containerd_version: active, vmm: qemu },
+         ]
+    uses: ./.github/workflows/run-cri-containerd-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+      runner: arm64-non-k8s
+      arch: arm64
+      containerd_version: ${{ matrix.params.containerd_version }}
+      vmm: ${{ matrix.params.vmm }}

.github/workflows/cleanup-resources.yaml

@@ -4,13 +4,13 @@ on:
     - cron: "0 0 * * *"
   workflow_dispatch:
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   cleanup-resources:
     runs-on: ubuntu-22.04
+    permissions:
+      id-token: write # Used for OIDC access to log into Azure
     environment: ci
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

.github/workflows/codeql.yml

@@ -19,8 +19,8 @@ on:
   schedule:
     - cron: '45 0 * * 1'
 
-permissions:
-  contents: read
+permissions: {}
+
 
 jobs:
   analyze:

.github/workflows/commit-message-check.yaml

@@ -6,8 +6,7 @@ on:
       - reopened
       - synchronize
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -42,7 +41,7 @@ jobs:
         filter_out_pattern: '^Revert "|^Reapply "'
 
     - name: DCO Check
-      uses: tim-actions/dco@2fd0504dc0d27b33f542867c300c60840c6dcb20 # master (2020-04-28)
+      uses: tim-actions/dco@f2279e6e62d5a7d9115b0cb8e837b777b1b02e21 # v1.1.0
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
 

.github/workflows/darwin-tests.yaml

@@ -6,8 +6,7 @@ on:
       - reopened
       - synchronize
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -18,13 +17,15 @@ jobs:
   test:
     runs-on: macos-latest
     steps:
-    - name: Install Go
-      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
-      with:
-        go-version: 1.23.10
     - name: Checkout code
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         persist-credentials: false
+
+    - name: Install golang
+      run: |
+        ./tests/install_go.sh -f -p
+        echo "/usr/local/go/bin" >> "${GITHUB_PATH}"
+
     - name: Build utils
       run: ./ci/darwin-test.sh

.github/workflows/docs-url-alive-check.yaml

@@ -2,8 +2,7 @@ on:
   schedule:
     - cron:  '0 23 * * 0'
 
-permissions:
-  contents: read
+permissions: {}
 
 name: Docs URL Alive Check
 jobs:
@@ -14,23 +13,21 @@ jobs:
     env:
       target_branch: ${{ github.base_ref }}
     steps:
-    - name: Install Go
-      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
-      with:
-        go-version: 1.23.10
-      env:
-        GOPATH: ${{ github.workspace }}/kata-containers
     - name: Set env
       run: |
         echo "GOPATH=${{ github.workspace }}" >> "$GITHUB_ENV"
-        echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Checkout code
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         fetch-depth: 0
         persist-credentials: false
         path: ./src/github.com/${{ github.repository }}
-    # docs url alive check
+
+    - name: Install golang
+      run: |
+        ./tests/install_go.sh -f -p
+        echo "/usr/local/go/bin" >> "${GITHUB_PATH}"
+
     - name: Docs URL Alive Check
       run: |
         cd "${GOPATH}/src/github.com/${{ github.repository }}" && make docs-url-alive-check

.github/workflows/gatekeeper-skipper.yaml

@@ -31,8 +31,7 @@ on:
       skip_static:
         value: ${{ jobs.skipper.outputs.skip_static }}
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   skipper:

.github/workflows/gatekeeper.yaml

@@ -12,8 +12,7 @@ on:
       - reopened
       - labeled
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/govulncheck.yaml

@@ -3,8 +3,7 @@ on:
 
 name: Govulncheck
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   govulncheck:
@@ -14,12 +13,12 @@ jobs:
         include:
           - binary: "kata-runtime"
             make_target: "runtime"
-          - binary: "containerd-shim-kata-v2" 
+          - binary: "containerd-shim-kata-v2"
             make_target: "containerd-shim-v2"
           - binary: "kata-monitor"
             make_target: "monitor"
       fail-fast: false
-    
+
     steps:
       - name: Checkout the code
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

.github/workflows/kata-runtime-classes-sync.yaml

@@ -6,8 +6,7 @@ on:
       - reopened
       - synchronize
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/osv-scanner.yaml

@@ -0,0 +1,43 @@
+# A sample workflow which sets up periodic OSV-Scanner scanning for vulnerabilities,
+# in addition to a PR check which fails if new vulnerabilities are introduced.
+#
+# For more examples and options, including how to ignore specific vulnerabilities,
+# see https://google.github.io/osv-scanner/github-action/
+
+name: OSV-Scanner
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: '0 1 * * 0'
+  push:
+    branches: [ "main" ]
+
+permissions: {}
+
+jobs:
+  scan-scheduled:
+    permissions:
+      actions: read # # Required to upload SARIF file to CodeQL
+      contents: read  # Read commit contents
+      security-events: write  # Require writing security events to upload SARIF file to security tab
+    if: ${{ github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
+    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@b00f71e051ddddc6e46a193c31c8c0bf283bf9e6" # v2.1.0
+    with:
+      scan-args: |-
+        -r
+        ./
+  scan-pr:
+    permissions:
+      actions: read # Required to upload SARIF file to CodeQL
+      contents: read  # Read commit contents
+      security-events: write  # Require writing security events to upload SARIF file to security tab
+    if: ${{ github.event_name == 'pull_request' }}
+    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@b00f71e051ddddc6e46a193c31c8c0bf283bf9e6" # v2.1.0
+    with:
+      # Example of specifying custom arguments
+      scan-args: |-
+        -r
+        ./

.github/workflows/payload-after-push.yaml

@@ -5,8 +5,7 @@ on:
       - main
   workflow_dispatch:
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -25,6 +24,7 @@ jobs:
       target-branch: ${{ github.ref_name }}
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+      KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }}
 
   build-assets-arm64:
     permissions:

.github/workflows/publish-kata-deploy-payload.yaml

@@ -34,8 +34,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   kata-payload:
@@ -85,6 +84,6 @@ jobs:
           TAG: ${{ inputs.tag }}
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          "$(pwd)/kata-static.tar.xz" \
+          "$(pwd)/kata-static.tar.zst" \
           "${REGISTRY}/${REPO}" \
           "${TAG}"

.github/workflows/release-amd64.yaml

@@ -8,9 +8,10 @@ on:
     secrets:
       QUAY_DEPLOYER_PASSWORD:
         required: true
+      KBUILD_SIGN_PIN:
+        required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-kata-static-tarball-amd64:
@@ -20,6 +21,7 @@ jobs:
       stage: release
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+      KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }}
     permissions:
       contents: read
       packages: write
@@ -71,9 +73,9 @@ jobs:
           fi
           for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "ghcr.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "ghcr.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
           done

.github/workflows/release-arm64.yaml

@@ -9,8 +9,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-kata-static-tarball-arm64:
@@ -71,9 +70,9 @@ jobs:
           fi
           for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "ghcr.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "ghcr.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
           done

.github/workflows/release-ppc64le.yaml

@@ -9,8 +9,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-kata-static-tarball-ppc64le:
@@ -71,9 +70,9 @@ jobs:
           fi
           for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "ghcr.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "ghcr.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
           done

.github/workflows/release-s390x.yaml

@@ -11,8 +11,7 @@ on:
       QUAY_DEPLOYER_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   build-kata-static-tarball-s390x:
@@ -75,9 +74,9 @@ jobs:
           fi
           for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "ghcr.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "ghcr.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.zst "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${TARGET_ARCH}"
           done

.github/workflows/release.yaml

@@ -2,8 +2,7 @@ name: Release Kata Containers
 on:
   workflow_dispatch
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   release:
@@ -35,6 +34,7 @@ jobs:
       target-arch: amd64
     secrets:
       QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+      KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }}
 
   build-and-push-assets-arm64:
     needs: release
@@ -126,7 +126,7 @@ jobs:
 
       - name: Set KATA_STATIC_TARBALL env var
         run: |
-          tarball=$(pwd)/kata-static.tar.xz
+          tarball=$(pwd)/kata-static.tar.zst
           echo "KATA_STATIC_TARBALL=${tarball}" >> "$GITHUB_ENV"
 
       - name: Download amd64 artifacts

.github/workflows/run-cri-containerd-tests.yaml

@@ -1,7 +1,6 @@
 name: CI | Run cri-containerd tests
 
-permissions:
-  contents: read
+permissions: {}
 
 on:
   workflow_call:
@@ -59,6 +58,8 @@ jobs:
       - name: Install dependencies
         timeout-minutes: 15
         run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball for ${{ inputs.arch }}
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -2,11 +2,6 @@ name: CI | Run kubernetes tests on AKS
 on:
   workflow_call:
     inputs:
-      artifact-run-id:
-        description: "The run id where the artifact was uploaded"
-        required: false
-        type: string
-        default: ${{ github.run_id }}
       tarball-suffix:
         required: false
         type: string
@@ -37,46 +32,9 @@ on:
        required: true
       AZ_SUBSCRIPTION_ID:
         required: true
-  workflow_dispatch:
-    inputs:
-      artifact-run-id:
-        description: "The workflow run id where the artifact was uploaded"
-        required: true
-        type: string
-      tarball-suffix:
-        description: "The suffix of the kata tarball to use"
-        required: false
-        type: string
-      registry:
-        description: "The oci container registry to install kata-deploy from"
-        required: true
-        type: string
-      repo:
-        description: "The oci container repository/image to install kata-deploy from"
-        required: true
-        type: string
-      tag:
-        description: "The oci container image tag to install kata-deploy using"
-        required: true
-        type: string
-      pr-number:
-        description: "Identifier used to distinguish between PRs/dev/nightly tests"
-        required: true
-        type: string
-      commit-hash:
-        description: "The code to checkout for testing"
-        required: false
-        type: string
-      target-branch:
-        description: "The target branch to rebase on and ensure the tests are up-to-date"
-        required: false
-        type: string
-        default: ""
 
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   run-k8s-tests:
@@ -111,6 +69,9 @@ jobs:
             instance-type: normal
             auto-generate-policy: yes
     runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      id-token: write # Used for OIDC access to log into Azure
     environment: ci
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
@@ -141,9 +102,6 @@ jobs:
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          run-id: ${{ inputs.artifact-run-id }}
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository}}
           path: kata-artifacts
 
       - name: Install kata
@@ -189,6 +147,13 @@ jobs:
         timeout-minutes: 60
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
+      - name: Refresh OIDC token in case access token expired
+        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
+        with:
+          client-id: ${{ secrets.AZ_APPID }}
+          tenant-id: ${{ secrets.AZ_TENANT_ID }}
+          subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
       - name: Delete AKS cluster
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-k8s-tests-on-amd64.yaml

@@ -22,8 +22,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-k8s-tests-amd64:
@@ -31,11 +30,7 @@ jobs:
       fail-fast: false
       matrix:
         vmm:
-          - clh #cloud-hypervisor
-          - dragonball
-          - fc #firecracker
           - qemu
-          - cloud-hypervisor
         container_runtime:
           - containerd
         snapshotter:
@@ -73,6 +68,25 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: Remove unnecessary directories to free up space
+        run: |
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf /usr/lib/jvm
+          sudo rm -rf /usr/share/swift
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /usr/local/julia*
+          sudo rm -rf /opt/az
+          sudo rm -rf /usr/local/share/chromium
+          sudo rm -rf /opt/microsoft
+          sudo rm -rf /opt/google
+          sudo rm -rf /usr/lib/firefox
+
       - name: Configure CRI-O
         if: matrix.container_runtime == 'crio'
         run: bash tests/integration/kubernetes/gha-run.sh setup-crio

.github/workflows/run-k8s-tests-on-arm64.yaml

@@ -22,8 +22,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-k8s-tests-on-arm64:

.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml

@@ -0,0 +1,89 @@
+name: CI | Run NVIDIA GPU kubernetes tests on arm64
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+    secrets:
+      NGC_API_KEY:
+        required: true
+
+permissions: {}
+
+jobs:
+  run-nvidia-gpu-tests-on-amd64:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-nvidia-gpu
+        k8s:
+          - kubeadm
+    runs-on: amd64-nvidia-a100
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: all
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-nv-tests
+        env:
+          NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
+      - name: Collect artifacts ${{ matrix.vmm }}
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh collect-artifacts
+        continue-on-error: true
+
+      - name: Archive artifacts ${{ matrix.vmm }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: k8s-tests-${{ matrix.vmm }}-${{ matrix.k8s }}-${{ inputs.tag }}
+          path: /tmp/artifacts
+          retention-days: 1
+
+      - name: Delete kata-deploy
+        if: always()
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup

.github/workflows/run-k8s-tests-on-ppc64le.yaml

@@ -22,8 +22,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-k8s-tests:

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -25,8 +25,7 @@ on:
       AUTHENTICATED_IMAGE_PASSWORD:
         required: true
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-k8s-tests:

.github/workflows/run-kata-coco-stability-tests.yaml

@@ -35,9 +35,7 @@ on:
       AUTHENTICATED_IMAGE_PASSWORD:
         required: true
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   # Generate jobs for testing CoCo on non-TEE environments
@@ -52,6 +50,9 @@ jobs:
         pull-type:
           - guest-pull
     runs-on: ubuntu-22.04
+    permissions:
+
+      id-token: write # Used for OIDC access to log into Azure
     environment: ci
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
@@ -91,9 +92,6 @@ jobs:
       - name: Install kata
         run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
 
-      - name: Download Azure CLI
-        run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
-
       - name: Log into the Azure account
         uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
         with:
@@ -141,6 +139,13 @@ jobs:
         timeout-minutes: 300
         run: bash tests/stability/gha-stability-run.sh run-tests
 
+      - name: Refresh OIDC token in case access token expired
+        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
+        with:
+          client-id: ${{ secrets.AZ_APPID }}
+          tenant-id: ${{ secrets.AZ_TENANT_ID }}
+          subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
       - name: Delete AKS cluster
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-kata-coco-tests.yaml

@@ -2,11 +2,6 @@ name: CI | Run kata coco tests
 on:
   workflow_call:
     inputs:
-      artifact-run-id:
-        description: "The run id where the artifact was uploaded"
-        required: false
-        type: string
-        default: ${{ github.run_id }}
       tarball-suffix:
         required: false
         type: string
@@ -40,216 +35,179 @@ on:
         required: true
       ITA_KEY:
         required: true
-  workflow_dispatch:
-    inputs:
-      artifact-run-id:
-        description: "The workflow run id where the artifact was uploaded"
-        required: true
-        type: string
-      tarball-suffix:
-        description: "The suffix of the kata tarball to use"
-        required: false
-        type: string
-      registry:
-        description: "The oci container registry to install kata-deploy from"
-        required: true
-        type: string
-      repo:
-        description: "The oci container repository/image to install kata-deploy from"
-        required: true
-        type: string
-      tag:
-        description: "The oci container image tag to install kata-deploy using"
-        required: true
-        type: string
-      pr-number:
-        description: "Identifier used to distinguish between PRs/dev/nightly tests"
-        required: true
-        type: string
-      commit-hash:
-        description: "The code to checkout for testing"
-        required: false
-        type: string
-      target-branch:
-        description: "The target branch to rebase on and ensure the tests are up-to-date"
-        required: false
-        type: string
-        default: ""
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
-  # run-k8s-tests-on-tdx:
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       vmm:
-  #         - qemu-tdx
-  #       snapshotter:
-  #         - nydus
-  #       pull-type:
-  #         - guest-pull
-  #   runs-on: tdx
-  #   env:
-  #     DOCKER_REGISTRY: ${{ inputs.registry }}
-  #     DOCKER_REPO: ${{ inputs.repo }}
-  #     DOCKER_TAG: ${{ inputs.tag }}
-  #     GH_PR_NUMBER: ${{ inputs.pr-number }}
-  #     KATA_HYPERVISOR: ${{ matrix.vmm }}
-  #     KUBERNETES: "vanilla"
-  #     USING_NFD: "true"
-  #     KBS: "true"
-  #     K8S_TEST_HOST_TYPE: "baremetal"
-  #     KBS_INGRESS: "nodeport"
-  #     SNAPSHOTTER: ${{ matrix.snapshotter }}
-  #     PULL_TYPE: ${{ matrix.pull-type }}
-  #     AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
-  #     AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
-  #     ITA_KEY: ${{ secrets.ITA_KEY }}
-  #     AUTO_GENERATE_POLICY: "yes"
-  #   steps:
-  #     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-  #       with:
-  #         ref: ${{ inputs.commit-hash }}
-  #         fetch-depth: 0
-  #         persist-credentials: false
+  run-k8s-tests-on-tdx:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+        snapshotter:
+          - nydus
+        pull-type:
+          - guest-pull
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
+      USING_NFD: "true"
+      KBS: "true"
+      K8S_TEST_HOST_TYPE: "baremetal"
+      KBS_INGRESS: "nodeport"
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      PULL_TYPE: ${{ matrix.pull-type }}
+      AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      ITA_KEY: ${{ secrets.ITA_KEY }}
+      AUTO_GENERATE_POLICY: "yes"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+          persist-credentials: false
 
-  #     - name: Rebase atop of the latest target branch
-  #       run: |
-  #         ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
-  #       env:
-  #         TARGET_BRANCH: ${{ inputs.target-branch }}
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
-  #     - name: Deploy Snapshotter
-  #       timeout-minutes: 5
-  #       run: bash tests/integration/kubernetes/gha-run.sh deploy-snapshotter
+      - name: Deploy Snapshotter
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-snapshotter
 
-  #     - name: Deploy Kata
-  #       timeout-minutes: 10
-  #       run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
 
-  #     - name: Uninstall previous `kbs-client`
-  #       timeout-minutes: 10
-  #       run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
+      - name: Uninstall previous `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
 
-  #     - name: Deploy CoCo KBS
-  #       timeout-minutes: 10
-  #       run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
 
-  #     - name: Install `kbs-client`
-  #       timeout-minutes: 10
-  #       run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
-  #     - name: Deploy CSI driver
-  #       timeout-minutes: 5
-  #       run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
 
-  #     - name: Run tests
-  #       timeout-minutes: 100
-  #       run: bash tests/integration/kubernetes/gha-run.sh run-tests
+      - name: Run tests
+        timeout-minutes: 100
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
-  #     - name: Delete kata-deploy
-  #       if: always()
-  #       run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx
 
-  #     - name: Delete Snapshotter
-  #       if: always()
-  #       run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
+      - name: Delete Snapshotter
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
-  #     - name: Delete CoCo KBS
-  #       if: always()
-  #       run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+      - name: Delete CoCo KBS
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
 
-  #     - name: Delete CSI driver
-  #       timeout-minutes: 5
-  #       run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+      - name: Delete CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
 
-  # run-k8s-tests-sev-snp:
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       vmm:
-  #         - qemu-snp
-  #       snapshotter:
-  #         - nydus
-  #       pull-type:
-  #         - guest-pull
-  #   runs-on: sev-snp
-  #   env:
-  #     DOCKER_REGISTRY: ${{ inputs.registry }}
-  #     DOCKER_REPO: ${{ inputs.repo }}
-  #     DOCKER_TAG: ${{ inputs.tag }}
-  #     GH_PR_NUMBER: ${{ inputs.pr-number }}
-  #     KATA_HYPERVISOR: ${{ matrix.vmm }}
-  #     KUBECONFIG: /home/kata/.kube/config
-  #     KUBERNETES: "vanilla"
-  #     USING_NFD: "false"
-  #     KBS: "true"
-  #     KBS_INGRESS: "nodeport"
-  #     K8S_TEST_HOST_TYPE: "baremetal"
-  #     SNAPSHOTTER: ${{ matrix.snapshotter }}
-  #     PULL_TYPE: ${{ matrix.pull-type }}
-  #     AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
-  #     AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
-  #     AUTO_GENERATE_POLICY: "yes"
-  #   steps:
-  #     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-  #       with:
-  #         ref: ${{ inputs.commit-hash }}
-  #         fetch-depth: 0
-  #         persist-credentials: false
+  run-k8s-tests-sev-snp:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-snp
+        snapshotter:
+          - nydus
+        pull-type:
+          - guest-pull
+    runs-on: sev-snp
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      KBS: "true"
+      KBS_INGRESS: "nodeport"
+      K8S_TEST_HOST_TYPE: "baremetal"
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      PULL_TYPE: ${{ matrix.pull-type }}
+      AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      AUTO_GENERATE_POLICY: "yes"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+          persist-credentials: false
 
-  #     - name: Rebase atop of the latest target branch
-  #       run: |
-  #         ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
-  #       env:
-  #         TARGET_BRANCH: ${{ inputs.target-branch }}
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
-  #     - name: Deploy Snapshotter
-  #       timeout-minutes: 5
-  #       run: bash tests/integration/kubernetes/gha-run.sh deploy-snapshotter
+      - name: Deploy Snapshotter
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-snapshotter
 
-  #     - name: Deploy Kata
-  #       timeout-minutes: 10
-  #       run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
 
-  #     - name: Uninstall previous `kbs-client`
-  #       timeout-minutes: 10
-  #       run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
+      - name: Uninstall previous `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
 
-  #     - name: Deploy CoCo KBS
-  #       timeout-minutes: 10
-  #       run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
 
-  #     - name: Install `kbs-client`
-  #       timeout-minutes: 10
-  #       run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
-  #     - name: Deploy CSI driver
-  #       timeout-minutes: 5
-  #       run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
 
-  #     - name: Run tests
-  #       timeout-minutes: 50
-  #       run: bash tests/integration/kubernetes/gha-run.sh run-tests
+      - name: Run tests
+        timeout-minutes: 50
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
-  #     - name: Delete kata-deploy
-  #       if: always()
-  #       run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp
 
-  #     - name: Delete Snapshotter
-  #       if: always()
-  #       run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
+      - name: Delete Snapshotter
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
-  #     - name: Delete CoCo KBS
-  #       if: always()
-  #       run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+      - name: Delete CoCo KBS
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
 
-  #     - name: Delete CSI driver
-  #       timeout-minutes: 5
-  #       run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+      - name: Delete CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
 
   # Generate jobs for testing CoCo on non-TEE environments
   run-k8s-tests-coco-nontee:
@@ -263,6 +221,8 @@ jobs:
         pull-type:
           - guest-pull
     runs-on: ubuntu-22.04
+    permissions:
+      id-token: write # Used for OIDC access to log into Azure
     environment: ci
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
@@ -303,17 +263,11 @@ jobs:
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          run-id: ${{ inputs.artifact-run-id }}
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository}}
           path: kata-artifacts
 
       - name: Install kata
         run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
 
-      - name: Download Azure CLI
-        run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
-
       - name: Log into the Azure account
         uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
         with:
@@ -369,6 +323,13 @@ jobs:
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh report-tests
 
+      - name: Refresh OIDC token in case access token expired
+        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
+        with:
+          client-id: ${{ secrets.AZ_APPID }}
+          tenant-id: ${{ secrets.AZ_TENANT_ID }}
+          subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
       - name: Delete AKS cluster
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-kata-deploy-tests-on-aks.yaml

@@ -29,9 +29,7 @@ on:
       AZ_SUBSCRIPTION_ID:
         required: true
 
-permissions:
-  contents: read
-  id-token: write
+permissions: {}
 
 jobs:
   run-kata-deploy-tests:
@@ -50,6 +48,8 @@ jobs:
             vmm: clh
     runs-on: ubuntu-22.04
     environment: ci
+    permissions:
+      id-token: write # Used for OIDC access to log into Azure
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
@@ -72,9 +72,6 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
-      - name: Download Azure CLI
-        run: bash tests/functional/kata-deploy/gha-run.sh install-azure-cli
-
       - name: Log into the Azure account
         uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
         with:
@@ -105,6 +102,13 @@ jobs:
       - name: Run tests
         run: bash tests/functional/kata-deploy/gha-run.sh run-tests
 
+      - name: Refresh OIDC token in case access token expired
+        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
+        with:
+          client-id: ${{ secrets.AZ_APPID }}
+          tenant-id: ${{ secrets.AZ_TENANT_ID }}
+          subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
       - name: Delete AKS cluster
         if: always()
         run: bash tests/functional/kata-deploy/gha-run.sh delete-cluster

.github/workflows/run-kata-deploy-tests.yaml

@@ -22,8 +22,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-kata-deploy-tests:
@@ -59,6 +58,25 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: Remove unnecessary directories to free up space
+        run: |
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf /usr/lib/jvm
+          sudo rm -rf /usr/share/swift
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /usr/local/julia*
+          sudo rm -rf /opt/az
+          sudo rm -rf /usr/local/share/chromium
+          sudo rm -rf /opt/microsoft
+          sudo rm -rf /opt/google
+          sudo rm -rf /usr/lib/firefox
+
       - name: Deploy ${{ matrix.k8s }}
         run:  bash tests/functional/kata-deploy/gha-run.sh deploy-k8s
 

.github/workflows/run-kata-monitor-tests.yaml

@@ -2,11 +2,6 @@ name: CI | Run kata-monitor tests
 on:
   workflow_call:
     inputs:
-      artifact-run-id:
-        description: "The run id where the artifact was uploaded"
-        required: false
-        type: string
-        default: ${{ github.run_id }}
       tarball-suffix:
         required: false
         type: string
@@ -17,28 +12,8 @@ on:
         required: false
         type: string
         default: ""
-  workflow_dispatch:
-    inputs:
-      artifact-run-id:
-        description: "The workflow run id where the artifact was uploaded"
-        required: true
-        type: string
-      tarball-suffix:
-        description: "Identifier used to distinguish between PRs/dev/nightly tests"
-        required: false
-        type: string
-      commit-hash:
-        description: "The code to checkout for testing"
-        required: false
-        type: string
-      target-branch:
-        description: "The target branch to rebase on and ensure the tests are up-to-date"
-        required: false
-        type: string
-        default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-monitor:
@@ -78,14 +53,13 @@ jobs:
 
       - name: Install dependencies
         run: bash tests/functional/kata-monitor/gha-run.sh install-dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          run-id: ${{ inputs.artifact-run-id }}
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository}}
           path: kata-artifacts
 
       - name: Install kata

.github/workflows/run-metrics.yaml

@@ -22,8 +22,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-metrics:

.github/workflows/run-runk-tests.yaml

@@ -13,8 +13,7 @@ on:
         type: string
         default: ""
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   run-runk:
@@ -38,6 +37,8 @@ jobs:
 
       - name: Install dependencies
         run: bash tests/integration/runk/gha-run.sh install-dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0

.github/workflows/shellcheck.yaml

@@ -10,8 +10,7 @@ on:
       - reopened
       - synchronize
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -27,6 +26,6 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
       - name: Run ShellCheck
-        uses: ludeeus/action-shellcheck@00b27aa7cb85167568cb48a3838b75f4265f2bca # master (2024-06-20)
+        uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # v2.0.0
         with:
           ignore_paths: "**/vendor/**"

.github/workflows/shellcheck_required.yaml

@@ -11,8 +11,7 @@ on:
       - reopened
       - synchronize
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -29,7 +28,7 @@ jobs:
           persist-credentials: false
 
       - name: Run ShellCheck
-        uses: ludeeus/action-shellcheck@00b27aa7cb85167568cb48a3838b75f4265f2bca # master (2024-06-20)
+        uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # v2.0.0
         with:
           severity: error
           ignore_paths: "**/vendor/**"

.github/workflows/stale.yaml

@@ -4,8 +4,7 @@ on:
     - cron: '0 0 * * *'
   workflow_dispatch:
 
-permissions:
-  contents: read
+permissions: {}
 
 jobs:
   stale:

.github/workflows/static-checks-self-hosted.yaml

@@ -6,8 +6,7 @@ on:
       - reopened
       - labeled # a workflow runs only when the 'ok-to-test' label is added
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/static-checks.yaml

@@ -7,8 +7,7 @@ on:
       - synchronize
   workflow_dispatch:
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -150,3 +149,36 @@ jobs:
     needs: skipper
     if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     uses: ./.github/workflows/govulncheck.yaml
+
+  codegen:
+    runs-on: ubuntu-22.04
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
+    permissions:
+      contents: read  # for checkout
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: generate
+        run: make -C src/agent generate-protocols
+      - name: check for diff
+        run: |
+          diff=$(git diff)
+          if [[ -z "${diff}" ]]; then
+            echo "No diff detected."
+            exit 0
+          fi
+
+          cat << EOF >> "${GITHUB_STEP_SUMMARY}"
+          Run \`make -C src/agent generate-protocols\` to update protobuf bindings.
+
+          \`\`\`diff
+          ${diff}
+          \`\`\`
+          EOF
+
+          echo "::error::Golang protobuf bindings need to be regenerated (see Github step summary for diff)."
+          exit 1

.github/workflows/zizmor.yaml

@@ -2,11 +2,9 @@ name: GHA security analysis
 
 on:
   push:
-    branches: ["main"]
   pull_request:
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -27,3 +25,5 @@ jobs:
 
       - name: Run zizmor
         uses: zizmorcore/zizmor-action@f52a838cfabf134edcbaa7c8b3677dde20045018 # v0.1.1
+        with:
+          persona: auditor

Makefile

@@ -42,7 +42,7 @@ generate-protocols:
 
 # Some static checks rely on generated source files of components.
 static-checks: static-checks-build
-	bash tests/static-checks.sh github.com/kata-containers/kata-containers
+	bash tests/static-checks.sh
 
 docs-url-alive-check:
 	bash ci/docs-url-alive-check.sh

VERSION

@@ -1 +1 @@
-3.18.0
+3.21.0

ci/README.md

@@ -306,7 +306,7 @@ tarball to the newly created VM that will be used for debugging purposes.
 > [!NOTE]
 > Those artifacts are only available (for 15 days) when all jobs are finished.
 
-Once you have the `kata-static.tar.xz` in your VM, you can login to the VM with
+Once you have the `kata-static.tar.zst` in your VM, you can login to the VM with
 `kcli ssh debug-nerdctl-pr8070`, go ahead and then clone your development branch
 
 ```bash
@@ -323,15 +323,15 @@ $ git config --global user.name "Your Name"
 $ git rebase upstream/main
 ```
 
-Now copy the `kata-static.tar.xz` into your `kata-containers/kata-artifacts` directory
+Now copy the `kata-static.tar.zst` into your `kata-containers/kata-artifacts` directory
 
 ```bash
 $ mkdir kata-artifacts
-$ cp ../kata-static.tar.xz kata-artifacts/
+$ cp ../kata-static.tar.zst kata-artifacts/
 ```
 
 > [!NOTE]
-> If you downloaded the .zip from GitHub you need to uncompress first to see `kata-static.tar.xz`
+> If you downloaded the .zip from GitHub you need to uncompress first to see `kata-static.tar.zst`
 
 And finally run the tests following what's in the yaml file for the test you're
 debugging.
@@ -363,11 +363,11 @@ and have fun debugging and hacking!
 
 Steps for debugging the Kubernetes tests are very similar to the ones for
 debugging non-Kubernetes tests, with the caveat that what you'll need, this
-time, is not the `kata-static.tar.xz` tarball, but rather a payload to be used
+time, is not the `kata-static.tar.zst` tarball, but rather a payload to be used
 with kata-deploy.
 
 In order to generate your own kata-deploy image you can generate your own
-`kata-static.tar.xz` and then take advantage of the following script.  Be aware
+`kata-static.tar.zst` and then take advantage of the following script.  Be aware
 that the image generated and uploaded must be accessible by the VM where you'll
 be performing your tests.
 

ci/openshift-ci/peer-pods-azure.sh

@@ -116,33 +116,44 @@ az network vnet subnet update \
 for NODE_NAME in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do [[ "${NODE_NAME}" =~ 'worker' ]] && kubectl label node "${NODE_NAME}" node.kubernetes.io/worker=; done
 
 # CAA artifacts
-CAA_IMAGE="quay.io/confidential-containers/cloud-api-adaptor"
-TAGS="$(curl https://quay.io/api/v1/repository/confidential-containers/cloud-api-adaptor/tag/?onlyActiveTags=true)"
-DIGEST=$(echo "${TAGS}" | jq -r '.tags[] | select(.name | contains("latest-amd64")) | .manifest_digest')
-CAA_TAG="$(echo "${TAGS}" | jq -r '.tags[] | select(.manifest_digest | contains("'"${DIGEST}"'")) | .name' | grep -v "latest")"
+if [[ -z "${CAA_TAG}" ]]; then
+	if [[ -n "${CAA_IMAGE}" ]]; then
+		echo "CAA_IMAGE (${CAA_IMAGE}) is set but CAA_TAG isn't, which is not supported. Please specify both or none"
+		exit 1
+	fi
+	TAGS="$(curl https://quay.io/api/v1/repository/confidential-containers/cloud-api-adaptor/tag/?onlyActiveTags=true)"
+	DIGEST=$(echo "${TAGS}" | jq -r '.tags[] | select(.name | contains("latest-amd64")) | .manifest_digest')
+	CAA_TAG="$(echo "${TAGS}" | jq -r '.tags[] | select(.manifest_digest | contains("'"${DIGEST}"'")) | .name' | grep -v "latest")"
+fi
+if [[ -z "${CAA_IMAGE}" ]]; then
+	CAA_IMAGE="quay.io/confidential-containers/cloud-api-adaptor"
+fi
 
 # Get latest PP image
-SUCCESS_TIME=$(curl -s \
-  -H "Accept: application/vnd.github+json" \
-  "https://api.github.com/repos/confidential-containers/cloud-api-adaptor/actions/workflows/azure-nightly-build.yml/runs?status=success" \
-  | jq -r '.workflow_runs[0].updated_at')
-PP_IMAGE_ID="/CommunityGalleries/cocopodvm-d0e4f35f-5530-4b9c-8596-112487cdea85/Images/podvm_image0/Versions/$(date -u -jf "%Y-%m-%dT%H:%M:%SZ" "${SUCCESS_TIME}" "+%Y.%m.%d" 2>/dev/null || date -d "${SUCCESS_TIME}" +%Y.%m.%d)"
+if [[ -z "${PP_IMAGE_ID}" ]]; then
+	SUCCESS_TIME=$(curl -s \
+	  -H "Accept: application/vnd.github+json" \
+	  "https://api.github.com/repos/confidential-containers/cloud-api-adaptor/actions/workflows/azure-nightly-build.yml/runs?status=success" \
+	  | jq -r '.workflow_runs[0].updated_at')
+	PP_IMAGE_ID="/CommunityGalleries/cocopodvm-d0e4f35f-5530-4b9c-8596-112487cdea85/Images/podvm_image0/Versions/$(date -u -jf "%Y-%m-%dT%H:%M:%SZ" "${SUCCESS_TIME}" "+%Y.%m.%d" 2>/dev/null || date -d "${SUCCESS_TIME}" +%Y.%m.%d)"
+fi
 
-echo "AZURE_REGION: \"${AZURE_REGION}\""
-echo "PP_REGION: \"${PP_REGION}\""
-echo "AZURE_RESOURCE_GROUP: \"${AZURE_RESOURCE_GROUP}\""
-echo "PP_RESOURCE_GROUP: \"${PP_RESOURCE_GROUP}\""
-echo "PP_SUBNET_ID: \"${PP_SUBNET_ID}\""
-echo "CAA_TAG: \"${CAA_TAG}\""
-echo "PP_IMAGE_ID: \"${PP_IMAGE_ID}\""
+echo "AZURE_REGION=\"${AZURE_REGION}\""
+echo "PP_REGION=\"${PP_REGION}\""
+echo "AZURE_RESOURCE_GROUP=\"${AZURE_RESOURCE_GROUP}\""
+echo "PP_RESOURCE_GROUP=\"${PP_RESOURCE_GROUP}\""
+echo "PP_SUBNET_ID=\"${PP_SUBNET_ID}\""
+echo "CAA_IMAGE=\"${CAA_IMAGE}\""
+echo "CAA_TAG=\"${CAA_TAG}\""
+echo "PP_IMAGE_ID=\"${PP_IMAGE_ID}\""
 
 # Clone and configure caa
-git clone --depth 1 --no-checkout https://github.com/confidential-containers/cloud-api-adaptor.git
+git clone --revision "${CAA_GIT_SHA:-main}" --depth 1 --no-checkout https://github.com/confidential-containers/cloud-api-adaptor.git
 pushd cloud-api-adaptor
 git sparse-checkout init --cone
 git sparse-checkout set src/cloud-api-adaptor/install/
 git checkout
-echo "CAA_GIT_SHA: \"$(git rev-parse HEAD)\""
+echo "CAA_GIT_SHA=\"$(git rev-parse HEAD)\""
 pushd src/cloud-api-adaptor
 cat <<EOF > install/overlays/azure/workload-identity.yaml
 apiVersion: apps/v1
@@ -208,12 +219,12 @@ echo "AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}" >> install/overlays/azure/serv
 echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" >> install/overlays/azure/service-principal.env
 
 # Deploy Operator
-git clone --depth 1 --no-checkout https://github.com/confidential-containers/operator
+git clone --revision "${OPERATOR_SHA:-main}" --depth 1 --no-checkout https://github.com/confidential-containers/operator
 pushd operator
 git sparse-checkout init --cone
 git sparse-checkout set "config/"
 git checkout
-echo "OPERATOR_SHA: \"$(git rev-parse HEAD)\""
+echo "OPERATOR_SHA=\"$(git rev-parse HEAD)\""
 oc apply -k "config/release"
 oc apply -k "config/samples/ccruntime/peer-pods"
 popd
@@ -227,7 +238,7 @@ popd
 SECONDS=0
 ( while [[ "${SECONDS}" -lt 360 ]]; do
     kubectl get runtimeclass | grep -q kata-remote && exit 0
-done; exit 1 ) || { echo "kata-remote runtimeclass not initialized in 60s"; kubectl -n confidential-containers-system get all; echo; echo CAA; kubectl -n confidential-containers-system logs daemonset.apps/cloud-api-adaptor-daemonset; echo pre-install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-pre-install-daemon; echo install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-daemon-install; exit 1; }
+done; exit 1 ) || { echo "kata-remote runtimeclass not initialized in 60s"; kubectl -n confidential-containers-system get all; echo; echo "kubectl -n confidential-containers-system describe all"; kubectl -n confidential-containers-system describe all; echo; echo CAA; kubectl -n confidential-containers-system logs daemonset.apps/cloud-api-adaptor-daemonset; echo pre-install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-pre-install-daemon; echo install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-daemon-install; exit 1; }
 
 
 ################

docs/how-to/how-to-run-kata-containers-with-SE-VMs.md

@@ -89,16 +89,16 @@ However, if any of these components are absent, they must be built from the
 $ # Assume that the project is cloned at $GOPATH/src/github.com/kata-containers
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers
 $ make rootfs-initrd-confidential-tarball
-$ tar -tf build/kata-static-kernel-confidential.tar.xz | grep vmlinuz
+$ tar --zstd -tf build/kata-static-kernel-confidential.tar.zst | grep vmlinuz
 ./opt/kata/share/kata-containers/vmlinuz-confidential.container
 ./opt/kata/share/kata-containers/vmlinuz-6.7-136-confidential
 $ kernel_version=6.7-136
-$ tar -tf build/kata-static-rootfs-initrd-confidential.tar.xz | grep initrd
+$ tar --zstd -tf build/kata-static-rootfs-initrd-confidential.tar.zst | grep initrd
 ./opt/kata/share/kata-containers/kata-containers-initrd-confidential.img
 ./opt/kata/share/kata-containers/kata-ubuntu-20.04-confidential.initrd
 $ mkdir artifacts
-$ tar -xvf build/kata-static-kernel-confidential.tar.xz -C artifacts ./opt/kata/share/kata-containers/vmlinuz-${kernel_version}-confidential
-$ tar -xvf build/kata-static-rootfs-initrd-confidential.tar.xz -C artifacts ./opt/kata/share/kata-containers/kata-ubuntu-20.04-confidential.initrd
+$ tar --zstd -xvf build/kata-static-kernel-confidential.tar.zst -C artifacts ./opt/kata/share/kata-containers/vmlinuz-${kernel_version}-confidential
+$ tar --zstd -xvf build/kata-static-rootfs-initrd-confidential.tar.zst -C artifacts ./opt/kata/share/kata-containers/kata-ubuntu-20.04-confidential.initrd
 $ ls artifacts/opt/kata/share/kata-containers/
 kata-ubuntu-20.04-confidential.initrd  vmlinuz-${kernel_version}-confidential
 ```
@@ -190,8 +190,8 @@ can be easily accomplished by issuing the following make target:
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers
 $ mkdir hkd_dir && cp $host_key_document hkd_dir
 $ HKD_PATH=hkd_dir SE_KERNEL_PARAMS="agent.log=debug" make boot-image-se-tarball
-$ ls build/kata-static-boot-image-se.tar.xz
-build/kata-static-boot-image-se.tar.xz
+$ ls build/kata-static-boot-image-se.tar.zst
+build/kata-static-boot-image-se.tar.zst
 ```
 
 `SE_KERNEL_PARAMS` could be used to add any extra kernel parameters. If no additional kernel configuration is required, this can be omitted.
@@ -344,18 +344,18 @@ $ make virtiofsd-tarball
 $ make shim-v2-tarball
 $ mkdir kata-artifacts
 $ build_dir=$(readlink -f build)
-$ cp -r $build_dir/*.tar.xz kata-artifacts
+$ cp -r $build_dir/*.tar.zst kata-artifacts
 $ ls -1 kata-artifacts
-kata-static-agent.tar.xz
-kata-static-boot-image-se.tar.xz
-kata-static-coco-guest-components.tar.xz
-kata-static-kernel-confidential-modules.tar.xz
-kata-static-kernel-confidential.tar.xz
-kata-static-pause-image.tar.xz
-kata-static-qemu.tar.xz
-kata-static-rootfs-initrd-confidential.tar.xz
-kata-static-shim-v2.tar.xz
-kata-static-virtiofsd.tar.xz
+kata-static-agent.tar.zst
+kata-static-boot-image-se.tar.zst
+kata-static-coco-guest-components.tar.zst
+kata-static-kernel-confidential-modules.tar.zst
+kata-static-kernel-confidential.tar.zst
+kata-static-pause-image.tar.zst
+kata-static-qemu.tar.zst
+kata-static-rootfs-initrd-confidential.tar.zst
+kata-static-shim-v2.tar.zst
+kata-static-virtiofsd.tar.zst
 $ ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
 ```
 
@@ -369,7 +369,7 @@ command before running `kata-deploy-merge-builds.sh`:
 $ make rootfs-image-tarball
 ```
 
-At this point, you should have an archive file named `kata-static.tar.xz` at the project root,
+At this point, you should have an archive file named `kata-static.tar.zst` at the project root,
 which will be used to build a payload image. If you are using a local container registry at
 `localhost:5000`, proceed with the following:
 
@@ -381,7 +381,7 @@ Build and push a payload image with the name `localhost:5000/build-kata-deploy`
 `latest` using the following:
 
 ```
-$ ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh kata-static.tar.xz localhost:5000/build-kata-deploy latest
+$ ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh kata-static.tar.zst localhost:5000/build-kata-deploy latest
 ... logs ...
 Pushing the image localhost:5000/build-kata-deploy:latest to the registry
 The push refers to repository [localhost:5000/build-kata-deploy]

docs/how-to/how-to-use-the-kata-agent-policy.md

@@ -32,11 +32,24 @@ Kubernetes users can encode in `base64` format their Policy documents, and add t
 
 ### Encode a Policy file
 
-For example, the [`allow-all-except-exec-process.rego`](../../src/kata-opa/allow-all-except-exec-process.rego) sample policy file is different from the [default Policy](../../src/kata-opa/allow-all.rego) because it rejects any `ExecProcess` requests. You can encode this policy file:
+For example, the [`allow-all-except-exec-process.rego`](../../src/kata-opa/allow-all-except-exec-process.rego) sample policy file is different from the [default Policy](../../src/kata-opa/allow-all.rego) because it rejects any `ExecProcess` requests. To encode this policy file, you need to:
+- Embed the policy inside an init data struct
+- Compress
+- Base64 encode 
+For example:
 
 ```bash
-$ base64 -w 0 allow-all-except-exec-process.rego
-cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEdldE1ldHJpY3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHZXRPT01FdmVudFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEd1ZXN0RGV0YWlsc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IExpc3RJbnRlcmZhY2VzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgTGlzdFJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE1lbUhvdHBsdWdCeVByb2JlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgT25saW5lQ1BVTWVtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUGF1c2VDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBQdWxsSW1hZ2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZWFkU3RyZWFtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlU3RhbGVWaXJ0aW9mc1NoYXJlTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVzZWVkUmFuZG9tRGV2UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVzdW1lQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2V0R3Vlc3REYXRlVGltZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNldFBvbGljeVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNpZ25hbFByb2Nlc3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdGFydENvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0VHJhY2luZ1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXRzQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RvcFRyYWNpbmdSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBUdHlXaW5SZXNpemVSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVFcGhlbWVyYWxNb3VudHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVJbnRlcmZhY2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXYWl0UHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdyaXRlU3RyZWFtUmVxdWVzdCA6PSB0cnVlCgpkZWZhdWx0IEV4ZWNQcm9jZXNzUmVxdWVzdCA6PSBmYWxzZQo=
+$ STRING="$(< allow-all-except-exec-process.rego)"
+$ cat <<EOF | gzip -c | base64 -w0
+version = "0.1.0"
+algorithm = "sha256"
+
+[data]
+"policy.rego" = '''
+$STRING
+'''
+EOF
+H4sIAAAAAAAAA42UTW/TQBCG7/4Vq/QQOCQKQXCo1ENIAkRqiGWnpBJCaGKP7RXrXTM7DnV/PRMiVUh07R582J3H8/XO7AnJa2fVjRrNpm+ms1EEpnSkuarPd76C+bv3oyj6lgPD92jUOKOzbkpYupEA4/E4ulJL13Sky4rVq+y1ms/mb9VWZ+S8K1iM1DgClijRlcBpvLqf3OoMrcfJJkfLutBI12rRQFbhZD6dCRfJ4SeUqOSz/OMSNopyLKA1rBZ5vkjiLyhBj458gr9a9KyubxRTi/9i6W9oQualcR5TzrUNElLZR20waCcExqWzDNoi9WMp2PzoHkLQSi7JdQPUJ+QtMuksWLQQu912fZK+BZHz7QolaRN0c6s9bywjFZBhL5W4lsPEFuvPjhvTlh+6mNwx2MudNdLDZXwnf4SYGFo/3O64NWZTy+SEgAQhT1lECQZKsHan4UgXLGUw+FWTzHjh0woIt661HGxJgh4xT0RoV6/w1IO19XAOKfJFTxmxva6DRQsX/12jIKBLC0Y0Er2DuUutxMM5nak9QaZt2cOwf4En1ww42nN3OK+w14/B4u+a/CWLesHWTYU1Eph+GS/w0470Y/1LcgDNA40/yKOMzw/tE7N+wOx/NwUYj9H5qf4DsX93tO4FAAA=
 ```
 
 ### Attach the Policy to a pod
@@ -49,7 +62,7 @@ kind: Pod
 metadata:
   name: policy-exec-rejected
   annotations:
-    io.katacontainers.config.agent.policy: cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEdldE1ldHJpY3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHZXRPT01FdmVudFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEd1ZXN0RGV0YWlsc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IExpc3RJbnRlcmZhY2VzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgTGlzdFJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE1lbUhvdHBsdWdCeVByb2JlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgT25saW5lQ1BVTWVtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUGF1c2VDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBQdWxsSW1hZ2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZWFkU3RyZWFtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlU3RhbGVWaXJ0aW9mc1NoYXJlTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVzZWVkUmFuZG9tRGV2UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVzdW1lQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2V0R3Vlc3REYXRlVGltZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNldFBvbGljeVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNpZ25hbFByb2Nlc3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdGFydENvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0VHJhY2luZ1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXRzQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RvcFRyYWNpbmdSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBUdHlXaW5SZXNpemVSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVFcGhlbWVyYWxNb3VudHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVJbnRlcmZhY2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXYWl0UHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdyaXRlU3RyZWFtUmVxdWVzdCA6PSB0cnVlCgpkZWZhdWx0IEV4ZWNQcm9jZXNzUmVxdWVzdCA6PSBmYWxzZQo=
+    io.katacontainers.config.hypervisor.cc_init_data: H4sIAAAAAAAAA42UTW/TQBCG7/4Vq/QQOCQKQXCo1ENIAkRqiGWnpBJCaGKP7RXrXTM7DnV/PRMiVUh07R582J3H8/XO7AnJa2fVjRrNpm+ms1EEpnSkuarPd76C+bv3oyj6lgPD92jUOKOzbkpYupEA4/E4ulJL13Sky4rVq+y1ms/mb9VWZ+S8K1iM1DgClijRlcBpvLqf3OoMrcfJJkfLutBI12rRQFbhZD6dCRfJ4SeUqOSz/OMSNopyLKA1rBZ5vkjiLyhBj458gr9a9KyubxRTi/9i6W9oQualcR5TzrUNElLZR20waCcExqWzDNoi9WMp2PzoHkLQSi7JdQPUJ+QtMuksWLQQu912fZK+BZHz7QolaRN0c6s9bywjFZBhL5W4lsPEFuvPjhvTlh+6mNwx2MudNdLDZXwnf4SYGFo/3O64NWZTy+SEgAQhT1lECQZKsHan4UgXLGUw+FWTzHjh0woIt661HGxJgh4xT0RoV6/w1IO19XAOKfJFTxmxva6DRQsX/12jIKBLC0Y0Er2DuUutxMM5nak9QaZt2cOwf4En1ww42nN3OK+w14/B4u+a/CWLesHWTYU1Eph+GS/w0470Y/1LcgDNA40/yKOMzw/tE7N+wOx/NwUYj9H5qf4DsX93tO4FAAA=
 spec:
   runtimeClassName: kata
   containers:
@@ -66,7 +79,7 @@ Create the pod:
 $ kubectl apply -f pod1.yaml
 ```
 
-While creating the Pod sandbox, the Kata Shim will notice the `io.katacontainers.config.agent.policy` annotation and will send the Policy document to the Kata Agent - by sending a `SetPolicy` request. Note that this request will fail if the default Policy, included in the Guest image, doesn't allow this `SetPolicy` request. If the `SetPolicy` request is rejected by the Guest, the Kata Shim will fail to start the Pod sandbox.
+While creating the Pod sandbox, the Kata Shim will notice the `io.katacontainers.config.hypervisor.cc_init_data` annotation and will create the init data device on the host and mount it on the guest as a block device. The agent then reads the init data struct from this device and sets the policy if present.
 
 # How is the Policy being enforced?
 

src/agent/Cargo.lock

@@ -508,6 +508,15 @@ dependencies = [
  "wyz",
 ]
 
+[[package]]
+name = "block-buffer"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -889,6 +898,16 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "crypto-mac"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
+dependencies = [
+ "generic-array",
+ "subtle",
+]
+
 [[package]]
 name = "darling"
 version = "0.14.4"
@@ -958,6 +977,19 @@ dependencies = [
  "syn 2.0.101",
 ]
 
+[[package]]
+name = "dashmap"
+version = "5.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
+dependencies = [
+ "cfg-if",
+ "hashbrown 0.14.5",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core",
+]
+
 [[package]]
 name = "deranged"
 version = "0.4.0"
@@ -1020,13 +1052,22 @@ dependencies = [
  "syn 2.0.101",
 ]
 
+[[package]]
+name = "digest"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
- "block-buffer",
+ "block-buffer 0.10.4",
  "crypto-common",
 ]
 
@@ -1479,6 +1520,12 @@ dependencies = [
  "ahash 0.7.8",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+
 [[package]]
 name = "hashbrown"
 version = "0.15.3"
@@ -1524,6 +1571,16 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
+[[package]]
+name = "hmac"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
+dependencies = [
+ "crypto-mac",
+ "digest 0.9.0",
+]
+
 [[package]]
 name = "home"
 version = "0.5.9"
@@ -1854,6 +1911,17 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "io-uring"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013"
+dependencies = [
+ "bitflags 2.9.0",
+ "cfg-if",
+ "libc",
+]
+
 [[package]]
 name = "iovec"
 version = "0.1.4"
@@ -1996,16 +2064,16 @@ dependencies = [
  "libc",
  "log",
  "logging",
- "mem-agent",
+ "mem-agent-lib",
  "netlink-packet-core",
  "netlink-packet-route",
  "netlink-sys 0.7.0",
- "nix 0.24.3",
+ "nix 0.26.4",
  "oci-spec",
  "opentelemetry",
  "procfs 0.12.0",
  "prometheus",
- "protobuf 3.7.2",
+ "protobuf",
  "protocols",
  "regex",
  "rstest",
@@ -2019,7 +2087,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serial_test",
- "sha2",
+ "sha2 0.10.9",
  "slog",
  "slog-scope",
  "slog-stdlog",
@@ -2069,7 +2137,7 @@ dependencies = [
  "lazy_static",
  "libc",
  "mockall",
- "nix 0.24.3",
+ "nix 0.26.4",
  "oci-spec",
  "once_cell",
  "pci-ids",
@@ -2103,7 +2171,7 @@ dependencies = [
  "serde",
  "serde-enum-str",
  "serde_json",
- "sha2",
+ "sha2 0.10.9",
  "slog",
  "slog-scope",
  "sysinfo",
@@ -2180,6 +2248,23 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a7cbbd4ad467251987c6e5b47d53b11a5a05add08f2447a9e2d70aef1e0d138"
 
+[[package]]
+name = "libsystemd"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f4f0b5b062ba67aa075e331de778082c09e66b5ef32970ea5a1e9c37c9555d1"
+dependencies = [
+ "hmac",
+ "libc",
+ "log",
+ "nix 0.23.2",
+ "once_cell",
+ "serde",
+ "sha2 0.9.9",
+ "thiserror 1.0.69",
+ "uuid 0.8.2",
+]
+
 [[package]]
 name = "libz-sys"
 version = "1.1.22"
@@ -2243,11 +2328,18 @@ dependencies = [
  "serde_json",
  "slog",
  "slog-async",
+ "slog-journald",
  "slog-json",
  "slog-scope",
  "slog-term",
 ]
 
+[[package]]
+name = "maplit"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
+
 [[package]]
 name = "matchers"
 version = "0.0.1"
@@ -2258,14 +2350,14 @@ dependencies = [
 ]
 
 [[package]]
-name = "mem-agent"
-version = "0.1.0"
+name = "mem-agent-lib"
+version = "0.2.0"
 dependencies = [
  "anyhow",
  "async-trait",
  "chrono",
- "lazy_static",
- "nix 0.23.2",
+ "maplit",
+ "nix 0.30.1",
  "page_size",
  "slog",
  "slog-scope",
@@ -2507,6 +2599,18 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nix"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
+dependencies = [
+ "bitflags 2.9.0",
+ "cfg-if",
+ "cfg_aliases",
+ "libc",
+]
+
 [[package]]
 name = "nom"
 version = "7.1.3"
@@ -2686,6 +2790,12 @@ version = "1.21.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 
+[[package]]
+name = "opaque-debug"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
+
 [[package]]
 name = "opentelemetry"
 version = "0.14.0"
@@ -2738,17 +2848,6 @@ version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
 
-[[package]]
-name = "parking_lot"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
-dependencies = [
- "instant",
- "lock_api",
- "parking_lot_core 0.8.6",
-]
-
 [[package]]
 name = "parking_lot"
 version = "0.12.3"
@@ -2756,21 +2855,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
 dependencies = [
  "lock_api",
- "parking_lot_core 0.9.10",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
-dependencies = [
- "cfg-if",
- "instant",
- "libc",
- "redox_syscall 0.2.16",
- "smallvec",
- "winapi",
+ "parking_lot_core",
 ]
 
 [[package]]
@@ -2781,7 +2866,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall 0.5.12",
+ "redox_syscall",
  "smallvec",
  "windows-targets 0.52.6",
 ]
@@ -3072,22 +3157,21 @@ dependencies = [
 
 [[package]]
 name = "procfs"
-version = "0.16.0"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4"
+checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f"
 dependencies = [
  "bitflags 2.9.0",
  "hex",
- "lazy_static",
  "procfs-core",
  "rustix 0.38.44",
 ]
 
 [[package]]
 name = "procfs-core"
-version = "0.16.0"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29"
+checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec"
 dependencies = [
  "bitflags 2.9.0",
  "hex",
@@ -3095,19 +3179,19 @@ dependencies = [
 
 [[package]]
 name = "prometheus"
-version = "0.13.4"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1"
+checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
 dependencies = [
  "cfg-if",
  "fnv",
  "lazy_static",
  "libc",
  "memchr",
- "parking_lot 0.12.3",
- "procfs 0.16.0",
- "protobuf 2.28.0",
- "thiserror 1.0.69",
+ "parking_lot",
+ "procfs 0.17.0",
+ "protobuf",
+ "thiserror 2.0.12",
 ]
 
 [[package]]
@@ -3161,12 +3245,6 @@ dependencies = [
  "prost",
 ]
 
-[[package]]
-name = "protobuf"
-version = "2.28.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
-
 [[package]]
 name = "protobuf"
 version = "3.7.2"
@@ -3178,15 +3256,6 @@ dependencies = [
  "thiserror 1.0.69",
 ]
 
-[[package]]
-name = "protobuf-codegen"
-version = "2.28.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6"
-dependencies = [
- "protobuf 2.28.0",
-]
-
 [[package]]
 name = "protobuf-codegen"
 version = "3.7.2"
@@ -3195,7 +3264,7 @@ checksum = "5d3976825c0014bbd2f3b34f0001876604fe87e0c86cd8fa54251530f1544ace"
 dependencies = [
  "anyhow",
  "once_cell",
- "protobuf 3.7.2",
+ "protobuf",
  "protobuf-parse",
  "regex",
  "tempfile",
@@ -3211,7 +3280,7 @@ dependencies = [
  "anyhow",
  "indexmap 2.9.0",
  "log",
- "protobuf 3.7.2",
+ "protobuf",
  "protobuf-support",
  "tempfile",
  "thiserror 1.0.69",
@@ -3233,7 +3302,7 @@ version = "0.1.0"
 dependencies = [
  "async-trait",
  "oci-spec",
- "protobuf 3.7.2",
+ "protobuf",
  "serde",
  "serde_json",
  "ttrpc",
@@ -3311,15 +3380,6 @@ dependencies = [
  "getrandom 0.2.16",
 ]
 
-[[package]]
-name = "redox_syscall"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
-dependencies = [
- "bitflags 1.3.2",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.5.12"
@@ -3369,7 +3429,7 @@ dependencies = [
  "ahash 0.8.12",
  "fluent-uri 0.3.2",
  "once_cell",
- "parking_lot 0.12.3",
+ "parking_lot",
  "percent-encoding",
  "serde_json",
 ]
@@ -3500,7 +3560,7 @@ dependencies = [
  "rkyv_derive",
  "seahash",
  "tinyvec",
- "uuid",
+ "uuid 1.16.0",
 ]
 
 [[package]]
@@ -3669,10 +3729,10 @@ dependencies = [
  "lazy_static",
  "libc",
  "libseccomp",
- "nix 0.24.3",
+ "nix 0.26.4",
  "oci-spec",
  "path-absolutize",
- "protobuf 3.7.2",
+ "protobuf",
  "protocols",
  "regex",
  "rlimit",
@@ -3882,20 +3942,23 @@ dependencies = [
 
 [[package]]
 name = "serial_test"
-version = "0.5.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0bccbcf40c8938196944a3da0e133e031a33f4d6b72db3bda3cc556e361905d"
+checksum = "1c789ec87f4687d022a2405cf46e0cd6284889f1839de292cadeb6c6019506f2"
 dependencies = [
+ "dashmap",
+ "futures",
  "lazy_static",
- "parking_lot 0.11.2",
+ "log",
+ "parking_lot",
  "serial_test_derive",
 ]
 
 [[package]]
 name = "serial_test_derive"
-version = "0.5.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5"
+checksum = "b64f9e531ce97c88b4778aad0ceee079216071cffec6ac9b904277f8f92e7fe3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3910,7 +3973,20 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
 dependencies = [
  "cfg-if",
  "cpufeatures",
- "digest",
+ "digest 0.10.7",
+]
+
+[[package]]
+name = "sha2"
+version = "0.9.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
+dependencies = [
+ "block-buffer 0.9.0",
+ "cfg-if",
+ "cpufeatures",
+ "digest 0.9.0",
+ "opaque-debug",
 ]
 
 [[package]]
@@ -3921,7 +3997,7 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
 dependencies = [
  "cfg-if",
  "cpufeatures",
- "digest",
+ "digest 0.10.7",
 ]
 
 [[package]]
@@ -3962,12 +4038,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
 
 [[package]]
 name = "slab"
-version = "0.4.9"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
-dependencies = [
- "autocfg",
-]
+checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
 
 [[package]]
 name = "slash-formatter"
@@ -3993,6 +4066,16 @@ dependencies = [
  "thread_local",
 ]
 
+[[package]]
+name = "slog-journald"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83e14eb8c2f5d0c8fc9fbac40e6391095e4dc5cb334f7dce99c75cb1919eb39c"
+dependencies = [
+ "libsystemd",
+ "slog",
+]
+
 [[package]]
 name = "slog-json"
 version = "2.6.1"
@@ -4132,6 +4215,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "subtle"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
+
 [[package]]
 name = "syn"
 version = "1.0.109"
@@ -4233,7 +4322,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
 name = "test-utils"
 version = "0.1.0"
 dependencies = [
- "nix 0.24.3",
+ "nix 0.26.4",
 ]
 
 [[package]]
@@ -4344,17 +4433,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.45.0"
+version = "1.46.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165"
+checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17"
 dependencies = [
  "backtrace",
  "bytes 1.10.1",
+ "io-uring",
  "libc",
  "mio",
- "parking_lot 0.12.3",
+ "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
+ "slab",
  "socket2 0.5.9",
  "tokio-macros",
  "windows-sys 0.52.0",
@@ -4580,8 +4671,8 @@ dependencies = [
  "libc",
  "log",
  "nix 0.26.4",
- "protobuf 3.7.2",
- "protobuf-codegen 3.7.2",
+ "protobuf",
+ "protobuf-codegen",
  "thiserror 1.0.69",
  "tokio",
  "tokio-vsock 0.4.0",
@@ -4590,30 +4681,28 @@ dependencies = [
 
 [[package]]
 name = "ttrpc-codegen"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdc0529f65223eca94fc5830e7d552d0d152ff42b74aff5c641edac39592f41f"
+checksum = "0e5c657ef5cea6f6c6073c1be0787ba4482f42a569d4821e467daec795271f86"
 dependencies = [
- "home",
- "protobuf 2.28.0",
- "protobuf-codegen 3.7.2",
+ "protobuf",
+ "protobuf-codegen",
  "protobuf-support",
  "ttrpc-compiler",
 ]
 
 [[package]]
 name = "ttrpc-compiler"
-version = "0.7.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be3fb2fe509cb9c0099b3b5551b759ae714f2dde56dfc713f2a5bda8c16064a"
+checksum = "3aa71f4a44711b3b9cc10ed0c7e239ff0fe4b8e6c900a142fb3bb26401385718"
 dependencies = [
  "derive-new",
- "home",
  "prost",
  "prost-build",
  "prost-types",
- "protobuf 2.28.0",
- "protobuf-codegen 2.28.0",
+ "protobuf",
+ "protobuf-codegen",
  "tempfile",
 ]
 
@@ -4693,6 +4782,15 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
+[[package]]
+name = "uuid"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "uuid"
 version = "1.16.0"
@@ -4706,7 +4804,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8"
 dependencies = [
  "outref",
- "uuid",
+ "uuid 1.16.0",
  "vsimd",
 ]
 
@@ -4930,7 +5028,7 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]

src/agent/Cargo.toml

@@ -13,13 +13,14 @@ lazy_static = "1.3.0"
 ttrpc = { version = "0.8.4", features = ["async"], default-features = false }
 protobuf = "3.7.2"
 libc = "0.2.94"
-nix = "0.24.2"
+# Notes: nix needs to stay in sync with libs
+nix = "0.26.4"
 capctl = "0.2.0"
 scan_fmt = "0.2.6"
 scopeguard = "1.0.0"
 thiserror = "1.0.26"
 regex = "1.10.5"
-serial_test = "0.5.1"
+serial_test = "0.10.0"
 url = "2.5.0"
 derivative = "2.2.0"
 const_format = "0.2.30"
@@ -30,7 +31,7 @@ async-recursion = "0.3.2"
 futures = "0.3.30"
 
 # Async runtime
-tokio = { version = "1.44.2", features = ["full"] }
+tokio = { version = "1.46.1", features = ["full"] }
 tokio-vsock = "0.3.4"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket"] }
@@ -49,7 +50,7 @@ slog-stdlog = "4.0.0"
 log = "0.4.11"
 
 cfg-if = "1.0.0"
-prometheus = { version = "0.13.0", features = ["process"] }
+prometheus = { version = "0.14.0", features = ["process"] }
 procfs = "0.12.0"
 
 anyhow = "1"

src/agent/Makefile

@@ -217,4 +217,11 @@ codecov-html: check_tarpaulin
 
 ##TARGET generate-protocols: generate/update grpc agent protocols
 generate-protocols:
+	image=$$(docker build -q \
+	  --build-arg GO_VERSION=$$(yq '.languages.golang.version' $(CURDIR)/../../versions.yaml) \
+	  --build-arg PROTOC_VERSION=$$(yq '.externals.protoc.version' $(CURDIR)/../../versions.yaml | grep -oE "[0-9.]+") \
+	  --build-arg PROTOC_GEN_GO_VERSION=$$(yq '.externals.protoc-gen-go.version' $(CURDIR)/../../versions.yaml) \
+	  --build-arg TTRPC_VERSION=$$(yq '.externals.ttrpc.version' $(CURDIR)/../../versions.yaml) \
+	  $(CURDIR)/../../tools/packaging/static-build/codegen) && \
+	docker run --rm --workdir /kata/src/agent -v $(CURDIR)/../..:/kata --user $(shell id -u) $$image \
 	../libs/protocols/hack/update-generated-proto.sh all

src/agent/rustjail/src/container.rs

@@ -32,6 +32,7 @@ use crate::cgroups::{DevicesCgroupInfo, Manager};
 use crate::console;
 use crate::log_child;
 use crate::process::Process;
+use crate::process::ProcessOperations;
 #[cfg(feature = "seccomp")]
 use crate::seccomp;
 use crate::selinux;
@@ -261,7 +262,7 @@ pub struct LinuxContainer {
     pub init_process_start_time: u64,
     pub uid_map_path: String,
     pub gid_map_path: String,
-    pub processes: HashMap<pid_t, Process>,
+    pub processes: HashMap<String, Process>,
     pub status: ContainerStatus,
     pub created: SystemTime,
     pub logger: Logger,
@@ -933,17 +934,13 @@ impl BaseContainer for LinuxContainer {
     }
 
     fn processes(&self) -> Result<Vec<i32>> {
-        Ok(self.processes.keys().cloned().collect())
+        Ok(self.processes.values().map(|p| p.pid).collect())
     }
 
     fn get_process(&mut self, eid: &str) -> Result<&mut Process> {
-        for (_, v) in self.processes.iter_mut() {
-            if eid == v.exec_id.as_str() {
-                return Ok(v);
-            }
-        }
-
-        Err(anyhow!("invalid eid {}", eid))
+        self.processes
+            .get_mut(eid)
+            .ok_or_else(|| anyhow!("invalid eid {}", eid))
     }
 
     fn stats(&self) -> Result<StatsContainerResponse> {
@@ -967,6 +964,12 @@ impl BaseContainer for LinuxContainer {
 
     async fn start(&mut self, mut p: Process) -> Result<()> {
         let logger = self.logger.new(o!("eid" => p.exec_id.clone()));
+
+        // Check if exec_id is already in use to prevent collisions
+        if self.processes.contains_key(p.exec_id.as_str()) {
+            return Err(anyhow!("exec_id '{}' already exists", p.exec_id));
+        }
+
         let tty = p.tty;
         let fifo_file = format!("{}/{}", &self.root, EXEC_FIFO_FILENAME);
         info!(logger, "enter container.start!");
@@ -1235,7 +1238,7 @@ impl BaseContainer for LinuxContainer {
             let spec = self.config.spec.as_mut().unwrap();
             update_namespaces(&self.logger, spec, p.pid)?;
         }
-        self.processes.insert(p.pid, p);
+        self.processes.insert(p.exec_id.clone(), p);
 
         info!(logger, "wait on child log handler");
         let _ = log_handler
@@ -1261,13 +1264,13 @@ impl BaseContainer for LinuxContainer {
         let spec = self.config.spec.as_ref().unwrap();
         let st = self.oci_state()?;
 
-        for pid in self.processes.keys() {
-            match signal::kill(Pid::from_raw(*pid), Some(Signal::SIGKILL)) {
+        for process in self.processes.values() {
+            match signal::kill(process.pid(), Some(Signal::SIGKILL)) {
                 Err(Errno::ESRCH) => {
                     info!(
                         self.logger,
                         "kill encounters ESRCH, pid: {}, container: {}",
-                        pid,
+                        process.pid(),
                         self.id.clone()
                     );
                     continue;
@@ -2084,10 +2087,11 @@ mod tests {
     #[tokio::test]
     async fn test_linuxcontainer_get_process() {
         let _ = new_linux_container_and_then(|mut c: LinuxContainer| {
-            c.processes.insert(
-                1,
-                Process::new(&sl(), &oci::Process::default(), "123", true, 1, None).unwrap(),
-            );
+            let process =
+                Process::new(&sl(), &oci::Process::default(), "123", true, 1, None).unwrap();
+            let exec_id = process.exec_id.clone();
+            c.processes.insert(exec_id, process);
+
             let p = c.get_process("123");
             assert!(p.is_ok(), "Expecting Ok, Got {:?}", p);
             Ok(())

src/agent/src/confidential_data_hub/mod.rs

@@ -22,6 +22,8 @@ use protocols::{
 };
 use safe_path::scoped_join;
 use std::fs;
+use std::fs::File;
+use std::io::{self, Read};
 use std::path::Path;
 use std::{os::unix::fs::symlink, path::PathBuf};
 use tokio::sync::OnceCell;
@@ -235,8 +237,8 @@ pub async fn unseal_file(path: &str) -> Result<()> {
         }
 
         let secret_name = entry.file_name();
-        let contents = fs::read_to_string(&target_path)?;
-        if contents.starts_with(SEALED_SECRET_PREFIX) {
+        if content_starts_with_prefix(&target_path, SEALED_SECRET_PREFIX).await? {
+            let contents = fs::read_to_string(&target_path)?;
             // Get the directory name of the sealed secret file
             let dir_name = target_path
                 .parent()
@@ -262,6 +264,17 @@ pub async fn unseal_file(path: &str) -> Result<()> {
     Ok(())
 }
 
+pub async fn content_starts_with_prefix(path: &Path, prefix: &str) -> io::Result<bool> {
+    let mut file = File::open(path)?;
+    let mut buffer = vec![0u8; prefix.len()];
+
+    match file.read_exact(&mut buffer) {
+        Ok(()) => Ok(buffer == prefix.as_bytes()),
+        Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => Ok(false),
+        Err(e) => Err(e),
+    }
+}
+
 pub async fn secure_mount(
     volume_type: &str,
     options: &std::collections::HashMap<String, String>,
@@ -294,7 +307,7 @@ mod tests {
     use std::fs::File;
     use std::io::{Read, Write};
     use std::sync::Arc;
-    use tempfile::tempdir;
+    use tempfile::{tempdir, NamedTempFile};
     use test_utils::skip_if_not_root;
     use tokio::signal::unix::{signal, SignalKind};
     struct TestService;
@@ -416,4 +429,34 @@ mod tests {
         rt.shutdown_background();
         std::thread::sleep(std::time::Duration::from_secs(2));
     }
+
+    #[tokio::test]
+    async fn test_content_starts_with_prefix() {
+        // Normal case: content matches the prefix
+        let mut f = NamedTempFile::new().unwrap();
+        write!(f, "sealed.hello_world").unwrap();
+        assert!(content_starts_with_prefix(f.path(), "sealed.")
+            .await
+            .unwrap());
+
+        // Does not match the prefix
+        let mut f2 = NamedTempFile::new().unwrap();
+        write!(f2, "notsealed.hello_world").unwrap();
+        assert!(!content_starts_with_prefix(f2.path(), "sealed.")
+            .await
+            .unwrap());
+
+        // File length < prefix.len()
+        let mut f3 = NamedTempFile::new().unwrap();
+        write!(f3, "seal").unwrap();
+        assert!(!content_starts_with_prefix(f3.path(), "sealed.")
+            .await
+            .unwrap());
+
+        // Empty file
+        let f4 = NamedTempFile::new().unwrap();
+        assert!(!content_starts_with_prefix(f4.path(), "sealed.")
+            .await
+            .unwrap());
+    }
 }

src/agent/src/config.rs

@@ -202,7 +202,7 @@ macro_rules! config_override {
         }
     };
 
-    ($builder:ident, $config:ident, $field:ident, $func: ident) => {
+    ($builder:ident, $config:ident, $field:ident, $func:ident) => {
         if let Some(v) = $builder.$field {
             $config.$field = $func(&v)?;
         }
@@ -661,8 +661,8 @@ impl AgentConfig {
             self.server_addr = addr;
         }
 
-        if let Ok(addr) = env::var(LOG_LEVEL_ENV_VAR) {
-            if let Ok(level) = logrus_to_slog_level(&addr) {
+        if let Ok(level) = env::var(LOG_LEVEL_ENV_VAR) {
+            if let Ok(level) = logrus_to_slog_level(&level) {
                 self.log_level = level;
             }
         }

src/agent/src/main.rs

@@ -30,6 +30,7 @@ use nix::unistd::{self, dup, sync, Pid};
 use std::env;
 use std::ffi::OsStr;
 use std::fs::{self, File};
+use std::io::ErrorKind;
 use std::os::unix::fs::{self as unixfs, FileTypeExt};
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
@@ -465,8 +466,17 @@ fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs)
         _ => vec![],
     };
     for binary in binaries.iter() {
-        if !Path::new(binary).exists() {
-            warn!(logger, "{} not found", binary);
+        let exists = Path::new(binary)
+            .try_exists()
+            .unwrap_or_else(|error| match error.kind() {
+                ErrorKind::NotFound => {
+                    warn!(logger, "{} not found", binary);
+                    false
+                }
+                _ => panic!("Path existence check failed for '{}': {}", binary, error),
+            });
+
+        if !exists {
             return false;
         }
     }

src/agent/src/rpc.rs

@@ -554,7 +554,7 @@ impl AgentService {
         req: protocols::agent::WaitProcessRequest,
     ) -> Result<protocols::agent::WaitProcessResponse> {
         let cid = req.container_id;
-        let eid = req.exec_id;
+        let mut eid = req.exec_id;
         let mut resp = WaitProcessResponse::new();
 
         info!(
@@ -587,7 +587,7 @@ impl AgentService {
             .get_container(&cid)
             .ok_or_else(|| anyhow!("Invalid container id"))?;
 
-        let p = match ctr.processes.get_mut(&pid) {
+        let p = match ctr.processes.values_mut().find(|p| p.pid == pid) {
             Some(p) => p,
             None => {
                 // Lost race, pick up exit code from channel
@@ -600,6 +600,8 @@ impl AgentService {
             }
         };
 
+        eid = p.exec_id.clone();
+
         // need to close all fd
         // ignore errors for some fd might be closed by stream
         p.cleanup_process_stream();
@@ -611,7 +613,7 @@ impl AgentService {
             let _ = s.send(p.exit_code).await;
         }
 
-        ctr.processes.remove(&pid);
+        ctr.processes.remove(&eid);
 
         Ok(resp)
     }
@@ -2670,7 +2672,7 @@ mod tests {
                 }
                 linux_container
                     .processes
-                    .insert(exec_process_id, exec_process);
+                    .insert(exec_process.exec_id.clone(), exec_process);
 
                 sandbox.add_container(linux_container);
             }

src/agent/src/sandbox.rs

@@ -272,8 +272,10 @@ impl Sandbox {
 
     pub fn find_process(&mut self, pid: pid_t) -> Option<&mut Process> {
         for (_, c) in self.containers.iter_mut() {
-            if let Some(p) = c.processes.get_mut(&pid) {
-                return Some(p);
+            for p in c.processes.values_mut() {
+                if p.pid == pid {
+                    return Some(p);
+                }
             }
         }
 
@@ -286,9 +288,11 @@ impl Sandbox {
             .ok_or_else(|| anyhow!(ERR_INVALID_CONTAINER_ID))?;
 
         if eid.is_empty() {
+            let init_pid = ctr.init_process_pid;
             return ctr
                 .processes
-                .get_mut(&ctr.init_process_pid)
+                .values_mut()
+                .find(|p| p.pid == init_pid)
                 .ok_or_else(|| anyhow!("cannot find init process!"));
         }
 
@@ -1014,23 +1018,26 @@ mod tests {
         linux_container.init_process_pid = 1;
         linux_container.id = cid.to_string();
         // add init process
-        linux_container.processes.insert(
-            1,
-            Process::new(&logger, &oci::Process::default(), "1", true, 1, None).unwrap(),
-        );
+        let mut init_process =
+            Process::new(&logger, &oci::Process::default(), "1", true, 1, None).unwrap();
+        init_process.pid = 1;
+        linux_container
+            .processes
+            .insert("1".to_string(), init_process);
         // add exec process
-        linux_container.processes.insert(
-            123,
-            Process::new(
-                &logger,
-                &oci::Process::default(),
-                "exec-123",
-                false,
-                1,
-                None,
-            )
-            .unwrap(),
-        );
+        let mut exec_process = Process::new(
+            &logger,
+            &oci::Process::default(),
+            "exec-123",
+            false,
+            1,
+            None,
+        )
+        .unwrap();
+        exec_process.pid = 123;
+        linux_container
+            .processes
+            .insert("exec-123".to_string(), exec_process);
 
         s.add_container(linux_container);
 
@@ -1081,8 +1088,8 @@ mod tests {
             .unwrap();
             // processes interally only have pids when manually set
             test_process.pid = test_pid;
-
-            linux_container.processes.insert(test_pid, test_process);
+            let test_exec_id = test_process.exec_id.clone();
+            linux_container.processes.insert(test_exec_id, test_process);
 
             s.add_container(linux_container);
 

src/dragonball/Cargo.lock

@@ -143,7 +143,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b"
 dependencies = [
  "libc",
- "thiserror",
+ "thiserror 1.0.48",
 ]
 
 [[package]]
@@ -266,7 +266,7 @@ dependencies = [
  "lazy_static",
  "libc",
  "nix 0.23.2",
- "thiserror",
+ "thiserror 1.0.48",
  "vm-memory",
  "vmm-sys-util",
 ]
@@ -275,7 +275,7 @@ dependencies = [
 name = "dbs-allocator"
 version = "0.1.1"
 dependencies = [
- "thiserror",
+ "thiserror 1.0.48",
 ]
 
 [[package]]
@@ -285,8 +285,8 @@ dependencies = [
  "kvm-bindings",
  "kvm-ioctls",
  "libc",
- "memoffset",
- "thiserror",
+ "memoffset 0.6.5",
+ "thiserror 1.0.48",
  "vm-memory",
  "vmm-sys-util",
 ]
@@ -302,7 +302,7 @@ dependencies = [
  "kvm-ioctls",
  "lazy_static",
  "libc",
- "thiserror",
+ "thiserror 1.0.48",
  "vm-fdt",
  "vm-memory",
 ]
@@ -311,7 +311,7 @@ dependencies = [
 name = "dbs-device"
 version = "0.2.0"
 dependencies = [
- "thiserror",
+ "thiserror 1.0.48",
 ]
 
 [[package]]
@@ -354,7 +354,7 @@ dependencies = [
  "kvm-ioctls",
  "libc",
  "log",
- "thiserror",
+ "thiserror 1.0.48",
  "vfio-bindings",
  "vfio-ioctls",
  "vm-memory",
@@ -366,7 +366,7 @@ version = "0.1.0"
 dependencies = [
  "kvm-bindings",
  "serde_json",
- "thiserror",
+ "thiserror 1.0.48",
  "vmm-sys-util",
 ]
 
@@ -378,7 +378,7 @@ dependencies = [
  "dbs-utils",
  "dbs-virtio-devices",
  "log",
- "thiserror",
+ "thiserror 1.0.48",
  "timerfd",
 ]
 
@@ -392,7 +392,7 @@ dependencies = [
  "log",
  "serde",
  "serde_json",
- "thiserror",
+ "thiserror 1.0.48",
  "timerfd",
  "vmm-sys-util",
 ]
@@ -423,7 +423,7 @@ dependencies = [
  "sendfd",
  "serde",
  "serde_json",
- "thiserror",
+ "thiserror 1.0.48",
  "threadpool",
  "timerfd",
  "vhost",
@@ -491,6 +491,17 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "displaydoc"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+]
+
 [[package]]
 name = "downcast-rs"
 version = "1.2.0"
@@ -536,7 +547,7 @@ dependencies = [
  "slog-scope",
  "slog-term",
  "test-utils",
- "thiserror",
+ "thiserror 1.0.48",
  "tracing",
  "vfio-bindings",
  "vfio-ioctls",
@@ -653,9 +664,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 
 [[package]]
 name = "form_urlencoded"
-version = "1.2.0"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652"
+checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
 dependencies = [
  "percent-encoding",
 ]
@@ -914,13 +925,110 @@ dependencies = [
 ]
 
 [[package]]
-name = "idna"
-version = "0.4.0"
+name = "icu_collections"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c"
+checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47"
 dependencies = [
- "unicode-bidi",
- "unicode-normalization",
+ "displaydoc",
+ "potential_utf",
+ "yoke",
+ "zerofrom",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_locale_core"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a"
+dependencies = [
+ "displaydoc",
+ "litemap",
+ "tinystr",
+ "writeable",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_normalizer"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979"
+dependencies = [
+ "displaydoc",
+ "icu_collections",
+ "icu_normalizer_data",
+ "icu_properties",
+ "icu_provider",
+ "smallvec",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_normalizer_data"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3"
+
+[[package]]
+name = "icu_properties"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b"
+dependencies = [
+ "displaydoc",
+ "icu_collections",
+ "icu_locale_core",
+ "icu_properties_data",
+ "icu_provider",
+ "potential_utf",
+ "zerotrie",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_properties_data"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632"
+
+[[package]]
+name = "icu_provider"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af"
+dependencies = [
+ "displaydoc",
+ "icu_locale_core",
+ "stable_deref_trait",
+ "tinystr",
+ "writeable",
+ "yoke",
+ "zerofrom",
+ "zerotrie",
+ "zerovec",
+]
+
+[[package]]
+name = "idna"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
+dependencies = [
+ "idna_adapter",
+ "smallvec",
+ "utf8_iter",
+]
+
+[[package]]
+name = "idna_adapter"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344"
+dependencies = [
+ "icu_normalizer",
+ "icu_properties",
 ]
 
 [[package]]
@@ -1050,12 +1158,6 @@ dependencies = [
  "vm-memory",
 ]
 
-[[package]]
-name = "linux-raw-sys"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.3.8"
@@ -1068,6 +1170,12 @@ version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
 
+[[package]]
+name = "litemap"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
+
 [[package]]
 name = "lock_api"
 version = "0.4.10"
@@ -1119,6 +1227,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -1185,7 +1302,7 @@ dependencies = [
  "cc",
  "cfg-if",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
 ]
 
 [[package]]
@@ -1197,7 +1314,20 @@ dependencies = [
  "bitflags 1.3.2",
  "cfg-if",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
+]
+
+[[package]]
+name = "nix"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "libc",
+ "memoffset 0.7.1",
+ "pin-utils",
 ]
 
 [[package]]
@@ -1331,9 +1461,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "openssl"
-version = "0.10.72"
+version = "0.10.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da"
+checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8"
 dependencies = [
  "bitflags 2.4.0",
  "cfg-if",
@@ -1352,7 +1482,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.32",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -1372,9 +1502,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.108"
+version = "0.9.109"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e145e1651e858e820e4860f7b9c5e169bc1d8ce1c86043be79fa7b7634821847"
+checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571"
 dependencies = [
  "cc",
  "libc",
@@ -1408,9 +1538,9 @@ dependencies = [
 
 [[package]]
 name = "percent-encoding"
-version = "2.3.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
+checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
 [[package]]
 name = "pin-project-lite"
@@ -1430,6 +1560,15 @@ version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
+[[package]]
+name = "potential_utf"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585"
+dependencies = [
+ "zerovec",
+]
+
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -1438,9 +1577,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.66"
+version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
+checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
 dependencies = [
  "unicode-ident",
 ]
@@ -1462,22 +1601,31 @@ dependencies = [
 
 [[package]]
 name = "procfs"
-version = "0.14.2"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1de8dacb0873f77e6aefc6d71e044761fcc68060290f5b1089fcdf84626bb69"
+checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f"
 dependencies = [
- "bitflags 1.3.2",
- "byteorder",
+ "bitflags 2.4.0",
+ "hex",
+ "procfs-core",
+ "rustix 0.38.25",
+]
+
+[[package]]
+name = "procfs-core"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec"
+dependencies = [
+ "bitflags 2.4.0",
  "hex",
- "lazy_static",
- "rustix 0.36.17",
 ]
 
 [[package]]
 name = "prometheus"
-version = "0.13.3"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"
+checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
 dependencies = [
  "cfg-if",
  "fnv",
@@ -1485,22 +1633,36 @@ dependencies = [
  "libc",
  "memchr",
  "parking_lot",
- "procfs 0.14.2",
+ "procfs 0.17.0",
  "protobuf",
- "thiserror",
+ "thiserror 2.0.12",
 ]
 
 [[package]]
 name = "protobuf"
-version = "2.28.0"
+version = "3.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
+checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
+dependencies = [
+ "once_cell",
+ "protobuf-support",
+ "thiserror 1.0.48",
+]
+
+[[package]]
+name = "protobuf-support"
+version = "3.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
+dependencies = [
+ "thiserror 1.0.48",
+]
 
 [[package]]
 name = "quote"
-version = "1.0.33"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
 dependencies = [
  "proc-macro2",
 ]
@@ -1537,7 +1699,7 @@ checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
 dependencies = [
  "getrandom 0.2.10",
  "redox_syscall 0.2.16",
- "thiserror",
+ "thiserror 1.0.48",
 ]
 
 [[package]]
@@ -1592,20 +1754,6 @@ version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
-[[package]]
-name = "rustix"
-version = "0.36.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "305efbd14fde4139eb501df5f136994bb520b033fa9fbdce287507dc23b8c7ed"
-dependencies = [
- "bitflags 1.3.2",
- "errno",
- "io-lifetimes",
- "libc",
- "linux-raw-sys 0.1.4",
- "windows-sys 0.45.0",
-]
-
 [[package]]
 name = "rustix"
 version = "0.37.28"
@@ -1718,7 +1866,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.32",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -1774,12 +1922,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "slab"
-version = "0.4.9"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
-dependencies = [
- "autocfg",
-]
+checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
 
 [[package]]
 name = "slog"
@@ -1825,9 +1970,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.11.0"
+version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
+checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 
 [[package]]
 name = "socket2"
@@ -1839,6 +1984,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
 [[package]]
 name = "subtle"
 version = "2.5.0"
@@ -1858,15 +2009,26 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.32"
+version = "2.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2"
+checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
 dependencies = [
  "proc-macro2",
  "quote",
  "unicode-ident",
 ]
 
+[[package]]
+name = "synstructure"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"
@@ -1912,7 +2074,7 @@ dependencies = [
 name = "test-utils"
 version = "0.1.0"
 dependencies = [
- "nix 0.24.3",
+ "nix 0.26.4",
 ]
 
 [[package]]
@@ -1921,7 +2083,16 @@ version = "1.0.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7"
 dependencies = [
- "thiserror-impl",
+ "thiserror-impl 1.0.48",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
+dependencies = [
+ "thiserror-impl 2.0.12",
 ]
 
 [[package]]
@@ -1932,7 +2103,18 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.32",
+ "syn 2.0.104",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -1995,20 +2177,15 @@ dependencies = [
 ]
 
 [[package]]
-name = "tinyvec"
-version = "1.6.0"
+name = "tinystr"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b"
 dependencies = [
- "tinyvec_macros",
+ "displaydoc",
+ "zerovec",
 ]
 
-[[package]]
-name = "tinyvec_macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
-
 [[package]]
 name = "tokio"
 version = "1.44.2"
@@ -2033,7 +2210,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.32",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -2094,7 +2271,7 @@ checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.32",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -2118,38 +2295,29 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 
-[[package]]
-name = "unicode-bidi"
-version = "0.3.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
-
 [[package]]
 name = "unicode-ident"
 version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
 
-[[package]]
-name = "unicode-normalization"
-version = "0.1.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
-dependencies = [
- "tinyvec",
-]
-
 [[package]]
 name = "url"
-version = "2.4.1"
+version = "2.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5"
+checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
 dependencies = [
  "form_urlencoded",
  "idna",
  "percent-encoding",
 ]
 
+[[package]]
+name = "utf8_iter"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
@@ -2179,7 +2347,7 @@ dependencies = [
  "kvm-ioctls",
  "libc",
  "log",
- "thiserror",
+ "thiserror 2.0.12",
  "vfio-bindings",
  "vm-memory",
  "vmm-sys-util",
@@ -2293,7 +2461,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.32",
+ "syn 2.0.104",
  "wasm-bindgen-shared",
 ]
 
@@ -2327,7 +2495,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.32",
+ "syn 2.0.104",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -2379,15 +2547,6 @@ dependencies = [
  "windows-targets 0.48.5",
 ]
 
-[[package]]
-name = "windows-sys"
-version = "0.45.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
-dependencies = [
- "windows-targets 0.42.2",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
@@ -2406,21 +2565,6 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
-[[package]]
-name = "windows-targets"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
-dependencies = [
- "windows_aarch64_gnullvm 0.42.2",
- "windows_aarch64_msvc 0.42.2",
- "windows_i686_gnu 0.42.2",
- "windows_i686_msvc 0.42.2",
- "windows_x86_64_gnu 0.42.2",
- "windows_x86_64_gnullvm 0.42.2",
- "windows_x86_64_msvc 0.42.2",
-]
-
 [[package]]
 name = "windows-targets"
 version = "0.48.5"
@@ -2452,12 +2596,6 @@ dependencies = [
  "windows_x86_64_msvc 0.52.6",
 ]
 
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
-
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.48.5"
@@ -2470,12 +2608,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
-
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.48.5"
@@ -2488,12 +2620,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
-[[package]]
-name = "windows_i686_gnu"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
-
 [[package]]
 name = "windows_i686_gnu"
 version = "0.48.5"
@@ -2512,12 +2638,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
-[[package]]
-name = "windows_i686_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
-
 [[package]]
 name = "windows_i686_msvc"
 version = "0.48.5"
@@ -2530,12 +2650,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
-
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.48.5"
@@ -2548,12 +2662,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
-
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.48.5"
@@ -2566,12 +2674,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
-
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.48.5"
@@ -2603,6 +2705,12 @@ dependencies = [
  "bitflags 2.4.0",
 ]
 
+[[package]]
+name = "writeable"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"
+
 [[package]]
 name = "xattr"
 version = "1.0.1"
@@ -2612,6 +2720,84 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "yoke"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc"
+dependencies = [
+ "serde",
+ "stable_deref_trait",
+ "yoke-derive",
+ "zerofrom",
+]
+
+[[package]]
+name = "yoke-derive"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+ "synstructure",
+]
+
+[[package]]
+name = "zerofrom"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
+dependencies = [
+ "zerofrom-derive",
+]
+
+[[package]]
+name = "zerofrom-derive"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+ "synstructure",
+]
+
+[[package]]
+name = "zerotrie"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595"
+dependencies = [
+ "displaydoc",
+ "yoke",
+ "zerofrom",
+]
+
+[[package]]
+name = "zerovec"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428"
+dependencies = [
+ "yoke",
+ "zerofrom",
+ "zerovec-derive",
+]
+
+[[package]]
+name = "zerovec-derive"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+]
+
 [[package]]
 name = "zstd"
 version = "0.11.2+zstd.1.5.2"

src/dragonball/Cargo.toml

@@ -87,7 +87,7 @@ linux-loader = {workspace = true}
 log = "0.4.14"
 nix = "0.24.2"
 procfs = "0.12.0"
-prometheus = { version = "0.13.0", features = ["process"] }
+prometheus = { version = "0.14.0", features = ["process"] }
 seccompiler = {workspace = true}
 serde = "1.0.27"
 serde_derive = "1.0.27"

src/dragonball/dbs_device/src/device_manager.rs

@@ -102,7 +102,7 @@ use crate::resources::Resource;
 use crate::{DeviceIo, IoAddress, IoSize, PioAddress};
 
 /// Error types for `IoManager` related operations.
-#[derive(Error, Debug)]
+#[derive(Error, Debug, PartialEq)]
 pub enum Error {
     /// The inserting device overlaps with a current device.
     #[error("device address conflicts with existing devices")]

src/dragonball/dbs_interrupt/src/notifier.rs

@@ -5,7 +5,9 @@
 
 use std::any::Any;
 use std::io::Error;
+use std::sync::atomic::{AtomicU16, Ordering};
 use std::sync::Arc;
+use std::sync::Mutex;
 
 use vmm_sys_util::eventfd::EventFd;
 
@@ -121,6 +123,77 @@ mod msi {
     }
 }
 
+/// Vector value used to disable MSI for a queue.
+pub const VIRTQ_MSI_NO_VECTOR: u16 = 0xffff;
+
+#[derive(Clone, PartialEq, Debug, Copy)]
+pub enum VirtioInterruptType {
+    Config,
+    Queue(u16),
+}
+
+#[derive(Clone)]
+pub struct VirtioNotifierMsix {
+    pub(crate) config_vector: Arc<AtomicU16>,
+    pub(crate) queues_vectors: Arc<Mutex<Vec<u16>>>,
+    pub(crate) interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+    pub(crate) interrupt_type: VirtioInterruptType,
+}
+
+impl VirtioNotifierMsix {
+    pub fn new(
+        config_vector: Arc<AtomicU16>,
+        queues_vectors: Arc<Mutex<Vec<u16>>>,
+        interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+        interrupt_type: VirtioInterruptType,
+    ) -> Self {
+        VirtioNotifierMsix {
+            config_vector,
+            queues_vectors,
+            interrupt_source_group,
+            interrupt_type,
+        }
+    }
+}
+
+impl InterruptNotifier for VirtioNotifierMsix {
+    fn notify(&self) -> std::result::Result<(), std::io::Error> {
+        let vector = match self.interrupt_type {
+            VirtioInterruptType::Config => self.config_vector.load(Ordering::Acquire),
+            VirtioInterruptType::Queue(queue_index) => {
+                self.queues_vectors.lock().unwrap()[queue_index as usize]
+            }
+        };
+        if vector == VIRTQ_MSI_NO_VECTOR {
+            return Ok(());
+        }
+
+        self.interrupt_source_group
+            .trigger(vector as InterruptIndex)
+    }
+    fn notifier(&self) -> Option<&EventFd> {
+        let vector = match self.interrupt_type {
+            VirtioInterruptType::Config => self.config_vector.load(Ordering::Acquire),
+            VirtioInterruptType::Queue(queue_index) => {
+                self.queues_vectors.lock().unwrap()[queue_index as usize]
+            }
+        };
+        if vector == VIRTQ_MSI_NO_VECTOR {
+            return None;
+        }
+
+        self.interrupt_source_group
+            .notifier(vector as InterruptIndex)
+    }
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn clone_boxed(&self) -> Box<dyn InterruptNotifier> {
+        Box::new(self.clone())
+    }
+}
+
 /// Struct to discard interrupts.
 #[derive(Copy, Clone, Debug, Default)]
 pub struct NoopNotifier {}

src/dragonball/dbs_pci/Cargo.toml

@@ -16,6 +16,8 @@ thiserror = "1"
 dbs-allocator = { workspace = true }
 dbs-boot = { workspace = true }
 dbs-device = { workspace = true }
+dbs-address-space = { workspace = true }
+dbs-virtio-devices = { workspace = true }
 dbs-interrupt = { workspace = true, features = [
     "kvm-irq",
     "kvm-legacy-irq",
@@ -23,12 +25,18 @@ dbs-interrupt = { workspace = true, features = [
 ] }
 downcast-rs = "1.2.0"
 byteorder = "1.4.3"
+serde = "1.0.27"
+
 vm-memory = {workspace = true}
 kvm-ioctls = {workspace = true}
 kvm-bindings = {workspace = true}
 vfio-ioctls = {workspace = true}
 vfio-bindings = {workspace = true}
 libc = "0.2.39"
+vmm-sys-util = {workspace = true}
+virtio-queue = {workspace = true}
+dbs-utils = {workspace = true}
+
 
 [dev-dependencies]
 dbs-arch = { workspace = true }

src/dragonball/dbs_pci/src/lib.rs

@@ -21,7 +21,7 @@
 //! - PCI configuration: a common framework to emulator PCI configuration space header.
 //! - PCI MSI/MSIx: structs to emulate PCI MSI/MSIx capabilities.
 
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 
 use dbs_device::device_manager::IoManagerContext;
 use dbs_interrupt::KvmIrqManager;
@@ -58,8 +58,17 @@ pub use msix::{MsixCap, MsixState, MSIX_TABLE_ENTRY_SIZE};
 mod vfio;
 pub use vfio::{VfioPciDevice, VfioPciError, VENDOR_NVIDIA};
 
+mod virtio_pci;
+pub use virtio_pci::{VirtioPciDevice, VirtioPciDeviceError, CAPABILITY_BAR_SIZE};
+
+mod pci_address;
+use dbs_virtio_devices::VirtioDevice;
+pub use pci_address::PciAddress;
+
+mod pci_common_config;
+
 /// Error codes related to PCI root/bus/device operations.
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, thiserror::Error, PartialEq)]
 pub enum Error {
     /// Failed to activate the PCI root/bus/device.
     #[error("failed to activate PCI device, {0:?}")]
@@ -110,6 +119,9 @@ pub enum Error {
     /// PCI ROM BAR address is invalid.
     #[error("address {0} size {1} too big")]
     RomBarAddressInvalid(u64, u64),
+    /// Invalid parameter
+    #[error("invalid pci device address")]
+    InvalidParameter,
 }
 
 /// Specialized `Result` for PCI related operations.
@@ -130,3 +142,8 @@ pub fn fill_config_data(data: &mut [u8]) {
         *pos = 0xff;
     }
 }
+
+/// we only support one pci bus
+pub const PCI_BUS_DEFAULT: u8 = 0;
+
+type ArcMutexBoxDynVirtioDevice<AS, Q, R> = Arc<Mutex<Box<dyn VirtioDevice<AS, Q, R>>>>;

src/dragonball/dbs_pci/src/pci_address.rs

@@ -0,0 +1,100 @@
+// Copyright (C) 2024 Alibaba Cloud. All rights reserved.
+//
+// Copyright (C) 2025 Ant Group. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
+
+use std::cmp::Ordering;
+use std::fmt;
+
+use crate::{Error, Result};
+
+const PCI_MAX_DEV_ID: u8 = 0x1f;
+const PCI_MAX_FUNC_ID: u8 = 0x7;
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+pub struct PciAddress {
+    /// Bus number, in the range [0, 0xff].
+    bus: u8,
+    /// Device id, in the range [0x0, 0x1f].
+    dev: u8,
+    /// Function id, in the range [0x0, 0x7].
+    func: u8,
+}
+
+impl PartialOrd for PciAddress {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for PciAddress {
+    fn cmp(&self, other: &Self) -> Ordering {
+        // Compare in the order of bus -> dev -> func.
+        self.bus
+            .cmp(&other.bus)
+            .then_with(|| self.dev.cmp(&other.dev))
+            .then_with(|| self.func.cmp(&other.func))
+    }
+}
+
+impl PciAddress {
+    /// Create a new PCI address from bus and device/function id.
+    ///
+    /// * `bus`: PCI bus number, in the range \[0x0, 0xff\].
+    /// * `dev`: PCI device id, in the range \[0x0, 0x1f\].
+    /// * `func`: PCI function id, in the range \[0x0, 0x7\].
+    pub fn new(bus: u8, dev: u8, func: u8) -> Result<Self> {
+        if dev > PCI_MAX_DEV_ID || func > PCI_MAX_FUNC_ID {
+            return Err(Error::InvalidParameter);
+        }
+
+        Ok(PciAddress { bus, dev, func })
+    }
+
+    /// Get PCI device id on the PCI bus, which is in [0x0, 0x1f]
+    pub fn dev_id(&self) -> u8 {
+        self.dev
+    }
+
+    /// Get PCI device function id, which is in [0x0, 0x7].
+    pub fn func_id(&self) -> u8 {
+        self.func
+    }
+
+    /// Get PCI device bus number, which is in [0x0, 0xff].
+    pub fn bus_id(&self) -> u8 {
+        self.bus
+    }
+}
+
+impl fmt::Debug for PciAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "PciAddress: {:02x}:{:02x}.{:02x}",
+            self.bus, self.dev, self.func
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_pci_address() {
+        // test invlaid device id
+        assert_eq!(PciAddress::new(0, 32, 0), Err(Error::InvalidParameter));
+
+        // test invalid function id
+        assert_eq!(PciAddress::new(0, 0, 8), Err(Error::InvalidParameter));
+
+        // test pci address
+        let (bus, dev, func) = (3, 5, 4);
+        let address = PciAddress::new(bus, dev, func).unwrap();
+        assert_eq!(address.bus_id(), bus);
+        assert_eq!(address.dev_id(), dev);
+        assert_eq!(address.func_id(), func);
+    }
+}

src/dragonball/dbs_pci/src/pci_common_config.rs

@@ -0,0 +1,507 @@
+// Copyright 2018 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE-BSD-3-Clause file.
+//
+// Copyright © 2019 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+//
+// Copyright (C) 2024 Alibaba Cloud. All rights reserved.
+//
+// Copyright (C) 2025 Ant Group. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0 or BSD-3-Clause
+
+use std::sync::atomic::{AtomicU16, Ordering};
+use std::sync::{Arc, Mutex};
+
+use byteorder::{ByteOrder, LittleEndian};
+use log::{error, trace, warn};
+use serde::{Deserialize, Serialize};
+use virtio_queue::QueueT;
+use vm_memory::{GuestAddressSpace, GuestMemoryRegion};
+
+use crate::ArcMutexBoxDynVirtioDevice;
+use dbs_virtio_devices::VirtioQueueConfig;
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct VirtioPciCommonConfigState {
+    pub driver_status: u8,
+    pub config_generation: u8,
+    pub device_feature_select: u32,
+    pub driver_feature_select: u32,
+    pub queue_select: u16,
+    pub msix_config: u16,
+    pub msix_queues: Vec<u16>,
+}
+
+/* The standard layout for the ring is a continuous chunk of memory which looks
+ * like this.  We assume num is a power of 2.
+ *
+ * struct vring
+ * {
+ *	// The actual descriptors (16 bytes each)
+ *	struct vring_desc desc[num];
+ *
+ *	// A ring of available descriptor heads with free-running index.
+ *	__virtio16 avail_flags;
+ *	__virtio16 avail_idx;
+ *	__virtio16 available[num];
+ *	__virtio16 used_event_idx;
+ *
+ *	// Padding to the next align boundary.
+ *	char pad[];
+ *
+ *	// A ring of used descriptor heads with free-running index.
+ *	__virtio16 used_flags;
+ *	__virtio16 used_idx;
+ *	struct vring_used_elem used[num];
+ *	__virtio16 avail_event_idx;
+ * };
+ * struct vring_desc {
+ *	__virtio64 addr;
+ *	__virtio32 len;
+ *	__virtio16 flags;
+ *	__virtio16 next;
+ * };
+ *
+ * struct vring_avail {
+ *	__virtio16 flags;
+ *	__virtio16 idx;
+ *	__virtio16 ring[];
+ * };
+ *
+ * // u32 is used here for ids for padding reasons.
+ * struct vring_used_elem {
+ *	// Index of start of used descriptor chain.
+ *	__virtio32 id;
+ *	// Total length of the descriptor chain which was used (written to)
+ *	__virtio32 len;
+ * };
+*
+ * Kernel header used for this reference: include/uapi/linux/virtio_ring.h
+ * Virtio Spec: https://docs.oasis-open.org/virtio/virtio/v1.2/csd01/virtio-v1.2-csd01.html
+ *
+ */
+
+/// Contains the data for reading and writing the common configuration structure of a virtio PCI
+/// device.
+///
+/// * Registers:
+///
+/// ** About the whole device.
+///    le32 device_feature_select;     // 0x00 // read-write
+///    le32 device_feature;            // 0x04 // read-only for driver
+///    le32 driver_feature_select;     // 0x08 // read-write
+///    le32 driver_feature;            // 0x0C // read-write
+///    le16 msix_config;               // 0x10 // read-write
+///    le16 num_queues;                // 0x12 // read-only for driver
+///    u8 device_status;               // 0x14 // read-write (driver_status)
+///    u8 config_generation;           // 0x15 // read-only for driver
+///
+/// ** About a specific virtqueue.
+///    le16 queue_select;              // 0x16 // read-write
+///    le16 queue_size;                // 0x18 // read-write, power of 2, or 0.
+///    le16 queue_msix_vector;         // 0x1A // read-write
+///    le16 queue_enable;              // 0x1C // read-write (Ready)
+///    le16 queue_notify_off;          // 0x1E // read-only for driver
+///    le64 queue_desc;                // 0x20 // read-write
+///    le64 queue_avail;               // 0x28 // read-write
+///    le64 queue_used;                // 0x30 // read-write
+pub struct VirtioPciCommonConfig {
+    pub driver_status: u8,
+    pub config_generation: u8,
+    pub device_feature_select: u32,
+    pub driver_feature_select: u32,
+    pub queue_select: u16,
+    pub msix_config: Arc<AtomicU16>,
+    pub msix_queues: Arc<Mutex<Vec<u16>>>,
+}
+
+impl VirtioPciCommonConfig {
+    pub fn new(state: VirtioPciCommonConfigState) -> Self {
+        VirtioPciCommonConfig {
+            driver_status: state.driver_status,
+            config_generation: state.config_generation,
+            device_feature_select: state.device_feature_select,
+            driver_feature_select: state.driver_feature_select,
+            queue_select: state.queue_select,
+            msix_config: Arc::new(AtomicU16::new(state.msix_config)),
+            msix_queues: Arc::new(Mutex::new(state.msix_queues)),
+        }
+    }
+
+    // TODO(fupan): use for live upgrade later
+    #[allow(dead_code)]
+    fn state(&self) -> VirtioPciCommonConfigState {
+        VirtioPciCommonConfigState {
+            driver_status: self.driver_status,
+            config_generation: self.config_generation,
+            device_feature_select: self.device_feature_select,
+            driver_feature_select: self.driver_feature_select,
+            queue_select: self.queue_select,
+            msix_config: self.msix_config.load(Ordering::Acquire),
+            msix_queues: self.msix_queues.lock().unwrap().clone(),
+        }
+    }
+
+    fn read_common_config_byte(&self, offset: u64) -> u8 {
+        trace!("read_common_config_byte: offset 0x{:x}", offset);
+        // The driver is only allowed to do aligned, properly sized access.
+        match offset {
+            0x14 => self.driver_status,
+            0x15 => self.config_generation,
+            _ => {
+                warn!("invalid virtio config byte read: 0x{:x}", offset);
+                0
+            }
+        }
+    }
+
+    fn write_common_config_byte(&mut self, offset: u64, value: u8) {
+        trace!(
+            "write_common_config_byte: offset 0x{:x} value 0x{:x}",
+            offset,
+            value
+        );
+        match offset {
+            0x14 => self.driver_status = value,
+            _ => {
+                warn!("invalid virtio config byte write: 0x{:x}", offset);
+            }
+        }
+    }
+
+    fn read_common_config_word<Q: QueueT + 'static>(
+        &self,
+        offset: u64,
+        queues: &[VirtioQueueConfig<Q>],
+    ) -> u16 {
+        trace!("read_common_config_word: offset 0x{:x}", offset);
+        match offset {
+            0x10 => self.msix_config.load(Ordering::Acquire),
+            0x12 => queues.len() as u16, // num_queues
+            0x16 => self.queue_select,
+            0x18 => self.with_queue(queues, |q| q.max_size()).unwrap_or(0),
+            0x1a => self.msix_queues.lock().unwrap()[self.queue_select as usize],
+            0x1c => u16::from(self.with_queue(queues, |q| q.ready()).unwrap_or(false)),
+            0x1e => self.queue_select, // notify_off
+            _ => {
+                warn!("invalid virtio register word read: 0x{:x}", offset);
+                0
+            }
+        }
+    }
+
+    fn write_common_config_word<Q: QueueT + 'static>(
+        &mut self,
+        offset: u64,
+        value: u16,
+        queues: &mut [VirtioQueueConfig<Q>],
+    ) {
+        trace!(
+            "write_common_config_word: offset 0x{:x} value 0x{:x}",
+            offset,
+            value
+        );
+        match offset {
+            0x10 => self.msix_config.store(value, Ordering::Release),
+            0x16 => self.queue_select = value,
+            0x18 => self.with_queue_mut(queues, |q| q.set_size(value)),
+            0x1a => self.msix_queues.lock().unwrap()[self.queue_select as usize] = value,
+            0x1c => self.with_queue_mut(queues, |q| {
+                let ready = value == 1;
+                q.set_ready(ready);
+            }),
+            _ => {
+                warn!("invalid virtio register word write: 0x{:x}", offset);
+            }
+        }
+    }
+
+    fn read_common_config_dword<
+        AS: GuestAddressSpace + 'static,
+        Q: QueueT + 'static,
+        R: 'static + GuestMemoryRegion,
+    >(
+        &self,
+        offset: u64,
+        device: ArcMutexBoxDynVirtioDevice<AS, Q, R>,
+    ) -> u32 {
+        trace!("read_common_config_dword: offset 0x{:x}", offset);
+        match offset {
+            0x00 => self.device_feature_select,
+            0x04 => {
+                // Only 64 bits of features (2 pages) are defined for now, so limit
+                // device_feature_select to avoid shifting by 64 or more bits.
+                let locked_device = device.lock().unwrap();
+                if self.device_feature_select < 2 {
+                    locked_device.get_avail_features(self.device_feature_select)
+                } else {
+                    0
+                }
+            }
+            0x08 => self.driver_feature_select,
+            _ => {
+                warn!("invalid virtio register dword read: 0x{:x}", offset);
+                0
+            }
+        }
+    }
+
+    fn write_common_config_dword<
+        AS: GuestAddressSpace + 'static,
+        Q: QueueT + 'static,
+        R: 'static + GuestMemoryRegion,
+    >(
+        &mut self,
+        offset: u64,
+        value: u32,
+        queues: &mut [VirtioQueueConfig<Q>],
+        device: ArcMutexBoxDynVirtioDevice<AS, Q, R>,
+    ) {
+        trace!(
+            "write_common_config_dword: offset 0x{:x} value 0x{:x}",
+            offset,
+            value
+        );
+
+        match offset {
+            0x00 => self.device_feature_select = value,
+            0x08 => self.driver_feature_select = value,
+            0x0c => {
+                if self.driver_feature_select < 2 {
+                    let mut locked_device = device.lock().unwrap();
+                    locked_device.set_acked_features(self.driver_feature_select, value);
+                } else {
+                    warn!(
+                        "invalid ack_features (page {}, value 0x{:x})",
+                        self.driver_feature_select, value
+                    );
+                }
+            }
+            0x20 => self.with_queue_mut(queues, |q| q.set_desc_table_address(Some(value), None)),
+            0x24 => self.with_queue_mut(queues, |q| q.set_desc_table_address(None, Some(value))),
+            0x28 => self.with_queue_mut(queues, |q| q.set_avail_ring_address(Some(value), None)),
+            0x2c => self.with_queue_mut(queues, |q| q.set_avail_ring_address(None, Some(value))),
+            0x30 => self.with_queue_mut(queues, |q| q.set_used_ring_address(Some(value), None)),
+            0x34 => self.with_queue_mut(queues, |q| q.set_used_ring_address(None, Some(value))),
+            _ => {
+                warn!("invalid virtio register dword write: 0x{:x}", offset);
+            }
+        }
+    }
+
+    fn read_common_config_qword(&self, _offset: u64) -> u64 {
+        trace!("read_common_config_qword: offset 0x{:x}", _offset);
+        0 // Assume the guest has no reason to read write-only registers.
+    }
+
+    fn write_common_config_qword<Q: QueueT + 'static>(
+        &mut self,
+        offset: u64,
+        value: u64,
+        queues: &mut [VirtioQueueConfig<Q>],
+    ) {
+        trace!(
+            "write_common_config_qword: offset 0x{:x}, value 0x{:x}",
+            offset,
+            value
+        );
+
+        let low = Some((value & 0xffff_ffff) as u32);
+        let high = Some((value >> 32) as u32);
+
+        match offset {
+            0x20 => self.with_queue_mut(queues, |q| q.set_desc_table_address(low, high)),
+            0x28 => self.with_queue_mut(queues, |q| q.set_avail_ring_address(low, high)),
+            0x30 => self.with_queue_mut(queues, |q| q.set_used_ring_address(low, high)),
+            _ => {
+                warn!("invalid virtio register qword write: 0x{:x}", offset);
+            }
+        }
+    }
+
+    fn with_queue<U, F, Q>(&self, queues: &[VirtioQueueConfig<Q>], f: F) -> Option<U>
+    where
+        F: FnOnce(&Q) -> U,
+        Q: QueueT + 'static,
+    {
+        queues.get(self.queue_select as usize).map(|q| f(&q.queue))
+    }
+
+    fn with_queue_mut<F: FnOnce(&mut Q), Q: QueueT + 'static>(
+        &self,
+        queues: &mut [VirtioQueueConfig<Q>],
+        f: F,
+    ) {
+        if let Some(queue) = queues.get_mut(self.queue_select as usize) {
+            f(&mut queue.queue);
+        }
+    }
+
+    pub fn read<
+        AS: GuestAddressSpace + 'static,
+        Q: QueueT + 'static,
+        R: 'static + GuestMemoryRegion,
+    >(
+        &self,
+        offset: u64,
+        data: &mut [u8],
+        queues: &[VirtioQueueConfig<Q>],
+        device: ArcMutexBoxDynVirtioDevice<AS, Q, R>,
+    ) {
+        assert!(data.len() <= 8);
+
+        match data.len() {
+            1 => {
+                let v = self.read_common_config_byte(offset);
+                data[0] = v;
+            }
+            2 => {
+                let v = self.read_common_config_word(offset, queues);
+                LittleEndian::write_u16(data, v);
+            }
+            4 => {
+                let v = self.read_common_config_dword(offset, device);
+                LittleEndian::write_u32(data, v);
+            }
+            8 => {
+                let v = self.read_common_config_qword(offset);
+                LittleEndian::write_u64(data, v);
+            }
+            _ => error!("invalid data length for virtio read: len {}", data.len()),
+        }
+    }
+
+    pub fn write<
+        AS: GuestAddressSpace + 'static,
+        Q: QueueT + 'static,
+        R: 'static + GuestMemoryRegion,
+    >(
+        &mut self,
+        offset: u64,
+        data: &[u8],
+        queues: &mut [VirtioQueueConfig<Q>],
+        device: ArcMutexBoxDynVirtioDevice<AS, Q, R>,
+    ) {
+        assert!(data.len() <= 8);
+
+        match data.len() {
+            1 => self.write_common_config_byte(offset, data[0]),
+            2 => self.write_common_config_word(offset, LittleEndian::read_u16(data), queues),
+            4 => {
+                self.write_common_config_dword(offset, LittleEndian::read_u32(data), queues, device)
+            }
+            8 => self.write_common_config_qword(offset, LittleEndian::read_u64(data), queues),
+            _ => error!("invalid data length for virtio write: len {}", data.len()),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::virtio_pci::tests::{DummyDevice, DUMMY_FEATURES};
+    use super::*;
+    use dbs_virtio_devices::VirtioDevice;
+    use virtio_queue::QueueSync;
+    use vm_memory::{GuestMemoryMmap, GuestRegionMmap};
+
+    #[test]
+    fn write_base_regs() {
+        let regs_state = VirtioPciCommonConfigState {
+            driver_status: 0xaa,
+            config_generation: 0x55,
+            device_feature_select: 0x0,
+            driver_feature_select: 0x0,
+            queue_select: 0xff,
+            msix_config: 0,
+            msix_queues: vec![0; 3],
+        };
+        let mut regs = VirtioPciCommonConfig::new(regs_state);
+
+        let dev: Arc<
+            Mutex<Box<dyn VirtioDevice<Arc<GuestMemoryMmap>, QueueSync, GuestRegionMmap>>>,
+        > = Arc::new(Mutex::new(Box::new(DummyDevice::new())));
+        let mut queues = Vec::new();
+        queues.push(VirtioQueueConfig::create(2, 0).unwrap());
+        queues.push(VirtioQueueConfig::create(2, 1).unwrap());
+
+        // Can set all bits of driver_status.
+        regs.write(0x14, &[0x55], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0x00];
+        regs.read(0x14, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0x55);
+
+        // The config generation register is read only.
+        regs.write(0x15, &[0xaa], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0x00];
+        regs.read(0x15, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0x55);
+
+        // Device features is read-only and passed through from the device.
+        regs.write(0x04, &[0, 0, 0, 0], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0, 0, 0, 0];
+        regs.read(0x04, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(LittleEndian::read_u32(&read_back), DUMMY_FEATURES as u32);
+
+        // Read device features with device_feature_select as 0
+        regs.write(0x00, &[0, 0, 0, 0], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0, 0, 0, 0];
+        regs.read(0x04, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(LittleEndian::read_u32(&read_back), DUMMY_FEATURES as u32);
+
+        // Read device features with device_feature_select as 1
+        regs.write(0x00, &[1, 0, 0, 0], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0, 0, 0, 0];
+        regs.read(0x04, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(
+            LittleEndian::read_u32(&read_back),
+            (DUMMY_FEATURES >> 32) as u32
+        );
+
+        // Feature select registers are read/write.
+        regs.write(0x00, &[1, 2, 3, 4], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0, 0, 0, 0];
+        regs.read(0x00, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(LittleEndian::read_u32(&read_back), 0x0403_0201);
+        regs.write(0x08, &[1, 2, 3, 4], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0, 0, 0, 0];
+        regs.read(0x08, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(LittleEndian::read_u32(&read_back), 0x0403_0201);
+
+        // 'queue_select' can be read and written.
+        regs.write(0x16, &[0xaa, 0x55], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0x00, 0x00];
+        regs.read(0x16, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0xaa);
+        assert_eq!(read_back[1], 0x55);
+
+        // write msix_queues by queue_select 2
+        regs.write(0x16, &[0x02, 0x00], &mut queues, Arc::clone(&dev));
+        regs.write(0x1a, &[0xbb, 0xcc], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0x00, 0x00];
+        regs.read(0x1a, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0xbb);
+        assert_eq!(read_back[1], 0xcc);
+
+        // 'msix_config' can be read and written.
+        regs.write(0x10, &[0xdd, 0xee], &mut queues, Arc::clone(&dev));
+        let mut read_back = vec![0x00, 0x00];
+        regs.read(0x10, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0xdd);
+        assert_eq!(read_back[1], 0xee);
+
+        // 'queue_size' can be read and set.
+        let mut read_back = vec![0x00, 0x00];
+        // queue_select is 2 and queues[2] is None, so queue_size is 0
+        regs.read(0x18, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0x00);
+        assert_eq!(read_back[1], 0x00);
+        // queue_select is 1, so queue_size is 2
+        regs.write(0x16, &[0x01, 0x00], &mut queues, Arc::clone(&dev));
+        regs.read(0x18, &mut read_back, &queues, Arc::clone(&dev));
+        assert_eq!(read_back[0], 0x02);
+        assert_eq!(read_back[1], 0x00);
+    }
+}

src/dragonball/dbs_pci/src/vfio.rs

@@ -888,7 +888,7 @@ pub struct VfioPciDeviceState<C: PciSystemContext> {
     vfio_path: String,
     interrupt: Interrupt,
     vfio_dev: Arc<VfioDevice>,
-    context: Weak<C>,
+    context: Arc<Mutex<C>>,
     configuration: PciConfiguration,
     device: Option<Weak<dyn DeviceIo>>,
     regions: Vec<Region>,
@@ -904,7 +904,7 @@ impl<C: PciSystemContext> VfioPciDeviceState<C> {
         vfio_path: String,
         vfio_dev: Arc<VfioDevice>,
         bus: Weak<PciBus>,
-        context: Weak<C>,
+        context: Arc<Mutex<C>>,
         vendor_device_id: u32,
         clique_id: Option<u8>,
         vfio_container: Arc<VfioContainer>,
@@ -1277,11 +1277,7 @@ impl<C: PciSystemContext> VfioPciDeviceState<C> {
     }
 
     fn register_regions(&mut self, vm: &Arc<VmFd>) -> Result<()> {
-        let ctx = self
-            .context
-            .upgrade()
-            .ok_or(VfioPciError::BusIsDropped)?
-            .get_device_manager_context();
+        let ctx = self.context.lock().unwrap().get_device_manager_context();
         let mut tx = ctx.begin_tx();
 
         for region in self.regions.iter_mut() {
@@ -1336,22 +1332,7 @@ impl<C: PciSystemContext> VfioPciDeviceState<C> {
     }
 
     fn unregister_regions(&mut self, vm: &Arc<VmFd>) -> Result<()> {
-        // This routine handle VfioPciDevice dropped but not unmap memory
-        if self.context.upgrade().is_none() {
-            for region in self.regions.iter_mut() {
-                if region.mappable() {
-                    region.unmap(vm, &self.vfio_container)?;
-                }
-            }
-
-            return Ok(());
-        }
-
-        let ctx = self
-            .context
-            .upgrade()
-            .ok_or(VfioPciError::BusIsDropped)?
-            .get_device_manager_context();
+        let ctx = self.context.lock().unwrap().get_device_manager_context();
         let mut tx = ctx.begin_tx();
 
         for region in self.regions.iter_mut() {
@@ -1380,11 +1361,8 @@ impl<C: PciSystemContext> VfioPciDeviceState<C> {
                 } else {
                     // Safe to unwrap because activate() has set self.device to a valid value.
                     let device = self.device.as_ref().unwrap().clone();
-                    let ctx: <C as PciSystemContext>::D = self
-                        .context
-                        .upgrade()
-                        .ok_or(VfioPciError::BusIsDropped)?
-                        .get_device_manager_context();
+                    let ctx: <C as PciSystemContext>::D =
+                        self.context.lock().unwrap().get_device_manager_context();
                     let mut tx = ctx.begin_tx();
 
                     if let Err(e) = region.retrap(
@@ -1561,7 +1539,7 @@ impl<C: PciSystemContext> VfioPciDevice<C> {
         path: String,
         bus: Weak<PciBus>,
         device: VfioDevice,
-        context: Weak<C>,
+        context: Arc<Mutex<C>>,
         vm_fd: Arc<VmFd>,
         vendor_device_id: u32,
         clique_id: Option<u8>,
@@ -1649,11 +1627,7 @@ impl<C: PciSystemContext> VfioPciDevice<C> {
             state.interrupt.add_msi_irq_resource(base, size);
         }
 
-        let irq_manager = state
-            .context
-            .upgrade()
-            .ok_or(VfioPciError::BusIsDropped)?
-            .get_interrupt_manager();
+        let irq_manager = state.context.lock().unwrap().get_interrupt_manager();
         state.interrupt.initialize(irq_manager)?;
         #[cfg(target_arch = "aarch64")]
         self.set_device_id(&mut state);

src/dragonball/dbs_utils/src/net/mac.rs

@@ -146,7 +146,6 @@ mod tests {
         assert!(MacAddr::from_bytes(&src3[..]).is_err());
     }
 
-    #[cfg(feature = "with-serde")]
     #[test]
     fn test_mac_addr_serialization_and_deserialization() {
         let mac: MacAddr =

src/dragonball/dbs_virtio_devices/src/lib.rs

@@ -145,6 +145,8 @@ pub enum ActivateError {
     #[cfg(feature = "vhost")]
     #[error("Vhost activate error")]
     VhostActivate(vhost_rs::Error),
+    #[error("VirtioPci error")]
+    VirtioPci,
 }
 
 impl std::convert::From<Error> for ActivateError {

src/dragonball/dbs_virtio_devices/src/mmio/mmio_state.rs

@@ -371,6 +371,7 @@ where
             }
 
             let _ = self.intr_mgr.reset();
+            self.unregister_ioevent_doorbell();
             self.unregister_ioevent();
             self.features_select = 0;
             self.acked_features_select = 0;

src/dragonball/dbs_virtio_devices/src/vsock/mod.rs

@@ -313,8 +313,8 @@ mod tests {
     pub struct TestContext {
         pub cid: u64,
         pub mem: GuestMemoryMmap,
-        pub mem_size: usize,
-        pub epoll_manager: EpollManager,
+        pub _mem_size: usize,
+        pub _epoll_manager: EpollManager,
         pub device: Vsock<Arc<GuestMemoryMmap>, TestMuxer>,
     }
 
@@ -327,8 +327,8 @@ mod tests {
             Self {
                 cid: CID,
                 mem,
-                mem_size: MEM_SIZE,
-                epoll_manager: epoll_manager.clone(),
+                _mem_size: MEM_SIZE,
+                _epoll_manager: epoll_manager.clone(),
                 device: Vsock::new_with_muxer(
                     CID,
                     Arc::new(defs::QUEUE_SIZES.to_vec()),
@@ -394,7 +394,7 @@ mod tests {
             EventHandlerContext {
                 guest_rxvq,
                 guest_txvq,
-                guest_evvq,
+                _guest_evvq: guest_evvq,
                 queues,
                 epoll_handler: None,
                 device: Vsock::new_with_muxer(
@@ -422,7 +422,7 @@ mod tests {
         pub queues: Vec<VirtioQueueConfig<QueueSync>>,
         pub guest_rxvq: GuestQ<'a>,
         pub guest_txvq: GuestQ<'a>,
-        pub guest_evvq: GuestQ<'a>,
+        pub _guest_evvq: GuestQ<'a>,
         pub mem: Arc<GuestMemoryMmap>,
     }
 

src/dragonball/src/api/v1/vmm_action.rs

@@ -17,7 +17,6 @@ use tracing::instrument;
 use crate::error::{Result, StartMicroVmError, StopMicrovmError};
 use crate::event_manager::EventManager;
 use crate::tracer::{DragonballTracer, TraceError, TraceInfo};
-use crate::vcpu::VcpuManagerError;
 use crate::vm::{CpuTopology, KernelConfigInfo, VmConfigInfo};
 use crate::vmm::Vmm;
 
@@ -55,6 +54,8 @@ pub use crate::device_manager::virtio_net_dev_mgr::{
 };
 #[cfg(feature = "virtio-vsock")]
 pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError};
+#[cfg(feature = "host-device")]
+use crate::vcpu::VcpuManagerError;
 #[cfg(feature = "hotplug")]
 pub use crate::vcpu::{VcpuResizeError, VcpuResizeInfo};
 
@@ -204,6 +205,10 @@ pub enum VmmAction {
     /// input. This action can only be called before the microVM has booted.
     InsertBlockDevice(BlockDeviceConfigInfo),
 
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
+    /// Prepare to remove a block device that already exists
+    PrepareRemoveBlockDevice(String),
+
     #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     /// Remove a new block device for according to given drive_id
     RemoveBlockDevice(String),
@@ -355,6 +360,10 @@ impl VmmService {
                 self.add_block_device(vmm, event_mgr, block_device_config)
             }
             #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
+            VmmAction::PrepareRemoveBlockDevice(blkdev_id) => {
+                self.prepare_remove_block_device(vmm, &blkdev_id)
+            }
+            #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
             VmmAction::UpdateBlockDevice(blk_update) => {
                 self.update_blk_rate_limiters(vmm, blk_update)
             }
@@ -663,10 +672,17 @@ impl VmmService {
                 VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot)
             })?;
 
+        let (sender, receiver) = mpsc::channel();
+        let vmm_data = if ctx.is_hotplug() {
+            VmmData::SyncHotplug((sender.clone(), receiver))
+        } else {
+            VmmData::Empty
+        };
+
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, config)
-            .map(|_| VmmData::Empty)
+            .insert_device(ctx, config, sender.clone())
+            .map(|_| vmm_data)
             .map_err(VmmActionError::Block)
     }
 
@@ -687,6 +703,38 @@ impl VmmService {
             .map_err(VmmActionError::Block)
     }
 
+    // using upcall to unplug the block device in the guest
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
+    fn prepare_remove_block_device(
+        &mut self,
+        vmm: &mut Vmm,
+        blockdev_id: &str,
+    ) -> VmmRequestResult {
+        let vm = vmm.get_vm_mut().ok_or(VmmActionError::HostDeviceConfig(
+            VfioDeviceError::InvalidVMID,
+        ))?;
+
+        info!("prepare_remove_block_device: {:?}", blockdev_id);
+        let ctx = vm.create_device_op_context(None).map_err(|e| {
+            info!("create device op context error: {:?}", e);
+            if let StartMicroVmError::MicroVMAlreadyRunning = e {
+                VmmActionError::HostDeviceConfig(VfioDeviceError::UpdateNotAllowedPostBoot)
+            } else if let StartMicroVmError::UpcallServerNotReady = e {
+                VmmActionError::UpcallServerNotReady
+            } else {
+                VmmActionError::StartMicroVm(e)
+            }
+        })?;
+
+        let (sender, receiver) = mpsc::channel();
+
+        vm.device_manager_mut()
+            .block_manager
+            .prepare_remove_device(&ctx, blockdev_id, sender.clone())
+            .map(|_| VmmData::SyncHotplug((sender, receiver)))
+            .map_err(VmmActionError::Block)
+    }
+
     #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     // Remove the device
     #[instrument(skip(self, event_mgr))]
@@ -1525,6 +1573,7 @@ mod tests {
                     queue_size: 256,
                     use_shared_irq: None,
                     use_generic_irq: None,
+                    use_pci_bus: Some(true),
                 }),
                 InstanceState::Uninitialized,
                 &|result| {

src/dragonball/src/device_manager/blk_dev_mgr.rs

@@ -7,19 +7,28 @@
 // found in the THIRD-PARTY file.
 
 //! Device manager for virtio-blk and vhost-user-blk devices.
-use std::collections::{vec_deque, VecDeque};
 use std::convert::TryInto;
 use std::fs::OpenOptions;
 use std::os::unix::fs::OpenOptionsExt;
 use std::os::unix::io::AsRawFd;
 use std::path::{Path, PathBuf};
+use std::sync::mpsc::Sender;
 use std::sync::Arc;
+use std::{
+    collections::{vec_deque, VecDeque},
+    sync::mpsc,
+};
 
+use dbs_device::DeviceIo;
+use dbs_pci::VirtioPciDevice;
+use dbs_upcall::{DevMgrResponse, UpcallClientResponse};
 use dbs_virtio_devices as virtio;
 use dbs_virtio_devices::block::{aio::Aio, io_uring::IoUring, Block, LocalFile, Ufile};
 #[cfg(feature = "vhost-user-blk")]
 use dbs_virtio_devices::vhost::vhost_user::block::VhostUserBlock;
 use serde_derive::{Deserialize, Serialize};
+use virtio_queue::QueueSync;
+use vm_memory::GuestRegionMmap;
 
 use crate::address_space_manager::GuestAddressSpaceImpl;
 use crate::config_manager::{ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo};
@@ -190,6 +199,8 @@ pub struct BlockDeviceConfigInfo {
     pub use_shared_irq: Option<bool>,
     /// Use generic irq
     pub use_generic_irq: Option<bool>,
+    /// Use pci bus
+    pub use_pci_bus: Option<bool>,
 }
 
 impl std::default::Default for BlockDeviceConfigInfo {
@@ -208,6 +219,7 @@ impl std::default::Default for BlockDeviceConfigInfo {
             rate_limiter: None,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: None,
         }
     }
 }
@@ -349,6 +361,7 @@ impl BlockDeviceMgr {
         &mut self,
         mut ctx: DeviceOpContext,
         config: BlockDeviceConfigInfo,
+        sender: mpsc::Sender<Option<i32>>,
     ) -> std::result::Result<(), BlockDeviceError> {
         if !cfg!(feature = "hotplug") && ctx.is_hotplug {
             return Err(BlockDeviceError::UpdateNotAllowedPostBoot);
@@ -380,31 +393,68 @@ impl BlockDeviceMgr {
                     return Ok(());
                 }
 
+                let mut slot = 0;
+
+                let use_generic_irq = config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ);
+
                 match config.device_type {
                     BlockDeviceType::RawBlock => {
                         let device = Self::create_blk_device(&config, &mut ctx)
                             .map_err(BlockDeviceError::Virtio)?;
-                        let dev = DeviceManager::create_mmio_virtio_device(
-                            device,
-                            &mut ctx,
-                            config.use_shared_irq.unwrap_or(self.use_shared_irq),
-                            config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
-                        )
-                        .map_err(BlockDeviceError::DeviceManager)?;
-                        self.update_device_by_index(index, Arc::clone(&dev))?;
+
+                        let dev = if let Some(true) = config.use_pci_bus {
+                            let pci_dev = DeviceManager::create_virtio_pci_device(
+                                device,
+                                &mut ctx,
+                                use_generic_irq,
+                            )
+                            .map_err(BlockDeviceError::DeviceManager)?;
+
+                            let (_, devfn) = DeviceManager::get_pci_device_info(&pci_dev)?;
+                            slot = devfn >> 3;
+
+                            pci_dev
+                        } else {
+                            DeviceManager::create_mmio_virtio_device(
+                                device,
+                                &mut ctx,
+                                config.use_shared_irq.unwrap_or(self.use_shared_irq),
+                                use_generic_irq,
+                            )
+                            .map_err(BlockDeviceError::DeviceManager)?
+                        };
+
+                        let callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>> =
+                            Some(Box::new(move |_| {
+                                // send the pci device slot to caller.
+                                let _ = sender.send(Some(slot as i32));
+                            }));
+
+                        self.update_device_by_index(index, dev.clone())?;
                         // live-upgrade need save/restore device from info.device.
                         self.info_list[index].set_device(dev.clone());
-                        ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| {
+
+                        let mut cleanup = |e, ctx: DeviceOpContext| -> BlockDeviceError {
                             let logger = ctx.logger().new(slog::o!());
                             self.remove_device(ctx, &config.drive_id).unwrap();
                             error!(
                                 logger,
-                                "failed to hot-add virtio block device {}, {:?}",
+                                "failed to hot-add pci virtio block device {}, {:?}",
                                 &config.drive_id,
                                 e
                             );
                             BlockDeviceError::DeviceManager(e)
-                        })
+                        };
+
+                        if let Some(true) = config.use_pci_bus {
+                            let _ = ctx
+                                .insert_hotplug_pci_device(&dev, callback)
+                                .map_err(|e| cleanup(e, ctx))?;
+                            Ok(())
+                        } else {
+                            ctx.insert_hotplug_mmio_device(&dev, callback)
+                                .map_err(|e| cleanup(e, ctx))
+                        }
                     }
                     #[cfg(feature = "vhost-user-blk")]
                     BlockDeviceType::Spool | BlockDeviceType::Spdk => {
@@ -417,8 +467,13 @@ impl BlockDeviceMgr {
                             config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
                         )
                         .map_err(BlockDeviceError::DeviceManager)?;
+                        let callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>> =
+                            Some(Box::new(move |_| {
+                                let _ = sender.send(None);
+                            }));
+
                         self.update_device_by_index(index, Arc::clone(&dev))?;
-                        ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| {
+                        ctx.insert_hotplug_mmio_device(&dev, callback).map_err(|e| {
                             let logger = ctx.logger().new(slog::o!());
                             self.remove_device(ctx, &config.drive_id).unwrap();
                             error!(
@@ -450,15 +505,25 @@ impl BlockDeviceMgr {
                         info.config.drive_id,
                         info.config.path_on_host.to_str().unwrap_or("<unknown>")
                     );
+
+                    let use_shared_irq = info.config.use_shared_irq.unwrap_or(self.use_shared_irq);
+                    let use_generic_irq = info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ);
                     let device = Self::create_blk_device(&info.config, ctx)
                         .map_err(BlockDeviceError::Virtio)?;
-                    let device = DeviceManager::create_mmio_virtio_device(
-                        device,
-                        ctx,
-                        info.config.use_shared_irq.unwrap_or(self.use_shared_irq),
-                        info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
-                    )
-                    .map_err(BlockDeviceError::RegisterBlockDevice)?;
+
+                    let device = if let Some(true) = info.config.use_pci_bus {
+                        DeviceManager::create_virtio_pci_device(device, ctx, use_generic_irq)
+                            .map_err(BlockDeviceError::RegisterBlockDevice)?
+                    } else {
+                        DeviceManager::create_mmio_virtio_device(
+                            device,
+                            ctx,
+                            use_shared_irq,
+                            use_generic_irq,
+                        )
+                        .map_err(BlockDeviceError::RegisterBlockDevice)?
+                    };
+
                     info.device = Some(device);
                 }
                 #[cfg(feature = "vhost-user-blk")]
@@ -496,7 +561,7 @@ impl BlockDeviceMgr {
         while let Some(mut info) = self.info_list.pop_back() {
             info!(ctx.logger(), "remove drive {}", info.config.drive_id);
             if let Some(device) = info.device.take() {
-                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+                DeviceManager::destroy_virtio_device(device, ctx)?;
             }
         }
 
@@ -510,6 +575,62 @@ impl BlockDeviceMgr {
         }
     }
 
+    /// prepare to remove device
+    pub fn prepare_remove_device(
+        &self,
+        ctx: &DeviceOpContext,
+        blockdev_id: &str,
+        result_sender: Sender<Option<i32>>,
+    ) -> Result<(), BlockDeviceError> {
+        if !cfg!(feature = "hotplug") {
+            return Err(BlockDeviceError::UpdateNotAllowedPostBoot);
+        }
+
+        info!(ctx.logger(), "prepare remove block device");
+
+        let callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>> =
+            Some(Box::new(move |result| match result {
+                UpcallClientResponse::DevMgr(response) => {
+                    if let DevMgrResponse::Other(resp) = response {
+                        if let Err(e) = result_sender.send(Some(resp.result)) {
+                            log::error!("send upcall result failed, due to {:?}!", e);
+                        }
+                    }
+                }
+                UpcallClientResponse::UpcallReset => {
+                    if let Err(e) = result_sender.send(None) {
+                        log::error!("send upcall result failed, due to {:?}!", e);
+                    }
+                }
+                #[allow(unreachable_patterns)]
+                _ => {
+                    log::debug!("this arm should only be triggered under test");
+                }
+            }));
+
+        let device_index = self
+            .get_index_of_drive_id(blockdev_id)
+            .ok_or(BlockDeviceError::InvalidDeviceId(blockdev_id.to_string()))?;
+
+        let info = &self.info_list[device_index];
+        if let Some(device) = info.device.as_ref() {
+            if let Some(_mmio_dev) = device.as_any().downcast_ref::<DbsMmioV2Device>() {
+                if callback.is_some() {
+                    ctx.remove_hotplug_mmio_device(device, callback)?;
+                }
+            } else if let Some(_pci_dev) = device.as_any().downcast_ref::<VirtioPciDevice<
+                GuestAddressSpaceImpl,
+                QueueSync,
+                GuestRegionMmap,
+            >>() {
+                if callback.is_some() {
+                    ctx.remove_hotplug_pci_device(device, callback)?;
+                }
+            }
+        }
+        Ok(())
+    }
+
     /// remove a block device, it basically is the inverse operation of `insert_device``
     pub fn remove_device(
         &mut self,
@@ -524,7 +645,7 @@ impl BlockDeviceMgr {
             Some(mut info) => {
                 info!(ctx.logger(), "remove drive {}", info.config.drive_id);
                 if let Some(device) = info.device.take() {
-                    DeviceManager::destroy_mmio_virtio_device(device, &mut ctx)
+                    DeviceManager::destroy_virtio_device(device, &mut ctx)
                         .map_err(BlockDeviceError::DeviceManager)?;
                 }
             }
@@ -783,7 +904,7 @@ impl BlockDeviceMgr {
     pub fn update_device_by_index(
         &mut self,
         index: usize,
-        device: Arc<DbsMmioV2Device>,
+        device: Arc<dyn DeviceIo>,
     ) -> Result<(), BlockDeviceError> {
         if let Some(info) = self.info_list.get_mut(index) {
             info.device = Some(device);
@@ -818,6 +939,21 @@ impl BlockDeviceMgr {
                             .map(|_p| ())
                             .map_err(|_e| BlockDeviceError::BlockEpollHanderSendFail);
                     }
+                } else if let Some(pci_dev) = device.as_any().downcast_ref::<VirtioPciDevice<
+                    GuestAddressSpaceImpl,
+                    QueueSync,
+                    GuestRegionMmap,
+                >>() {
+                    let inner_dev = pci_dev.device();
+                    if let Some(blk_dev) = inner_dev
+                        .as_any()
+                        .downcast_ref::<virtio::block::Block<GuestAddressSpaceImpl>>()
+                    {
+                        return blk_dev
+                            .set_patch_rate_limiters(new_cfg.bytes(), new_cfg.ops())
+                            .map(|_p| ())
+                            .map_err(|_e| BlockDeviceError::BlockEpollHanderSendFail);
+                    }
                 }
                 Ok(())
             }
@@ -848,6 +984,7 @@ mod tests {
     use super::*;
     use crate::device_manager::tests::create_address_space;
     use crate::test_utils::tests::create_vm_for_test;
+    use std::sync::mpsc::channel;
 
     #[test]
     fn test_block_device_type() {
@@ -887,14 +1024,16 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        let (sender, _receiver) = channel();
         assert!(vm
             .device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device.clone(),)
+            .insert_device(ctx, dummy_block_device.clone(), sender)
             .is_ok());
 
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
@@ -961,10 +1100,12 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
+        let (sender, _receiver) = channel();
         vm.device_manager_mut()
             .block_manager
-            .insert_device(device_op_ctx, dummy_block_device)
+            .insert_device(device_op_ctx, dummy_block_device, sender)
             .unwrap();
 
         let cfg = BlockDeviceConfigUpdateInfo {
@@ -1037,14 +1178,16 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        let (sender, _receiver) = channel();
         assert!(vm
             .device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device.clone(),)
+            .insert_device(ctx, dummy_block_device.clone(), sender)
             .is_ok());
 
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
@@ -1077,6 +1220,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_2 = TempFile::new().unwrap();
@@ -1095,19 +1239,21 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        let (sender, _receiver) = channel();
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device_1)
+            .insert_device(ctx, root_block_device_1, sender.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         assert!(vm
             .device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device_2)
+            .insert_device(ctx, root_block_device_2, sender)
             .is_err());
     }
 
@@ -1131,6 +1277,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_2 = TempFile::new().unwrap();
@@ -1149,6 +1296,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_3 = TempFile::new().unwrap();
@@ -1167,6 +1315,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
@@ -1186,23 +1335,24 @@ mod tests {
         assert!(vm.device_manager().block_manager.has_root_block_device(),);
         assert!(!vm.device_manager().block_manager.has_part_uuid_root());
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 3);
+        let (sender, _receiver) = channel();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device)
+            .insert_device(ctx, root_block_device, sender.clone())
             .unwrap();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2)
+            .insert_device(ctx, dummy_block_device_2, sender.clone())
             .unwrap();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_3)
+            .insert_device(ctx, dummy_block_device_3, sender.clone())
             .unwrap();
     }
 
@@ -1226,6 +1376,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_2 = TempFile::new().unwrap();
@@ -1244,6 +1395,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_3 = TempFile::new().unwrap();
@@ -1262,24 +1414,26 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        let (sender, _receiver) = channel();
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone())
+            .insert_device(ctx, dummy_block_device_2.clone(), sender.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_3.clone())
+            .insert_device(ctx, dummy_block_device_3.clone(), sender.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device.clone())
+            .insert_device(ctx, root_block_device.clone(), sender.clone())
             .unwrap();
 
         assert!(vm.device_manager().block_manager.has_root_block_device(),);
@@ -1322,6 +1476,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let dummy_file_2 = TempFile::new().unwrap();
@@ -1340,20 +1495,22 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
 
         let mut vm = crate::vm::tests::create_vm_instance();
+        let (sender, _receiver) = channel();
 
         // Add 2 block devices.
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device)
+            .insert_device(ctx, root_block_device, sender.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone())
+            .insert_device(ctx, dummy_block_device_2.clone(), sender.clone())
             .unwrap();
 
         // Get index zero.
@@ -1384,7 +1541,7 @@ mod tests {
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone())
+            .insert_device(ctx, dummy_block_device_2.clone(), sender.clone())
             .unwrap();
 
         let index = vm
@@ -1407,7 +1564,7 @@ mod tests {
         assert!(vm
             .device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone(),)
+            .insert_device(ctx, dummy_block_device_2.clone(), sender.clone())
             .is_err());
 
         // Update with 2 root block devices.
@@ -1417,7 +1574,7 @@ mod tests {
         assert!(vm
             .device_manager_mut()
             .block_manager
-            .insert_device(ctx, dummy_block_device_2,)
+            .insert_device(ctx, dummy_block_device_2, sender.clone())
             .is_err(),);
 
         // Switch roots and add a PARTUUID for the new one.
@@ -1435,6 +1592,7 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
         let root_block_device_new = BlockDeviceConfigInfo {
             path_on_host: dummy_path_2,
@@ -1450,16 +1608,17 @@ mod tests {
             queue_size: 128,
             use_shared_irq: None,
             use_generic_irq: None,
+            use_pci_bus: Some(true),
         };
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device_old)
+            .insert_device(ctx, root_block_device_old, sender.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
         vm.device_manager_mut()
             .block_manager
-            .insert_device(ctx, root_block_device_new)
+            .insert_device(ctx, root_block_device_new, sender.clone())
             .unwrap();
         assert!(vm.device_manager().block_manager.has_part_uuid_root);
     }

src/dragonball/src/device_manager/mem_dev_mgr.rs

@@ -310,7 +310,7 @@ impl MemDeviceMgr {
     pub fn remove_devices(&self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> {
         for info in self.info_list.iter() {
             if let Some(device) = &info.device {
-                DeviceManager::destroy_mmio_virtio_device(device.clone(), ctx)?;
+                DeviceManager::destroy_mmio_device(device.clone(), ctx)?;
             }
         }
 

src/dragonball/src/device_manager/mod.rs

@@ -13,17 +13,19 @@ use arc_swap::ArcSwap;
 use dbs_address_space::AddressSpace;
 #[cfg(target_arch = "aarch64")]
 use dbs_arch::{DeviceType, MMIODeviceInfo};
+use dbs_boot::layout::MMIO_LOW_END;
 use dbs_device::device_manager::{Error as IoManagerError, IoManager, IoManagerContext};
-#[cfg(target_arch = "aarch64")]
 use dbs_device::resources::DeviceResources;
 use dbs_device::resources::Resource;
 use dbs_device::DeviceIo;
 use dbs_interrupt::KvmIrqManager;
 use dbs_legacy_devices::ConsoleHandler;
-#[cfg(all(feature = "host-device", target_arch = "aarch64"))]
-use dbs_pci::PciBusResources;
+#[cfg(feature = "dbs-virtio-devices")]
+use dbs_pci::CAPABILITY_BAR_SIZE;
 use dbs_utils::epoll_manager::EpollManager;
 use kvm_ioctls::VmFd;
+use log::error;
+use virtio_queue::QueueSync;
 
 #[cfg(feature = "dbs-virtio-devices")]
 use dbs_device::resources::ResourceConstraint;
@@ -40,6 +42,7 @@ use dbs_virtio_devices::{
 
 #[cfg(feature = "host-device")]
 use dbs_pci::VfioPciDevice;
+use dbs_pci::VirtioPciDevice;
 #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
 use dbs_upcall::{
     DevMgrRequest, DevMgrService, MmioDevRequest, PciDevRequest, UpcallClient, UpcallClientError,
@@ -57,6 +60,8 @@ use crate::resource_manager::ResourceManager;
 use crate::vm::{KernelConfigInfo, Vm, VmConfigInfo};
 use crate::IoManagerCached;
 
+use vm_memory::GuestRegionMmap;
+
 /// Virtual machine console device manager.
 pub mod console_manager;
 /// Console Manager for virtual machines console device.
@@ -131,6 +136,9 @@ macro_rules! info(
     };
 );
 
+// The flag of whether to use the shared irq.
+const USE_SHARED_IRQ: bool = true;
+
 /// Errors related to device manager operations.
 #[derive(Debug, thiserror::Error)]
 pub enum DeviceMgrError {
@@ -173,13 +181,28 @@ pub enum DeviceMgrError {
     HotplugDevice(#[source] UpcallClientError),
 
     /// Failed to free device resource.
-    #[error("failed to free device resources: {0}")]
+    #[error("failed to allocate/free device resources: {0}")]
     ResourceError(#[source] crate::resource_manager::ResourceError),
 
     #[cfg(feature = "host-device")]
     /// Error from Vfio Pci
     #[error("failed to do vfio pci operation: {0:?}")]
     VfioPci(#[source] dbs_pci::VfioPciError),
+    /// Error from Virtio Pci
+    #[error("failed to do virtio pci operation")]
+    VirtioPci,
+    /// PCI system manager error
+    #[error("Pci system manager error")]
+    PciSystemManager,
+    /// Dragonball pci system error
+    #[error("pci error: {0:?}")]
+    PciError(#[source] dbs_pci::Error),
+    /// Virtio Pci system error
+    #[error("virtio pci error: {0:?}")]
+    VirtioPciError(#[source] dbs_pci::VirtioPciDeviceError),
+    /// Unsupported pci device type
+    #[error("unsupported pci device type")]
+    InvalidPciDeviceType,
 }
 
 /// Specialized version of `std::result::Result` for device manager operations.
@@ -290,9 +313,10 @@ pub struct DeviceOpContext {
     #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
     upcall_client: Option<Arc<UpcallClient<DevMgrService>>>,
     #[cfg(feature = "dbs-virtio-devices")]
-    virtio_devices: Vec<Arc<DbsMmioV2Device>>,
+    virtio_devices: Vec<Arc<dyn DeviceIo>>,
     #[cfg(feature = "host-device")]
     vfio_manager: Option<Arc<Mutex<VfioDeviceMgr>>>,
+    pci_system_manager: Arc<Mutex<PciSystemManager>>,
     vm_config: Option<VmConfigInfo>,
     shared_info: Arc<RwLock<InstanceInfo>>,
 }
@@ -343,6 +367,7 @@ impl DeviceOpContext {
             shared_info,
             #[cfg(feature = "host-device")]
             vfio_manager: None,
+            pci_system_manager: device_mgr.pci_system_manager.clone(),
         }
     }
 
@@ -444,6 +469,11 @@ impl DeviceOpContext {
         }
         Ok(dev_info)
     }
+
+    /// check the hotplug context
+    pub fn is_hotplug(&self) -> bool {
+        self.is_hotplug
+    }
 }
 
 #[cfg(all(feature = "hotplug", not(feature = "dbs-upcall")))]
@@ -513,7 +543,7 @@ impl DeviceOpContext {
 
     pub(crate) fn insert_hotplug_mmio_device(
         &self,
-        dev: &Arc<DbsMmioV2Device>,
+        dev: &Arc<dyn DeviceIo>,
         callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>>,
     ) -> Result<()> {
         if !self.is_hotplug {
@@ -532,7 +562,7 @@ impl DeviceOpContext {
 
     pub(crate) fn remove_hotplug_mmio_device(
         &self,
-        dev: &Arc<DbsMmioV2Device>,
+        dev: &Arc<dyn DeviceIo>,
         callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>>,
     ) -> Result<()> {
         if !self.is_hotplug {
@@ -630,6 +660,7 @@ pub struct DeviceManager {
     vhost_user_net_manager: VhostUserNetDeviceMgr,
     #[cfg(feature = "host-device")]
     pub(crate) vfio_manager: Arc<Mutex<VfioDeviceMgr>>,
+    pub(crate) pci_system_manager: Arc<Mutex<PciSystemManager>>,
 }
 
 impl DeviceManager {
@@ -640,11 +671,25 @@ impl DeviceManager {
         epoll_manager: EpollManager,
         logger: &slog::Logger,
         shared_info: Arc<RwLock<InstanceInfo>>,
-    ) -> Self {
-        DeviceManager {
-            io_manager: Arc::new(ArcSwap::new(Arc::new(IoManager::new()))),
-            io_lock: Arc::new(Mutex::new(())),
-            irq_manager: Arc::new(KvmIrqManager::new(vm_fd.clone())),
+    ) -> Result<Self> {
+        let irq_manager = Arc::new(KvmIrqManager::new(vm_fd.clone()));
+        let io_manager = Arc::new(ArcSwap::new(Arc::new(IoManager::new())));
+        let io_lock = Arc::new(Mutex::new(()));
+        let io_context = DeviceManagerContext::new(io_manager.clone(), io_lock.clone());
+        let mut mgr = PciSystemManager::new(irq_manager.clone(), io_context, res_manager.clone())?;
+
+        let requirements = mgr.resource_requirements();
+        let resources = res_manager
+            .allocate_device_resources(&requirements, USE_SHARED_IRQ)
+            .map_err(DeviceMgrError::ResourceError)?;
+        mgr.activate(resources)?;
+
+        let pci_system_manager = Arc::new(Mutex::new(mgr));
+
+        Ok(DeviceManager {
+            io_manager,
+            io_lock,
+            irq_manager,
             res_manager,
             vm_fd: vm_fd.clone(),
             logger: logger.new(slog::o!()),
@@ -671,8 +716,13 @@ impl DeviceManager {
             #[cfg(feature = "vhost-user-net")]
             vhost_user_net_manager: VhostUserNetDeviceMgr::default(),
             #[cfg(feature = "host-device")]
-            vfio_manager: Arc::new(Mutex::new(VfioDeviceMgr::new(vm_fd, logger))),
-        }
+            vfio_manager: Arc::new(Mutex::new(VfioDeviceMgr::new(
+                vm_fd,
+                pci_system_manager.clone(),
+                logger,
+            ))),
+            pci_system_manager,
+        })
     }
 
     /// Get the underlying IoManager to dispatch IO read/write requests.
@@ -879,7 +929,7 @@ impl DeviceManager {
     /// Start all registered devices when booting the associated virtual machine.
     pub fn start_devices(
         &mut self,
-        vm_as: &GuestAddressSpaceImpl,
+        #[allow(unused)] vm_as: &GuestAddressSpaceImpl,
     ) -> std::result::Result<(), StartMicroVmError> {
         // It is safe because we don't expect poison lock.
         #[cfg(feature = "host-device")]
@@ -899,6 +949,7 @@ impl DeviceManager {
         address_space: Option<&AddressSpace>,
     ) -> Result<()> {
         // create context for removing devices
+        #[allow(unused)]
         let mut ctx = DeviceOpContext::new(
             Some(epoll_mgr),
             self,
@@ -1023,7 +1074,7 @@ impl DeviceManager {
     }
 
     #[cfg(feature = "dbs-virtio-devices")]
-    fn get_virtio_mmio_device_info(device: &Arc<DbsMmioV2Device>) -> Result<(u64, u64, u32)> {
+    fn get_virtio_mmio_device_info(device: &Arc<dyn DeviceIo>) -> Result<(u64, u64, u32)> {
         let resources = device.get_assigned_resources();
         let irq = resources
             .get_legacy_irq()
@@ -1037,26 +1088,11 @@ impl DeviceManager {
 
         Err(DeviceMgrError::GetDeviceResource)
     }
-
-    /// Get pci bus resources for creating fdt.
-    #[cfg(feature = "host-device")]
-    pub fn get_pci_bus_resources(&self) -> Option<PciBusResources> {
-        let mut vfio_dev_mgr = self.vfio_manager.lock().unwrap();
-        let vfio_pci_mgr = vfio_dev_mgr.get_pci_manager();
-        vfio_pci_mgr.as_ref()?;
-        let pci_manager = vfio_pci_mgr.unwrap();
-        let ecam_space = pci_manager.get_ecam_space();
-        let bar_space = pci_manager.get_bar_space();
-        Some(PciBusResources {
-            ecam_space,
-            bar_space,
-        })
-    }
 }
 
 #[cfg(feature = "dbs-virtio-devices")]
 impl DeviceManager {
-    fn get_virtio_device_info(device: &Arc<DbsMmioV2Device>) -> Result<(u64, u64, u32)> {
+    fn get_virtio_device_info(device: &Arc<dyn DeviceIo>) -> Result<(u64, u64, u32)> {
         let resources = device.get_assigned_resources();
         let irq = resources
             .get_legacy_irq()
@@ -1078,7 +1114,7 @@ impl DeviceManager {
         ctx: &mut DeviceOpContext,
         use_shared_irq: bool,
         use_generic_irq: bool,
-    ) -> std::result::Result<Arc<DbsMmioV2Device>, DeviceMgrError> {
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
         let features = DRAGONBALL_FEATURE_INTR_USED | DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY;
         DeviceManager::create_mmio_virtio_device_with_features(
             device,
@@ -1096,7 +1132,7 @@ impl DeviceManager {
         ctx: &mut DeviceOpContext,
         use_shared_irq: bool,
         use_generic_irq: bool,
-    ) -> std::result::Result<Arc<DbsMmioV2Device>, DeviceMgrError> {
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
         let features = DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY;
         DeviceManager::create_mmio_virtio_device_with_features(
             device,
@@ -1115,7 +1151,7 @@ impl DeviceManager {
         features: Option<u32>,
         use_shared_irq: bool,
         use_generic_irq: bool,
-    ) -> std::result::Result<Arc<DbsMmioV2Device>, DeviceMgrError> {
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
         // Every emulated Virtio MMIO device needs a 4K configuration space,
         // and another 4K space for per queue notification.
         const MMIO_ADDRESS_DEFAULT: ResourceConstraint = ResourceConstraint::MmioAddress {
@@ -1146,12 +1182,18 @@ impl DeviceManager {
         Self::register_mmio_virtio_device(Arc::new(virtio_dev), ctx)
     }
 
-    /// Teardown the Virtio MMIO transport layer device associated with the virtio backend device.
-    pub fn destroy_mmio_virtio_device(
+    fn destroy_mmio_device(
         device: Arc<dyn DeviceIo>,
         ctx: &mut DeviceOpContext,
     ) -> std::result::Result<(), DeviceMgrError> {
-        Self::destroy_mmio_device(device.clone(), ctx)?;
+        // unregister IoManager
+        Self::deregister_virtio_device(&device, ctx)?;
+
+        // unregister Resource manager
+        let resources = device.get_assigned_resources();
+        ctx.res_manager
+            .free_device_resources(&resources)
+            .map_err(DeviceMgrError::ResourceError)?;
 
         let mmio_dev = device
             .as_any()
@@ -1163,27 +1205,11 @@ impl DeviceManager {
         Ok(())
     }
 
-    fn destroy_mmio_device(
-        device: Arc<dyn DeviceIo>,
-        ctx: &mut DeviceOpContext,
-    ) -> std::result::Result<(), DeviceMgrError> {
-        // unregister IoManager
-        Self::deregister_mmio_virtio_device(&device, ctx)?;
-
-        // unregister Resource manager
-        let resources = device.get_assigned_resources();
-        ctx.res_manager
-            .free_device_resources(&resources)
-            .map_err(DeviceMgrError::ResourceError)?;
-
-        Ok(())
-    }
-
     /// Create an Virtio MMIO transport layer device for the virtio backend device.
     pub fn register_mmio_virtio_device(
-        device: Arc<DbsMmioV2Device>,
+        device: Arc<dyn DeviceIo>,
         ctx: &mut DeviceOpContext,
-    ) -> std::result::Result<Arc<DbsMmioV2Device>, DeviceMgrError> {
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
         let (mmio_base, mmio_size, irq) = Self::get_virtio_device_info(&device)?;
         info!(
             ctx.logger(),
@@ -1225,8 +1251,210 @@ impl DeviceManager {
         }
     }
 
+    /// Create an Virtio PCI transport layer device for the virtio backend device.
+    pub fn create_virtio_pci_device(
+        mut device: DbsVirtioDevice,
+        ctx: &mut DeviceOpContext,
+        use_generic_irq: bool,
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
+        let pci_system_manager = ctx.pci_system_manager.lock().unwrap();
+
+        // We always use 64bit bars, we don't support 32bit bar now
+        // We aligned to the size of the bar itself, refers to cloud-hypervisor
+        // https://github.com/cloud-hypervisor/cloud-hypervisor/commit/bfc65bff2a5bdb9aca7dcd3284a0ced0e5cc7db8
+        //
+        // Allocate virtio-pci config bar below MMIO_LOW_END.
+        // Each bridge PCI bridge only has two bridge windows:
+        // - One is non-prefetchable and located below `MMIO_LOW_END`.
+        // - The other is prefetchable and located above `MMIO_LOW_END`.
+        // In reference to `clh`, the config BAR is set as non-prefetchable.
+        // Therefore, it must be allocated below `MMIO_LOW_END`.
+        const DEFAULE_VIRTIO_PCI_CONFIG_BAR: ResourceConstraint = ResourceConstraint::MmioAddress {
+            range: Some((0, MMIO_LOW_END)),
+            align: CAPABILITY_BAR_SIZE,
+            size: CAPABILITY_BAR_SIZE,
+        };
+
+        // Virtio pci device always use msi-x, extend irq resources to other_requests
+        let mut other_requests = vec![];
+        VirtioPciDevice::get_interrupt_requirements(device.as_ref(), &mut other_requests);
+
+        // allocate device resources by pci_bus, MmioAddress + KvmSlot?
+        let mut device_requests = vec![];
+        device.get_resource_requirements(&mut device_requests, use_generic_irq);
+
+        // Extend KvmSlot resources to other_requests
+        for req in device_requests.iter() {
+            if !matches!(
+                req,
+                ResourceConstraint::PioAddress { .. } | ResourceConstraint::MmioAddress { .. }
+            ) {
+                other_requests.push(*req);
+            }
+        }
+
+        // allocate PciMsixIrq and KvmSlot by res_manager
+        let other_resources = ctx
+            .res_manager
+            .allocate_device_resources(&other_requests, false)
+            .map_err(DeviceMgrError::ResourceError)?;
+
+        let pci_bus = pci_system_manager.pci_root_bus();
+        let dev_id = pci_system_manager
+            .new_device_id(None)
+            .ok_or(DeviceMgrError::VirtioPci)?;
+
+        // Allocate config bar resources by pci_bus
+        let default_config_req = vec![DEFAULE_VIRTIO_PCI_CONFIG_BAR];
+        let default_config_res = pci_bus
+            .allocate_resources(&default_config_req)
+            .map_err(DeviceMgrError::PciError)?;
+        assert!(default_config_res.get_all_resources().len() == 1);
+
+        // Allocate MmioAddress and PioAddress resource by pci bus, other resourece type will skip
+        let mut device_resource = pci_bus
+            .allocate_resources(&device_requests)
+            .map_err(DeviceMgrError::PciError)?;
+
+        // Extend PciMsixIrq and KvmSlot resources to device_resource
+        other_resources.get_all_resources().iter().for_each(|res| {
+            device_resource.append(res.clone());
+        });
+
+        // Do map for virtio share memory region by set_resource, this will use KvmSlot + MmioAddress resources, which should be allocated before
+        let _virito_shared_mem_list = device
+            .as_mut()
+            .set_resource(ctx.vm_fd.clone(), device_resource.clone())
+            .map_err(DeviceMgrError::Virtio)?;
+
+        // Extend config bar resources to device_resource
+        // Now device_resource contains all resources
+        default_config_res
+            .get_all_resources()
+            .iter()
+            .for_each(|res| {
+                device_resource.append(res.clone());
+            });
+
+        drop(pci_system_manager);
+
+        // new a virtio pci device
+        let mut virtio_dev = VirtioPciDevice::new(
+            ctx.vm_fd.clone(),
+            ctx.get_vm_as()?,
+            ctx.get_address_space()?,
+            ctx.irq_manager.clone(),
+            device_resource,
+            dev_id,
+            device,
+            true,
+            Arc::downgrade(&pci_bus),
+            default_config_res.get_all_resources()[0].clone(),
+        )
+        .map_err(DeviceMgrError::VirtioPciError)?;
+
+        virtio_dev
+            .alloc_bars()
+            .map_err(DeviceMgrError::VirtioPciError)?;
+
+        let arc_dev = Arc::new(virtio_dev);
+
+        pci_bus
+            .register_device(arc_dev.clone())
+            .map_err(DeviceMgrError::PciError)?;
+
+        Self::register_virtio_pci_device(arc_dev, ctx)
+    }
+
+    /// Create an Virtio PCI transport layer device for the virtio backend device.
+    pub fn register_virtio_pci_device(
+        device: Arc<dyn DeviceIo>,
+        ctx: &DeviceOpContext,
+    ) -> std::result::Result<Arc<dyn DeviceIo>, DeviceMgrError> {
+        let resources = device.get_trapped_io_resources();
+        let mut tx = ctx.io_context.begin_tx();
+        if let Err(e) = ctx
+            .io_context
+            .register_device_io(&mut tx, device.clone(), &resources)
+        {
+            ctx.io_context.cancel_tx(tx);
+            Err(DeviceMgrError::IoManager(e))
+        } else {
+            ctx.io_context.commit_tx(tx);
+            Ok(device)
+        }
+    }
+
+    /// Deregister Virtio device from IoManager
+    pub fn deregister_virtio_device(
+        device: &Arc<dyn DeviceIo>,
+        ctx: &mut DeviceOpContext,
+    ) -> std::result::Result<(), DeviceMgrError> {
+        let resources = device.get_trapped_io_resources();
+        info!(
+            ctx.logger(),
+            "unregister pci virtio device: {:?}", resources
+        );
+        let mut tx = ctx.io_context.begin_tx();
+        if let Err(e) = ctx.io_context.unregister_device_io(&mut tx, &resources) {
+            ctx.io_context.cancel_tx(tx);
+            Err(DeviceMgrError::IoManager(e))
+        } else {
+            ctx.io_context.commit_tx(tx);
+            Ok(())
+        }
+    }
+
+    /// Destroy/Deregister resources for a Virtio PCI
+    fn destroy_pci_device(
+        device: Arc<dyn DeviceIo>,
+        ctx: &mut DeviceOpContext,
+        dev_id: u8,
+    ) -> std::result::Result<(), DeviceMgrError> {
+        // unregister IoManager
+        Self::deregister_virtio_device(&device, ctx)?;
+        // unregister Resource manager
+        let resources = device.get_assigned_resources();
+        let mut system_resources = DeviceResources::new();
+        resources.iter().for_each(|res| {
+            if !matches!(
+                res,
+                Resource::PioAddressRange { .. } | Resource::MmioAddressRange { .. }
+            ) {
+                system_resources.append(res.clone());
+            }
+        });
+        info!(
+            ctx.logger(),
+            "unregister resource {:?} from system resource manager for pci device",
+            system_resources
+        );
+        ctx.res_manager
+            .free_device_resources(&system_resources)
+            .map_err(DeviceMgrError::ResourceError)?;
+        let pci_system_manager = ctx.pci_system_manager.lock().unwrap();
+        let pci_bus = pci_system_manager.pci_root_bus();
+        info!(
+            ctx.logger(),
+            "unregister resource {:?} from pci bus resource manager for pci device", resources
+        );
+        pci_bus.free_resources(resources);
+        let _ = pci_system_manager.free_device_id(dev_id as u32);
+
+        let pci_dev = device
+            .as_any()
+            .downcast_ref::<VirtioPciDevice<GuestAddressSpaceImpl, QueueSync, GuestRegionMmap>>()
+            .ok_or(DeviceMgrError::InvalidOperation)?;
+
+        pci_dev.remove();
+
+        Ok(())
+    }
+
     #[cfg(feature = "host-device")]
     fn get_pci_device_info(device: &Arc<dyn DeviceIo>) -> Result<(u8, u8)> {
+        use virtio_queue::QueueSync;
+
         if let Some(pci_dev) = device
             .as_any()
             .downcast_ref::<VfioPciDevice<PciSystemManager>>()
@@ -1243,10 +1471,41 @@ impl DeviceManager {
             // together those 8 bits combined as devfn value
             let devfn = (((slot) & 0x1f) << 3) | ((func) & 0x07);
 
+            return Ok((busno, devfn));
+        } else if let Some(pci_dev) = device.as_any().downcast_ref::<VirtioPciDevice<
+            GuestAddressSpaceImpl,
+            QueueSync,
+            GuestRegionMmap,
+        >>() {
+            // reference from kernel: include/uapi/linux/pci.h
+            let busno = pci_dev.bus_id().map_err(DeviceMgrError::VirtioPciError)?;
+            let slot = pci_dev.device_id();
+            let func = 0;
+            let devfn = (((slot) & 0x1f) << 3) | ((func) & 0x07);
+
             return Ok((busno, devfn));
         }
 
-        Err(DeviceMgrError::GetDeviceResource)
+        Err(DeviceMgrError::InvalidPciDeviceType)
+    }
+
+    /// Teardown the Virtio PCI or MMIO transport layer device associated with the virtio backend device.
+    pub fn destroy_virtio_device(
+        device: Arc<dyn DeviceIo>,
+        ctx: &mut DeviceOpContext,
+    ) -> std::result::Result<(), DeviceMgrError> {
+        if let Some(mmio_dev) = device.as_any().downcast_ref::<DbsMmioV2Device>() {
+            Self::destroy_mmio_device(device.clone(), ctx)?;
+            mmio_dev.remove();
+        } else if let Some(pci_dev) = device.as_any().downcast_ref::<VirtioPciDevice<
+            GuestAddressSpaceImpl,
+            QueueSync,
+            GuestRegionMmap,
+        >>() {
+            Self::destroy_pci_device(device.clone(), ctx, pci_dev.device_id())?;
+        }
+
+        Ok(())
     }
 }
 
@@ -1275,7 +1534,9 @@ mod tests {
     use dbs_address_space::{AddressSpaceLayout, AddressSpaceRegion, AddressSpaceRegionType};
     use kvm_ioctls::Kvm;
     use test_utils::skip_if_not_root;
-    use vm_memory::{GuestAddress, GuestUsize, MmapRegion};
+    #[cfg(feature = "virtio-fs")]
+    use vm_memory::MmapRegion;
+    use vm_memory::{GuestAddress, GuestUsize};
 
     use super::*;
     #[cfg(target_arch = "x86_64")]
@@ -1309,12 +1570,29 @@ mod tests {
                 String::from("1"),
             )));
 
+            let irq_manager = Arc::new(KvmIrqManager::new(vm_fd.clone()));
+            let io_manager = Arc::new(ArcSwap::new(Arc::new(IoManager::new())));
+            let io_lock = Arc::new(Mutex::new(()));
+            let io_context = DeviceManagerContext::new(io_manager.clone(), io_lock.clone());
+            let mut mgr =
+                PciSystemManager::new(irq_manager.clone(), io_context, res_manager.clone())
+                    .unwrap();
+
+            let requirements = mgr.resource_requirements();
+            let resources = res_manager
+                .allocate_device_resources(&requirements, USE_SHARED_IRQ)
+                .map_err(DeviceMgrError::ResourceError)
+                .unwrap();
+            mgr.activate(resources).unwrap();
+
+            let pci_system_manager = Arc::new(Mutex::new(mgr));
+
             DeviceManager {
                 vm_fd: Arc::clone(&vm_fd),
                 con_manager: ConsoleManager::new(epoll_manager, &logger),
-                io_manager: Arc::new(ArcSwap::new(Arc::new(IoManager::new()))),
-                io_lock: Arc::new(Mutex::new(())),
-                irq_manager: Arc::new(KvmIrqManager::new(vm_fd.clone())),
+                io_manager,
+                io_lock,
+                irq_manager,
                 res_manager,
 
                 legacy_manager: None,
@@ -1337,7 +1615,12 @@ mod tests {
                 #[cfg(feature = "vhost-user-net")]
                 vhost_user_net_manager: VhostUserNetDeviceMgr::default(),
                 #[cfg(feature = "host-device")]
-                vfio_manager: Arc::new(Mutex::new(VfioDeviceMgr::new(vm_fd, &logger))),
+                vfio_manager: Arc::new(Mutex::new(VfioDeviceMgr::new(
+                    vm_fd,
+                    pci_system_manager.clone(),
+                    &logger,
+                ))),
+                pci_system_manager,
 
                 logger,
                 shared_info,

src/dragonball/src/device_manager/vfio_dev_mgr/mod.rs

@@ -17,12 +17,11 @@ use std::ops::Deref;
 use std::os::fd::RawFd;
 use std::path::Path;
 use std::sync::mpsc::Sender;
-use std::sync::{Arc, Weak};
+use std::sync::{Arc, Mutex, Weak};
 
 use dbs_device::resources::Resource::LegacyIrq;
 use dbs_device::resources::{DeviceResources, Resource, ResourceConstraint};
 use dbs_device::DeviceIo;
-use dbs_interrupt::KvmIrqManager;
 use dbs_pci::{VfioPciDevice, VENDOR_NVIDIA};
 use dbs_upcall::{DevMgrResponse, UpcallClientResponse};
 use kvm_ioctls::{DeviceFd, VmFd};
@@ -37,8 +36,8 @@ use vm_memory::{
 use super::StartMicroVmError;
 use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl};
 use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos};
-use crate::device_manager::{DeviceManagerContext, DeviceMgrError, DeviceOpContext};
-use crate::resource_manager::{ResourceError, ResourceManager};
+use crate::device_manager::{DeviceMgrError, DeviceOpContext};
+use crate::resource_manager::ResourceError;
 
 // The flag of whether to use the shared irq.
 const USE_SHARED_IRQ: bool = true;
@@ -230,7 +229,7 @@ pub struct VfioDeviceMgr {
     info_list: DeviceConfigInfos<HostDeviceConfig>,
     locked_vm_size: u64,
     vfio_container: Option<Arc<VfioContainer>>,
-    pci_vfio_manager: Option<Arc<PciSystemManager>>,
+    pci_system_manager: Arc<Mutex<PciSystemManager>>,
     pci_legacy_irqs: Option<HashMap<u8, u8>>,
     nvidia_shared_irq: Option<u32>,
     logger: slog::Logger,
@@ -238,13 +237,17 @@ pub struct VfioDeviceMgr {
 
 impl VfioDeviceMgr {
     /// Create a new VFIO device manager.
-    pub fn new(vm_fd: Arc<VmFd>, logger: &slog::Logger) -> Self {
+    pub fn new(
+        vm_fd: Arc<VmFd>,
+        pci_system_manager: Arc<Mutex<PciSystemManager>>,
+        logger: &slog::Logger,
+    ) -> Self {
         VfioDeviceMgr {
             vm_fd,
             info_list: DeviceConfigInfos::new(),
             locked_vm_size: 0,
             vfio_container: None,
-            pci_vfio_manager: None,
+            pci_system_manager,
             pci_legacy_irqs: Some(HashMap::new()),
             nvidia_shared_irq: None,
             logger: logger.new(slog::o!()),
@@ -288,17 +291,6 @@ impl VfioDeviceMgr {
         &mut self,
         ctx: &mut DeviceOpContext,
     ) -> std::result::Result<(), StartMicroVmError> {
-        // create and attach pci root bus
-        #[cfg(all(feature = "hotplug", feature = "host-device"))]
-        if ctx.pci_hotplug_enabled {
-            let _ = self
-                .create_pci_manager(
-                    ctx.irq_manager.clone(),
-                    ctx.io_context.clone(),
-                    ctx.res_manager.clone(),
-                )
-                .map_err(StartMicroVmError::CreateVfioDevice)?;
-        }
         for (idx, info) in self.info_list.clone().iter().enumerate() {
             self.create_device(&info.config, ctx, idx)
                 .map_err(StartMicroVmError::CreateVfioDevice)?;
@@ -574,12 +566,9 @@ impl VfioDeviceMgr {
             "subsystem" => "vfio_dev_mgr",
              "host_bdf" => &cfg.bus_slot_func,
         );
-        // safe to get pci_manager
-        let pci_manager = self.create_pci_manager(
-            ctx.irq_manager.clone(),
-            ctx.io_context.clone(),
-            ctx.res_manager.clone(),
-        )?;
+
+        let pci_manager = self.get_pci_manager();
+        let pci_manager = pci_manager.lock().unwrap();
         let pci_bus = pci_manager.pci_root_bus();
         let id = pci_manager
             .new_device_id(cfg.guest_dev_id)
@@ -607,7 +596,7 @@ impl VfioDeviceMgr {
                 sysfs_path,
                 Arc::downgrade(&pci_bus),
                 vfio_dev,
-                Arc::downgrade(self.get_pci_manager().unwrap()),
+                self.get_pci_manager(),
                 ctx.vm_fd.clone(),
                 cfg.vendor_device_id,
                 cfg.clique_id,
@@ -665,8 +654,8 @@ impl VfioDeviceMgr {
 
         // safe to unwrap because pci vfio manager is already created
         let _ = self
-            .pci_vfio_manager
-            .as_mut()
+            .pci_system_manager
+            .lock()
             .unwrap()
             .free_device_id(device_id)
             .ok_or(VfioDeviceError::InvalidDeviceID(device_id))?;
@@ -698,27 +687,9 @@ impl VfioDeviceMgr {
         Ok(())
     }
 
-    pub(crate) fn create_pci_manager(
-        &mut self,
-        irq_manager: Arc<KvmIrqManager>,
-        io_context: DeviceManagerContext,
-        res_manager: Arc<ResourceManager>,
-    ) -> Result<&mut Arc<PciSystemManager>> {
-        if self.pci_vfio_manager.is_none() {
-            let mut mgr = PciSystemManager::new(irq_manager, io_context, res_manager.clone())?;
-            let requirements = mgr.resource_requirements();
-            let resources = res_manager
-                .allocate_device_resources(&requirements, USE_SHARED_IRQ)
-                .or(Err(VfioDeviceError::NoResource))?;
-            mgr.activate(resources)?;
-            self.pci_vfio_manager = Some(Arc::new(mgr));
-        }
-        Ok(self.pci_vfio_manager.as_mut().unwrap())
-    }
-
     /// Get the PCI manager to support PCI device passthrough
-    pub fn get_pci_manager(&mut self) -> Option<&mut Arc<PciSystemManager>> {
-        self.pci_vfio_manager.as_mut()
+    pub fn get_pci_manager(&mut self) -> Arc<Mutex<PciSystemManager>> {
+        self.pci_system_manager.clone()
     }
 }
 

src/dragonball/src/device_manager/vfio_dev_mgr/pci_vfio.rs

@@ -12,14 +12,13 @@ use dbs_interrupt::KvmIrqManager;
 use dbs_pci::ECAM_SPACE_LENGTH;
 use dbs_pci::{create_pci_root_bus, PciBus, PciDevice, PciRootDevice, PciSystemContext};
 
-use super::{Result, VfioDeviceError};
+use super::DeviceMgrError;
 #[cfg(target_arch = "aarch64")]
 use crate::device_manager::vfio_dev_mgr::USE_SHARED_IRQ;
 use crate::device_manager::DeviceManagerContext;
 use crate::resource_manager::ResourceManager;
+use dbs_pci::PCI_BUS_DEFAULT;
 
-/// we only support one pci bus
-pub const PCI_BUS_DEFAULT: u8 = 0;
 /// The default mmio size for pci root bus.
 const PCI_MMIO_DEFAULT_SIZE: u64 = 2048u64 << 30;
 
@@ -38,13 +37,13 @@ impl PciSystemManager {
         irq_manager: Arc<KvmIrqManager>,
         io_context: DeviceManagerContext,
         res_manager: Arc<ResourceManager>,
-    ) -> std::result::Result<Self, VfioDeviceError> {
+    ) -> std::result::Result<Self, DeviceMgrError> {
         let resources = PciSystemManager::allocate_root_device_resources(res_manager)?;
         let pci_root = Arc::new(
-            PciRootDevice::create(PCI_BUS_DEFAULT, resources).map_err(VfioDeviceError::PciError)?,
+            PciRootDevice::create(PCI_BUS_DEFAULT, resources).map_err(DeviceMgrError::PciError)?,
         );
         let pci_root_bus =
-            create_pci_root_bus(PCI_BUS_DEFAULT).map_err(VfioDeviceError::PciError)?;
+            create_pci_root_bus(PCI_BUS_DEFAULT).map_err(DeviceMgrError::PciError)?;
 
         Ok(PciSystemManager {
             irq_manager,
@@ -58,7 +57,7 @@ impl PciSystemManager {
     #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
     fn allocate_root_device_resources(
         _res_manager: Arc<ResourceManager>,
-    ) -> Result<DeviceResources> {
+    ) -> std::result::Result<DeviceResources, DeviceMgrError> {
         let mut resources = DeviceResources::new();
         resources.append(Resource::PioAddressRange {
             // PCI CONFIG_ADDRESS port address 0xcf8 and uses 32 bits
@@ -76,7 +75,7 @@ impl PciSystemManager {
     #[cfg(target_arch = "aarch64")]
     fn allocate_root_device_resources(
         res_manager: Arc<ResourceManager>,
-    ) -> Result<DeviceResources> {
+    ) -> std::result::Result<DeviceResources, DeviceMgrError> {
         let requests = vec![ResourceConstraint::MmioAddress {
             range: Some((0x0, 0xffff_ffff)),
             align: 4096,
@@ -84,23 +83,26 @@ impl PciSystemManager {
         }];
         let resources = res_manager
             .allocate_device_resources(&requests, USE_SHARED_IRQ)
-            .map_err(VfioDeviceError::AllocateDeviceResource)?;
+            .map_err(DeviceMgrError::ResourceError)?;
         Ok(resources)
     }
 
     /// Activate the PCI subsystem.
-    pub fn activate(&mut self, resources: DeviceResources) -> Result<()> {
+    pub fn activate(
+        &mut self,
+        resources: DeviceResources,
+    ) -> std::result::Result<(), DeviceMgrError> {
         let bus_id = self.pci_root_bus.bus_id();
 
         self.pci_root
             .add_bus(self.pci_root_bus.clone(), bus_id)
-            .map_err(VfioDeviceError::PciError)?;
+            .map_err(DeviceMgrError::PciError)?;
         PciRootDevice::activate(self.pci_root.clone(), &mut self.io_context)
-            .map_err(VfioDeviceError::PciError)?;
+            .map_err(DeviceMgrError::PciError)?;
 
         self.pci_root_bus
             .assign_resources(resources)
-            .map_err(VfioDeviceError::PciError)?;
+            .map_err(DeviceMgrError::PciError)?;
 
         Ok(())
     }

src/dragonball/src/device_manager/vhost_net_dev_mgr.rs

@@ -256,7 +256,7 @@ impl VhostNetDeviceMgr {
                 info.config.iface_id
             );
             if let Some(device) = info.device.take() {
-                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+                DeviceManager::destroy_mmio_device(device, ctx)?;
             }
         }
 

src/dragonball/src/device_manager/virtio_net_dev_mgr.rs

@@ -382,7 +382,7 @@ impl VirtioNetDeviceMgr {
                 info.config.iface_id
             );
             if let Some(device) = info.device.take() {
-                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+                DeviceManager::destroy_mmio_device(device, ctx)?;
             }
         }
         Ok(())

src/dragonball/src/device_manager/vsock_dev_mgr.rs

@@ -294,7 +294,7 @@ impl VsockDeviceMgr {
                 info.config.id
             );
             if let Some(device) = info.device.take() {
-                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+                DeviceManager::destroy_mmio_device(device, ctx)?;
             }
         }
         Ok(())

src/dragonball/src/error.rs

@@ -77,6 +77,10 @@ pub enum Error {
     /// Cannot open the VM file descriptor.
     #[error(transparent)]
     Vm(vm::VmError),
+
+    /// Fail to create device manager system
+    #[error("failed to create device manager system: {0}")]
+    DeviceMgrError(#[source] device_manager::DeviceMgrError),
 }
 
 /// Errors associated with starting the instance.

src/dragonball/src/vm/mod.rs

@@ -35,7 +35,7 @@ use crate::address_space_manager::{
 use crate::api::v1::{InstanceInfo, InstanceState};
 use crate::device_manager::console_manager::DmesgWriter;
 use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext};
-use crate::error::{LoadInitrdError, Result, StartMicroVmError, StopMicrovmError};
+use crate::error::{Error, LoadInitrdError, Result, StartMicroVmError, StopMicrovmError};
 use crate::event_manager::EventManager;
 use crate::kvm_context::KvmContext;
 use crate::resource_manager::ResourceManager;
@@ -228,7 +228,8 @@ impl Vm {
             epoll_manager.clone(),
             &logger,
             api_shared_info.clone(),
-        );
+        )
+        .map_err(Error::DeviceMgrError)?;
 
         Ok(Vm {
             epoll_manager,

src/libs/kata-sys-util/Cargo.toml

@@ -18,7 +18,7 @@ common-path = "=1.0.0"
 fail = "0.5.0"
 lazy_static = "1.4.0"
 libc = "0.2.100"
-nix = "0.24.2"
+nix = "0.26.4"
 once_cell = "1.9.0"
 serde = { version = "1.0.138", features = ["derive"] }
 serde_json = "1.0.73"

src/libs/kata-sys-util/src/hooks.rs

@@ -375,7 +375,11 @@ mod tests {
         fn build_oci_hook(self) -> oci::Hook {
             let mut hook = oci::Hook::default();
             hook.set_path(PathBuf::from(self.path));
-            hook.set_args(Some(self.args));
+            if self.args.is_empty() {
+                hook.set_args(None);
+            } else {
+                hook.set_args(Some(self.args));
+            }
             hook.set_env(Some(self.env));
             hook.set_timeout(self.timeout);