Merge pull request #8292 from fidencio/topic/release-ensure-gh-is-used-from-a-git-repo

release: Always use actions/checkout to ensure we're in a git repo

.github/workflows/add-backport-label.yaml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install hub extension script
         run: |

.github/workflows/add-issues-to-project.yaml

@@ -39,7 +39,7 @@ jobs:
           popd &>/dev/null
 
       - name: Checkout code to allow hub to communicate with the project
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Add issue to issue backlog
         env:

.github/workflows/add-pr-sizing-label.yaml

@@ -21,7 +21,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Install PR sizing label script
         run: |

.github/workflows/basic-ci-amd64.yaml

@@ -0,0 +1,200 @@
+name: CI | Basic amd64 tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-cri-containerd:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
+
+      - name: Run cri-containerd tests
+        run: bash tests/integration/cri-containerd/gha-run.sh run
+
+  run-containerd-stability:
+    strategy:
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/stability/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/stability/gha-run.sh install-kata kata-artifacts
+
+      - name: Run containerd-stability tests
+        run: bash tests/stability/gha-run.sh run
+
+  run-nydus:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu', 'dragonball']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nydus/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nydus/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nydus tests
+        run: bash tests/integration/nydus/gha-run.sh run
+
+  run-runk:
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/runk/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
+
+      - name: Run tracing tests
+        run: bash tests/integration/runk/gha-run.sh run
+
+  run-vfio:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2304
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/vfio/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Run vfio tests
+        timeout-minutes: 15
+        run: bash tests/functional/vfio/gha-run.sh run

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -16,6 +16,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-asset:
@@ -23,9 +27,13 @@ jobs:
     strategy:
       matrix:
         asset:
+          - agent
+          - agent-opa
+          - agent-ctl
           - cloud-hypervisor
           - cloud-hypervisor-glibc
           - firecracker
+          - kata-ctl
           - kernel
           - kernel-sev
           - kernel-dragonball-experimental
@@ -33,6 +41,7 @@ jobs:
           - kernel-nvidia-gpu
           - kernel-nvidia-gpu-snp
           - kernel-nvidia-gpu-tdx-experimental
+          - log-parser-rs
           - nydus
           - ovmf
           - ovmf-sev
@@ -44,12 +53,18 @@ jobs:
           - rootfs-initrd
           - rootfs-initrd-mariner
           - rootfs-initrd-sev
+          - runk
           - shim-v2
           - tdvf
+          - trace-forwarder
           - virtiofsd
         stage:
           - ${{ inputs.stage }}
         exclude:
+          - asset: agent
+            stage: release
+          - asset: agent-opa
+            stage: release
           - asset: cloud-hypervisor-glibc
             stage: release
     steps:
@@ -61,11 +76,17 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
 
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -76,6 +97,10 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -89,9 +114,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-asset
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -16,10 +16,14 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-asset:
-    runs-on: arm64
+    runs-on: arm64-builder
     strategy:
       matrix:
         asset:
@@ -48,10 +52,17 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -62,6 +73,10 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -72,16 +87,22 @@ jobs:
           if-no-files-found: error
 
   create-kata-tarball:
-    runs-on: arm64
+    runs-on: arm64-builder
     needs: build-asset
     steps:
       - name: Adjust a permission for repo
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -16,6 +16,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-asset:
@@ -44,10 +48,17 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -59,6 +70,10 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -76,9 +91,15 @@ jobs:
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:

.github/workflows/cargo-deny-runner.yaml

@@ -19,7 +19,7 @@ jobs:
     steps:
       - name: Checkout Code
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Generate Action
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: bash cargo-deny-generator.sh

.github/workflows/ci-nightly.yaml

@@ -15,4 +15,5 @@ jobs:
       commit-hash: ${{ github.sha }}
       pr-number: "nightly"
       tag: ${{ github.sha }}-nightly
+      target-branch: ${{ github.ref_name }}
     secrets: inherit

.github/workflows/ci-on-push.yaml

@@ -3,6 +3,7 @@ on:
   pull_request_target:
     branches:
       - 'main'
+      - 'stable-*'
     types:
       # Adding 'labeled' to the list of activity types that trigger this event
       # (default: opened, synchronize, reopened) so that we can run this
@@ -27,4 +28,5 @@ jobs:
       commit-hash: ${{ github.event.pull_request.head.sha }}
       pr-number: ${{ github.event.pull_request.number }}
       tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
     secrets: inherit

.github/workflows/ci.yaml

@@ -11,6 +11,10 @@ on:
       tag:
         required: true
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-kata-static-tarball-amd64:
@@ -18,6 +22,7 @@ jobs:
     with:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
 
   publish-kata-deploy-payload-amd64:
     needs: build-kata-static-tarball-amd64
@@ -28,8 +33,94 @@ jobs:
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
     secrets: inherit
 
+  build-and-publish-tee-confidential-unencrypted-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Kata Containers ghcr.io
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker build and push
+        uses: docker/build-push-action@v4
+        with:
+          tags: ghcr.io/kata-containers/test-images:unencrypted-${{ inputs.pr-number }}
+          push: true
+          context: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/
+          platforms: linux/amd64, linux/s390x
+          file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
+
+  run-docker-tests-on-garm:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-docker-tests-on-garm.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-nerdctl-tests-on-garm:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-nerdctl-tests-on-garm.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-kata-deploy-tests-on-aks:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-kata-deploy-tests-on-aks.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-kata-deploy-tests-on-garm:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-kata-deploy-tests-on-garm.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-kata-monitor-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-kata-monitor-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
   run-k8s-tests-on-aks:
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
@@ -39,34 +130,43 @@ jobs:
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
     secrets: inherit
 
-  run-k8s-tests-on-sev:
+  run-k8s-tests-on-garm:
     needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-sev.yaml
+    uses: ./.github/workflows/run-k8s-tests-on-garm.yaml
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
 
-  run-k8s-tests-on-snp:
+  run-k8s-tests-with-crio-on-garm:
     needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-snp.yaml
+    uses: ./.github/workflows/run-k8s-tests-with-crio-on-garm.yaml
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
 
-  run-k8s-tests-on-tdx:
-    needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml
+  run-kata-coco-tests:
+    needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
+    uses: ./.github/workflows/run-kata-coco-tests.yaml
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
 
   run-metrics-tests:
     needs: build-kata-static-tarball-amd64
@@ -74,24 +174,12 @@ jobs:
     with:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
 
-  run-cri-containerd-tests:
+  run-basic-amd64-tests:
     needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/run-cri-containerd-tests.yaml
-    with:
-      tarball-suffix: -${{ inputs.tag }}
-      commit-hash: ${{ inputs.commit-hash }}
-
-  run-nydus-tests:
-    needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/run-nydus-tests.yaml
-    with:
-      tarball-suffix: -${{ inputs.tag }}
-      commit-hash: ${{ inputs.commit-hash }}
-
-  run-vfio-tests:
-    needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/run-vfio-tests.yaml
+    uses: ./.github/workflows/basic-ci-amd64.yaml
     with:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}

.github/workflows/darwin-tests.yaml

@@ -21,6 +21,6 @@ jobs:
       with:
         go-version: 1.19.3
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Build utils
       run: ./ci/darwin-test.sh

.github/workflows/docs-url-alive-check.yaml

@@ -22,7 +22,7 @@ jobs:
         echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}

.github/workflows/kata-runtime-classes-sync.yaml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Ensure the split out runtime classes match the all-in-one file
       run: |
         pushd tools/packaging/kata-deploy/runtimeclasses/

.github/workflows/move-issues-to-in-progress.yaml

@@ -38,7 +38,17 @@ jobs:
 
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Move issue to "In progress"
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

.github/workflows/payload-after-push.yaml

@@ -4,6 +4,7 @@ on:
     branches:
       - main
       - stable-*
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -15,6 +16,7 @@ jobs:
     with:
       commit-hash: ${{ github.sha }}
       push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   build-assets-arm64:
@@ -22,6 +24,7 @@ jobs:
     with:
       commit-hash: ${{ github.sha }}
       push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   build-assets-s390x:
@@ -29,6 +32,7 @@ jobs:
     with:
       commit-hash: ${{ github.sha }}
       push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-kata-deploy-payload-amd64:
@@ -39,6 +43,7 @@ jobs:
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-amd64
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-kata-deploy-payload-arm64:
@@ -49,6 +54,7 @@ jobs:
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-arm64
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-kata-deploy-payload-s390x:
@@ -59,6 +65,7 @@ jobs:
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-s390x
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-manifest:
@@ -66,7 +73,7 @@ jobs:
     needs: [publish-kata-deploy-payload-amd64, publish-kata-deploy-payload-arm64, publish-kata-deploy-payload-s390x]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Login to Kata Containers quay.io
         uses: docker/login-action@v2

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -17,14 +17,25 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   kata-payload:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -17,18 +17,29 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   kata-payload:
-    runs-on: arm64
+    runs-on: arm64-builder
     steps:
       - name: Adjust a permission for repo
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -17,6 +17,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   kata-payload:
@@ -26,9 +30,16 @@ jobs:
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/release-amd64.yaml

@@ -29,7 +29,7 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:

.github/workflows/release-arm64.yaml

@@ -14,7 +14,7 @@ jobs:
 
   kata-deploy:
     needs: build-kata-static-tarball-arm64
-    runs-on: arm64
+    runs-on: arm64-builder
     steps:
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v2
@@ -29,7 +29,7 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:

.github/workflows/release-s390x.yaml

@@ -29,7 +29,7 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:

.github/workflows/release.yaml

@@ -32,7 +32,7 @@ jobs:
     needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v2
@@ -73,11 +73,7 @@ jobs:
     needs: publish-multi-arch-images
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: install hub
-        run: |
-          wget -q -O- https://github.com/mislav/hub/releases/download/v2.14.2/hub-linux-amd64-2.14.2.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
+      - uses: actions/checkout@v4
 
       - name: download-artifacts-amd64
         uses: actions/download-artifact@v3
@@ -90,7 +86,7 @@ jobs:
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
       - name: download-artifacts-arm64
@@ -104,7 +100,7 @@ jobs:
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
       - name: download-artifacts-s390x
@@ -118,13 +114,13 @@ jobs:
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
   upload-versions-yaml:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: upload versions.yaml
         env:
           GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
@@ -133,28 +129,28 @@ jobs:
           pushd $GITHUB_WORKSPACE
           versions_file="kata-containers-$tag-versions.yaml"
           cp versions.yaml ${versions_file}
-          hub release edit -m "" -a "${versions_file}" "${tag}"
+          gh release upload "${tag}" "${versions_file}"
           popd
 
   upload-cargo-vendored-tarball:
     needs: upload-multi-arch-static-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: generate-and-upload-tarball
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="kata-containers-$tag-vendor.tar.gz"
           pushd $GITHUB_WORKSPACE
           bash -c "tools/packaging/release/generate_vendor.sh ${tarball}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
   upload-libseccomp-tarball:
     needs: upload-cargo-vendored-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: download-and-upload-tarball
         env:
           GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
@@ -174,6 +170,6 @@ jobs:
           # "-m" option should be empty to re-use the existing release title
           # without opening a text editor.
           # For the details, check https://hub.github.com/hub-release.1.html.
-          hub release edit -m "" -a "${tarball}" "${tag}"
-          hub release edit -m "" -a "${asc}" "${tag}"
+          gh release upload "${tag}" "${tarball}"
+          gh release upload "${tag}" "${asc}"
           popd

.github/workflows/require-pr-porting-labels.yaml

@@ -36,7 +36,17 @@ jobs:
 
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Install porting checker script
         run: |

.github/workflows/run-cri-containerd-tests.yaml

@@ -1,42 +0,0 @@
-name: CI | Run cri-containerd tests
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-cri-containerd:
-    strategy:
-      fail-fast: true
-      matrix:
-        containerd_version: ['lts', 'active']
-        vmm: ['clh', 'qemu']
-    runs-on: garm-ubuntu-2204
-    env:
-      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Install dependencies
-        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Install kata
-        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
-
-      - name: Run cri-containerd tests
-        run: bash tests/integration/cri-containerd/gha-run.sh run

.github/workflows/run-docker-tests-on-garm.yaml

@@ -0,0 +1,56 @@
+name: CI | Run docker integration tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-docker-tests:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # all the tests are not flaky, otherwise we'll fail them
+      # all due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+    runs-on: garm-ubuntu-2304-smaller
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/docker/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/docker/gha-run.sh install-kata kata-artifacts
+
+      - name: Run docker smoke test
+        timeout-minutes: 5
+        run: bash tests/integration/docker/gha-run.sh run

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -17,6 +17,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   run-k8s-tests:
@@ -29,6 +33,9 @@ jobs:
           - clh
           - dragonball
           - qemu
+        instance-type:
+          - small
+          - normal
         include:
           - host_os: cbl-mariner
             vmm: clh
@@ -40,11 +47,20 @@ jobs:
       GH_PR_NUMBER: ${{ inputs.pr-number }}
       KATA_HOST_OS: ${{ matrix.host_os }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
       USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli

.github/workflows/run-k8s-tests-on-garm.yaml

@@ -0,0 +1,88 @@
+name: CI | Run kubernetes tests on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh #cloud-hypervisor
+          - fc #firecracker
+          - qemu
+        snapshotter:
+          - devmapper
+        k8s:
+          - k3s
+        instance:
+          - garm-ubuntu-2004
+          - garm-ubuntu-2004-smaller
+        include:
+          - instance: garm-ubuntu-2004
+            instance-type: normal
+          - instance: garm-ubuntu-2004-smaller
+            instance-type: small
+    runs-on: ${{ matrix.instance }}
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Configure the ${{ matrix.snapshotter }} snapshotter
+        run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+  
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+  
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -1,48 +0,0 @@
-name: CI | Run kubernetes tests on SEV
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-sev
-    runs-on: sev
-    env:
-      DOCKER_REGISTRY: ${{ inputs.registry }}
-      DOCKER_REPO: ${{ inputs.repo }}
-      DOCKER_TAG: ${{ inputs.tag }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-      KUBECONFIG: /home/kata/.kube/config
-      USING_NFD: "false"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Deploy Kata
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
-  
-      - name: Run tests
-        timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests
-
-      - name: Delete kata-deploy
-        if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh cleanup-sev

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -1,48 +0,0 @@
-name: CI | Run kubernetes tests on SEV-SNP
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-snp
-    runs-on: sev-snp
-    env:
-      DOCKER_REGISTRY: ${{ inputs.registry }}
-      DOCKER_REPO: ${{ inputs.repo }}
-      DOCKER_TAG: ${{ inputs.tag }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-      KUBECONFIG: /home/kata/.kube/config
-      USING_NFD: "false"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Deploy Kata
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
-  
-      - name: Run tests
-        timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests
-      
-      - name: Delete kata-deploy
-        if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -1,47 +0,0 @@
-name: CI | Run kubernetes tests on TDX
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-tdx
-    runs-on: tdx
-    env:
-      DOCKER_REGISTRY: ${{ inputs.registry }}
-      DOCKER_REPO: ${{ inputs.repo }}
-      DOCKER_TAG: ${{ inputs.tag }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-      USING_NFD: "true"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Deploy Kata
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
-  
-      - name: Run tests
-        timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests
-  
-      - name: Delete kata-deploy
-        if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx

.github/workflows/run-k8s-tests-with-crio-on-garm.yaml

@@ -0,0 +1,86 @@
+name: CI | Run kubernetes tests, using CRI-O, on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        k8s:
+          - k0s
+        instance:
+          - garm-ubuntu-2004
+          - garm-ubuntu-2004-smaller
+        include:
+          - instance: garm-ubuntu-2004
+            instance-type: normal
+          - instance: garm-ubuntu-2004-smaller
+            instance-type: small
+          - k8s: k0s
+            k8s-extra-params: '--cri-socket remote:unix:///var/run/crio/crio.sock --kubelet-extra-args --cgroup-driver="systemd"'
+    runs-on: ${{ matrix.instance }}
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      KUBERNETES_EXTRA_PARAMS: ${{ matrix.k8s-extra-params }}
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Configure CRI-O
+        run: bash tests/integration/kubernetes/gha-run.sh setup-crio
+
+      - name: Deploy ${{ matrix.k8s }}
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+  
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+  
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm

.github/workflows/run-kata-coco-tests.yaml

@@ -0,0 +1,176 @@
+name: CI | Run kata coco tests
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests-on-tdx:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "k3s"
+      USING_NFD: "true"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests
+
+  run-k8s-tests-on-tdx:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "k3s"
+      USING_NFD: "true"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx
+
+  run-k8s-tests-on-sev:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-sev
+    runs-on: sev
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-sev
+
+  run-k8s-tests-sev-snp:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-snp
+    runs-on: sev-snp
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp

.github/workflows/run-kata-deploy-tests-on-aks.yaml

@@ -0,0 +1,89 @@
+name: CI | Run kata-deploy tests on AKS
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        host_os:
+          - ubuntu
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+        include:
+          - host_os: cbl-mariner
+            vmm: clh
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Download Azure CLI
+        run: bash tests/functional/kata-deploy/gha-run.sh install-azure-cli
+
+      - name: Log into the Azure account
+        run: bash tests/functional/kata-deploy/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+
+      - name: Create AKS cluster
+        timeout-minutes: 10
+        run: bash tests/functional/kata-deploy/gha-run.sh create-cluster
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Install `kubectl`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-kubectl
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: bash tests/functional/kata-deploy/gha-run.sh get-cluster-credentials
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests
+      
+      - name: Delete AKS cluster
+        if: always()
+        run: bash tests/functional/kata-deploy/gha-run.sh delete-cluster

.github/workflows/run-kata-deploy-tests-on-garm.yaml

@@ -0,0 +1,65 @@
+name: CI | Run kata-deploy tests on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+        k8s:
+          - k0s
+          - k3s
+          - rke2
+    runs-on: garm-ubuntu-2004-smaller
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run:  bash tests/functional/kata-deploy/gha-run.sh deploy-k8s
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests

.github/workflows/run-kata-monitor-tests.yaml

@@ -0,0 +1,59 @@
+name: CI | Run kata-monitor tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-monitor:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        container_engine:
+          - crio
+          - containerd
+        include:
+          - container_engine: containerd
+            containerd_version: lts
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINER_ENGINE: ${{ matrix.container_engine }}
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/kata-monitor/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/functional/kata-monitor/gha-run.sh install-kata kata-artifacts
+
+      - name: Run kata-monitor tests
+        run: bash tests/functional/kata-monitor/gha-run.sh run

.github/workflows/run-metrics.yaml

@@ -8,22 +8,28 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
-  run-metrics:
-    strategy:
-      fail-fast: true
-      matrix:
-        vmm: ['clh', 'qemu']
-      max-parallel: 1
+  setup-kata:
+    name: Kata Setup
     runs-on: metrics
     env:
       GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
@@ -34,6 +40,24 @@ jobs:
       - name: Install kata
         run: bash tests/metrics/gha-run.sh install-kata kata-artifacts
 
+  run-metrics:
+    needs: setup-kata
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm: ['clh', 'qemu']
+      max-parallel: 1
+    runs-on: metrics
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - name: enabling the hypervisor
+        run: bash tests/metrics/gha-run.sh enabling-hypervisor
+
       - name: run launch times test
         run: bash tests/metrics/gha-run.sh run-test-launchtimes
 
@@ -49,9 +73,18 @@ jobs:
       - name: run tensorflow test
         run:  bash tests/metrics/gha-run.sh run-test-tensorflow
 
+      - name: run fio test
+        run:  bash tests/metrics/gha-run.sh run-test-fio
+
+      - name: run iperf test
+        run:  bash tests/metrics/gha-run.sh run-test-iperf
+
+      - name: run latency test
+        run:  bash tests/metrics/gha-run.sh run-test-latency
+
       - name: make metrics tarball ${{ matrix.vmm }}
         run: bash tests/metrics/gha-run.sh make-tarball-results
-          
+
       - name: archive metrics results ${{ matrix.vmm }}
         uses: actions/upload-artifact@v3
         with:

.github/workflows/run-nerdctl-tests-on-garm.yaml

@@ -0,0 +1,57 @@
+name: CI | Run nerdctl integration tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-nerdctl-tests:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # all the tests are not flaky, otherwise we'll fail them
+      # all due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+    runs-on: garm-ubuntu-2304-smaller
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nerdctl/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nerdctl/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nerdctl smoke test
+        timeout-minutes: 5
+        run: bash tests/integration/nerdctl/gha-run.sh run

.github/workflows/run-nydus-tests.yaml

@@ -1,42 +0,0 @@
-name: CI | Run nydus tests
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-nydus:
-    strategy:
-      fail-fast: true
-      matrix:
-        containerd_version: ['lts', 'active']
-        vmm: ['clh', 'qemu', 'dragonball']
-    runs-on: garm-ubuntu-2204
-    env:
-      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Install dependencies
-        run: bash tests/integration/nydus/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Install kata
-        run: bash tests/integration/nydus/gha-run.sh install-kata kata-artifacts
-
-      - name: Run nydus tests
-        run: bash tests/integration/nydus/gha-run.sh run

.github/workflows/run-runk-tests.yaml

@@ -0,0 +1,46 @@
+name: CI | Run runk tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-runk:
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/runk/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
+
+      - name: Run tracing tests
+        run: bash tests/integration/runk/gha-run.sh run

.github/workflows/run-vfio-tests.yaml

@@ -1,37 +0,0 @@
-name: CI | Run vfio tests
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-vfio:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm: ['clh', 'qemu']
-    runs-on: garm-ubuntu-2204
-    env:
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Install dependencies
-        run: bash tests/functional/vfio/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Run vfio tests
-        run: bash tests/functional/vfio/gha-run.sh run

.github/workflows/static-checks-dragonball.yaml

@@ -1,37 +0,0 @@
-on:
-  pull_request:
-    types:
-      - opened
-      - edited
-      - reopened
-      - synchronize
-    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-name: Static checks dragonball
-jobs:
-  test-dragonball:
-    runs-on: dragonball
-    env:
-      RUST_BACKTRACE: "1"
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set env
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-      - name: Install Rust
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          ./ci/install_rust.sh
-          echo PATH="$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
-      - name: Run Unit Test
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          cd src/dragonball
-          cargo version
-          rustc --version
-          sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test

.github/workflows/static-checks.yaml

@@ -12,74 +12,183 @@ concurrency:
 
 name: Static checks
 jobs:
-  static-checks:
+  check-kernel-config-version:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Ensure the kernel config version has been updated
+        run: |
+          kernel_dir="tools/packaging/kernel/"
+          kernel_version_file="${kernel_dir}kata_config_version"
+          modified_files=$(git diff --name-only origin/$GITHUB_BASE_REF..HEAD)
+          if git diff --name-only origin/$GITHUB_BASE_REF..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+            echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
+            if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
+              echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
+            else
+              echo "Readme file changed, no need for kernel config version update."
+            fi
+            echo "Check passed"
+          fi
+
+  build-checks:
     runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false
       matrix:
-        cmd:
+        component:
+          - agent
+          - dragonball
+          - runtime
+          - runtime-rs
+          - agent-ctl
+          - kata-ctl
+          - log-parser-rs
+          - runk
+          - trace-forwarder
+        command:
           - "make vendor"
-          - "make static-checks"
           - "make check"
           - "make test"
           - "sudo -E PATH=\"$PATH\" make test"
+        include:
+          - component: agent
+            component-path: src/agent
+          - component: dragonball
+            component-path: src/dragonball
+          - component: runtime
+            component-path: src/runtime
+          - component: runtime-rs
+            component-path: src/runtime-rs
+          - component: agent-ctl
+            component-path: src/tools/agent-ctl
+          - component: kata-ctl
+            component-path: src/tools/kata-ctl
+          - component: log-parser-rs
+            component-path: src/tools/log-parser-rs
+          - component: runk
+            component-path: src/tools/runk
+          - component: trace-forwarder
+            component-path: src/tools/trace-forwarder
+          - install-libseccomp: no
+          - component: agent
+            install-libseccomp: yes
+          - component: runk
+            install-libseccomp: yes
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install yq
+        run: |
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        if: ${{ matrix.component == 'runtime' }}
+        run: |
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+      - name: Install rust
+        if: ${{ matrix.component != 'runtime' }}
+        run: |
+          ./tests/install_rust.sh
+          echo "${HOME}/.cargo/bin" >> $GITHUB_PATH
+      - name: Install musl-tools
+        if: ${{ matrix.component != 'runtime' }}
+        run: sudo apt-get -y install musl-tools
+      - name: Install libseccomp
+        if: ${{ matrix.command != 'make vendor'  &&  matrix.command != 'make check' &&  matrix.install-libseccomp == 'yes' }}
+        run: |
+          libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+          gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+          ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+          echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+          echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+          echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
+      - name: Setup XDG_RUNTIME_DIR for the `runtime` tests
+        if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }}
+        run: |
+          XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700))
+          echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> $GITHUB_ENV
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  build-checks-depending-on-kvm:
+    runs-on: garm-ubuntu-2004-smaller
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - runtime-rs
+        include:
+          - component: runtime-rs
+            command: "sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test"
+          - component: runtime-rs
+            component-path: src/dragonball
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install system deps
+        run: |
+          sudo apt-get install -y build-essential musl-tools
+      - name: Install yq
+        run: |
+          sudo -E ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install rust
+        run: |
+          export PATH="$PATH:/usr/local/bin"
+          ./tests/install_rust.sh
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          export PATH="$PATH:${HOME}/.cargo/bin"
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  static-checks:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        cmd:
+          - "make static-checks"
     env:
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
       GOPATH: ${{ github.workspace }}
     steps:
-    - name: Free disk space
-      run: |
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-    - name: Checkout code
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Install Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19.3
-    - name: Check kernel config version
-      run: |
-        cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
-        kernel_dir="tools/packaging/kernel/"
-        kernel_version_file="${kernel_dir}kata_config_version"
-        modified_files=$(git diff --name-only origin/main..HEAD)
-        if git diff --name-only origin/main..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
-          echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
-          if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
-            echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
-          else
-            echo "Readme file changed, no need for kernel config version update."
-          fi
-          echo "Check passed"
-        fi
-    - name: Set PATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    - name: Run check
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: ./src/github.com/${{ github.repository }}
+      - name: Install yq
+        run: |
+          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        run: |
+          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+      - name: Install system dependencies
+        run: |
+          sudo apt-get -y install moreutils hunspell pandoc
+      - name: Run check
+        run: |
+          export PATH=${PATH}:${GOPATH}/bin
+          cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}

VERSION

@@ -1 +1 @@
-3.2.0-alpha4
+3.3.0-alpha0

ci/install_libseccomp.sh

@@ -7,12 +7,10 @@
 
 set -o errexit
 
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+script_name="$(basename "${BASH_SOURCE[0]}")"
 
-clone_tests_repo
-
-source "${tests_repo_dir}/.ci/lib.sh"
+source "${script_dir}/../tests/common.bash"
 
 # The following variables if set on the environment will change the behavior
 # of gperf and libseccomp configure scripts, that may lead this script to
@@ -25,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 # Variables for libseccomp
 libseccomp_version="${LIBSECCOMP_VERSION:-""}"
 if [ -z "${libseccomp_version}" ]; then
-    libseccomp_version=$(get_version "externals.libseccomp.version")
+    libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version")
 fi
 libseccomp_url="${LIBSECCOMP_URL:-""}"
 if [ -z "${libseccomp_url}" ]; then
-    libseccomp_url=$(get_version "externals.libseccomp.url")
+    libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url")
 fi
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
@@ -38,11 +36,11 @@ cflags="-O2"
 # Variables for gperf
 gperf_version="${GPERF_VERSION:-""}"
 if [ -z "${gperf_version}" ]; then
-    gperf_version=$(get_version "externals.gperf.version")
+    gperf_version=$(get_from_kata_deps "externals.gperf.version")
 fi
 gperf_url="${GPERF_URL:-""}"
 if [ -z "${gperf_url}" ]; then
-    gperf_url=$(get_version "externals.gperf.url")
+    gperf_url=$(get_from_kata_deps "externals.gperf.url")
 fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"
@@ -87,7 +85,8 @@ build_and_install_libseccomp() {
     curl -sLO "${libseccomp_tarball_url}"
     tar -xf "${libseccomp_tarball}"
     pushd "libseccomp-${libseccomp_version}"
-    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
+    [ "${arch}" == $(uname -m) ] && cc_name="" || cc_name="${arch}-linux-gnu-gcc"
+    CC=${cc_name} ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
     make
     make install
     popd

docs/design/README.md

@@ -14,6 +14,7 @@ Kata Containers design documents:
 - [`Inotify` support](inotify.md)
 - [`Hooks` support](hooks-handling.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
+- [Metrics in Rust Runtime(runtime-rs)](kata-metrics-in-runtime-rs.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 - [Design for direct-assigned volume](direct-blk-device-assignment.md)
 - [Design for core-scheduling](core-scheduling.md)

docs/design/architecture/kubernetes.md

@@ -12,7 +12,7 @@ only needs to run a container runtime and a container agent (called a
 Kata Containers represents a Kubelet pod as a VM.
 
 A Kubernetes cluster runs a control plane where a scheduler (typically
-running on a dedicated master node) calls into a compute Kubelet. This
+running on a dedicated control-plane node) calls into a compute Kubelet. This
 Kubelet instance is responsible for managing the lifecycle of pods
 within the nodes and eventually relies on a container runtime to
 handle execution. The Kubelet architecture decouples lifecycle

docs/design/kata-metrics-in-runtime-rs.md

@@ -0,0 +1,50 @@
+# Kata Metrics in Rust Runtime(runtime-rs)
+
+Rust Runtime(runtime-rs) is responsible for:
+
+- Gather metrics about `shim`.
+- Gather metrics from `hypervisor` (through `channel`).
+- Get metrics from `agent` (through `ttrpc`).
+
+---
+
+Here are listed all the metrics gathered by `runtime-rs`.
+
+> * Current status of each entry is marked as:
+>  * ✅：DONE
+>   * 🚧：TODO
+
+### Kata Shim
+
+| STATUS | Metric name                                                  | Type        | Units          | Labels                                                       |
+| ------ | ------------------------------------------------------------ | ----------- | -------------- | ------------------------------------------------------------ |
+| 🚧      | `kata_shim_agent_rpc_durations_histogram_milliseconds`: <br> RPC latency distributions. | `HISTOGRAM` | `milliseconds` | <ul><li>`action` (RPC actions of Kata agent)<ul><li>`grpc.CheckRequest`</li><li>`grpc.CloseStdinRequest`</li><li>`grpc.CopyFileRequest`</li><li>`grpc.CreateContainerRequest`</li><li>`grpc.CreateSandboxRequest`</li><li>`grpc.DestroySandboxRequest`</li><li>`grpc.ExecProcessRequest`</li><li>`grpc.GetMetricsRequest`</li><li>`grpc.GuestDetailsRequest`</li><li>`grpc.ListInterfacesRequest`</li><li>`grpc.ListProcessesRequest`</li><li>`grpc.ListRoutesRequest`</li><li>`grpc.MemHotplugByProbeRequest`</li><li>`grpc.OnlineCPUMemRequest`</li><li>`grpc.PauseContainerRequest`</li><li>`grpc.RemoveContainerRequest`</li><li>`grpc.ReseedRandomDevRequest`</li><li>`grpc.ResumeContainerRequest`</li><li>`grpc.SetGuestDateTimeRequest`</li><li>`grpc.SignalProcessRequest`</li><li>`grpc.StartContainerRequest`</li><li>`grpc.StatsContainerRequest`</li><li>`grpc.TtyWinResizeRequest`</li><li>`grpc.UpdateContainerRequest`</li><li>`grpc.UpdateInterfaceRequest`</li><li>`grpc.UpdateRoutesRequest`</li><li>`grpc.WaitProcessRequest`</li><li>`grpc.WriteStreamRequest`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_fds`: <br> Kata containerd shim v2 open FDs.      | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+| ✅      | `kata_shim_io_stat`: <br> Kata containerd shim v2 process IO statistics. | `GAUGE`     |                | <ul><li>`item` (see `/proc/<pid>/io`)<ul><li>`cancelledwritebytes`</li><li>`rchar`</li><li>`readbytes`</li><li>`syscr`</li><li>`syscw`</li><li>`wchar`</li><li>`writebytes`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_netdev`: <br> Kata containerd shim v2 network devices statistics. | `GAUGE`     |                | <ul><li>`interface` (network device name)</li><li>`item` (see `/proc/net/dev`)<ul><li>`recv_bytes`</li><li>`recv_compressed`</li><li>`recv_drop`</li><li>`recv_errs`</li><li>`recv_fifo`</li><li>`recv_frame`</li><li>`recv_multicast`</li><li>`recv_packets`</li><li>`sent_bytes`</li><li>`sent_carrier`</li><li>`sent_colls`</li><li>`sent_compressed`</li><li>`sent_drop`</li><li>`sent_errs`</li><li>`sent_fifo`</li><li>`sent_packets`</li></ul></li><li>`sandbox_id`</li></ul> |
+| 🚧      | `kata_shim_pod_overhead_cpu`: <br> Kata Pod overhead for CPU resources(percent). | `GAUGE`     | percent        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_pod_overhead_memory_in_bytes`: <br> Kata Pod overhead for memory resources(bytes). | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| ✅      | `kata_shim_proc_stat`: <br> Kata containerd shim v2 process statistics. | `GAUGE`     |                | <ul><li>`item` (see `/proc/<pid>/stat`)<ul><li>`cstime`</li><li>`cutime`</li><li>`stime`</li><li>`utime`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_proc_status`: <br> Kata containerd shim v2 process status. | `GAUGE`     |                | <ul><li>`item` (see `/proc/<pid>/status`)<ul><li>`hugetlbpages`</li><li>`nonvoluntary_ctxt_switches`</li><li>`rssanon`</li><li>`rssfile`</li><li>`rssshmem`</li><li>`vmdata`</li><li>`vmexe`</li><li>`vmhwm`</li><li>`vmlck`</li><li>`vmlib`</li><li>`vmpeak`</li><li>`vmpin`</li><li>`vmpmd`</li><li>`vmpte`</li><li>`vmrss`</li><li>`vmsize`</li><li>`vmstk`</li><li>`vmswap`</li><li>`voluntary_ctxt_switches`</li></ul></li><li>`sandbox_id`</li></ul> |
+| 🚧      | `kata_shim_process_cpu_seconds_total`: <br> Total user and system CPU time spent in seconds. | `COUNTER`   | `seconds`      | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_max_fds`: <br> Maximum number of open file descriptors. | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_open_fds`: <br> Number of open file descriptors. | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_resident_memory_bytes`: <br> Resident memory size in bytes. | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_start_time_seconds`: <br> Start time of the process since `unix` epoch in seconds. | `GAUGE`     | `seconds`      | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_virtual_memory_bytes`: <br> Virtual memory size in bytes. | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_virtual_memory_max_bytes`: <br> Maximum amount of virtual memory available in bytes. | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_rpc_durations_histogram_milliseconds`: <br> RPC latency distributions. | `HISTOGRAM` | `milliseconds` | <ul><li>`action` (Kata shim v2 actions)<ul><li>`checkpoint`</li><li>`close_io`</li><li>`connect`</li><li>`create`</li><li>`delete`</li><li>`exec`</li><li>`kill`</li><li>`pause`</li><li>`pids`</li><li>`resize_pty`</li><li>`resume`</li><li>`shutdown`</li><li>`start`</li><li>`state`</li><li>`stats`</li><li>`update`</li><li>`wait`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_threads`: <br> Kata containerd shim v2 process threads. | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+
+### Kata Hypervisor
+
+Different from golang runtime, hypervisor and shim in runtime-rs belong to the **same process**, so all previous metrics for hypervisor and shim only need to be gathered once. Thus, we currently only collect previous metrics in kata shim.
+
+At the same time, we added the interface(`VmmAction::GetHypervisorMetrics`) to gather hypervisor metrics, in case we design tailor-made metrics for hypervisor in the future. Here're metrics exposed from [src/dragonball/src/metric.rs](https://github.com/kata-containers/kata-containers/blob/main/src/dragonball/src/metric.rs).
+
+| Metric name                                                  | Type       | Units | Labels                                                       |
+| ------------------------------------------------------------ | ---------- | ----- | ------------------------------------------------------------ |
+| `kata_hypervisor_scrape_count`: <br> Metrics scrape count    | `COUNTER`  |       | <ul><li>`sandbox_id`</li></ul>                               |
+| `kata_hypervisor_vcpu`: <br>Hypervisor metrics specific to VCPUs' mode of functioning. | `IntGauge` |       | <ul><li>`item`<ul><li>`exit_io_in`</li><li>`exit_io_out`</li><li>`exit_mmio_read`</li><li>`exit_mmio_write`</li><li>`failures`</li><li>`filter_cpuid`</li></ul></li><li>`sandbox_id`</li></ul> |
+| `kata_hypervisor_seccomp`: <br> Hypervisor metrics for the seccomp filtering. | `IntGauge` |       | <ul><li>`item`<ul><li>`num_faults`</li></ul></li><li>`sandbox_id`</li></ul> |
+| `kata_hypervisor_seccomp`: <br> Hypervisor metrics for the seccomp filtering. | `IntGauge` |       | <ul><li>`item`<ul><li>`sigbus`</li><li>`sigsegv`</li></ul></li><li>`sandbox_id`</li></ul> |

docs/design/kata-vra.md

@@ -43,7 +43,7 @@ and perform DMA transactions _anywhere_.
 
 The second feature is ACS (Access Control Services), which controls which
 devices are allowed to communicate with one another and thus avoids improper
-routing of packets irrespectively of whether IOMMU is enabled or not.
+routing of packets `irrespectively` of whether IOMMU is enabled or not.
 
 When IOMMU is enabled, ACS is normally configured to force all PCI Express DMA
 to go through the root complex so IOMMU can translate it, impacting performance
@@ -126,7 +126,7 @@ efficient P2P communication.
 ## PCI Express Virtual P2P Approval Capability
 
 Most of the time, the PCI Express topology is flattened and obfuscated to ensure
-easy migration of the VM image between different physical hardware topologies.
+easy migration of the VM image between different physical hardware `topologies`.
 In Kata, we can configure the hypervisor to use PCI Express root ports to
 hotplug the VFIO  devices one is passing through. A user can select how many PCI
 Express root ports to allocate depending on how many devices are passed through.
@@ -220,7 +220,7 @@ containers that he wants to run with Kata. The goal is to make such things as
 transparent as possible, so we also introduced
 [CDI](https://github.com/container-orchestrated-devices/container-device-interface)
 (Container Device Interface) to Kata. CDI is a[
-specification](https://github.com/container-orchestrated-devices/container-device-interface/blob/master/SPEC.md)
+specification](https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md)
 for container runtimes to support third-party devices.
 
 As written before, we can provide a clique ID for the devices that belong
@@ -300,7 +300,7 @@ pcie_switch_port = 8
 ```
 
 Each device that is passed through is attached to a PCI Express downstream port
-as illustrated below. We can even replicate the host’s two DPUs topologies with
+as illustrated below. We can even replicate the host’s two DPUs `topologies` with
 added metadata through the CDI. Most of the time, a container only needs one
 pair of GPU and NIC for GPUDirect RDMA. This is more of a showcase of what we
 can do with the power of Kata and CDI. One could even think of adding groups of
@@ -328,7 +328,7 @@ $ lspci -tv
 ```
 
 The configuration of using either the root port or switch port can be applied on
-a per Container or Pod basis, meaning we can switch PCI Express topologies on
+a per Container or Pod basis, meaning we can switch PCI Express `topologies` on
 each run of an application.
 
 ## Hypervisor Resource Limits

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -28,10 +28,10 @@ __Steps from the Developer Guide:__
 __SNP-specific steps:__
 - Build the SNP-specific kernel as shown below (see this [guide](../../tools/packaging/kernel/README.md#build-kata-containers-kernel) for more information)
 ```bash
-$ pushd kata-containers/tools/packaging/kernel/
-$ ./build-kernel.sh -a x86_64 -x snp setup
-$ ./build-kernel.sh -a x86_64 -x snp build
-$ sudo -E PATH="${PATH}" ./build-kernel.sh -x snp install
+$ pushd kata-containers/tools/packaging/
+$ ./kernel/build-kernel.sh -a x86_64 -x snp setup
+$ ./kernel/build-kernel.sh -a x86_64 -x snp build
+$ sudo -E PATH="${PATH}" ./kernel/build-kernel.sh -x snp install
 $ popd
 ```
 - Build a current OVMF capable of SEV-SNP:

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -27,6 +27,8 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
+  uid: $(uuidgen)
+  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/how-to/how-to-use-k8s-with-containerd-and-kata.md

@@ -139,12 +139,12 @@ By default the CNI plugin binaries is installed under `/opt/cni/bin` (in package
   EOF
   ```
 
-## Allow pods to run in the master node
+## Allow pods to run in the control-plane node
 
-By default, the cluster will not schedule pods in the master node. To enable master node scheduling:
+By default, the cluster will not schedule pods in the control-plane node. To enable control-plane node scheduling:
 
 ```bash
-$ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master-
+$ sudo -E kubectl taint nodes --all node-role.kubernetes.io/control-plane-
 ```
 
 ## Create runtime class for Kata Containers

docs/how-to/how-to-use-kata-containers-with-firecracker.md

@@ -19,12 +19,14 @@ This document requires the presence of Kata Containers on your system. Install u
 
 ## Install AWS Firecracker
 
-Kata Containers only support AWS Firecracker v0.23.4 ([yet](https://github.com/kata-containers/kata-containers/pull/1519)).
+For information about the supported version of Firecracker, see the Kata Containers
+[`versions.yaml`](../../versions.yaml).
+
 To install Firecracker we need to get the `firecracker` and `jailer` binaries:
 
 ```bash
 $ release_url="https://github.com/firecracker-microvm/firecracker/releases"
-$ version="v0.23.1"
+$ version=$(yq read <kata-repository>/versions.yaml assets.hypervisor.firecracker.version)
 $ arch=`uname -m`
 $ curl ${release_url}/download/${version}/firecracker-${version}-${arch} -o firecracker
 $ curl ${release_url}/download/${version}/jailer-${version}-${arch} -o jailer

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -32,6 +32,7 @@ The `nydus-sandbox.yaml` looks like below:
 metadata:
   attempt: 1
   name: nydus-sandbox
+  uid: nydus-uid
   namespace: default
 log_directory: /tmp
 linux:

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -29,7 +29,7 @@ Then you can build and install the guest kernel image as shown [here](../../tool
 
 ## Run a Kata Container utilizing `virtio-mem`
 
-Use following command to enable memory overcommitment of a Linux kernel.  Because QEMU `virtio-mem` device need to allocate a lot of memory.
+Use following command to enable memory over-commitment of a Linux kernel.  Because QEMU `virtio-mem` device need to allocate a lot of memory.
 ```
 $ echo 1 | sudo tee /proc/sys/vm/overcommit_memory
 ```
@@ -42,6 +42,8 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
+  uid: $(uuidgen)
+  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/how-to/run-kata-with-k8s.md

@@ -115,11 +115,11 @@ $ sudo kubeadm init --ignore-preflight-errors=all --config kubeadm-config.yaml
 $ export KUBECONFIG=/etc/kubernetes/admin.conf
 ```
 
-### Allow pods to run in the master node
+### Allow pods to run in the control-plane node
 
-By default, the cluster will not schedule pods in the master node. To enable master node scheduling:
+By default, the cluster will not schedule pods in the control-plane node. To enable control-plane node scheduling:
 ```bash
-$ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master-
+$ sudo -E kubectl taint nodes --all node-role.kubernetes.io/control-plane-
 ```
 
 ### Create runtime class for Kata Containers

docs/install/minikube-installation-guide.md

@@ -91,7 +91,7 @@ Before you install Kata Containers, check that your Minikube is operating. On yo
 $ kubectl get nodes
 ```
 
-You should see your `master` node listed as being `Ready`.
+You should see your `control-plane` node listed as being `Ready`.
 
 Check you have virtualization enabled inside your Minikube. The following should return
 a number larger than `0` if you have either of the `vmx` or `svm` nested virtualization features

src/agent/Cargo.toml

@@ -8,7 +8,7 @@ license = "Apache-2.0"
 [dependencies]
 oci = { path = "../libs/oci" }
 rustjail = { path = "rustjail" }
-protocols = { path = "../libs/protocols", features = ["async"] }
+protocols = { path = "../libs/protocols", features = ["async", "with-serde"] }
 lazy_static = "1.3.0"
 ttrpc = { version = "0.7.1", features = ["async"], default-features = false }
 protobuf = "3.2.0"
@@ -67,6 +67,12 @@ serde = { version = "1.0.129", features = ["derive"] }
 toml = "0.5.8"
 clap = { version = "3.0.1", features = ["derive"] }
 
+# Communication with the OPA service
+http = { version = "0.2.8", optional = true }
+reqwest = { version = "0.11.14", optional = true }
+# The "vendored" feature for openssl is required for musl build
+openssl = { version = "0.10.54", features = ["vendored"], optional = true }
+
 [dev-dependencies]
 tempfile = "3.1.0"
 test-utils = { path = "../libs/test-utils" }
@@ -83,6 +89,7 @@ lto = true
 [features]
 seccomp = ["rustjail/seccomp"]
 standard-oci-runtime = ["rustjail/standard-oci-runtime"]
+agent-policy = ["http", "openssl", "reqwest"]
 
 [[bin]]
 name = "kata-agent"

src/agent/Makefile

@@ -33,6 +33,14 @@ ifeq ($(SECCOMP),yes)
     override EXTRA_RUSTFEATURES += seccomp
 endif
 
+##VAR AGENT_POLICY=yes|no define if agent enables the policy feature
+AGENT_POLICY ?= no
+
+# Enable the policy feature of rust build
+ifeq ($(AGENT_POLICY),yes)
+    override EXTRA_RUSTFEATURES += agent-policy
+endif
+
 include ../../utils.mk
 
 ifeq ($(ARCH), ppc64le)
@@ -54,7 +62,7 @@ endif
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
 ##VAR DESTDIR=<path> is a directory prepended to each installed target file
-DESTDIR :=
+DESTDIR ?=
 ##VAR BINDIR=<path> is a directory for installing executable programs
 BINDIR := /usr/bin
 
@@ -140,7 +148,7 @@ vendor:
 
 
 #TARGET test: run cargo tests
-test:
+test: $(GENERATED_FILES)
 	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test

src/agent/rustjail/Cargo.toml

@@ -34,7 +34,7 @@ futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"
 libseccomp = { version = "0.3.0", optional = true }
-zbus = "2.3.0"
+zbus = "3.12.0"
 bit-vec= "0.6.3"
 xattr = "0.2.3"
 

src/agent/rustjail/src/cgroups/systemd/common.rs

@@ -6,7 +6,10 @@
 pub const DEFAULT_SLICE: &str = "system.slice";
 pub const SLICE_SUFFIX: &str = ".slice";
 pub const SCOPE_SUFFIX: &str = ".scope";
-pub const UNIT_MODE: &str = "replace";
+pub const WHO_ENUM_ALL: &str = "all";
+pub const SIGNAL_KILL: i32 = nix::sys::signal::SIGKILL as i32;
+pub const UNIT_MODE_REPLACE: &str = "replace";
+pub const NO_SUCH_UNIT_ERROR: &str = "org.freedesktop.systemd1.NoSuchUnit";
 
 pub type Properties<'a> = Vec<(&'a str, zbus::zvariant::Value<'a>)>;
 

src/agent/rustjail/src/cgroups/systemd/dbus_client.rs

@@ -1,56 +1,50 @@
-// Copyright 2021-2022 Kata Contributors
+// Copyright 2021-2023 Kata Contributors
 //
 // SPDX-License-Identifier: Apache-2.0
 //
 
 use std::vec;
 
-use super::common::CgroupHierarchy;
-use super::common::{Properties, SLICE_SUFFIX, UNIT_MODE};
+use super::common::{
+    CgroupHierarchy, Properties, NO_SUCH_UNIT_ERROR, SIGNAL_KILL, SLICE_SUFFIX, UNIT_MODE_REPLACE,
+    WHO_ENUM_ALL,
+};
 use super::interface::system::ManagerProxyBlocking as SystemManager;
-use anyhow::{Context, Result};
+use anyhow::{anyhow, Context, Result};
 use zbus::zvariant::Value;
 
 pub trait SystemdInterface {
-    fn start_unit(
-        &self,
-        pid: i32,
-        parent: &str,
-        unit_name: &str,
-        cg_hierarchy: &CgroupHierarchy,
-    ) -> Result<()>;
-
-    fn set_properties(&self, unit_name: &str, properties: &Properties) -> Result<()>;
-
-    fn stop_unit(&self, unit_name: &str) -> Result<()>;
-
+    fn start_unit(&self, pid: i32, parent: &str, cg_hierarchy: &CgroupHierarchy) -> Result<()>;
+    fn set_properties(&self, properties: &Properties) -> Result<()>;
+    fn kill_unit(&self) -> Result<()>;
+    fn freeze_unit(&self) -> Result<()>;
+    fn thaw_unit(&self) -> Result<()>;
+    fn add_process(&self, pid: i32) -> Result<()>;
     fn get_version(&self) -> Result<String>;
-
-    fn unit_exists(&self, unit_name: &str) -> Result<bool>;
-
-    fn add_process(&self, pid: i32, unit_name: &str) -> Result<()>;
+    fn unit_exists(&self) -> Result<bool>;
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct DBusClient {}
+pub struct DBusClient {
+    unit_name: String,
+}
 
 impl DBusClient {
+    pub fn new(unit_name: String) -> Self {
+        Self { unit_name }
+    }
+
     fn build_proxy(&self) -> Result<SystemManager<'static>> {
         let connection =
             zbus::blocking::Connection::system().context("Establishing a D-Bus connection")?;
         let proxy = SystemManager::new(&connection).context("Building a D-Bus proxy manager")?;
+
         Ok(proxy)
     }
 }
 
 impl SystemdInterface for DBusClient {
-    fn start_unit(
-        &self,
-        pid: i32,
-        parent: &str,
-        unit_name: &str,
-        cg_hierarchy: &CgroupHierarchy,
-    ) -> Result<()> {
+    fn start_unit(&self, pid: i32, parent: &str, cg_hierarchy: &CgroupHierarchy) -> Result<()> {
         let proxy = self.build_proxy()?;
 
         // enable CPUAccounting & MemoryAccounting & (Block)IOAccounting by default
@@ -68,7 +62,7 @@ impl SystemdInterface for DBusClient {
             CgroupHierarchy::Unified => properties.push(("BlockIOAccounting", Value::Bool(true))),
         }
 
-        if unit_name.ends_with(SLICE_SUFFIX) {
+        if self.unit_name.ends_with(SLICE_SUFFIX) {
             properties.push(("Wants", Value::Str(parent.into())));
         } else {
             properties.push(("Slice", Value::Str(parent.into())));
@@ -76,27 +70,57 @@ impl SystemdInterface for DBusClient {
         }
 
         proxy
-            .start_transient_unit(unit_name, UNIT_MODE, &properties, &[])
-            .with_context(|| format!("failed to start transient unit {}", unit_name))?;
-        Ok(())
-    }
-
-    fn set_properties(&self, unit_name: &str, properties: &Properties) -> Result<()> {
-        let proxy = self.build_proxy()?;
-
-        proxy
-            .set_unit_properties(unit_name, true, properties)
-            .with_context(|| format!("failed to set unit properties {}", unit_name))?;
+            .start_transient_unit(&self.unit_name, UNIT_MODE_REPLACE, &properties, &[])
+            .context(format!("failed to start transient unit {}", self.unit_name))?;
 
         Ok(())
     }
 
-    fn stop_unit(&self, unit_name: &str) -> Result<()> {
+    fn set_properties(&self, properties: &Properties) -> Result<()> {
         let proxy = self.build_proxy()?;
 
         proxy
-            .stop_unit(unit_name, UNIT_MODE)
-            .with_context(|| format!("failed to stop unit {}", unit_name))?;
+            .set_unit_properties(&self.unit_name, true, properties)
+            .context(format!("failed to set unit {} properties", self.unit_name))?;
+
+        Ok(())
+    }
+
+    fn kill_unit(&self) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .kill_unit(&self.unit_name, WHO_ENUM_ALL, SIGNAL_KILL)
+            .or_else(|e| match e {
+                zbus::Error::MethodError(error_name, _, _)
+                    if error_name.as_str() == NO_SUCH_UNIT_ERROR =>
+                {
+                    Ok(())
+                }
+                _ => Err(e),
+            })
+            .context(format!("failed to kill unit {}", self.unit_name))?;
+
+        Ok(())
+    }
+
+    fn freeze_unit(&self) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .freeze_unit(&self.unit_name)
+            .context(format!("failed to freeze unit {}", self.unit_name))?;
+
+        Ok(())
+    }
+
+    fn thaw_unit(&self) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .thaw_unit(&self.unit_name)
+            .context(format!("failed to thaw unit {}", self.unit_name))?;
+
         Ok(())
     }
 
@@ -105,24 +129,37 @@ impl SystemdInterface for DBusClient {
 
         let systemd_version = proxy
             .version()
-            .with_context(|| "failed to get systemd version".to_string())?;
+            .context("failed to get systemd version".to_string())?;
+
         Ok(systemd_version)
     }
 
-    fn unit_exists(&self, unit_name: &str) -> Result<bool> {
-        let proxy = self
-            .build_proxy()
-            .with_context(|| format!("Checking if systemd unit {} exists", unit_name))?;
+    fn unit_exists(&self) -> Result<bool> {
+        let proxy = self.build_proxy()?;
 
-        Ok(proxy.get_unit(unit_name).is_ok())
+        match proxy.get_unit(&self.unit_name) {
+            Ok(_) => Ok(true),
+            Err(zbus::Error::MethodError(error_name, _, _))
+                if error_name.as_str() == NO_SUCH_UNIT_ERROR =>
+            {
+                Ok(false)
+            }
+            Err(e) => Err(anyhow!(format!(
+                "failed to check if unit {} exists: {:?}",
+                self.unit_name, e
+            ))),
+        }
     }
 
-    fn add_process(&self, pid: i32, unit_name: &str) -> Result<()> {
+    fn add_process(&self, pid: i32) -> Result<()> {
         let proxy = self.build_proxy()?;
 
         proxy
-            .attach_processes_to_unit(unit_name, "/", &[pid as u32])
-            .with_context(|| format!("failed to add process {}", unit_name))?;
+            .attach_processes_to_unit(&self.unit_name, "/", &[pid as u32])
+            .context(format!(
+                "failed to add process into unit {}",
+                self.unit_name
+            ))?;
 
         Ok(())
     }

src/agent/rustjail/src/cgroups/systemd/interface/system.rs

@@ -1,4 +1,4 @@
-// Copyright 2021-2022 Kata Contributors
+// Copyright 2021-2023 Kata Contributors
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -8,7 +8,7 @@
 
 //! # DBus interface proxy for: `org.freedesktop.systemd1.Manager`
 //!
-//! This code was generated by `zbus-xmlgen` `2.0.1` from DBus introspection data.
+//! This code was generated by `zbus-xmlgen` `3.1.1` from DBus introspection data.
 //! Source: `Interface '/org/freedesktop/systemd1' from service 'org.freedesktop.systemd1' on system bus`.
 //!
 //! You may prefer to adapt it, instead of using it verbatim.
@@ -189,12 +189,14 @@ trait Manager {
     ) -> zbus::Result<zbus::zvariant::OwnedObjectPath>;
 
     /// GetUnitByInvocationID method
+    #[dbus_proxy(name = "GetUnitByInvocationID")]
     fn get_unit_by_invocation_id(
         &self,
         invocation_id: &[u8],
     ) -> zbus::Result<zbus::zvariant::OwnedObjectPath>;
 
     /// GetUnitByPID method
+    #[dbus_proxy(name = "GetUnitByPID")]
     fn get_unit_by_pid(&self, pid: u32) -> zbus::Result<zbus::zvariant::OwnedObjectPath>;
 
     /// GetUnitFileLinks method
@@ -210,6 +212,7 @@ trait Manager {
     fn halt(&self) -> zbus::Result<()>;
 
     /// KExec method
+    #[dbus_proxy(name = "KExec")]
     fn kexec(&self) -> zbus::Result<()>;
 
     /// KillUnit method
@@ -330,6 +333,7 @@ trait Manager {
     fn lookup_dynamic_user_by_name(&self, name: &str) -> zbus::Result<u32>;
 
     /// LookupDynamicUserByUID method
+    #[dbus_proxy(name = "LookupDynamicUserByUID")]
     fn lookup_dynamic_user_by_uid(&self, uid: u32) -> zbus::Result<String>;
 
     /// MaskUnitFiles method
@@ -571,139 +575,139 @@ trait Manager {
     fn ctrl_alt_del_burst_action(&self) -> zbus::Result<String>;
 
     /// DefaultBlockIOAccounting property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultBlockIOAccounting")]
     fn default_block_ioaccounting(&self) -> zbus::Result<bool>;
 
     /// DefaultCPUAccounting property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultCPUAccounting")]
     fn default_cpuaccounting(&self) -> zbus::Result<bool>;
 
     /// DefaultLimitAS property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitAS")]
     fn default_limit_as(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitASSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitASSoft")]
     fn default_limit_assoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitCORE property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitCORE")]
     fn default_limit_core(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitCORESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitCORESoft")]
     fn default_limit_coresoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitCPU property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitCPU")]
     fn default_limit_cpu(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitCPUSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitCPUSoft")]
     fn default_limit_cpusoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitDATA property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitDATA")]
     fn default_limit_data(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitDATASoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitDATASoft")]
     fn default_limit_datasoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitFSIZE property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitFSIZE")]
     fn default_limit_fsize(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitFSIZESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitFSIZESoft")]
     fn default_limit_fsizesoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitLOCKS property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitLOCKS")]
     fn default_limit_locks(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitLOCKSSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitLOCKSSoft")]
     fn default_limit_lockssoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitMEMLOCK property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitMEMLOCK")]
     fn default_limit_memlock(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitMEMLOCKSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitMEMLOCKSoft")]
     fn default_limit_memlocksoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitMSGQUEUE property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitMSGQUEUE")]
     fn default_limit_msgqueue(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitMSGQUEUESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitMSGQUEUESoft")]
     fn default_limit_msgqueuesoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNICE property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNICE")]
     fn default_limit_nice(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNICESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNICESoft")]
     fn default_limit_nicesoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNOFILE property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNOFILE")]
     fn default_limit_nofile(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNOFILESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNOFILESoft")]
     fn default_limit_nofilesoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNPROC property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNPROC")]
     fn default_limit_nproc(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNPROCSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNPROCSoft")]
     fn default_limit_nprocsoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRSS property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRSS")]
     fn default_limit_rss(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRSSSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRSSSoft")]
     fn default_limit_rsssoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRTPRIO property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRTPRIO")]
     fn default_limit_rtprio(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRTPRIOSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRTPRIOSoft")]
     fn default_limit_rtpriosoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRTTIME property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRTTIME")]
     fn default_limit_rttime(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRTTIMESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRTTIMESoft")]
     fn default_limit_rttimesoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitSIGPENDING property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitSIGPENDING")]
     fn default_limit_sigpending(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitSIGPENDINGSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitSIGPENDINGSoft")]
     fn default_limit_sigpendingsoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitSTACK property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitSTACK")]
     fn default_limit_stack(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitSTACKSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitSTACKSoft")]
     fn default_limit_stacksoft(&self) -> zbus::Result<u64>;
 
     /// DefaultMemoryAccounting property
@@ -711,11 +715,11 @@ trait Manager {
     fn default_memory_accounting(&self) -> zbus::Result<bool>;
 
     /// DefaultOOMPolicy property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultOOMPolicy")]
     fn default_oompolicy(&self) -> zbus::Result<String>;
 
     /// DefaultRestartUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultRestartUSec")]
     fn default_restart_usec(&self) -> zbus::Result<u64>;
 
     /// DefaultStandardError property
@@ -731,7 +735,7 @@ trait Manager {
     fn default_start_limit_burst(&self) -> zbus::Result<u32>;
 
     /// DefaultStartLimitIntervalUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultStartLimitIntervalUSec")]
     fn default_start_limit_interval_usec(&self) -> zbus::Result<u64>;
 
     /// DefaultTasksAccounting property
@@ -743,19 +747,19 @@ trait Manager {
     fn default_tasks_max(&self) -> zbus::Result<u64>;
 
     /// DefaultTimeoutAbortUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultTimeoutAbortUSec")]
     fn default_timeout_abort_usec(&self) -> zbus::Result<u64>;
 
     /// DefaultTimeoutStartUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultTimeoutStartUSec")]
     fn default_timeout_start_usec(&self) -> zbus::Result<u64>;
 
     /// DefaultTimeoutStopUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultTimeoutStopUSec")]
     fn default_timeout_stop_usec(&self) -> zbus::Result<u64>;
 
     /// DefaultTimerAccuracyUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultTimerAccuracyUSec")]
     fn default_timer_accuracy_usec(&self) -> zbus::Result<u64>;
 
     /// Environment property
@@ -803,65 +807,64 @@ trait Manager {
     fn generators_start_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDGeneratorsFinishTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDGeneratorsFinishTimestamp")]
     fn init_rdgenerators_finish_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDGeneratorsFinishTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDGeneratorsFinishTimestampMonotonic")]
     fn init_rdgenerators_finish_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDGeneratorsStartTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDGeneratorsStartTimestamp")]
     fn init_rdgenerators_start_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDGeneratorsStartTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDGeneratorsStartTimestampMonotonic")]
     fn init_rdgenerators_start_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDSecurityFinishTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDSecurityFinishTimestamp")]
     fn init_rdsecurity_finish_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDSecurityFinishTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDSecurityFinishTimestampMonotonic")]
     fn init_rdsecurity_finish_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDSecurityStartTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDSecurityStartTimestamp")]
     fn init_rdsecurity_start_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDSecurityStartTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDSecurityStartTimestampMonotonic")]
     fn init_rdsecurity_start_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDTimestamp")]
     fn init_rdtimestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDTimestampMonotonic")]
     fn init_rdtimestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDUnitsLoadFinishTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDUnitsLoadFinishTimestamp")]
     fn init_rdunits_load_finish_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDUnitsLoadFinishTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDUnitsLoadFinishTimestampMonotonic")]
     fn init_rdunits_load_finish_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDUnitsLoadStartTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDUnitsLoadStartTimestamp")]
     fn init_rdunits_load_start_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDUnitsLoadStartTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDUnitsLoadStartTimestampMonotonic")]
     fn init_rdunits_load_start_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// KExecWatchdogUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "KExecWatchdogUSec")]
     fn kexec_watchdog_usec(&self) -> zbus::Result<u64>;
-    #[dbus_proxy(property)]
     fn set_kexec_watchdog_usec(&self, value: u64) -> zbus::Result<()>;
 
     /// KernelTimestamp property
@@ -883,33 +886,31 @@ trait Manager {
     /// LogLevel property
     #[dbus_proxy(property)]
     fn log_level(&self) -> zbus::Result<String>;
-    #[dbus_proxy(property)]
     fn set_log_level(&self, value: &str) -> zbus::Result<()>;
 
     /// LogTarget property
     #[dbus_proxy(property)]
     fn log_target(&self) -> zbus::Result<String>;
-    #[dbus_proxy(property)]
     fn set_log_target(&self, value: &str) -> zbus::Result<()>;
 
     /// NFailedJobs property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "NFailedJobs")]
     fn nfailed_jobs(&self) -> zbus::Result<u32>;
 
     /// NFailedUnits property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "NFailedUnits")]
     fn nfailed_units(&self) -> zbus::Result<u32>;
 
     /// NInstalledJobs property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "NInstalledJobs")]
     fn ninstalled_jobs(&self) -> zbus::Result<u32>;
 
     /// NJobs property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "NJobs")]
     fn njobs(&self) -> zbus::Result<u32>;
 
     /// NNames property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "NNames")]
     fn nnames(&self) -> zbus::Result<u32>;
 
     /// Progress property
@@ -917,15 +918,13 @@ trait Manager {
     fn progress(&self) -> zbus::Result<f64>;
 
     /// RebootWatchdogUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "RebootWatchdogUSec")]
     fn reboot_watchdog_usec(&self) -> zbus::Result<u64>;
-    #[dbus_proxy(property)]
     fn set_reboot_watchdog_usec(&self, value: u64) -> zbus::Result<()>;
 
     /// RuntimeWatchdogUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "RuntimeWatchdogUSec")]
     fn runtime_watchdog_usec(&self) -> zbus::Result<u64>;
-    #[dbus_proxy(property)]
     fn set_runtime_watchdog_usec(&self, value: u64) -> zbus::Result<()>;
 
     /// SecurityFinishTimestamp property
@@ -947,7 +946,6 @@ trait Manager {
     /// ServiceWatchdogs property
     #[dbus_proxy(property)]
     fn service_watchdogs(&self) -> zbus::Result<bool>;
-    #[dbus_proxy(property)]
     fn set_service_watchdogs(&self, value: bool) -> zbus::Result<()>;
 
     /// ShowStatus property
@@ -963,7 +961,7 @@ trait Manager {
     fn tainted(&self) -> zbus::Result<String>;
 
     /// TimerSlackNSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "TimerSlackNSec")]
     fn timer_slack_nsec(&self) -> zbus::Result<u64>;
 
     /// UnitPath property

src/agent/rustjail/src/cgroups/systemd/manager.rs

@@ -5,7 +5,7 @@
 
 use crate::cgroups::Manager as CgroupManager;
 use crate::protocols::agent::CgroupStats;
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use cgroups::freezer::FreezerState;
 use libc::{self, pid_t};
 use oci::LinuxResources;
@@ -29,7 +29,6 @@ pub struct Manager {
     pub mounts: HashMap<String, String>,
     pub cgroups_path: CgroupsPath,
     pub cpath: String,
-    pub unit_name: String,
     // dbus client for set properties
     dbus_client: DBusClient,
     // fs manager for get properties
@@ -40,14 +39,12 @@ pub struct Manager {
 
 impl CgroupManager for Manager {
     fn apply(&self, pid: pid_t) -> Result<()> {
-        let unit_name = self.unit_name.as_str();
-        if self.dbus_client.unit_exists(unit_name)? {
-            self.dbus_client.add_process(pid, self.unit_name.as_str())?;
+        if self.dbus_client.unit_exists()? {
+            self.dbus_client.add_process(pid)?;
         } else {
             self.dbus_client.start_unit(
                 (pid as u32).try_into().unwrap(),
                 self.cgroups_path.slice.as_str(),
-                self.unit_name.as_str(),
                 &self.cg_hierarchy,
             )?;
         }
@@ -66,8 +63,7 @@ impl CgroupManager for Manager {
         Pids::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
         CpuSet::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
 
-        self.dbus_client
-            .set_properties(self.unit_name.as_str(), &properties)?;
+        self.dbus_client.set_properties(&properties)?;
 
         Ok(())
     }
@@ -77,11 +73,15 @@ impl CgroupManager for Manager {
     }
 
     fn freeze(&self, state: FreezerState) -> Result<()> {
-        self.fs_manager.freeze(state)
+        match state {
+            FreezerState::Thawed => self.dbus_client.thaw_unit(),
+            FreezerState::Frozen => self.dbus_client.freeze_unit(),
+            _ => Err(anyhow!("Invalid FreezerState")),
+        }
     }
 
     fn destroy(&mut self) -> Result<()> {
-        self.dbus_client.stop_unit(self.unit_name.as_str())?;
+        self.dbus_client.kill_unit()?;
         self.fs_manager.destroy()
     }
 
@@ -120,8 +120,7 @@ impl Manager {
             mounts: fs_manager.mounts.clone(),
             cgroups_path,
             cpath,
-            unit_name,
-            dbus_client: DBusClient {},
+            dbus_client: DBusClient::new(unit_name),
             fs_manager,
             cg_hierarchy: if cgroups::hierarchies::is_cgroup2_unified_mode() {
                 CgroupHierarchy::Unified

src/agent/rustjail/src/container.rs

@@ -80,6 +80,7 @@ const CLOG_FD: &str = "CLOG_FD";
 const FIFO_FD: &str = "FIFO_FD";
 const HOME_ENV_KEY: &str = "HOME";
 const PIDNS_FD: &str = "PIDNS_FD";
+const PIDNS_ENABLED: &str = "PIDNS_ENABLED";
 const CONSOLE_SOCKET_FD: &str = "CONSOLE_SOCKET_FD";
 
 #[derive(Debug)]
@@ -280,6 +281,17 @@ pub struct SyncPc {
     pid: pid_t,
 }
 
+#[derive(Debug, Clone)]
+pub struct PidNs {
+    enabled: bool,
+    fd: Option<i32>,
+}
+impl PidNs {
+    pub fn new(enabled: bool, fd: Option<i32>) -> Self {
+        Self { enabled, fd }
+    }
+}
+
 pub trait Container: BaseContainer {
     fn pause(&mut self) -> Result<()>;
     fn resume(&mut self) -> Result<()>;
@@ -339,16 +351,20 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let crfd = std::env::var(CRFD_FD)?.parse::<i32>().unwrap();
     let cfd_log = std::env::var(CLOG_FD)?.parse::<i32>().unwrap();
 
-    // get the pidns fd from parent, if parent had passed the pidns fd,
-    // then get it and join in this pidns; otherwise, create a new pidns
-    // by unshare from the parent pidns.
-    match std::env::var(PIDNS_FD) {
-        Ok(fd) => {
-            let pidns_fd = fd.parse::<i32>().context("get parent pidns fd")?;
-            sched::setns(pidns_fd, CloneFlags::CLONE_NEWPID).context("failed to join pidns")?;
-            let _ = unistd::close(pidns_fd);
+    if std::env::var(PIDNS_ENABLED)?.eq(format!("{}", true).as_str()) {
+        // get the pidns fd from parent, if parent had passed the pidns fd,
+        // then get it and join in this pidns; otherwise, create a new pidns
+        // by unshare from the parent pidns.
+        match std::env::var(PIDNS_FD) {
+            Ok(fd) => {
+                let pidns_fd = fd.parse::<i32>().context("get parent pidns fd")?;
+                sched::setns(pidns_fd, CloneFlags::CLONE_NEWPID).context("failed to join pidns")?;
+                let _ = unistd::close(pidns_fd);
+            }
+            Err(_e) => {
+                sched::unshare(CloneFlags::CLONE_NEWPID)?;
+            }
         }
-        Err(_e) => sched::unshare(CloneFlags::CLONE_NEWPID)?,
     }
 
     match unsafe { fork() } {
@@ -983,9 +999,13 @@ impl BaseContainer for LinuxContainer {
         }
 
         let pidns = get_pid_namespace(&self.logger, linux)?;
+        #[cfg(not(feature = "standard-oci-runtime"))]
+        if !pidns.enabled {
+            return Err(anyhow!("cannot find the pid ns"));
+        }
 
-        defer!(if let Some(pid) = pidns {
-            let _ = unistd::close(pid);
+        defer!(if let Some(fd) = pidns.fd {
+            let _ = unistd::close(fd);
         });
 
         let exec_path = std::env::current_exe()?;
@@ -1008,14 +1028,15 @@ impl BaseContainer for LinuxContainer {
             .env(CRFD_FD, format!("{}", crfd))
             .env(CWFD_FD, format!("{}", cwfd))
             .env(CLOG_FD, format!("{}", cfd_log))
-            .env(CONSOLE_SOCKET_FD, console_name);
+            .env(CONSOLE_SOCKET_FD, console_name)
+            .env(PIDNS_ENABLED, format!("{}", pidns.enabled));
 
         if p.init {
             child = child.env(FIFO_FD, format!("{}", fifofd));
         }
 
-        if pidns.is_some() {
-            child = child.env(PIDNS_FD, format!("{}", pidns.unwrap()));
+        if pidns.fd.is_some() {
+            child = child.env(PIDNS_FD, format!("{}", pidns.fd.unwrap()));
         }
 
         child.spawn()?;
@@ -1249,11 +1270,11 @@ pub fn update_namespaces(logger: &Logger, spec: &mut Spec, init_pid: RawFd) -> R
     Ok(())
 }
 
-fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result<Option<RawFd>> {
+fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result<PidNs> {
     for ns in &linux.namespaces {
         if ns.r#type == "pid" {
             if ns.path.is_empty() {
-                return Ok(None);
+                return Ok(PidNs::new(true, None));
             }
 
             let fd =
@@ -1269,11 +1290,11 @@ fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result<Option<RawFd>> {
                     e
                 })?;
 
-            return Ok(Some(fd));
+            return Ok(PidNs::new(true, Some(fd)));
         }
     }
 
-    Err(anyhow!("cannot find the pid ns"))
+    Ok(PidNs::new(false, None))
 }
 
 fn is_userns_enabled(linux: &Linux) -> bool {

src/agent/rustjail/src/lib.rs

@@ -423,12 +423,18 @@ fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
         let mut r = Vec::new();
 
         for d in l.Devices.iter() {
+            // if the filemode for the device is 0 (unset), use a default value as runc does
+            let filemode = if d.FileMode != 0 {
+                Some(d.FileMode)
+            } else {
+                Some(0o666)
+            };
             r.push(oci::LinuxDevice {
                 path: d.Path.clone(),
                 r#type: d.Type.clone(),
                 major: d.Major,
                 minor: d.Minor,
-                file_mode: Some(d.FileMode),
+                file_mode: filemode,
                 uid: Some(d.UID),
                 gid: Some(d.GID),
             });

src/agent/src/config.rs

@@ -5,7 +5,6 @@
 use crate::rpc;
 use anyhow::{bail, ensure, Context, Result};
 use serde::Deserialize;
-use std::collections::HashSet;
 use std::env;
 use std::fs;
 use std::str::FromStr;
@@ -52,17 +51,6 @@ const ERR_INVALID_CONTAINER_PIPE_SIZE_PARAM: &str = "unable to parse container p
 const ERR_INVALID_CONTAINER_PIPE_SIZE_KEY: &str = "invalid container pipe size key name";
 const ERR_INVALID_CONTAINER_PIPE_NEGATIVE: &str = "container pipe size should not be negative";
 
-#[derive(Debug, Default, Deserialize)]
-pub struct EndpointsConfig {
-    pub allowed: Vec<String>,
-}
-
-#[derive(Debug, Default)]
-pub struct AgentEndpoints {
-    pub allowed: HashSet<String>,
-    pub all_allowed: bool,
-}
-
 #[derive(Debug)]
 pub struct AgentConfig {
     pub debug_console: bool,
@@ -75,7 +63,6 @@ pub struct AgentConfig {
     pub server_addr: String,
     pub unified_cgroup_hierarchy: bool,
     pub tracing: bool,
-    pub endpoints: AgentEndpoints,
     pub supports_seccomp: bool,
 }
 
@@ -91,7 +78,6 @@ pub struct AgentConfigBuilder {
     pub server_addr: Option<String>,
     pub unified_cgroup_hierarchy: Option<bool>,
     pub tracing: Option<bool>,
-    pub endpoints: Option<EndpointsConfig>,
 }
 
 macro_rules! config_override {
@@ -151,7 +137,6 @@ impl Default for AgentConfig {
             server_addr: format!("{}:{}", VSOCK_ADDR, DEFAULT_AGENT_VSOCK_PORT),
             unified_cgroup_hierarchy: false,
             tracing: false,
-            endpoints: Default::default(),
             supports_seccomp: rpc::have_seccomp(),
         }
     }
@@ -182,19 +167,13 @@ impl FromStr for AgentConfig {
         config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
         config_override!(agent_config_builder, agent_config, tracing);
 
-        // Populate the allowed endpoints hash set, if we got any from the config file.
-        if let Some(endpoints) = agent_config_builder.endpoints {
-            for ep in endpoints.allowed {
-                agent_config.endpoints.allowed.insert(ep);
-            }
-        }
-
         Ok(agent_config)
     }
 }
 
 impl AgentConfig {
     #[instrument]
+    #[allow(clippy::redundant_closure_call)]
     pub fn from_cmdline(file: &str, args: Vec<String>) -> Result<AgentConfig> {
         // If config file specified in the args, generate our config from it
         let config_position = args.iter().position(|a| a == "--config" || a == "-c");
@@ -297,9 +276,6 @@ impl AgentConfig {
             config.tracing = get_bool_value(&name_value)?;
         }
 
-        // We did not get a configuration file: allow all endpoints.
-        config.endpoints.all_allowed = true;
-
         Ok(config)
     }
 
@@ -309,10 +285,6 @@ impl AgentConfig {
             .with_context(|| format!("Failed to read config file {}", file))?;
         AgentConfig::from_str(&config)
     }
-
-    pub fn is_allowed_endpoint(&self, ep: &str) -> bool {
-        self.endpoints.all_allowed || self.endpoints.allowed.contains(ep)
-    }
 }
 
 #[instrument]
@@ -1377,26 +1349,13 @@ Caused by:
             r#"
                dev_mode = true
                server_addr = 'vsock://8:2048'
-
-               [endpoints]
-               allowed = ["CreateContainer", "StartContainer"]
               "#,
         )
         .unwrap();
 
-        // Verify that the all_allowed flag is false
-        assert!(!config.endpoints.all_allowed);
-
         // Verify that the override worked
         assert!(config.dev_mode);
         assert_eq!(config.server_addr, "vsock://8:2048");
-        assert_eq!(
-            config.endpoints.allowed,
-            vec!["CreateContainer".to_string(), "StartContainer".to_string()]
-                .iter()
-                .cloned()
-                .collect()
-        );
 
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);

src/agent/src/device.rs

@@ -35,9 +35,9 @@ const VM_ROOTFS: &str = "/";
 const BLOCK: &str = "block";
 pub const DRIVER_9P_TYPE: &str = "9p";
 pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
-pub const DRIVER_BLK_TYPE: &str = "blk";
+pub const DRIVER_BLK_PCI_TYPE: &str = "blk";
 pub const DRIVER_BLK_CCW_TYPE: &str = "blk-ccw";
-pub const DRIVER_MMIO_BLK_TYPE: &str = "mmioblk";
+pub const DRIVER_BLK_MMIO_TYPE: &str = "mmioblk";
 pub const DRIVER_SCSI_TYPE: &str = "scsi";
 pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
 pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
@@ -935,9 +935,9 @@ async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<Sp
     }
 
     match device.type_.as_str() {
-        DRIVER_BLK_TYPE => virtio_blk_device_handler(device, sandbox).await,
+        DRIVER_BLK_PCI_TYPE => virtio_blk_device_handler(device, sandbox).await,
         DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, sandbox).await,
-        DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, sandbox).await,
+        DRIVER_BLK_MMIO_TYPE => virtiommio_blk_device_handler(device, sandbox).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, sandbox).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, sandbox).await,
         DRIVER_VFIO_PCI_GK_TYPE | DRIVER_VFIO_PCI_TYPE => {
@@ -1467,6 +1467,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[allow(clippy::redundant_clone)]
     async fn test_virtio_blk_matcher() {
         let root_bus = create_pci_root_bus_path();
         let devname = "vda";
@@ -1551,6 +1552,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[allow(clippy::redundant_clone)]
     async fn test_scsi_block_matcher() {
         let root_bus = create_pci_root_bus_path();
         let devname = "sda";
@@ -1581,6 +1583,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[allow(clippy::redundant_clone)]
     async fn test_vfio_matcher() {
         let grpa = IommuGroup(1);
         let grpb = IommuGroup(22);
@@ -1602,6 +1605,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[allow(clippy::redundant_clone)]
     async fn test_mmio_block_matcher() {
         let devname_a = "vda";
         let devname_b = "vdb";

src/agent/src/main.rs

@@ -48,6 +48,7 @@ mod pci;
 pub mod random;
 mod sandbox;
 mod signal;
+mod storage;
 mod uevent;
 mod util;
 mod version;
@@ -73,6 +74,9 @@ use tokio::{
 mod rpc;
 mod tracer;
 
+#[cfg(feature = "agent-policy")]
+mod policy;
+
 cfg_if! {
     if #[cfg(target_arch = "s390x")] {
         mod ap;
@@ -90,6 +94,11 @@ lazy_static! {
         AgentConfig::from_cmdline("/proc/cmdline", env::args().collect()).unwrap();
 }
 
+#[cfg(feature = "agent-policy")]
+lazy_static! {
+    static ref AGENT_POLICY: Mutex<policy::AgentPolicy> = Mutex::new(AgentPolicy::new());
+}
+
 #[derive(Parser)]
 // The default clap version info doesn't match our form, so we need to override it
 #[clap(global_setting(AppSettings::DisableVersionFlag))]
@@ -326,6 +335,19 @@ async fn start_sandbox(
         s.rtnl.handle_localhost().await?;
     }
 
+    // - When init_mode is true, enabling the localhost link during the
+    //   handle_localhost call above is required before starting OPA with the
+    //   initialize_policy call below.
+    // - When init_mode is false, the Policy could be initialized earlier,
+    //   because initialize_policy doesn't start OPA. OPA is started by
+    //   systemd after localhost has been enabled.
+    #[cfg(feature = "agent-policy")]
+    if let Err(e) = initialize_policy(init_mode).await {
+        error!(logger, "Failed to initialize agent policy: {:?}", e);
+        // Continuing execution without a security policy could be dangerous.
+        std::process::abort();
+    }
+
     let sandbox = Arc::new(Mutex::new(s));
 
     let signal_handler_task = tokio::spawn(setup_signal_handler(
@@ -387,6 +409,18 @@ fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result
     Ok(())
 }
 
+#[cfg(feature = "agent-policy")]
+async fn initialize_policy(init_mode: bool) -> Result<()> {
+    let opa_addr = "localhost:8181";
+    let agent_policy_path = "/agent_policy";
+    let default_agent_policy = "/etc/kata-opa/default-policy.rego";
+    AGENT_POLICY
+        .lock()
+        .await
+        .initialize(init_mode, opa_addr, agent_policy_path, default_agent_policy)
+        .await
+}
+
 // The Rust standard library had suppressed the default SIGPIPE behavior,
 // see https://github.com/rust-lang/rust/pull/13158.
 // Since the parent's signal handler would be inherited by it's child process,
@@ -401,6 +435,9 @@ fn reset_sigpipe() {
 use crate::config::AgentConfig;
 use std::os::unix::io::{FromRawFd, RawFd};
 
+#[cfg(feature = "agent-policy")]
+use crate::policy::AgentPolicy;
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/agent/src/namespace.rs

@@ -7,14 +7,14 @@ use anyhow::{anyhow, Result};
 use nix::mount::MsFlags;
 use nix::sched::{unshare, CloneFlags};
 use nix::unistd::{getpid, gettid};
+use slog::Logger;
 use std::fmt;
 use std::fs;
 use std::fs::File;
 use std::path::{Path, PathBuf};
 use tracing::instrument;
 
-use crate::mount::{baremount, FLAGS};
-use slog::Logger;
+use crate::mount::baremount;
 
 const PERSISTENT_NS_DIR: &str = "/var/run/sandbox-ns";
 pub const NSTYPEIPC: &str = "ipc";
@@ -116,15 +116,7 @@ impl Namespace {
                 // Bind mount the new namespace from the current thread onto the mount point to persist it.
 
                 let mut flags = MsFlags::empty();
-
-                if let Some(x) = FLAGS.get("rbind") {
-                    let (clear, f) = *x;
-                    if clear {
-                        flags &= !f;
-                    } else {
-                        flags |= f;
-                    }
-                };
+                flags |= MsFlags::MS_BIND | MsFlags::MS_REC;
 
                 baremount(source, destination, "none", flags, "", &logger).map_err(|e| {
                     anyhow!(

src/agent/src/network.rs

@@ -29,7 +29,7 @@ impl Network {
     }
 }
 
-pub fn setup_guest_dns(logger: Logger, dns_list: Vec<String>) -> Result<()> {
+pub fn setup_guest_dns(logger: Logger, dns_list: &[String]) -> Result<()> {
     do_setup_guest_dns(
         logger,
         dns_list,
@@ -38,7 +38,7 @@ pub fn setup_guest_dns(logger: Logger, dns_list: Vec<String>) -> Result<()> {
     )
 }
 
-fn do_setup_guest_dns(logger: Logger, dns_list: Vec<String>, src: &str, dst: &str) -> Result<()> {
+fn do_setup_guest_dns(logger: Logger, dns_list: &[String], src: &str, dst: &str) -> Result<()> {
     let logger = logger.new(o!( "subsystem" => "network"));
 
     if dns_list.is_empty() {
@@ -124,7 +124,7 @@ mod tests {
             .expect("failed to write file contents");
 
         // call do_setup_guest_dns
-        let result = do_setup_guest_dns(logger, dns.clone(), src_filename, dst_filename);
+        let result = do_setup_guest_dns(logger, &dns, src_filename, dst_filename);
 
         assert!(result.is_ok(), "result should be ok, but {:?}", result);
 

src/agent/src/policy.rs

@@ -0,0 +1,267 @@
+// Copyright (c) 2023 Microsoft Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{bail, Result};
+use serde::{Deserialize, Serialize};
+use slog::Drain;
+use tokio::io::AsyncWriteExt;
+use tokio::time::{sleep, Duration};
+
+static EMPTY_JSON_INPUT: &str = "{\"input\":{}}";
+
+static OPA_DATA_PATH: &str = "/data";
+static OPA_POLICIES_PATH: &str = "/policies";
+
+static POLICY_LOG_FILE: &str = "/tmp/policy.txt";
+
+/// Convenience macro to obtain the scope logger
+macro_rules! sl {
+    () => {
+        slog_scope::logger()
+    };
+}
+
+/// Example of HTTP response from OPA: {"result":true}
+#[derive(Debug, Serialize, Deserialize)]
+struct AllowResponse {
+    result: bool,
+}
+
+/// Singleton policy object.
+#[derive(Debug, Default)]
+pub struct AgentPolicy {
+    /// When true policy errors are ignored, for debug purposes.
+    allow_failures: bool,
+
+    /// OPA path used to query if an Agent gRPC request should be allowed.
+    /// The request name (e.g., CreateContainerRequest) must be added to
+    /// this path.
+    query_path: String,
+
+    /// OPA path used to add or delete a rego format Policy.
+    policy_path: String,
+
+    /// Client used to connect a single time to the OPA service and reused
+    /// for all the future communication with OPA.
+    opa_client: Option<reqwest::Client>,
+
+    /// "/tmp/policy.txt" log file for policy activity.
+    log_file: Option<tokio::fs::File>,
+}
+
+impl AgentPolicy {
+    /// Create AgentPolicy object.
+    pub fn new() -> Self {
+        Self {
+            allow_failures: false,
+            ..Default::default()
+        }
+    }
+
+    /// Wait for OPA to start and connect to it.
+    pub async fn initialize(
+        &mut self,
+        launch_opa: bool,
+        opa_addr: &str,
+        policy_name: &str,
+        default_policy: &str,
+    ) -> Result<()> {
+        if sl!().is_enabled(slog::Level::Debug) {
+            self.log_file = Some(
+                tokio::fs::OpenOptions::new()
+                    .write(true)
+                    .truncate(true)
+                    .create(true)
+                    .open(POLICY_LOG_FILE)
+                    .await?,
+            );
+            debug!(sl!(), "policy: log file: {}", POLICY_LOG_FILE);
+        }
+
+        if launch_opa {
+            start_opa(opa_addr)?;
+        }
+
+        let opa_uri = format!("http://{opa_addr}/v1");
+        self.query_path = format!("{opa_uri}{OPA_DATA_PATH}{policy_name}/");
+        self.policy_path = format!("{opa_uri}{OPA_POLICIES_PATH}{policy_name}");
+        let opa_client = reqwest::Client::builder().http1_only().build()?;
+        let policy = tokio::fs::read_to_string(default_policy).await?;
+
+        // This loop is necessary to get the opa_client connected to the
+        // OPA service while that service is starting. Future requests to
+        // OPA are expected to work without retrying, after connecting
+        // successfully for the first time.
+        for i in 0..50 {
+            if i > 0 {
+                sleep(Duration::from_millis(100)).await;
+                debug!(sl!(), "policy initialize: PUT failed, retrying");
+            }
+
+            // Set-up the default policy.
+            if opa_client
+                .put(&self.policy_path)
+                .body(policy.clone())
+                .send()
+                .await
+                .is_ok()
+            {
+                self.opa_client = Some(opa_client);
+
+                // Check if requests causing policy errors should actually
+                // be allowed. That is an insecure configuration but is
+                // useful for allowing insecure pods to start, then connect to
+                // them and inspect Guest logs for the root cause of a failure.
+                //
+                // Note that post_query returns Ok(false) in case
+                // AllowRequestsFailingPolicy was not defined in the policy.
+                self.allow_failures = self
+                    .post_query("AllowRequestsFailingPolicy", EMPTY_JSON_INPUT)
+                    .await?;
+                return Ok(());
+            }
+        }
+        bail!("Failed to connect to OPA")
+    }
+
+    /// Ask OPA to check if an API call should be allowed or not.
+    pub async fn is_allowed_endpoint(&mut self, ep: &str, request: &str) -> bool {
+        let post_input = format!("{{\"input\":{request}}}");
+        self.log_opa_input(ep, &post_input).await;
+        match self.post_query(ep, &post_input).await {
+            Err(e) => {
+                debug!(
+                    sl!(),
+                    "policy: failed to query endpoint {}: {:?}. Returning false.", ep, e
+                );
+                false
+            }
+            Ok(allowed) => allowed,
+        }
+    }
+
+    /// Replace the Policy in OPA.
+    pub async fn set_policy(&mut self, policy: &str) -> Result<()> {
+        if let Some(opa_client) = &mut self.opa_client {
+            // Delete the old rules.
+            opa_client.delete(&self.policy_path).send().await?;
+
+            // Put the new rules.
+            opa_client
+                .put(&self.policy_path)
+                .body(policy.to_string())
+                .send()
+                .await?;
+
+            // Check if requests causing policy errors should actually be allowed.
+            // That is an insecure configuration but is useful for allowing insecure
+            // pods to start, then connect to them and inspect Guest logs for the
+            // root cause of a failure.
+            //
+            // Note that post_query returns Ok(false) in case
+            // AllowRequestsFailingPolicy was not defined in the policy.
+            self.allow_failures = self
+                .post_query("AllowRequestsFailingPolicy", EMPTY_JSON_INPUT)
+                .await?;
+
+            Ok(())
+        } else {
+            bail!("Agent Policy is not initialized")
+        }
+    }
+
+    // Post query to OPA.
+    async fn post_query(&mut self, ep: &str, post_input: &str) -> Result<bool> {
+        debug!(sl!(), "policy check: {ep}");
+
+        if let Some(opa_client) = &mut self.opa_client {
+            let uri = format!("{}{ep}", &self.query_path);
+            let response = opa_client
+                .post(uri)
+                .body(post_input.to_string())
+                .send()
+                .await?;
+
+            if response.status() != http::StatusCode::OK {
+                bail!("policy: POST {} response status {}", ep, response.status());
+            }
+
+            let http_response = response.text().await?;
+            let opa_response: serde_json::Result<AllowResponse> =
+                serde_json::from_str(&http_response);
+
+            match opa_response {
+                Ok(resp) => {
+                    if !resp.result {
+                        if self.allow_failures {
+                            warn!(
+                                sl!(),
+                                "policy: POST {} response <{}>. Ignoring error!", ep, http_response
+                            );
+                            return Ok(true);
+                        } else {
+                            error!(sl!(), "policy: POST {} response <{}>", ep, http_response);
+                        }
+                    }
+                    Ok(resp.result)
+                }
+                Err(_) => {
+                    warn!(
+                        sl!(),
+                        "policy: endpoint {} not found in policy. Returning false.", ep,
+                    );
+                    Ok(false)
+                }
+            }
+        } else {
+            bail!("Agent Policy is not initialized")
+        }
+    }
+
+    async fn log_opa_input(&mut self, ep: &str, input: &str) {
+        if let Some(log_file) = &mut self.log_file {
+            match ep {
+                "StatsContainerRequest" | "ReadStreamRequest" | "SetPolicyRequest" => {
+                    // - StatsContainerRequest and ReadStreamRequest are called
+                    //   relatively often, so we're not logging them, to avoid
+                    //   growing this log file too much.
+                    // - Confidential Containers Policy documents are relatively
+                    //   large, so we're not logging them here, for SetPolicyRequest.
+                    //   The Policy text can be obtained directly from the pod YAML.
+                }
+                _ => {
+                    let log_entry = format!("[\"ep\":\"{ep}\",{input}],\n\n");
+
+                    if let Err(e) = log_file.write_all(log_entry.as_bytes()).await {
+                        warn!(sl!(), "policy: log_opa_input: write_all failed: {}", e);
+                    } else if let Err(e) = log_file.flush().await {
+                        warn!(sl!(), "policy: log_opa_input: flush failed: {}", e);
+                    }
+                }
+            }
+        }
+    }
+}
+
+fn start_opa(opa_addr: &str) -> Result<()> {
+    let bin_dirs = vec!["/bin", "/usr/bin", "/usr/local/bin"];
+    for bin_dir in &bin_dirs {
+        let opa_path = bin_dir.to_string() + "/opa";
+        if std::fs::metadata(&opa_path).is_ok() {
+            // args copied from kata-opa.service.in.
+            std::process::Command::new(&opa_path)
+                .arg("run")
+                .arg("--server")
+                .arg("--disable-telemetry")
+                .arg("--addr")
+                .arg(opa_addr)
+                .arg("--log-level")
+                .arg("info")
+                .spawn()?;
+            return Ok(());
+        }
+    }
+    bail!("OPA binary not found in {:?}", &bin_dirs);
+}

src/agent/src/sandbox.rs

@@ -3,16 +3,20 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::linux_abi::*;
-use crate::mount::{get_mount_fs_type, remove_mounts, TYPE_ROOTFS};
-use crate::namespace::Namespace;
-use crate::netlink::Handle;
-use crate::network::Network;
-use crate::pci;
-use crate::uevent::{Uevent, UeventMatcher};
-use crate::watcher::BindWatcher;
+use std::collections::hash_map::Entry;
+use std::collections::HashMap;
+use std::fmt::{Debug, Formatter};
+use std::fs;
+use std::os::unix::fs::PermissionsExt;
+use std::path::Path;
+use std::str::FromStr;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+use std::{thread, time};
+
 use anyhow::{anyhow, Context, Result};
 use kata_types::cpu::CpuSet;
+use kata_types::mount::StorageDevice;
 use libc::pid_t;
 use oci::{Hook, Hooks};
 use protocols::agent::OnlineCPUMemRequest;
@@ -22,22 +26,69 @@ use rustjail::container::BaseContainer;
 use rustjail::container::LinuxContainer;
 use rustjail::process::Process;
 use slog::Logger;
-use std::collections::HashMap;
-use std::fs;
-use std::os::unix::fs::PermissionsExt;
-use std::path::Path;
-use std::str::FromStr;
-use std::sync::Arc;
-use std::{thread, time};
 use tokio::sync::mpsc::{channel, Receiver, Sender};
 use tokio::sync::oneshot;
 use tokio::sync::Mutex;
 use tracing::instrument;
 
+use crate::linux_abi::*;
+use crate::mount::{get_mount_fs_type, TYPE_ROOTFS};
+use crate::namespace::Namespace;
+use crate::netlink::Handle;
+use crate::network::Network;
+use crate::pci;
+use crate::storage::StorageDeviceGeneric;
+use crate::uevent::{Uevent, UeventMatcher};
+use crate::watcher::BindWatcher;
+
 pub const ERR_INVALID_CONTAINER_ID: &str = "Invalid container id";
 
 type UeventWatcher = (Box<dyn UeventMatcher>, oneshot::Sender<Uevent>);
 
+#[derive(Clone)]
+pub struct StorageState {
+    count: Arc<AtomicU32>,
+    device: Arc<dyn StorageDevice>,
+}
+
+impl Debug for StorageState {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("StorageState").finish()
+    }
+}
+
+impl StorageState {
+    fn new() -> Self {
+        StorageState {
+            count: Arc::new(AtomicU32::new(1)),
+            device: Arc::new(StorageDeviceGeneric::default()),
+        }
+    }
+
+    pub fn from_device(device: Arc<dyn StorageDevice>) -> Self {
+        Self {
+            count: Arc::new(AtomicU32::new(1)),
+            device,
+        }
+    }
+
+    pub fn path(&self) -> Option<&str> {
+        self.device.path()
+    }
+
+    pub async fn ref_count(&self) -> u32 {
+        self.count.load(Ordering::Relaxed)
+    }
+
+    async fn inc_ref_count(&self) {
+        self.count.fetch_add(1, Ordering::Acquire);
+    }
+
+    async fn dec_and_test_ref_count(&self) -> bool {
+        self.count.fetch_sub(1, Ordering::AcqRel) == 1
+    }
+}
+
 #[derive(Debug)]
 pub struct Sandbox {
     pub logger: Logger,
@@ -52,7 +103,7 @@ pub struct Sandbox {
     pub shared_utsns: Namespace,
     pub shared_ipcns: Namespace,
     pub sandbox_pidns: Option<Namespace>,
-    pub storages: HashMap<String, u32>,
+    pub storages: HashMap<String, StorageState>,
     pub running: bool,
     pub no_pivot_root: bool,
     pub sender: Option<tokio::sync::oneshot::Sender<i32>>,
@@ -98,85 +149,60 @@ impl Sandbox {
         })
     }
 
-    // set_sandbox_storage sets the sandbox level reference
-    // counter for the sandbox storage.
-    // This method also returns a boolean to let
-    // callers know if the storage already existed or not.
-    // It will return true if storage is new.
-    //
-    // It's assumed that caller is calling this method after
-    // acquiring a lock on sandbox.
+    /// Add a new storage object or increase reference count of existing one.
+    /// The caller may detect new storage object by checking `StorageState.refcount == 1`.
     #[instrument]
-    pub fn set_sandbox_storage(&mut self, path: &str) -> bool {
-        match self.storages.get_mut(path) {
-            None => {
-                self.storages.insert(path.to_string(), 1);
-                true
+    pub async fn add_sandbox_storage(&mut self, path: &str) -> StorageState {
+        match self.storages.entry(path.to_string()) {
+            Entry::Occupied(e) => {
+                let state = e.get().clone();
+                state.inc_ref_count().await;
+                state
             }
-            Some(count) => {
-                *count += 1;
-                false
+            Entry::Vacant(e) => {
+                let state = StorageState::new();
+                e.insert(state.clone());
+                state
             }
         }
     }
 
-    // unset_sandbox_storage will decrement the sandbox storage
-    // reference counter. If there aren't any containers using
-    // that sandbox storage, this method will remove the
-    // storage reference from the sandbox and return 'true' to
-    // let the caller know that they can clean up the storage
-    // related directories by calling remove_sandbox_storage
-    //
-    // It's assumed that caller is calling this method after
-    // acquiring a lock on sandbox.
+    /// Update the storage device associated with a path.
+    pub fn update_sandbox_storage(
+        &mut self,
+        path: &str,
+        device: Arc<dyn StorageDevice>,
+    ) -> std::result::Result<Arc<dyn StorageDevice>, Arc<dyn StorageDevice>> {
+        if !self.storages.contains_key(path) {
+            return Err(device);
+        }
+
+        let state = StorageState::from_device(device);
+        // Safe to unwrap() because we have just ensured existence of entry.
+        let state = self.storages.insert(path.to_string(), state).unwrap();
+        Ok(state.device)
+    }
+
+    /// Decrease reference count and destroy the storage object if reference count reaches zero.
+    /// Returns `Ok(true)` if the reference count has reached zero and the storage object has been
+    /// removed.
     #[instrument]
-    pub fn unset_sandbox_storage(&mut self, path: &str) -> Result<bool> {
-        match self.storages.get_mut(path) {
+    pub async fn remove_sandbox_storage(&mut self, path: &str) -> Result<bool> {
+        match self.storages.get(path) {
             None => Err(anyhow!("Sandbox storage with path {} not found", path)),
-            Some(count) => {
-                *count -= 1;
-                if *count < 1 {
-                    self.storages.remove(path);
-                    return Ok(true);
+            Some(state) => {
+                if state.dec_and_test_ref_count().await {
+                    if let Some(storage) = self.storages.remove(path) {
+                        storage.device.cleanup()?;
+                    }
+                    Ok(true)
+                } else {
+                    Ok(false)
                 }
-                Ok(false)
             }
         }
     }
 
-    // remove_sandbox_storage removes the sandbox storage if no
-    // containers are using that storage.
-    //
-    // It's assumed that caller is calling this method after
-    // acquiring a lock on sandbox.
-    #[instrument]
-    pub fn remove_sandbox_storage(&self, path: &str) -> Result<()> {
-        let mounts = vec![path.to_string()];
-        remove_mounts(&mounts)?;
-        // "remove_dir" will fail if the mount point is backed by a read-only filesystem.
-        // This is the case with the device mapper snapshotter, where we mount the block device directly
-        // at the underlying sandbox path which was provided from the base RO kataShared path from the host.
-        if let Err(err) = fs::remove_dir(path) {
-            warn!(self.logger, "failed to remove dir {}, {:?}", path, err);
-        }
-        Ok(())
-    }
-
-    // unset_and_remove_sandbox_storage unsets the storage from sandbox
-    // and if there are no containers using this storage it will
-    // remove it from the sandbox.
-    //
-    // It's assumed that caller is calling this method after
-    // acquiring a lock on sandbox.
-    #[instrument]
-    pub fn unset_and_remove_sandbox_storage(&mut self, path: &str) -> Result<()> {
-        if self.unset_sandbox_storage(path)? {
-            return self.remove_sandbox_storage(path);
-        }
-
-        Ok(())
-    }
-
     #[instrument]
     pub async fn setup_shared_namespaces(&mut self) -> Result<bool> {
         // Set up shared IPC namespace
@@ -184,22 +210,18 @@ impl Sandbox {
             .get_ipc()
             .setup()
             .await
-            .context("Failed to setup persistent IPC namespace")?;
+            .context("setup persistent IPC namespace")?;
 
         // // Set up shared UTS namespace
         self.shared_utsns = Namespace::new(&self.logger)
             .get_uts(self.hostname.as_str())
             .setup()
             .await
-            .context("Failed to setup persistent UTS namespace")?;
+            .context("setup persistent UTS namespace")?;
 
         Ok(true)
     }
 
-    pub fn add_container(&mut self, c: LinuxContainer) {
-        self.containers.insert(c.id.clone(), c);
-    }
-
     #[instrument]
     pub fn update_shared_pidns(&mut self, c: &LinuxContainer) -> Result<()> {
         // Populate the shared pid path only if this is an infra container and
@@ -224,14 +246,18 @@ impl Sandbox {
         Ok(())
     }
 
+    pub fn add_container(&mut self, c: LinuxContainer) {
+        self.containers.insert(c.id.clone(), c);
+    }
+
     pub fn get_container(&mut self, id: &str) -> Option<&mut LinuxContainer> {
         self.containers.get_mut(id)
     }
 
     pub fn find_process(&mut self, pid: pid_t) -> Option<&mut Process> {
         for (_, c) in self.containers.iter_mut() {
-            if c.processes.get(&pid).is_some() {
-                return c.processes.get_mut(&pid);
+            if let Some(p) = c.processes.get_mut(&pid) {
+                return Some(p);
             }
         }
 
@@ -280,25 +306,17 @@ impl Sandbox {
         let guest_cpuset = rustjail_cgroups::fs::get_guest_cpuset()?;
 
         for (_, ctr) in self.containers.iter() {
-            let cpu = ctr
-                .config
-                .spec
-                .as_ref()
-                .unwrap()
-                .linux
-                .as_ref()
-                .unwrap()
-                .resources
-                .as_ref()
-                .unwrap()
-                .cpu
-                .as_ref();
-            let container_cpust = if let Some(c) = cpu { &c.cpus } else { "" };
-
-            info!(self.logger, "updating {}", ctr.id.as_str());
-            ctr.cgroup_manager
-                .as_ref()
-                .update_cpuset_path(guest_cpuset.as_str(), container_cpust)?;
+            if let Some(spec) = ctr.config.spec.as_ref() {
+                if let Some(linux) = spec.linux.as_ref() {
+                    if let Some(resources) = linux.resources.as_ref() {
+                        if let Some(cpus) = resources.cpu.as_ref() {
+                            info!(self.logger, "updating {}", ctr.id.as_str());
+                            ctr.cgroup_manager
+                                .update_cpuset_path(guest_cpuset.as_str(), &cpus.cpus)?;
+                        }
+                    }
+                }
+            }
         }
 
         Ok(())
@@ -360,31 +378,28 @@ impl Sandbox {
     #[instrument]
     pub async fn run_oom_event_monitor(&self, mut rx: Receiver<String>, container_id: String) {
         let logger = self.logger.clone();
-
-        if self.event_tx.is_none() {
-            error!(
-                logger,
-                "sandbox.event_tx not found in run_oom_event_monitor"
-            );
-            return;
-        }
-
-        let tx = self.event_tx.as_ref().unwrap().clone();
+        let tx = match self.event_tx.as_ref() {
+            Some(v) => v.clone(),
+            None => {
+                error!(
+                    logger,
+                    "sandbox.event_tx not found in run_oom_event_monitor"
+                );
+                return;
+            }
+        };
 
         tokio::spawn(async move {
             loop {
                 let event = rx.recv().await;
-                // None means the container has exited,
-                // and sender in OOM notifier is dropped.
+                // None means the container has exited, and sender in OOM notifier is dropped.
                 if event.is_none() {
                     return;
                 }
                 info!(logger, "got an OOM event {:?}", event);
-
-                let _ = tx
-                    .send(container_id.clone())
-                    .await
-                    .map_err(|e| error!(logger, "failed to send message: {:?}", e));
+                if let Err(e) = tx.send(container_id.clone()).await {
+                    error!(logger, "failed to send message: {:?}", e);
+                }
             }
         });
     }
@@ -397,39 +412,36 @@ fn online_resources(logger: &Logger, path: &str, pattern: &str, num: i32) -> Res
 
     for e in fs::read_dir(path)? {
         let entry = e?;
-        let tmpname = entry.file_name();
-        let name = tmpname.to_str().unwrap();
-        let p = entry.path();
-
-        if re.is_match(name) {
-            let file = format!("{}/{}", p.to_str().unwrap(), SYSFS_ONLINE_FILE);
-            info!(logger, "{}", file.as_str());
-
-            let c = fs::read_to_string(file.as_str());
-            if c.is_err() {
-                continue;
-            }
-            let c = c.unwrap();
-
-            if c.trim().contains('0') {
-                let r = fs::write(file.as_str(), "1");
-                if r.is_err() {
+        // Skip direntry which doesn't match the pattern.
+        match entry.file_name().to_str() {
+            None => continue,
+            Some(v) => {
+                if !re.is_match(v) {
                     continue;
                 }
-                count += 1;
+            }
+        };
 
-                if num > 0 && count == num {
+        let p = entry.path().join(SYSFS_ONLINE_FILE);
+        if let Ok(c) = fs::read_to_string(&p) {
+            // Try to online the object in offline state.
+            if c.trim().contains('0') && fs::write(&p, "1").is_ok() && num > 0 {
+                count += 1;
+                if count == num {
                     break;
                 }
             }
         }
     }
 
-    if num > 0 {
-        return Ok(count);
-    }
+    Ok(count)
+}
 
-    Ok(0)
+#[instrument]
+fn online_memory(logger: &Logger) -> Result<()> {
+    online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1)
+        .context("online memory resource")?;
+    Ok(())
 }
 
 // max wait for all CPUs to online will use 50 * 100 = 5 seconds.
@@ -473,13 +485,6 @@ fn online_cpus(logger: &Logger, num: i32) -> Result<i32> {
     ))
 }
 
-#[instrument]
-fn online_memory(logger: &Logger) -> Result<()> {
-    online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1)
-        .context("online memory resource")?;
-    Ok(())
-}
-
 fn onlined_cpus() -> Result<i32> {
     let content =
         fs::read_to_string(SYSFS_CPU_ONLINE_PATH).context("read sysfs cpu online file")?;
@@ -524,24 +529,22 @@ mod tests {
         let tmpdir_path = tmpdir.path().to_str().unwrap();
 
         // Add a new sandbox storage
-        let new_storage = s.set_sandbox_storage(tmpdir_path);
+        let new_storage = s.add_sandbox_storage(tmpdir_path).await;
 
         // Check the reference counter
-        let ref_count = s.storages[tmpdir_path];
+        let ref_count = new_storage.ref_count().await;
         assert_eq!(
             ref_count, 1,
             "Invalid refcount, got {} expected 1.",
             ref_count
         );
-        assert!(new_storage);
 
         // Use the existing sandbox storage
-        let new_storage = s.set_sandbox_storage(tmpdir_path);
-        assert!(!new_storage, "Should be false as already exists.");
+        let new_storage = s.add_sandbox_storage(tmpdir_path).await;
 
         // Since we are using existing storage, the reference counter
         // should be 2 by now.
-        let ref_count = s.storages[tmpdir_path];
+        let ref_count = new_storage.ref_count().await;
         assert_eq!(
             ref_count, 2,
             "Invalid refcount, got {} expected 2.",
@@ -549,52 +552,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    #[serial]
-    async fn remove_sandbox_storage() {
-        skip_if_not_root!();
-
-        let logger = slog::Logger::root(slog::Discard, o!());
-        let s = Sandbox::new(&logger).unwrap();
-
-        let tmpdir = Builder::new().tempdir().unwrap();
-        let tmpdir_path = tmpdir.path().to_str().unwrap();
-
-        let srcdir = Builder::new()
-            .prefix("src")
-            .tempdir_in(tmpdir_path)
-            .unwrap();
-        let srcdir_path = srcdir.path().to_str().unwrap();
-
-        let destdir = Builder::new()
-            .prefix("dest")
-            .tempdir_in(tmpdir_path)
-            .unwrap();
-        let destdir_path = destdir.path().to_str().unwrap();
-
-        let emptydir = Builder::new()
-            .prefix("empty")
-            .tempdir_in(tmpdir_path)
-            .unwrap();
-
-        assert!(
-            s.remove_sandbox_storage(srcdir_path).is_err(),
-            "Expect Err as the directory is not a mountpoint"
-        );
-
-        assert!(s.remove_sandbox_storage("").is_err());
-
-        let invalid_dir = emptydir.path().join("invalid");
-
-        assert!(s
-            .remove_sandbox_storage(invalid_dir.to_str().unwrap())
-            .is_err());
-
-        assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
-
-        assert!(s.remove_sandbox_storage(destdir_path).is_ok());
-    }
-
     #[tokio::test]
     #[serial]
     async fn unset_and_remove_sandbox_storage() {
@@ -604,8 +561,7 @@ mod tests {
         let mut s = Sandbox::new(&logger).unwrap();
 
         assert!(
-            s.unset_and_remove_sandbox_storage("/tmp/testEphePath")
-                .is_err(),
+            s.remove_sandbox_storage("/tmp/testEphePath").await.is_err(),
             "Should fail because sandbox storage doesn't exist"
         );
 
@@ -626,8 +582,12 @@ mod tests {
 
         assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
 
-        assert!(s.set_sandbox_storage(destdir_path));
-        assert!(s.unset_and_remove_sandbox_storage(destdir_path).is_ok());
+        s.add_sandbox_storage(destdir_path).await;
+        let storage = StorageDeviceGeneric::new(destdir_path.to_string());
+        assert!(s
+            .update_sandbox_storage(destdir_path, Arc::new(storage))
+            .is_ok());
+        assert!(s.remove_sandbox_storage(destdir_path).await.is_ok());
 
         let other_dir_str;
         {
@@ -640,10 +600,14 @@ mod tests {
             let other_dir_path = other_dir.path().to_str().unwrap();
             other_dir_str = other_dir_path.to_string();
 
-            assert!(s.set_sandbox_storage(other_dir_path));
+            s.add_sandbox_storage(other_dir_path).await;
+            let storage = StorageDeviceGeneric::new(other_dir_path.to_string());
+            assert!(s
+                .update_sandbox_storage(other_dir_path, Arc::new(storage))
+                .is_ok());
         }
 
-        assert!(s.unset_and_remove_sandbox_storage(&other_dir_str).is_err());
+        assert!(s.remove_sandbox_storage(&other_dir_str).await.is_ok());
     }
 
     #[tokio::test]
@@ -655,28 +619,30 @@ mod tests {
         let storage_path = "/tmp/testEphe";
 
         // Add a new sandbox storage
-        assert!(s.set_sandbox_storage(storage_path));
+        s.add_sandbox_storage(storage_path).await;
         // Use the existing sandbox storage
+        let state = s.add_sandbox_storage(storage_path).await;
         assert!(
-            !s.set_sandbox_storage(storage_path),
+            state.ref_count().await > 1,
             "Expects false as the storage is not new."
         );
 
         assert!(
-            !s.unset_sandbox_storage(storage_path).unwrap(),
+            !s.remove_sandbox_storage(storage_path).await.unwrap(),
             "Expects false as there is still a storage."
         );
 
         // Reference counter should decrement to 1.
-        let ref_count = s.storages[storage_path];
+        let storage = &s.storages[storage_path];
+        let refcount = storage.ref_count().await;
         assert_eq!(
-            ref_count, 1,
+            refcount, 1,
             "Invalid refcount, got {} expected 1.",
-            ref_count
+            refcount
         );
 
         assert!(
-            s.unset_sandbox_storage(storage_path).unwrap(),
+            s.remove_sandbox_storage(storage_path).await.unwrap(),
             "Expects true as there is still a storage."
         );
 
@@ -692,7 +658,7 @@ mod tests {
         // If no container is using the sandbox storage, the reference
         // counter for it should not exist.
         assert!(
-            s.unset_sandbox_storage(storage_path).is_err(),
+            s.remove_sandbox_storage(storage_path).await.is_err(),
             "Expects false as the reference counter should no exist."
         );
     }

src/agent/src/storage/bind_watcher_handler.rs

@@ -0,0 +1,37 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::Result;
+use kata_types::mount::StorageDevice;
+use protocols::agent::Storage;
+use std::iter;
+use std::sync::Arc;
+use tracing::instrument;
+
+use crate::storage::{new_device, StorageContext, StorageHandler};
+
+#[derive(Debug)]
+pub struct BindWatcherHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for BindWatcherHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        if let Some(cid) = ctx.cid {
+            ctx.sandbox
+                .lock()
+                .await
+                .bind_watcher
+                .add_container(cid.to_string(), iter::once(storage.clone()), ctx.logger)
+                .await?;
+        }
+        new_device("".to_string())
+    }
+}

src/agent/src/storage/block_handler.rs

@@ -0,0 +1,146 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fs;
+use std::os::unix::fs::PermissionsExt;
+use std::path::Path;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use kata_types::mount::StorageDevice;
+use protocols::agent::Storage;
+use tracing::instrument;
+
+use crate::device::{
+    get_scsi_device_name, get_virtio_blk_pci_device_name, get_virtio_mmio_device_name,
+    wait_for_pmem_device,
+};
+use crate::pci;
+use crate::storage::{common_storage_handler, new_device, StorageContext, StorageHandler};
+#[cfg(target_arch = "s390x")]
+use crate::{ccw, device::get_virtio_blk_ccw_device_name};
+
+#[derive(Debug)]
+pub struct VirtioBlkMmioHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for VirtioBlkMmioHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        if !Path::new(&storage.source).exists() {
+            get_virtio_mmio_device_name(ctx.sandbox, &storage.source)
+                .await
+                .context("failed to get mmio device name")?;
+        }
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}
+
+#[derive(Debug)]
+pub struct VirtioBlkPciHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for VirtioBlkPciHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        // If hot-plugged, get the device node path based on the PCI path
+        // otherwise use the virt path provided in Storage Source
+        if storage.source.starts_with("/dev") {
+            let metadata = fs::metadata(&storage.source)
+                .context(format!("get metadata on file {:?}", &storage.source))?;
+            let mode = metadata.permissions().mode();
+            if mode & libc::S_IFBLK == 0 {
+                return Err(anyhow!("Invalid device {}", &storage.source));
+            }
+        } else {
+            let pcipath = pci::Path::from_str(&storage.source)?;
+            let dev_path = get_virtio_blk_pci_device_name(ctx.sandbox, &pcipath).await?;
+            storage.source = dev_path;
+        }
+
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}
+
+#[derive(Debug)]
+pub struct VirtioBlkCcwHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for VirtioBlkCcwHandler {
+    #[cfg(target_arch = "s390x")]
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        let ccw_device = ccw::Device::from_str(&storage.source)?;
+        let dev_path = get_virtio_blk_ccw_device_name(ctx.sandbox, &ccw_device).await?;
+        storage.source = dev_path;
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+
+    #[cfg(not(target_arch = "s390x"))]
+    #[instrument]
+    async fn create_device(
+        &self,
+        _storage: Storage,
+        _ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Err(anyhow!("CCW is only supported on s390x"))
+    }
+}
+
+#[derive(Debug)]
+pub struct ScsiHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for ScsiHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        // Retrieve the device path from SCSI address.
+        let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?;
+        storage.source = dev_path;
+
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}
+
+#[derive(Debug)]
+pub struct PmemHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for PmemHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        // Retrieve the device for pmem storage
+        wait_for_pmem_device(ctx.sandbox, &storage.source).await?;
+
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}

src/agent/src/storage/ephemeral_handler.rs

@@ -0,0 +1,293 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fs;
+use std::fs::OpenOptions;
+use std::io::Write;
+use std::os::unix::fs::{MetadataExt, PermissionsExt};
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use kata_sys_util::mount::parse_mount_options;
+use kata_types::mount::{StorageDevice, KATA_MOUNT_OPTION_FS_GID};
+use nix::unistd::Gid;
+use protocols::agent::Storage;
+use slog::Logger;
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+use crate::device::{DRIVER_EPHEMERAL_TYPE, FS_TYPE_HUGETLB};
+use crate::mount::baremount;
+use crate::sandbox::Sandbox;
+use crate::storage::{
+    common_storage_handler, new_device, parse_options, StorageContext, StorageHandler, MODE_SETGID,
+};
+
+const FS_GID_EQ: &str = "fsgid=";
+const SYS_FS_HUGEPAGES_PREFIX: &str = "/sys/kernel/mm/hugepages";
+
+#[derive(Debug)]
+pub struct EphemeralHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for EphemeralHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        // hugetlbfs
+        if storage.fstype == FS_TYPE_HUGETLB {
+            info!(ctx.logger, "handle hugetlbfs storage");
+            // Allocate hugepages before mount
+            // /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
+            // /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+            // options eg "pagesize=2097152,size=524288000"(2M, 500M)
+            Self::allocate_hugepages(ctx.logger, &storage.options.to_vec())
+                .context("allocate hugepages")?;
+            common_storage_handler(ctx.logger, &storage)?;
+        } else if !storage.options.is_empty() {
+            // By now we only support one option field: "fsGroup" which
+            // isn't an valid mount option, thus we should remove it when
+            // do mount.
+            let opts = parse_options(&storage.options);
+            storage.options = Default::default();
+            common_storage_handler(ctx.logger, &storage)?;
+
+            // ephemeral_storage didn't support mount options except fsGroup.
+            if let Some(fsgid) = opts.get(KATA_MOUNT_OPTION_FS_GID) {
+                let gid = fsgid.parse::<u32>()?;
+
+                nix::unistd::chown(storage.mount_point.as_str(), None, Some(Gid::from_raw(gid)))?;
+
+                let meta = fs::metadata(&storage.mount_point)?;
+                let mut permission = meta.permissions();
+
+                let o_mode = meta.mode() | MODE_SETGID;
+                permission.set_mode(o_mode);
+                fs::set_permissions(&storage.mount_point, permission)?;
+            }
+        } else {
+            common_storage_handler(ctx.logger, &storage)?;
+        }
+
+        new_device("".to_string())
+    }
+}
+
+impl EphemeralHandler {
+    // Allocate hugepages by writing to sysfs
+    fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> {
+        info!(logger, "mounting hugePages storage options: {:?}", options);
+
+        let (pagesize, size) = Self::get_pagesize_and_size_from_option(options)
+            .context(format!("parse mount options: {:?}", &options))?;
+
+        info!(
+            logger,
+            "allocate hugepages. pageSize: {}, size: {}", pagesize, size
+        );
+
+        // sysfs entry is always of the form hugepages-${pagesize}kB
+        // Ref: https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
+        let path = Path::new(SYS_FS_HUGEPAGES_PREFIX)
+            .join(format!("hugepages-{}kB", pagesize / 1024))
+            .join("nr_hugepages");
+
+        // write numpages to nr_hugepages file.
+        let numpages = format!("{}", size / pagesize);
+        info!(logger, "write {} pages to {:?}", &numpages, &path);
+
+        let mut file = OpenOptions::new()
+            .write(true)
+            .open(&path)
+            .context(format!("open nr_hugepages directory {:?}", &path))?;
+
+        file.write_all(numpages.as_bytes())
+            .context(format!("write nr_hugepages failed: {:?}", &path))?;
+
+        // Even if the write succeeds, the kernel isn't guaranteed to be
+        // able to allocate all the pages we requested.  Verify that it
+        // did.
+        let verify = fs::read_to_string(&path).context(format!("reading {:?}", &path))?;
+        let allocated = verify
+            .trim_end()
+            .parse::<u64>()
+            .map_err(|_| anyhow!("Unexpected text {:?} in {:?}", &verify, &path))?;
+        if allocated != size / pagesize {
+            return Err(anyhow!(
+                "Only allocated {} of {} hugepages of size {}",
+                allocated,
+                numpages,
+                pagesize
+            ));
+        }
+
+        Ok(())
+    }
+
+    // Parse filesystem options string to retrieve hugepage details
+    // options eg "pagesize=2048,size=107374182"
+    fn get_pagesize_and_size_from_option(options: &[String]) -> Result<(u64, u64)> {
+        let mut pagesize_str: Option<&str> = None;
+        let mut size_str: Option<&str> = None;
+
+        for option in options {
+            let vars: Vec<&str> = option.trim().split(',').collect();
+
+            for var in vars {
+                if let Some(stripped) = var.strip_prefix("pagesize=") {
+                    pagesize_str = Some(stripped);
+                } else if let Some(stripped) = var.strip_prefix("size=") {
+                    size_str = Some(stripped);
+                }
+
+                if pagesize_str.is_some() && size_str.is_some() {
+                    break;
+                }
+            }
+        }
+
+        if pagesize_str.is_none() || size_str.is_none() {
+            return Err(anyhow!("no pagesize/size options found"));
+        }
+
+        let pagesize = pagesize_str
+            .unwrap()
+            .parse::<u64>()
+            .context(format!("parse pagesize: {:?}", &pagesize_str))?;
+        let size = size_str
+            .unwrap()
+            .parse::<u64>()
+            .context(format!("parse size: {:?}", &pagesize_str))?;
+
+        Ok((pagesize, size))
+    }
+}
+
+// update_ephemeral_mounts takes a list of ephemeral mounts and remounts them
+// with mount options passed by the caller
+#[instrument]
+pub async fn update_ephemeral_mounts(
+    logger: Logger,
+    storages: &[Storage],
+    _sandbox: &Arc<Mutex<Sandbox>>,
+) -> Result<()> {
+    for storage in storages {
+        let handler_name = &storage.driver;
+        let logger = logger.new(o!(
+            "msg" => "updating tmpfs storage",
+            "subsystem" => "storage",
+            "storage-type" => handler_name.to_owned()));
+
+        match handler_name.as_str() {
+            DRIVER_EPHEMERAL_TYPE => {
+                fs::create_dir_all(&storage.mount_point)?;
+
+                if storage.options.is_empty() {
+                    continue;
+                } else {
+                    // assume that fsGid has already been set
+                    let mount_path = Path::new(&storage.mount_point);
+                    let src_path = Path::new(&storage.source);
+                    let opts: Vec<&String> = storage
+                        .options
+                        .iter()
+                        .filter(|&opt| !opt.starts_with(FS_GID_EQ))
+                        .collect();
+                    let (flags, options) = parse_mount_options(&opts)?;
+
+                    info!(logger, "mounting storage";
+                        "mount-source" => src_path.display(),
+                        "mount-destination" => mount_path.display(),
+                        "mount-fstype"  => storage.fstype.as_str(),
+                        "mount-options" => options.as_str(),
+                    );
+
+                    baremount(
+                        src_path,
+                        mount_path,
+                        storage.fstype.as_str(),
+                        flags,
+                        options.as_str(),
+                        &logger,
+                    )?;
+                }
+            }
+            _ => {
+                return Err(anyhow!(
+                    "Unsupported storage type for syncing mounts {}. Only ephemeral storage update is supported",
+                    storage.driver
+                ));
+            }
+        };
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_pagesize_and_size_from_option() {
+        let expected_pagesize = 2048;
+        let expected_size = 107374182;
+        let expected = (expected_pagesize, expected_size);
+
+        let data = vec![
+            // (input, expected, is_ok)
+            ("size-1=107374182,pagesize-1=2048", expected, false),
+            ("size-1=107374182,pagesize=2048", expected, false),
+            ("size=107374182,pagesize-1=2048", expected, false),
+            ("size=107374182,pagesize=abc", expected, false),
+            ("size=abc,pagesize=2048", expected, false),
+            ("size=,pagesize=2048", expected, false),
+            ("size=107374182,pagesize=", expected, false),
+            ("size=107374182,pagesize=2048", expected, true),
+            ("pagesize=2048,size=107374182", expected, true),
+            ("foo=bar,pagesize=2048,size=107374182", expected, true),
+            (
+                "foo=bar,pagesize=2048,foo1=bar1,size=107374182",
+                expected,
+                true,
+            ),
+            (
+                "pagesize=2048,foo1=bar1,foo=bar,size=107374182",
+                expected,
+                true,
+            ),
+            (
+                "foo=bar,pagesize=2048,foo1=bar1,size=107374182,foo2=bar2",
+                expected,
+                true,
+            ),
+            (
+                "foo=bar,size=107374182,foo1=bar1,pagesize=2048",
+                expected,
+                true,
+            ),
+        ];
+
+        for case in data {
+            let input = case.0;
+            let r = EphemeralHandler::get_pagesize_and_size_from_option(&[input.to_string()]);
+
+            let is_ok = case.2;
+            if is_ok {
+                let expected = case.1;
+                let (pagesize, size) = r.unwrap();
+                assert_eq!(expected.0, pagesize);
+                assert_eq!(expected.1, size);
+            } else {
+                assert!(r.is_err());
+            }
+        }
+    }
+}

src/agent/src/storage/fs_handler.rs

@@ -0,0 +1,89 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fs;
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use kata_types::mount::StorageDevice;
+use protocols::agent::Storage;
+use tracing::instrument;
+
+use crate::storage::{common_storage_handler, new_device, StorageContext, StorageHandler};
+
+#[derive(Debug)]
+pub struct OverlayfsHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for OverlayfsHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        if storage
+            .options
+            .iter()
+            .any(|e| e == "io.katacontainers.fs-opt.overlay-rw")
+        {
+            let cid = ctx
+                .cid
+                .clone()
+                .ok_or_else(|| anyhow!("No container id in rw overlay"))?;
+            let cpath = Path::new(crate::rpc::CONTAINER_BASE).join(cid);
+            let work = cpath.join("work");
+            let upper = cpath.join("upper");
+
+            fs::create_dir_all(&work).context("Creating overlay work directory")?;
+            fs::create_dir_all(&upper).context("Creating overlay upper directory")?;
+
+            storage.fstype = "overlay".into();
+            storage
+                .options
+                .push(format!("upperdir={}", upper.to_string_lossy()));
+            storage
+                .options
+                .push(format!("workdir={}", work.to_string_lossy()));
+        }
+
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}
+
+#[derive(Debug)]
+pub struct Virtio9pHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for Virtio9pHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}
+
+#[derive(Debug)]
+pub struct VirtioFsHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for VirtioFsHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}

src/agent/src/storage/local_handler.rs

@@ -0,0 +1,61 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fs;
+use std::os::unix::fs::PermissionsExt;
+use std::sync::Arc;
+
+use anyhow::{Context, Result};
+use kata_types::mount::{StorageDevice, KATA_MOUNT_OPTION_FS_GID};
+use nix::unistd::Gid;
+use protocols::agent::Storage;
+use tracing::instrument;
+
+use crate::storage::{new_device, parse_options, StorageContext, StorageHandler, MODE_SETGID};
+
+#[derive(Debug)]
+pub struct LocalHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for LocalHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        storage: Storage,
+        _ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        fs::create_dir_all(&storage.mount_point).context(format!(
+            "failed to create dir all {:?}",
+            &storage.mount_point
+        ))?;
+
+        let opts = parse_options(&storage.options);
+
+        let mut need_set_fsgid = false;
+        if let Some(fsgid) = opts.get(KATA_MOUNT_OPTION_FS_GID) {
+            let gid = fsgid.parse::<u32>()?;
+
+            nix::unistd::chown(storage.mount_point.as_str(), None, Some(Gid::from_raw(gid)))?;
+            need_set_fsgid = true;
+        }
+
+        if let Some(mode) = opts.get("mode") {
+            let mut permission = fs::metadata(&storage.mount_point)?.permissions();
+
+            let mut o_mode = u32::from_str_radix(mode, 8)?;
+
+            if need_set_fsgid {
+                // set SetGid mode mask.
+                o_mode |= MODE_SETGID;
+            }
+            permission.set_mode(o_mode);
+
+            fs::set_permissions(&storage.mount_point, permission)?;
+        }
+
+        new_device("".to_string())
+    }
+}

src/agent/src/storage/mod.rs

@@ -0,0 +1,789 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::collections::HashMap;
+use std::fs;
+use std::os::unix::fs::{MetadataExt, PermissionsExt};
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use kata_sys_util::mount::{create_mount_destination, parse_mount_options};
+use kata_types::mount::{StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG};
+use nix::unistd::{Gid, Uid};
+use protocols::agent::Storage;
+use protocols::types::FSGroupChangePolicy;
+use slog::Logger;
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+use self::bind_watcher_handler::BindWatcherHandler;
+use self::block_handler::{PmemHandler, ScsiHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler};
+use self::ephemeral_handler::EphemeralHandler;
+use self::fs_handler::{OverlayfsHandler, Virtio9pHandler, VirtioFsHandler};
+use self::local_handler::LocalHandler;
+use crate::device::{
+    DRIVER_9P_TYPE, DRIVER_BLK_MMIO_TYPE, DRIVER_BLK_PCI_TYPE, DRIVER_EPHEMERAL_TYPE,
+    DRIVER_LOCAL_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE,
+    DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE,
+};
+use crate::mount::{baremount, is_mounted, remove_mounts};
+use crate::sandbox::Sandbox;
+
+pub use self::ephemeral_handler::update_ephemeral_mounts;
+
+mod bind_watcher_handler;
+mod block_handler;
+mod ephemeral_handler;
+mod fs_handler;
+mod local_handler;
+
+const RW_MASK: u32 = 0o660;
+const RO_MASK: u32 = 0o440;
+const EXEC_MASK: u32 = 0o110;
+const MODE_SETGID: u32 = 0o2000;
+
+#[derive(Debug)]
+pub struct StorageContext<'a> {
+    cid: &'a Option<String>,
+    logger: &'a Logger,
+    sandbox: &'a Arc<Mutex<Sandbox>>,
+}
+
+/// An implementation of generic storage device.
+#[derive(Default, Debug)]
+pub struct StorageDeviceGeneric {
+    path: Option<String>,
+}
+
+impl StorageDeviceGeneric {
+    /// Create a new instance of `StorageStateCommon`.
+    pub fn new(path: String) -> Self {
+        StorageDeviceGeneric { path: Some(path) }
+    }
+}
+
+impl StorageDevice for StorageDeviceGeneric {
+    fn path(&self) -> Option<&str> {
+        self.path.as_deref()
+    }
+
+    fn cleanup(&self) -> Result<()> {
+        let path = match self.path() {
+            None => return Ok(()),
+            Some(v) => {
+                if v.is_empty() {
+                    // TODO: Bind watch, local, ephemeral volume has empty path, which will get leaked.
+                    return Ok(());
+                } else {
+                    v
+                }
+            }
+        };
+        if !Path::new(path).exists() {
+            return Ok(());
+        }
+
+        if matches!(is_mounted(path), Ok(true)) {
+            let mounts = vec![path.to_string()];
+            remove_mounts(&mounts)?;
+        }
+        if matches!(is_mounted(path), Ok(true)) {
+            return Err(anyhow!("failed to umount mountpoint {}", path));
+        }
+
+        let p = Path::new(path);
+        if p.is_dir() {
+            let is_empty = p.read_dir()?.next().is_none();
+            if !is_empty {
+                return Err(anyhow!("directory is not empty when clean up storage"));
+            }
+            // "remove_dir" will fail if the mount point is backed by a read-only filesystem.
+            // This is the case with the device mapper snapshotter, where we mount the block device
+            // directly at the underlying sandbox path which was provided from the base RO kataShared
+            // path from the host.
+            let _ = fs::remove_dir(p);
+        } else if !p.is_file() {
+            // TODO: should we remove the file for bind mount?
+            return Err(anyhow!(
+                "storage path {} is neither directory nor file",
+                path
+            ));
+        }
+
+        Ok(())
+    }
+}
+
+/// Trait object to handle storage device.
+#[async_trait::async_trait]
+pub trait StorageHandler: Send + Sync {
+    /// Create a new storage device.
+    async fn create_device(
+        &self,
+        storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>>;
+}
+
+#[rustfmt::skip]
+lazy_static! {
+    pub static ref STORAGE_HANDLERS: StorageHandlerManager<Arc<dyn StorageHandler>> = {
+        let mut manager: StorageHandlerManager<Arc<dyn StorageHandler>> = StorageHandlerManager::new();
+        manager.add_handler(DRIVER_9P_TYPE, Arc::new(Virtio9pHandler{})).unwrap();
+        #[cfg(target_arch = "s390x")]
+        manager.add_handler(crate::device::DRIVER_BLK_CCW_TYPE, Arc::new(self::block_handler::VirtioBlkCcwHandler{})).unwrap();
+        manager.add_handler(DRIVER_BLK_MMIO_TYPE, Arc::new(VirtioBlkMmioHandler{})).unwrap();
+        manager.add_handler(DRIVER_BLK_PCI_TYPE, Arc::new(VirtioBlkPciHandler{})).unwrap();
+        manager.add_handler(DRIVER_EPHEMERAL_TYPE, Arc::new(EphemeralHandler{})).unwrap();
+        manager.add_handler(DRIVER_LOCAL_TYPE, Arc::new(LocalHandler{})).unwrap();
+        manager.add_handler(DRIVER_NVDIMM_TYPE, Arc::new(PmemHandler{})).unwrap();
+        manager.add_handler(DRIVER_OVERLAYFS_TYPE, Arc::new(OverlayfsHandler{})).unwrap();
+        manager.add_handler(DRIVER_SCSI_TYPE, Arc::new(ScsiHandler{})).unwrap();
+        manager.add_handler(DRIVER_VIRTIOFS_TYPE, Arc::new(VirtioFsHandler{})).unwrap();
+        manager.add_handler(DRIVER_WATCHABLE_BIND_TYPE, Arc::new(BindWatcherHandler{})).unwrap();
+        manager
+    };
+}
+
+// add_storages takes a list of storages passed by the caller, and perform the
+// associated operations such as waiting for the device to show up, and mount
+// it to a specific location, according to the type of handler chosen, and for
+// each storage.
+#[instrument]
+pub async fn add_storages(
+    logger: Logger,
+    storages: Vec<Storage>,
+    sandbox: &Arc<Mutex<Sandbox>>,
+    cid: Option<String>,
+) -> Result<Vec<String>> {
+    let mut mount_list = Vec::new();
+
+    for storage in storages {
+        let path = storage.mount_point.clone();
+        let state = sandbox.lock().await.add_sandbox_storage(&path).await;
+        if state.ref_count().await > 1 {
+            if let Some(path) = state.path() {
+                if !path.is_empty() {
+                    mount_list.push(path.to_string());
+                }
+            }
+            // The device already exists.
+            continue;
+        }
+
+        if let Some(handler) = STORAGE_HANDLERS.handler(&storage.driver) {
+            let logger =
+                logger.new(o!( "subsystem" => "storage", "storage-type" => storage.driver.clone()));
+            let mut ctx = StorageContext {
+                cid: &cid,
+                logger: &logger,
+                sandbox,
+            };
+
+            match handler.create_device(storage, &mut ctx).await {
+                Ok(device) => {
+                    match sandbox
+                        .lock()
+                        .await
+                        .update_sandbox_storage(&path, device.clone())
+                    {
+                        Ok(d) => {
+                            if let Some(path) = device.path() {
+                                if !path.is_empty() {
+                                    mount_list.push(path.to_string());
+                                }
+                            }
+                            drop(d);
+                        }
+                        Err(device) => {
+                            error!(logger, "failed to update device for storage");
+                            if let Err(e) = sandbox.lock().await.remove_sandbox_storage(&path).await
+                            {
+                                warn!(logger, "failed to remove dummy sandbox storage {:?}", e);
+                            }
+                            if let Err(e) = device.cleanup() {
+                                error!(
+                                    logger,
+                                    "failed to clean state for storage device {}, {}", path, e
+                                );
+                            }
+                            return Err(anyhow!("failed to update device for storage"));
+                        }
+                    }
+                }
+                Err(e) => {
+                    error!(logger, "failed to create device for storage, error: {e:?}");
+                    if let Err(e) = sandbox.lock().await.remove_sandbox_storage(&path).await {
+                        warn!(logger, "failed to remove dummy sandbox storage {e:?}");
+                    }
+                    return Err(e);
+                }
+            }
+        } else {
+            return Err(anyhow!(
+                "Failed to find the storage handler {}",
+                storage.driver
+            ));
+        }
+    }
+
+    Ok(mount_list)
+}
+
+pub(crate) fn new_device(path: String) -> Result<Arc<dyn StorageDevice>> {
+    let device = StorageDeviceGeneric::new(path);
+    Ok(Arc::new(device))
+}
+
+#[instrument]
+pub(crate) fn common_storage_handler(logger: &Logger, storage: &Storage) -> Result<String> {
+    mount_storage(logger, storage)?;
+    set_ownership(logger, storage)?;
+    Ok(storage.mount_point.clone())
+}
+
+// mount_storage performs the mount described by the storage structure.
+#[instrument]
+fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "mount"));
+
+    // There's a special mechanism to create mountpoint from a `sharedfs` instance before
+    // starting the kata-agent. Check for such cases.
+    if storage.source == KATA_SHAREDFS_GUEST_PREMOUNT_TAG && is_mounted(&storage.mount_point)? {
+        warn!(
+            logger,
+            "{} already mounted on {}, ignoring...",
+            KATA_SHAREDFS_GUEST_PREMOUNT_TAG,
+            &storage.mount_point
+        );
+        return Ok(());
+    }
+
+    let (flags, options) = parse_mount_options(&storage.options)?;
+    let mount_path = Path::new(&storage.mount_point);
+    let src_path = Path::new(&storage.source);
+    create_mount_destination(src_path, mount_path, "", &storage.fstype)
+        .context("Could not create mountpoint")?;
+
+    info!(logger, "mounting storage";
+        "mount-source" => src_path.display(),
+        "mount-destination" => mount_path.display(),
+        "mount-fstype"  => storage.fstype.as_str(),
+        "mount-options" => options.as_str(),
+    );
+
+    baremount(
+        src_path,
+        mount_path,
+        storage.fstype.as_str(),
+        flags,
+        options.as_str(),
+        &logger,
+    )
+}
+
+#[instrument]
+pub(crate) fn parse_options(option_list: &[String]) -> HashMap<String, String> {
+    let mut options = HashMap::new();
+    for opt in option_list {
+        let fields: Vec<&str> = opt.split('=').collect();
+        if fields.len() == 2 {
+            options.insert(fields[0].to_string(), fields[1].to_string());
+        }
+    }
+    options
+}
+
+#[instrument]
+pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "mount", "fn" => "set_ownership"));
+
+    // If fsGroup is not set, skip performing ownership change
+    if storage.fs_group.is_none() {
+        return Ok(());
+    }
+
+    let fs_group = storage.fs_group();
+    let read_only = storage.options.contains(&String::from("ro"));
+    let mount_path = Path::new(&storage.mount_point);
+    let metadata = mount_path.metadata().map_err(|err| {
+        error!(logger, "failed to obtain metadata for mount path";
+            "mount-path" => mount_path.to_str(),
+            "error" => err.to_string(),
+        );
+        err
+    })?;
+
+    if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch.into()
+        && metadata.gid() == fs_group.group_id
+    {
+        let mut mask = if read_only { RO_MASK } else { RW_MASK };
+        mask |= EXEC_MASK;
+
+        // With fsGroup change policy to OnRootMismatch, if the current
+        // gid of the mount path root directory matches the desired gid
+        // and the current permission of mount path root directory is correct,
+        // then ownership change will be skipped.
+        let current_mode = metadata.permissions().mode();
+        if (mask & current_mode == mask) && (current_mode & MODE_SETGID != 0) {
+            info!(logger, "skipping ownership change for volume";
+                "mount-path" => mount_path.to_str(),
+                "fs-group" => fs_group.group_id.to_string(),
+            );
+            return Ok(());
+        }
+    }
+
+    info!(logger, "performing recursive ownership change";
+        "mount-path" => mount_path.to_str(),
+        "fs-group" => fs_group.group_id.to_string(),
+    );
+    recursive_ownership_change(
+        mount_path,
+        None,
+        Some(Gid::from_raw(fs_group.group_id)),
+        read_only,
+    )
+}
+
+#[instrument]
+pub fn recursive_ownership_change(
+    path: &Path,
+    uid: Option<Uid>,
+    gid: Option<Gid>,
+    read_only: bool,
+) -> Result<()> {
+    let mut mask = if read_only { RO_MASK } else { RW_MASK };
+    if path.is_dir() {
+        for entry in fs::read_dir(path)? {
+            recursive_ownership_change(entry?.path().as_path(), uid, gid, read_only)?;
+        }
+        mask |= EXEC_MASK;
+        mask |= MODE_SETGID;
+    }
+
+    // We do not want to change the permission of the underlying file
+    // using symlink. Hence we skip symlinks from recursive ownership
+    // and permission changes.
+    if path.is_symlink() {
+        return Ok(());
+    }
+
+    nix::unistd::chown(path, uid, gid)?;
+
+    if gid.is_some() {
+        let metadata = path.metadata()?;
+        let mut permission = metadata.permissions();
+        let target_mode = metadata.mode() | mask;
+        permission.set_mode(target_mode);
+        fs::set_permissions(path, permission)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use anyhow::Error;
+    use nix::mount::MsFlags;
+    use protocols::agent::FSGroup;
+    use std::fs::File;
+    use tempfile::{tempdir, Builder};
+    use test_utils::{
+        skip_if_not_root, skip_loop_by_user, skip_loop_if_not_root, skip_loop_if_root, TestUserType,
+    };
+
+    #[test]
+    fn test_mount_storage() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            test_user: TestUserType,
+            storage: Storage,
+            error_contains: &'a str,
+
+            make_source_dir: bool,
+            make_mount_dir: bool,
+            deny_mount_permission: bool,
+        }
+
+        impl Default for TestData<'_> {
+            fn default() -> Self {
+                TestData {
+                    test_user: TestUserType::Any,
+                    storage: Storage {
+                        mount_point: "mnt".to_string(),
+                        source: "src".to_string(),
+                        fstype: "tmpfs".to_string(),
+                        ..Default::default()
+                    },
+                    make_source_dir: true,
+                    make_mount_dir: false,
+                    deny_mount_permission: false,
+                    error_contains: "",
+                }
+            }
+        }
+
+        let tests = &[
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                error_contains: "EPERM: Operation not permitted",
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::RootOnly,
+                ..Default::default()
+            },
+            TestData {
+                storage: Storage {
+                    mount_point: "mnt".to_string(),
+                    source: "src".to_string(),
+                    fstype: "bind".to_string(),
+                    ..Default::default()
+                },
+                make_source_dir: false,
+                make_mount_dir: true,
+                error_contains: "Could not create mountpoint",
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                deny_mount_permission: true,
+                error_contains: "Could not create mountpoint",
+                ..Default::default()
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            skip_loop_by_user!(msg, d.test_user);
+
+            let drain = slog::Discard;
+            let logger = slog::Logger::root(drain, o!());
+
+            let tempdir = tempdir().unwrap();
+
+            let source = tempdir.path().join(&d.storage.source);
+            let mount_point = tempdir.path().join(&d.storage.mount_point);
+
+            let storage = Storage {
+                source: source.to_str().unwrap().to_string(),
+                mount_point: mount_point.to_str().unwrap().to_string(),
+                ..d.storage.clone()
+            };
+
+            if d.make_source_dir {
+                fs::create_dir_all(&storage.source).unwrap();
+            }
+            if d.make_mount_dir {
+                fs::create_dir_all(&storage.mount_point).unwrap();
+            }
+
+            if d.deny_mount_permission {
+                fs::set_permissions(
+                    mount_point.parent().unwrap(),
+                    fs::Permissions::from_mode(0o000),
+                )
+                .unwrap();
+            }
+
+            let result = mount_storage(&logger, &storage);
+
+            // restore permissions so tempdir can be cleaned up
+            if d.deny_mount_permission {
+                fs::set_permissions(
+                    mount_point.parent().unwrap(),
+                    fs::Permissions::from_mode(0o755),
+                )
+                .unwrap();
+            }
+
+            if result.is_ok() {
+                nix::mount::umount(&mount_point).unwrap();
+            }
+
+            let msg = format!("{}: result: {:?}", msg, result);
+            if d.error_contains.is_empty() {
+                assert!(result.is_ok(), "{}", msg);
+            } else {
+                assert!(result.is_err(), "{}", msg);
+                let error_msg = format!("{}", result.unwrap_err());
+                assert!(error_msg.contains(d.error_contains), "{}", msg);
+            }
+        }
+    }
+
+    #[test]
+    fn test_set_ownership() {
+        skip_if_not_root!();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        #[derive(Debug)]
+        struct TestData<'a> {
+            mount_path: &'a str,
+            fs_group: Option<FSGroup>,
+            read_only: bool,
+            expected_group_id: u32,
+            expected_permission: u32,
+        }
+
+        let tests = &[
+            TestData {
+                mount_path: "foo",
+                fs_group: None,
+                read_only: false,
+                expected_group_id: 0,
+                expected_permission: 0,
+            },
+            TestData {
+                mount_path: "rw_mount",
+                fs_group: Some(FSGroup {
+                    group_id: 3000,
+                    group_change_policy: FSGroupChangePolicy::Always.into(),
+                    ..Default::default()
+                }),
+                read_only: false,
+                expected_group_id: 3000,
+                expected_permission: RW_MASK | EXEC_MASK | MODE_SETGID,
+            },
+            TestData {
+                mount_path: "ro_mount",
+                fs_group: Some(FSGroup {
+                    group_id: 3000,
+                    group_change_policy: FSGroupChangePolicy::OnRootMismatch.into(),
+                    ..Default::default()
+                }),
+                read_only: true,
+                expected_group_id: 3000,
+                expected_permission: RO_MASK | EXEC_MASK | MODE_SETGID,
+            },
+        ];
+
+        let tempdir = tempdir().expect("failed to create tmpdir");
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mount_dir = tempdir.path().join(d.mount_path);
+            fs::create_dir(&mount_dir)
+                .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg));
+
+            let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
+            let mut storage_data = Storage::new();
+            if d.read_only {
+                storage_data.set_options(vec!["foo".to_string(), "ro".to_string()]);
+            }
+            if let Some(fs_group) = d.fs_group.clone() {
+                storage_data.set_fs_group(fs_group);
+            }
+            storage_data.mount_point = mount_dir.clone().into_os_string().into_string().unwrap();
+
+            let result = set_ownership(&logger, &storage_data);
+            assert!(result.is_ok());
+
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().gid(),
+                d.expected_group_id
+            );
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().permissions().mode(),
+                (directory_mode | d.expected_permission)
+            );
+        }
+    }
+
+    #[test]
+    fn test_recursive_ownership_change() {
+        skip_if_not_root!();
+
+        const COUNT: usize = 5;
+
+        #[derive(Debug)]
+        struct TestData<'a> {
+            // Directory where the recursive ownership change should be performed on
+            path: &'a str,
+
+            // User ID for ownership change
+            uid: u32,
+
+            // Group ID for ownership change
+            gid: u32,
+
+            // Set when the permission should be read-only
+            read_only: bool,
+
+            // The expected permission of all directories after ownership change
+            expected_permission_directory: u32,
+
+            // The expected permission of all files after ownership change
+            expected_permission_file: u32,
+        }
+
+        let tests = &[
+            TestData {
+                path: "no_gid_change",
+                uid: 0,
+                gid: 0,
+                read_only: false,
+                expected_permission_directory: 0,
+                expected_permission_file: 0,
+            },
+            TestData {
+                path: "rw_gid_change",
+                uid: 0,
+                gid: 3000,
+                read_only: false,
+                expected_permission_directory: RW_MASK | EXEC_MASK | MODE_SETGID,
+                expected_permission_file: RW_MASK,
+            },
+            TestData {
+                path: "ro_gid_change",
+                uid: 0,
+                gid: 3000,
+                read_only: true,
+                expected_permission_directory: RO_MASK | EXEC_MASK | MODE_SETGID,
+                expected_permission_file: RO_MASK,
+            },
+        ];
+
+        let tempdir = tempdir().expect("failed to create tmpdir");
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mount_dir = tempdir.path().join(d.path);
+            fs::create_dir(&mount_dir)
+                .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg));
+
+            let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
+            let mut file_mode: u32 = 0;
+
+            // create testing directories and files
+            for n in 1..COUNT {
+                let nest_dir = mount_dir.join(format!("nested{}", n));
+                fs::create_dir(&nest_dir)
+                    .unwrap_or_else(|_| panic!("{}: failed to create nest directory", msg));
+
+                for f in 1..COUNT {
+                    let filename = nest_dir.join(format!("file{}", f));
+                    File::create(&filename)
+                        .unwrap_or_else(|_| panic!("{}: failed to create file", msg));
+                    file_mode = filename.as_path().metadata().unwrap().permissions().mode();
+                }
+            }
+
+            let uid = if d.uid > 0 {
+                Some(Uid::from_raw(d.uid))
+            } else {
+                None
+            };
+            let gid = if d.gid > 0 {
+                Some(Gid::from_raw(d.gid))
+            } else {
+                None
+            };
+            let result = recursive_ownership_change(&mount_dir, uid, gid, d.read_only);
+
+            assert!(result.is_ok());
+
+            assert_eq!(mount_dir.as_path().metadata().unwrap().gid(), d.gid);
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().permissions().mode(),
+                (directory_mode | d.expected_permission_directory)
+            );
+
+            for n in 1..COUNT {
+                let nest_dir = mount_dir.join(format!("nested{}", n));
+                for f in 1..COUNT {
+                    let filename = nest_dir.join(format!("file{}", f));
+                    let file = Path::new(&filename);
+
+                    assert_eq!(file.metadata().unwrap().gid(), d.gid);
+                    assert_eq!(
+                        file.metadata().unwrap().permissions().mode(),
+                        (file_mode | d.expected_permission_file)
+                    );
+                }
+
+                let dir = Path::new(&nest_dir);
+                assert_eq!(dir.metadata().unwrap().gid(), d.gid);
+                assert_eq!(
+                    dir.metadata().unwrap().permissions().mode(),
+                    (directory_mode | d.expected_permission_directory)
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn cleanup_storage() {
+        skip_if_not_root!();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        let tmpdir = Builder::new().tempdir().unwrap();
+        let tmpdir_path = tmpdir.path().to_str().unwrap();
+
+        let srcdir = Builder::new()
+            .prefix("src")
+            .tempdir_in(tmpdir_path)
+            .unwrap();
+        let srcdir_path = srcdir.path().to_str().unwrap();
+        let empty_file = Path::new(srcdir_path).join("emptyfile");
+        fs::write(&empty_file, "test").unwrap();
+
+        let destdir = Builder::new()
+            .prefix("dest")
+            .tempdir_in(tmpdir_path)
+            .unwrap();
+        let destdir_path = destdir.path().to_str().unwrap();
+
+        let emptydir = Builder::new()
+            .prefix("empty")
+            .tempdir_in(tmpdir_path)
+            .unwrap();
+
+        let s = StorageDeviceGeneric::default();
+        assert!(s.cleanup().is_ok());
+
+        let s = StorageDeviceGeneric::new("".to_string());
+        assert!(s.cleanup().is_ok());
+
+        let invalid_dir = emptydir
+            .path()
+            .join("invalid")
+            .to_str()
+            .unwrap()
+            .to_string();
+        let s = StorageDeviceGeneric::new(invalid_dir);
+        assert!(s.cleanup().is_ok());
+
+        assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
+
+        let s = StorageDeviceGeneric::new(destdir_path.to_string());
+        assert!(s.cleanup().is_ok());
+
+        // fail to remove non-empty directory
+        let s = StorageDeviceGeneric::new(srcdir_path.to_string());
+        s.cleanup().unwrap_err();
+
+        // remove a directory without umount
+        fs::remove_file(&empty_file).unwrap();
+        s.cleanup().unwrap();
+    }
+
+    fn bind_mount(src: &str, dst: &str, logger: &Logger) -> Result<(), Error> {
+        let src_path = Path::new(src);
+        let dst_path = Path::new(dst);
+
+        baremount(src_path, dst_path, "bind", MsFlags::MS_BIND, "", logger)
+    }
+}

src/dragonball/.gitignore

@@ -1,3 +1,2 @@
 target
-Cargo.lock
 .idea

src/dragonball/Cargo.toml

@@ -10,6 +10,7 @@ license = "Apache-2.0"
 edition = "2018"
 
 [dependencies]
+anyhow = "1.0.32"
 arc-swap = "1.5.0"
 bytes = "1.1.0"
 dbs-address-space =  { path = "./src/dbs_address_space" }
@@ -26,9 +27,11 @@ kvm-bindings = "0.6.0"
 kvm-ioctls = "0.12.0"
 lazy_static = "1.2"
 libc = "0.2.39"
-linux-loader = "0.6.0"
+linux-loader = "0.8.0"
 log = "0.4.14"
 nix = "0.24.2"
+procfs = "0.12.0"
+prometheus = { version = "0.13.0", features = ["process"] }
 seccompiler = "0.2.0"
 serde = "1.0.27"
 serde_derive = "1.0.27"
@@ -37,13 +40,14 @@ slog = "2.5.2"
 slog-scope = "4.4.0"
 thiserror = "1"
 vmm-sys-util = "0.11.0"
-virtio-queue = { version = "0.6.0", optional = true }
-vm-memory = { version = "0.9.0", features = ["backend-mmap"] }
+virtio-queue = { version = "0.7.0", optional = true }
+vm-memory = { version = "0.10.0", features = ["backend-mmap"] }
 crossbeam-channel = "0.5.6"
+fuse-backend-rs = "0.10.5"
 
 [dev-dependencies]
-slog-term = "2.9.0"
 slog-async = "2.7.0"
+slog-term = "2.9.0"
 test-utils = { path = "../libs/test-utils" }
 
 [features]
@@ -54,6 +58,6 @@ virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"]
 virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]
 virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"]
 # virtio-fs only work on atomic-guest-memory
-virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"]
+virtio-fs = ["dbs-virtio-devices/virtio-fs-pro", "virtio-queue", "atomic-guest-memory"]
 virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"]
 virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"]

src/dragonball/src/api/v1/vmm_action.rs

@@ -16,6 +16,8 @@ use crate::event_manager::EventManager;
 use crate::vm::{CpuTopology, KernelConfigInfo, VmConfigInfo};
 use crate::vmm::Vmm;
 
+use crate::hypervisor_metrics::get_hypervisor_metrics;
+
 use self::VmConfigError::*;
 use self::VmmActionError::MachineConfig;
 
@@ -58,6 +60,11 @@ pub enum VmmActionError {
     #[error("Upcall not ready, can't hotplug device.")]
     UpcallServerNotReady,
 
+    /// Error when get prometheus metrics.
+    /// Currently does not distinguish between error types for metrics.
+    #[error("failed to get hypervisor metrics")]
+    GetHypervisorMetrics,
+
     /// The action `ConfigureBootSource` failed either because of bad user input or an internal
     /// error.
     #[error("failed to configure boot source for VM: {0}")]
@@ -135,6 +142,9 @@ pub enum VmmAction {
     /// Get the configuration of the microVM.
     GetVmConfiguration,
 
+    /// Get Prometheus Metrics.
+    GetHypervisorMetrics,
+
     /// Set the microVM configuration (memory & vcpu) using `VmConfig` as input. This
     /// action can only be called before the microVM has booted.
     SetVmConfiguration(VmConfigInfo),
@@ -208,6 +218,8 @@ pub enum VmmData {
     Empty,
     /// The microVM configuration represented by `VmConfigInfo`.
     MachineConfiguration(Box<VmConfigInfo>),
+    /// Prometheus Metrics represented by String.
+    HypervisorMetrics(String),
 }
 
 /// Request data type used to communicate between the API and the VMM.
@@ -262,6 +274,7 @@ impl VmmService {
             VmmAction::GetVmConfiguration => Ok(VmmData::MachineConfiguration(Box::new(
                 self.machine_config.clone(),
             ))),
+            VmmAction::GetHypervisorMetrics => self.get_hypervisor_metrics(),
             VmmAction::SetVmConfiguration(machine_config) => {
                 self.set_vm_configuration(vmm, machine_config)
             }
@@ -345,7 +358,8 @@ impl VmmService {
             Some(ref path) => Some(File::open(path).map_err(|e| BootSource(InvalidInitrdPath(e)))?),
         };
 
-        let mut cmdline = linux_loader::cmdline::Cmdline::new(dbs_boot::layout::CMDLINE_MAX_SIZE);
+        let mut cmdline = linux_loader::cmdline::Cmdline::new(dbs_boot::layout::CMDLINE_MAX_SIZE)
+            .map_err(|err| BootSource(InvalidKernelCommandLine(err)))?;
         let boot_args = boot_source_config
             .boot_args
             .unwrap_or_else(|| String::from(DEFAULT_KERNEL_CMDLINE));
@@ -381,6 +395,13 @@ impl VmmService {
         Ok(VmmData::Empty)
     }
 
+    /// Get prometheus metrics.
+    fn get_hypervisor_metrics(&self) -> VmmRequestResult {
+        get_hypervisor_metrics()
+            .map_err(|_| VmmActionError::GetHypervisorMetrics)
+            .map(VmmData::HypervisorMetrics)
+    }
+
     /// Set virtual machine configuration.
     pub fn set_vm_configuration(
         &mut self,
@@ -671,6 +692,7 @@ impl VmmService {
             ));
         }
 
+        #[cfg(feature = "dbs-upcall")]
         vm.resize_vcpu(config, None).map_err(|e| {
             if let VcpuResizeError::UpcallServerNotReady = e {
                 return VmmActionError::UpcallServerNotReady;

src/dragonball/src/dbs_address_space/Cargo.toml

@@ -17,4 +17,4 @@ nix = "0.23.1"
 lazy_static = "1"
 thiserror = "1"
 vmm-sys-util = "0.11.0"
-vm-memory = { version = "0.9", features = ["backend-mmap", "backend-atomic"] }
+vm-memory = { version = "0.10", features = ["backend-mmap", "backend-atomic"] }

src/dragonball/src/dbs_arch/Cargo.toml

@@ -15,12 +15,12 @@ memoffset = "0.6"
 kvm-bindings = { version = "0.6.0", features = ["fam-wrappers"] }
 kvm-ioctls = "0.12.0"
 thiserror = "1"
-vm-memory = { version = "0.9" }
+vm-memory = { version = "0.10" }
 vmm-sys-util = "0.11.0"
 libc = ">=0.2.39"
 
 [dev-dependencies]
-vm-memory = { version = "0.9", features = ["backend-mmap"] }
+vm-memory = { version = "0.10", features = ["backend-mmap"] }
 
 [package.metadata.docs.rs]
 all-features = true

src/dragonball/src/dbs_boot/Cargo.toml

@@ -17,10 +17,10 @@ kvm-ioctls = "0.12.0"
 lazy_static = "1"
 libc = "0.2.39"
 thiserror = "1"
-vm-memory = "0.9.0"
+vm-memory = "0.10.0"
 vm-fdt = "0.2.0"
 
 [dev-dependencies]
-vm-memory = { version = "0.9.0", features = ["backend-mmap"] }
+vm-memory = { version = "0.10.0", features = ["backend-mmap"] }
 device_tree = ">=1.1.0"
 dbs-device = { path = "../dbs_device" }

src/dragonball/src/dbs_utils/src/net/net_gen/mod.rs

@@ -10,6 +10,7 @@
 #![allow(non_snake_case)]
 #![allow(missing_docs)]
 #![allow(deref_nullptr)]
+#![allow(ambiguous_glob_reexports)]
 
 // generated with bindgen /usr/include/linux/if.h --no-unstable-rust
 // --constified-enum '*' --with-derive-default -- -D __UAPI_DEF_IF_IFNAMSIZ -D

src/dragonball/src/dbs_virtio_devices/Cargo.toml

@@ -18,28 +18,28 @@ dbs-interrupt = { path = "../dbs_interrupt", features = ["kvm-legacy-irq", "kvm-
 dbs-utils = { path = "../dbs_utils" }
 epoll = ">=4.3.1, <4.3.2"
 io-uring = "0.5.2"
-fuse-backend-rs = { version = "0.10.0", optional = true }
+fuse-backend-rs = { version = "0.10.5", optional = true }
 kvm-bindings = "0.6.0"
 kvm-ioctls = "0.12.0"
 libc = "0.2.119"
 log = "0.4.14"
 nix = "0.24.3"
-nydus-api = "0.3.0"
-nydus-rafs = "0.3.1"
-nydus-storage = "0.6.3"
+nydus-api = "0.3.1"
+nydus-rafs = "0.3.2"
+nydus-storage = "0.6.4"
 rlimit = "0.7.0"
 serde = "1.0.27"
 serde_json = "1.0.9"
 thiserror = "1"
 threadpool = "1"
 virtio-bindings = "0.1.0"
-virtio-queue = "0.6.0"
+virtio-queue = "0.7.0"
 vmm-sys-util = "0.11.0"
-vm-memory = { version = "0.9.0", features = [ "backend-mmap" ] }
+vm-memory = { version = "0.10.0", features = [ "backend-mmap" ] }
 sendfd = "0.4.3"
 
 [dev-dependencies]
-vm-memory = { version = "0.9.0", features = [ "backend-mmap", "backend-atomic" ] }
+vm-memory = { version = "0.10.0", features = [ "backend-mmap", "backend-atomic" ] }
 
 [features]
 virtio-mmio = []

src/dragonball/src/dbs_virtio_devices/src/fs/device.rs

@@ -475,7 +475,7 @@ impl<AS: GuestAddressSpace> VirtioFs<AS> {
         let (mut rafs, rafs_cfg) = match config.as_ref() {
             Some(cfg) => {
                 let rafs_conf: Arc<ConfigV2> = Arc::new(
-                    serde_json::from_str(cfg).map_err(|e| FsError::BackendFs(e.to_string()))?,
+                    ConfigV2::from_str(cfg).map_err(|e| FsError::BackendFs(e.to_string()))?,
                 );
 
                 (

src/dragonball/src/dbs_virtio_devices/src/vsock/backend/hybrid_stream.rs

@@ -0,0 +1,94 @@
+// Copyright 2023 Ant Group. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::any::Any;
+use std::io::{Error, Read, Write};
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::time::Duration;
+
+use log::error;
+use nix::errno::Errno;
+
+use super::{VsockBackendType, VsockStream};
+
+pub struct HybridStream {
+    pub hybrid_stream: std::fs::File,
+    pub slave_stream: Option<Box<dyn VsockStream>>,
+}
+
+impl AsRawFd for HybridStream {
+    fn as_raw_fd(&self) -> RawFd {
+        self.hybrid_stream.as_raw_fd()
+    }
+}
+
+impl Read for HybridStream {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        self.hybrid_stream.read(buf)
+    }
+}
+
+impl Write for HybridStream {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        // The slave stream was only used to reply the connect result "ok <port>",
+        // thus it was only used once here, and the data would be replied by the
+        // main stream.
+        if let Some(mut stream) = self.slave_stream.take() {
+            stream.write(buf)
+        } else {
+            self.hybrid_stream.write(buf)
+        }
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.hybrid_stream.flush()
+    }
+}
+
+impl VsockStream for HybridStream {
+    fn backend_type(&self) -> VsockBackendType {
+        VsockBackendType::HybridStream
+    }
+
+    fn set_nonblocking(&mut self, nonblocking: bool) -> std::io::Result<()> {
+        let fd = self.hybrid_stream.as_raw_fd();
+        let mut flag = unsafe { libc::fcntl(fd, libc::F_GETFL) };
+
+        if nonblocking {
+            flag = flag | libc::O_NONBLOCK;
+        } else {
+            flag = flag & !libc::O_NONBLOCK;
+        }
+
+        let ret = unsafe { libc::fcntl(fd, libc::F_SETFL, flag) };
+
+        if ret < 0 {
+            error!("failed to set fcntl for fd {} with ret {}", fd, ret);
+            return Err(Error::last_os_error());
+        }
+
+        Ok(())
+    }
+
+    fn set_read_timeout(&mut self, _dur: Option<Duration>) -> std::io::Result<()> {
+        error!("unsupported!");
+        Err(Errno::ENOPROTOOPT.into())
+    }
+
+    fn set_write_timeout(&mut self, _dur: Option<Duration>) -> std::io::Result<()> {
+        error!("unsupported!");
+        Err(Errno::ENOPROTOOPT.into())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn recv_data_fd(
+        &self,
+        _bytes: &mut [u8],
+        _fds: &mut [RawFd],
+    ) -> std::io::Result<(usize, usize)> {
+        Err(Errno::ENOPROTOOPT.into())
+    }
+}

src/dragonball/src/dbs_virtio_devices/src/vsock/backend/mod.rs

@@ -9,13 +9,14 @@ use std::io::{Read, Write};
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::time::Duration;
 
+mod hybrid_stream;
 mod inner;
 mod tcp;
 mod unix_stream;
 
+pub use self::hybrid_stream::HybridStream;
 pub use self::inner::{VsockInnerBackend, VsockInnerConnector, VsockInnerStream};
 pub use self::tcp::VsockTcpBackend;
-pub use self::unix_stream::HybridUnixStreamBackend;
 pub use self::unix_stream::VsockUnixStreamBackend;
 
 /// The type of vsock backend.
@@ -27,6 +28,8 @@ pub enum VsockBackendType {
     Tcp,
     /// Inner backend
     Inner,
+    /// Fd passed hybrid stream backend
+    HybridStream,
     /// For test purpose
     #[cfg(test)]
     Test,

src/dragonball/src/dbs_virtio_devices/src/vsock/backend/unix_stream.rs

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::any::Any;
-use std::io::{Read, Write};
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::os::unix::net::{UnixListener, UnixStream};
 use std::time::Duration;
@@ -13,66 +12,6 @@ use sendfd::RecvWithFd;
 use super::super::{Result, VsockError};
 use super::{VsockBackend, VsockBackendType, VsockStream};
 
-pub struct HybridUnixStreamBackend {
-    pub unix_stream: Box<dyn VsockStream>,
-    pub slave_stream: Option<Box<dyn VsockStream>>,
-}
-
-impl VsockStream for HybridUnixStreamBackend {
-    fn backend_type(&self) -> VsockBackendType {
-        self.unix_stream.backend_type()
-    }
-
-    fn set_nonblocking(&mut self, nonblocking: bool) -> std::io::Result<()> {
-        self.unix_stream.set_nonblocking(nonblocking)
-    }
-
-    fn set_read_timeout(&mut self, dur: Option<Duration>) -> std::io::Result<()> {
-        self.unix_stream.set_read_timeout(dur)
-    }
-
-    fn set_write_timeout(&mut self, dur: Option<Duration>) -> std::io::Result<()> {
-        self.unix_stream.set_write_timeout(dur)
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self.unix_stream.as_any()
-    }
-
-    fn recv_data_fd(&self, bytes: &mut [u8], fds: &mut [RawFd]) -> std::io::Result<(usize, usize)> {
-        self.unix_stream.recv_data_fd(bytes, fds)
-    }
-}
-
-impl AsRawFd for HybridUnixStreamBackend {
-    fn as_raw_fd(&self) -> RawFd {
-        self.unix_stream.as_raw_fd()
-    }
-}
-
-impl Read for HybridUnixStreamBackend {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        self.unix_stream.read(buf)
-    }
-}
-
-impl Write for HybridUnixStreamBackend {
-    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        // The slave stream was only used to reply the connect result "ok <port>",
-        // thus it was only used once here, and the data would be replied by the
-        // main stream.
-        if let Some(mut stream) = self.slave_stream.take() {
-            stream.write(buf)
-        } else {
-            self.unix_stream.write(buf)
-        }
-    }
-
-    fn flush(&mut self) -> std::io::Result<()> {
-        self.unix_stream.flush()
-    }
-}
-
 impl VsockStream for UnixStream {
     fn backend_type(&self) -> VsockBackendType {
         VsockBackendType::UnixStream

src/dragonball/src/dbs_virtio_devices/src/vsock/muxer/mod.rs

@@ -60,6 +60,10 @@ pub enum Error {
     #[error("error connecting to a backend: {0}")]
     BackendConnect(#[source] std::io::Error),
 
+    /// Error set nonblock to a backend stream.
+    #[error("error set nonblocking to a backend: {0}")]
+    BackendSetNonBlock(#[source] std::io::Error),
+
     /// Error reading from backend.
     #[error("error reading from backend: {0}")]
     BackendRead(#[source] std::io::Error),

src/dragonball/src/dbs_virtio_devices/src/vsock/muxer/muxer_impl.rs

@@ -36,14 +36,14 @@
 ///       route all these events to their handlers, the muxer uses another
 ///       `HashMap` object, mapping `RawFd`s to `EpollListener`s.
 use std::collections::{HashMap, HashSet};
+use std::fs::File;
 use std::io::Read;
 use std::os::fd::FromRawFd;
 use std::os::unix::io::{AsRawFd, RawFd};
-use std::os::unix::net::UnixStream;
 
 use log::{debug, error, info, trace, warn};
 
-use super::super::backend::{HybridUnixStreamBackend, VsockBackend, VsockBackendType, VsockStream};
+use super::super::backend::{HybridStream, VsockBackend, VsockBackendType, VsockStream};
 
 use super::super::csm::{ConnState, VsockConnection};
 use super::super::defs::uapi;
@@ -480,13 +480,17 @@ impl VsockMuxer {
                         .and_then(|(nfd, local_port, peer_port)| {
                             // Here we should make sure the nfd the sole owner to convert it
                             // into an UnixStream object, otherwise, it could cause memory unsafety.
-                            let nstream = unsafe { UnixStream::from_raw_fd(nfd) };
+                            let nstream = unsafe { File::from_raw_fd(nfd) };
 
-                            let hybridstream = HybridUnixStreamBackend {
-                                unix_stream: Box::new(nstream),
+                            let mut hybridstream = HybridStream {
+                                hybrid_stream: nstream,
                                 slave_stream: Some(stream),
                             };
 
+                            hybridstream
+                                .set_nonblocking(true)
+                                .map_err(Error::BackendSetNonBlock)?;
+
                             self.add_connection(
                                 ConnMapKey {
                                     local_port,

src/dragonball/src/device_manager/mod.rs

@@ -1195,7 +1195,7 @@ mod tests {
         let mut cmdline = crate::vm::KernelConfigInfo::new(
             kernel_file,
             None,
-            linux_loader::cmdline::Cmdline::new(0x1000),
+            linux_loader::cmdline::Cmdline::new(0x1000).unwrap(),
         );
 
         let address_space = vm.vm_address_space().cloned();

src/dragonball/src/hypervisor_metrics.rs

@@ -0,0 +1,110 @@
+// Copyright 2021-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+extern crate procfs;
+
+use std::sync::Mutex;
+
+use anyhow::{anyhow, Result};
+use dbs_utils::metric::IncMetric;
+use prometheus::{Encoder, IntCounter, IntGaugeVec, Opts, Registry, TextEncoder};
+
+use crate::metric::METRICS;
+
+const NAMESPACE_KATA_HYPERVISOR: &str = "kata_hypervisor";
+
+lazy_static! {
+    static ref REGISTERED: Mutex<bool> = Mutex::new(false);
+
+    // custom registry
+    static ref REGISTRY: Registry = Registry::new();
+
+    // hypervisor metrics
+    static ref HYPERVISOR_SCRAPE_COUNT: IntCounter =
+    IntCounter::new(format!("{}_{}",NAMESPACE_KATA_HYPERVISOR,"scrape_count"), "Hypervisor metrics scrape count.").unwrap();
+
+    static ref HYPERVISOR_VCPU: IntGaugeVec =
+    IntGaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_HYPERVISOR,"vcpu"), "Hypervisor metrics specific to VCPUs' mode of functioning."), &["cpu_id", "item"]).unwrap();
+
+    static ref HYPERVISOR_SECCOMP: IntGaugeVec =
+    IntGaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_HYPERVISOR,"seccomp"), "Hypervisor metrics for the seccomp filtering."), &["item"]).unwrap();
+
+    static ref HYPERVISOR_SIGNALS: IntGaugeVec =
+    IntGaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_HYPERVISOR,"signals"), "Hypervisor metrics related to signals."), &["item"]).unwrap();
+}
+
+/// get prometheus metrics
+pub fn get_hypervisor_metrics() -> Result<String> {
+    let mut registered = REGISTERED
+        .lock()
+        .map_err(|e| anyhow!("failed to check hypervisor metrics register status {:?}", e))?;
+
+    if !(*registered) {
+        register_hypervisor_metrics()?;
+        *registered = true;
+    }
+
+    update_hypervisor_metrics()?;
+
+    // gather all metrics and return as a String
+    let metric_families = REGISTRY.gather();
+
+    let mut buffer = Vec::new();
+    let encoder = TextEncoder::new();
+    encoder.encode(&metric_families, &mut buffer)?;
+
+    Ok(String::from_utf8(buffer)?)
+}
+
+fn register_hypervisor_metrics() -> Result<()> {
+    REGISTRY.register(Box::new(HYPERVISOR_SCRAPE_COUNT.clone()))?;
+    REGISTRY.register(Box::new(HYPERVISOR_VCPU.clone()))?;
+    REGISTRY.register(Box::new(HYPERVISOR_SECCOMP.clone()))?;
+    REGISTRY.register(Box::new(HYPERVISOR_SIGNALS.clone()))?;
+
+    Ok(())
+}
+
+fn update_hypervisor_metrics() -> Result<()> {
+    HYPERVISOR_SCRAPE_COUNT.inc();
+
+    set_intgauge_vec_vcpu(&HYPERVISOR_VCPU);
+    set_intgauge_vec_seccomp(&HYPERVISOR_SECCOMP);
+    set_intgauge_vec_signals(&HYPERVISOR_SIGNALS);
+
+    Ok(())
+}
+
+fn set_intgauge_vec_vcpu(icv: &prometheus::IntGaugeVec) {
+    let metric_guard = METRICS.read().unwrap();
+    for (cpu_id, metrics) in metric_guard.vcpu.iter() {
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "exit_io_in"])
+            .set(metrics.exit_io_in.count() as i64);
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "exit_io_out"])
+            .set(metrics.exit_io_out.count() as i64);
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "exit_mmio_read"])
+            .set(metrics.exit_mmio_read.count() as i64);
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "exit_mmio_write"])
+            .set(metrics.exit_mmio_write.count() as i64);
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "failures"])
+            .set(metrics.failures.count() as i64);
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "filter_cpuid"])
+            .set(metrics.filter_cpuid.count() as i64);
+    }
+}
+
+fn set_intgauge_vec_seccomp(icv: &prometheus::IntGaugeVec) {
+    let metric_guard = METRICS.read().unwrap();
+    icv.with_label_values(&["num_faults"])
+        .set(metric_guard.seccomp.num_faults.count() as i64);
+}
+
+fn set_intgauge_vec_signals(icv: &prometheus::IntGaugeVec) {
+    let metric_guard = METRICS.read().unwrap();
+    icv.with_label_values(&["sigbus"])
+        .set(metric_guard.signals.sigbus.count() as i64);
+    icv.with_label_values(&["sigsegv"])
+        .set(metric_guard.signals.sigsegv.count() as i64);
+}

src/dragonball/src/lib.rs

@@ -9,6 +9,9 @@
 //TODO: Remove this, after the rest of dragonball has been committed.
 #![allow(dead_code)]
 
+#[macro_use]
+extern crate lazy_static;
+
 /// Address space manager for virtual machines.
 pub mod address_space_manager;
 /// API to handle vmm requests.
@@ -19,6 +22,8 @@ pub mod config_manager;
 pub mod device_manager;
 /// Errors related to Virtual machine manager.
 pub mod error;
+/// Prometheus Metrics.
+pub mod hypervisor_metrics;
 /// KVM operation context for virtual machines.
 pub mod kvm_context;
 /// Metrics system.