Merge pull request #8267 from gkurz/3.2.0-branch-bump

# Kata Containers 3.2.0

.github/workflows/add-backport-label.yaml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install hub extension script
         run: |

.github/workflows/add-issues-to-project.yaml

@@ -39,7 +39,7 @@ jobs:
           popd &>/dev/null
 
       - name: Checkout code to allow hub to communicate with the project
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Add issue to issue backlog
         env:

.github/workflows/add-pr-sizing-label.yaml

@@ -21,7 +21,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Install PR sizing label script
         run: |

.github/workflows/basic-ci-amd64.yaml

@@ -0,0 +1,200 @@
+name: CI | Basic amd64 tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-cri-containerd:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
+
+      - name: Run cri-containerd tests
+        run: bash tests/integration/cri-containerd/gha-run.sh run
+
+  run-containerd-stability:
+    strategy:
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/stability/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/stability/gha-run.sh install-kata kata-artifacts
+
+      - name: Run containerd-stability tests
+        run: bash tests/stability/gha-run.sh run
+
+  run-nydus:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu', 'dragonball']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nydus/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nydus/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nydus tests
+        run: bash tests/integration/nydus/gha-run.sh run
+
+  run-runk:
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/runk/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
+
+      - name: Run tracing tests
+        run: bash tests/integration/runk/gha-run.sh run
+
+  run-vfio:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2304
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/vfio/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Run vfio tests
+        timeout-minutes: 15
+        run: bash tests/functional/vfio/gha-run.sh run

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -16,6 +16,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-asset:
@@ -23,9 +27,12 @@ jobs:
     strategy:
       matrix:
         asset:
+          - agent
+          - agent-ctl
           - cloud-hypervisor
           - cloud-hypervisor-glibc
           - firecracker
+          - kata-ctl
           - kernel
           - kernel-sev
           - kernel-dragonball-experimental
@@ -33,6 +40,7 @@ jobs:
           - kernel-nvidia-gpu
           - kernel-nvidia-gpu-snp
           - kernel-nvidia-gpu-tdx-experimental
+          - log-parser-rs
           - nydus
           - ovmf
           - ovmf-sev
@@ -44,12 +52,16 @@ jobs:
           - rootfs-initrd
           - rootfs-initrd-mariner
           - rootfs-initrd-sev
+          - runk
           - shim-v2
           - tdvf
+          - trace-forwarder
           - virtiofsd
         stage:
           - ${{ inputs.stage }}
         exclude:
+          - asset: agent
+            stage: release
           - asset: cloud-hypervisor-glibc
             stage: release
     steps:
@@ -61,11 +73,17 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
 
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -76,6 +94,10 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -89,9 +111,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-asset
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -16,10 +16,14 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-asset:
-    runs-on: arm64
+    runs-on: arm64-builder
     strategy:
       matrix:
         asset:
@@ -48,10 +52,17 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -62,6 +73,10 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -72,16 +87,22 @@ jobs:
           if-no-files-found: error
 
   create-kata-tarball:
-    runs-on: arm64
+    runs-on: arm64-builder
     needs: build-asset
     steps:
       - name: Adjust a permission for repo
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -16,6 +16,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-asset:
@@ -44,10 +48,17 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -59,6 +70,10 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -76,9 +91,15 @@ jobs:
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:

.github/workflows/cargo-deny-runner.yaml

@@ -19,7 +19,7 @@ jobs:
     steps:
       - name: Checkout Code
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Generate Action
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: bash cargo-deny-generator.sh

.github/workflows/ci-nightly.yaml

@@ -15,4 +15,5 @@ jobs:
       commit-hash: ${{ github.sha }}
       pr-number: "nightly"
       tag: ${{ github.sha }}-nightly
+      target-branch: ${{ github.ref_name }}
     secrets: inherit

.github/workflows/ci-on-push.yaml

@@ -28,4 +28,5 @@ jobs:
       commit-hash: ${{ github.event.pull_request.head.sha }}
       pr-number: ${{ github.event.pull_request.number }}
       tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
     secrets: inherit

.github/workflows/ci.yaml

@@ -11,6 +11,10 @@ on:
       tag:
         required: true
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-kata-static-tarball-amd64:
@@ -18,6 +22,7 @@ jobs:
     with:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
 
   publish-kata-deploy-payload-amd64:
     needs: build-kata-static-tarball-amd64
@@ -28,8 +33,94 @@ jobs:
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
     secrets: inherit
 
+  build-and-publish-tee-confidential-unencrypted-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Kata Containers ghcr.io
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker build and push
+        uses: docker/build-push-action@v4
+        with:
+          tags: ghcr.io/kata-containers/test-images:unencrypted-${{ inputs.pr-number }}
+          push: true
+          context: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/
+          platforms: linux/amd64, linux/s390x
+          file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
+
+  run-docker-tests-on-garm:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-docker-tests-on-garm.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-nerdctl-tests-on-garm:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-nerdctl-tests-on-garm.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-kata-deploy-tests-on-aks:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-kata-deploy-tests-on-aks.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-kata-deploy-tests-on-garm:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-kata-deploy-tests-on-garm.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-kata-monitor-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-kata-monitor-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
   run-k8s-tests-on-aks:
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
@@ -39,34 +130,43 @@ jobs:
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
     secrets: inherit
 
-  run-k8s-tests-on-sev:
+  run-k8s-tests-on-garm:
     needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-sev.yaml
+    uses: ./.github/workflows/run-k8s-tests-on-garm.yaml
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
 
-  run-k8s-tests-on-snp:
+  run-k8s-tests-with-crio-on-garm:
     needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-snp.yaml
+    uses: ./.github/workflows/run-k8s-tests-with-crio-on-garm.yaml
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
 
-  run-k8s-tests-on-tdx:
-    needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml
+  run-kata-coco-tests:
+    needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
+    uses: ./.github/workflows/run-kata-coco-tests.yaml
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
 
   run-metrics-tests:
     needs: build-kata-static-tarball-amd64
@@ -74,24 +174,12 @@ jobs:
     with:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
 
-  run-cri-containerd-tests:
+  run-basic-amd64-tests:
     needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/run-cri-containerd-tests.yaml
-    with:
-      tarball-suffix: -${{ inputs.tag }}
-      commit-hash: ${{ inputs.commit-hash }}
-
-  run-nydus-tests:
-    needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/run-nydus-tests.yaml
-    with:
-      tarball-suffix: -${{ inputs.tag }}
-      commit-hash: ${{ inputs.commit-hash }}
-
-  run-vfio-tests:
-    needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/run-vfio-tests.yaml
+    uses: ./.github/workflows/basic-ci-amd64.yaml
     with:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}

.github/workflows/darwin-tests.yaml

@@ -21,6 +21,6 @@ jobs:
       with:
         go-version: 1.19.3
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Build utils
       run: ./ci/darwin-test.sh

.github/workflows/docs-url-alive-check.yaml

@@ -22,7 +22,7 @@ jobs:
         echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}

.github/workflows/kata-runtime-classes-sync.yaml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Ensure the split out runtime classes match the all-in-one file
       run: |
         pushd tools/packaging/kata-deploy/runtimeclasses/

.github/workflows/move-issues-to-in-progress.yaml

@@ -38,7 +38,17 @@ jobs:
 
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Move issue to "In progress"
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

.github/workflows/payload-after-push.yaml

@@ -4,6 +4,7 @@ on:
     branches:
       - main
       - stable-*
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -15,6 +16,7 @@ jobs:
     with:
       commit-hash: ${{ github.sha }}
       push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   build-assets-arm64:
@@ -22,6 +24,7 @@ jobs:
     with:
       commit-hash: ${{ github.sha }}
       push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   build-assets-s390x:
@@ -29,6 +32,7 @@ jobs:
     with:
       commit-hash: ${{ github.sha }}
       push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-kata-deploy-payload-amd64:
@@ -39,6 +43,7 @@ jobs:
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-amd64
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-kata-deploy-payload-arm64:
@@ -49,6 +54,7 @@ jobs:
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-arm64
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-kata-deploy-payload-s390x:
@@ -59,6 +65,7 @@ jobs:
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-s390x
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-manifest:
@@ -66,7 +73,7 @@ jobs:
     needs: [publish-kata-deploy-payload-amd64, publish-kata-deploy-payload-arm64, publish-kata-deploy-payload-s390x]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Login to Kata Containers quay.io
         uses: docker/login-action@v2

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -17,14 +17,25 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   kata-payload:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -17,18 +17,29 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   kata-payload:
-    runs-on: arm64
+    runs-on: arm64-builder
     steps:
       - name: Adjust a permission for repo
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -17,6 +17,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   kata-payload:
@@ -26,9 +30,16 @@ jobs:
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/release-amd64.yaml

@@ -29,7 +29,7 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:

.github/workflows/release-arm64.yaml

@@ -14,7 +14,7 @@ jobs:
 
   kata-deploy:
     needs: build-kata-static-tarball-arm64
-    runs-on: arm64
+    runs-on: arm64-builder
     steps:
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v2
@@ -29,7 +29,7 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:

.github/workflows/release-s390x.yaml

@@ -29,7 +29,7 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:

.github/workflows/release.yaml

@@ -32,7 +32,7 @@ jobs:
     needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v2
@@ -73,11 +73,7 @@ jobs:
     needs: publish-multi-arch-images
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: install hub
-        run: |
-          wget -q -O- https://github.com/mislav/hub/releases/download/v2.14.2/hub-linux-amd64-2.14.2.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
+      - uses: actions/checkout@v4
 
       - name: download-artifacts-amd64
         uses: actions/download-artifact@v3
@@ -90,7 +86,7 @@ jobs:
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
       - name: download-artifacts-arm64
@@ -104,7 +100,7 @@ jobs:
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
       - name: download-artifacts-s390x
@@ -118,13 +114,13 @@ jobs:
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
   upload-versions-yaml:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: upload versions.yaml
         env:
           GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
@@ -133,28 +129,28 @@ jobs:
           pushd $GITHUB_WORKSPACE
           versions_file="kata-containers-$tag-versions.yaml"
           cp versions.yaml ${versions_file}
-          hub release edit -m "" -a "${versions_file}" "${tag}"
+          gh release upload "${tag}" "${versions_file}"
           popd
 
   upload-cargo-vendored-tarball:
     needs: upload-multi-arch-static-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: generate-and-upload-tarball
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="kata-containers-$tag-vendor.tar.gz"
           pushd $GITHUB_WORKSPACE
           bash -c "tools/packaging/release/generate_vendor.sh ${tarball}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
   upload-libseccomp-tarball:
     needs: upload-cargo-vendored-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: download-and-upload-tarball
         env:
           GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
@@ -174,6 +170,6 @@ jobs:
           # "-m" option should be empty to re-use the existing release title
           # without opening a text editor.
           # For the details, check https://hub.github.com/hub-release.1.html.
-          hub release edit -m "" -a "${tarball}" "${tag}"
-          hub release edit -m "" -a "${asc}" "${tag}"
+          gh release upload "${tag}" "${tarball}"
+          gh release upload "${tag}" "${asc}"
           popd

.github/workflows/require-pr-porting-labels.yaml

@@ -36,7 +36,17 @@ jobs:
 
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Install porting checker script
         run: |

.github/workflows/run-cri-containerd-tests.yaml

@@ -1,42 +0,0 @@
-name: CI | Run cri-containerd tests
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-cri-containerd:
-    strategy:
-      fail-fast: true
-      matrix:
-        containerd_version: ['lts', 'active']
-        vmm: ['clh', 'qemu']
-    runs-on: garm-ubuntu-2204
-    env:
-      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Install dependencies
-        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Install kata
-        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
-
-      - name: Run cri-containerd tests
-        run: bash tests/integration/cri-containerd/gha-run.sh run

.github/workflows/run-docker-tests-on-garm.yaml

@@ -0,0 +1,56 @@
+name: CI | Run docker integration tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-docker-tests:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # all the tests are not flaky, otherwise we'll fail them
+      # all due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+    runs-on: garm-ubuntu-2304-smaller
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/docker/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/docker/gha-run.sh install-kata kata-artifacts
+
+      - name: Run docker smoke test
+        timeout-minutes: 5
+        run: bash tests/integration/docker/gha-run.sh run

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -17,6 +17,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   run-k8s-tests:
@@ -29,6 +33,9 @@ jobs:
           - clh
           - dragonball
           - qemu
+        instance-type:
+          - small
+          - normal
         include:
           - host_os: cbl-mariner
             vmm: clh
@@ -40,11 +47,20 @@ jobs:
       GH_PR_NUMBER: ${{ inputs.pr-number }}
       KATA_HOST_OS: ${{ matrix.host_os }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
       USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli

.github/workflows/run-k8s-tests-on-garm.yaml

@@ -0,0 +1,88 @@
+name: CI | Run kubernetes tests on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh #cloud-hypervisor
+          - fc #firecracker
+          - qemu
+        snapshotter:
+          - devmapper
+        k8s:
+          - k3s
+        instance:
+          - garm-ubuntu-2004
+          - garm-ubuntu-2004-smaller
+        include:
+          - instance: garm-ubuntu-2004
+            instance-type: normal
+          - instance: garm-ubuntu-2004-smaller
+            instance-type: small
+    runs-on: ${{ matrix.instance }}
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Configure the ${{ matrix.snapshotter }} snapshotter
+        run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+  
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+  
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -1,48 +0,0 @@
-name: CI | Run kubernetes tests on SEV
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-sev
-    runs-on: sev
-    env:
-      DOCKER_REGISTRY: ${{ inputs.registry }}
-      DOCKER_REPO: ${{ inputs.repo }}
-      DOCKER_TAG: ${{ inputs.tag }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-      KUBECONFIG: /home/kata/.kube/config
-      USING_NFD: "false"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Deploy Kata
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
-  
-      - name: Run tests
-        timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests
-
-      - name: Delete kata-deploy
-        if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh cleanup-sev

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -1,48 +0,0 @@
-name: CI | Run kubernetes tests on SEV-SNP
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-snp
-    runs-on: sev-snp
-    env:
-      DOCKER_REGISTRY: ${{ inputs.registry }}
-      DOCKER_REPO: ${{ inputs.repo }}
-      DOCKER_TAG: ${{ inputs.tag }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-      KUBECONFIG: /home/kata/.kube/config
-      USING_NFD: "false"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Deploy Kata
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
-  
-      - name: Run tests
-        timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests
-      
-      - name: Delete kata-deploy
-        if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -1,47 +0,0 @@
-name: CI | Run kubernetes tests on TDX
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-tdx
-    runs-on: tdx
-    env:
-      DOCKER_REGISTRY: ${{ inputs.registry }}
-      DOCKER_REPO: ${{ inputs.repo }}
-      DOCKER_TAG: ${{ inputs.tag }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-      USING_NFD: "true"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Deploy Kata
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
-  
-      - name: Run tests
-        timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests
-  
-      - name: Delete kata-deploy
-        if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx

.github/workflows/run-k8s-tests-with-crio-on-garm.yaml

@@ -0,0 +1,86 @@
+name: CI | Run kubernetes tests, using CRI-O, on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        k8s:
+          - k0s
+        instance:
+          - garm-ubuntu-2004
+          - garm-ubuntu-2004-smaller
+        include:
+          - instance: garm-ubuntu-2004
+            instance-type: normal
+          - instance: garm-ubuntu-2004-smaller
+            instance-type: small
+          - k8s: k0s
+            k8s-extra-params: '--cri-socket remote:unix:///var/run/crio/crio.sock --kubelet-extra-args --cgroup-driver="systemd"'
+    runs-on: ${{ matrix.instance }}
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      KUBERNETES_EXTRA_PARAMS: ${{ matrix.k8s-extra-params }}
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Configure CRI-O
+        run: bash tests/integration/kubernetes/gha-run.sh setup-crio
+
+      - name: Deploy ${{ matrix.k8s }}
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+  
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+  
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm

.github/workflows/run-kata-coco-tests.yaml

@@ -0,0 +1,176 @@
+name: CI | Run kata coco tests
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests-on-tdx:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "k3s"
+      USING_NFD: "true"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests
+
+  run-k8s-tests-on-tdx:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "k3s"
+      USING_NFD: "true"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx
+
+  run-k8s-tests-on-sev:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-sev
+    runs-on: sev
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-sev
+
+  run-k8s-tests-sev-snp:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-snp
+    runs-on: sev-snp
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp

.github/workflows/run-kata-deploy-tests-on-aks.yaml

@@ -0,0 +1,89 @@
+name: CI | Run kata-deploy tests on AKS
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        host_os:
+          - ubuntu
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+        include:
+          - host_os: cbl-mariner
+            vmm: clh
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Download Azure CLI
+        run: bash tests/functional/kata-deploy/gha-run.sh install-azure-cli
+
+      - name: Log into the Azure account
+        run: bash tests/functional/kata-deploy/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+
+      - name: Create AKS cluster
+        timeout-minutes: 10
+        run: bash tests/functional/kata-deploy/gha-run.sh create-cluster
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Install `kubectl`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-kubectl
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: bash tests/functional/kata-deploy/gha-run.sh get-cluster-credentials
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests
+      
+      - name: Delete AKS cluster
+        if: always()
+        run: bash tests/functional/kata-deploy/gha-run.sh delete-cluster

.github/workflows/run-kata-deploy-tests-on-garm.yaml

@@ -0,0 +1,65 @@
+name: CI | Run kata-deploy tests on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+        k8s:
+          - k0s
+          - k3s
+          - rke2
+    runs-on: garm-ubuntu-2004-smaller
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run:  bash tests/functional/kata-deploy/gha-run.sh deploy-k8s
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests

.github/workflows/run-kata-monitor-tests.yaml

@@ -0,0 +1,59 @@
+name: CI | Run kata-monitor tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-monitor:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        container_engine:
+          - crio
+          - containerd
+        include:
+          - container_engine: containerd
+            containerd_version: lts
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINER_ENGINE: ${{ matrix.container_engine }}
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/kata-monitor/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/functional/kata-monitor/gha-run.sh install-kata kata-artifacts
+
+      - name: Run kata-monitor tests
+        run: bash tests/functional/kata-monitor/gha-run.sh run

.github/workflows/run-metrics.yaml

@@ -8,22 +8,28 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
-  run-metrics:
-    strategy:
-      fail-fast: true
-      matrix:
-        vmm: ['clh', 'qemu']
-      max-parallel: 1
+  setup-kata:
+    name: Kata Setup
     runs-on: metrics
     env:
       GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
@@ -34,6 +40,24 @@ jobs:
       - name: Install kata
         run: bash tests/metrics/gha-run.sh install-kata kata-artifacts
 
+  run-metrics:
+    needs: setup-kata
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm: ['clh', 'qemu']
+      max-parallel: 1
+    runs-on: metrics
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - name: enabling the hypervisor
+        run: bash tests/metrics/gha-run.sh enabling-hypervisor
+
       - name: run launch times test
         run: bash tests/metrics/gha-run.sh run-test-launchtimes
 
@@ -52,6 +76,12 @@ jobs:
       - name: run fio test
         run:  bash tests/metrics/gha-run.sh run-test-fio
 
+      - name: run iperf test
+        run:  bash tests/metrics/gha-run.sh run-test-iperf
+
+      - name: run latency test
+        run:  bash tests/metrics/gha-run.sh run-test-latency
+
       - name: make metrics tarball ${{ matrix.vmm }}
         run: bash tests/metrics/gha-run.sh make-tarball-results
 

.github/workflows/run-nerdctl-tests-on-garm.yaml

@@ -0,0 +1,57 @@
+name: CI | Run nerdctl integration tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-nerdctl-tests:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # all the tests are not flaky, otherwise we'll fail them
+      # all due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+    runs-on: garm-ubuntu-2304-smaller
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nerdctl/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nerdctl/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nerdctl smoke test
+        timeout-minutes: 5
+        run: bash tests/integration/nerdctl/gha-run.sh run

.github/workflows/run-nydus-tests.yaml

@@ -1,42 +0,0 @@
-name: CI | Run nydus tests
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-nydus:
-    strategy:
-      fail-fast: true
-      matrix:
-        containerd_version: ['lts', 'active']
-        vmm: ['clh', 'qemu', 'dragonball']
-    runs-on: garm-ubuntu-2204
-    env:
-      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Install dependencies
-        run: bash tests/integration/nydus/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Install kata
-        run: bash tests/integration/nydus/gha-run.sh install-kata kata-artifacts
-
-      - name: Run nydus tests
-        run: bash tests/integration/nydus/gha-run.sh run

.github/workflows/run-runk-tests.yaml

@@ -0,0 +1,46 @@
+name: CI | Run runk tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-runk:
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/runk/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
+
+      - name: Run tracing tests
+        run: bash tests/integration/runk/gha-run.sh run

.github/workflows/run-vfio-tests.yaml

@@ -1,37 +0,0 @@
-name: CI | Run vfio tests
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-vfio:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm: ['clh', 'qemu']
-    runs-on: garm-ubuntu-2204
-    env:
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Install dependencies
-        run: bash tests/functional/vfio/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Run vfio tests
-        run: bash tests/functional/vfio/gha-run.sh run

.github/workflows/static-checks-dragonball.yaml

@@ -1,37 +0,0 @@
-on:
-  pull_request:
-    types:
-      - opened
-      - edited
-      - reopened
-      - synchronize
-    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-name: Static checks dragonball
-jobs:
-  test-dragonball:
-    runs-on: dragonball
-    env:
-      RUST_BACKTRACE: "1"
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set env
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-      - name: Install Rust
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          ./ci/install_rust.sh
-          echo PATH="$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
-      - name: Run Unit Test
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          cd src/dragonball
-          cargo version
-          rustc --version
-          sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test

.github/workflows/static-checks.yaml

@@ -12,74 +12,183 @@ concurrency:
 
 name: Static checks
 jobs:
-  static-checks:
+  check-kernel-config-version:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Ensure the kernel config version has been updated
+        run: |
+          kernel_dir="tools/packaging/kernel/"
+          kernel_version_file="${kernel_dir}kata_config_version"
+          modified_files=$(git diff --name-only origin/$GITHUB_BASE_REF..HEAD)
+          if git diff --name-only origin/$GITHUB_BASE_REF..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+            echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
+            if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
+              echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
+            else
+              echo "Readme file changed, no need for kernel config version update."
+            fi
+            echo "Check passed"
+          fi
+
+  build-checks:
     runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false
       matrix:
-        cmd:
+        component:
+          - agent
+          - dragonball
+          - runtime
+          - runtime-rs
+          - agent-ctl
+          - kata-ctl
+          - log-parser-rs
+          - runk
+          - trace-forwarder
+        command:
           - "make vendor"
-          - "make static-checks"
           - "make check"
           - "make test"
           - "sudo -E PATH=\"$PATH\" make test"
+        include:
+          - component: agent
+            component-path: src/agent
+          - component: dragonball
+            component-path: src/dragonball
+          - component: runtime
+            component-path: src/runtime
+          - component: runtime-rs
+            component-path: src/runtime-rs
+          - component: agent-ctl
+            component-path: src/tools/agent-ctl
+          - component: kata-ctl
+            component-path: src/tools/kata-ctl
+          - component: log-parser-rs
+            component-path: src/tools/log-parser-rs
+          - component: runk
+            component-path: src/tools/runk
+          - component: trace-forwarder
+            component-path: src/tools/trace-forwarder
+          - install-libseccomp: no
+          - component: agent
+            install-libseccomp: yes
+          - component: runk
+            install-libseccomp: yes
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install yq
+        run: |
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        if: ${{ matrix.component == 'runtime' }}
+        run: |
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+      - name: Install rust
+        if: ${{ matrix.component != 'runtime' }}
+        run: |
+          ./tests/install_rust.sh
+          echo "${HOME}/.cargo/bin" >> $GITHUB_PATH
+      - name: Install musl-tools
+        if: ${{ matrix.component != 'runtime' }}
+        run: sudo apt-get -y install musl-tools
+      - name: Install libseccomp
+        if: ${{ matrix.command != 'make vendor'  &&  matrix.command != 'make check' &&  matrix.install-libseccomp == 'yes' }}
+        run: |
+          libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+          gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+          ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+          echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+          echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+          echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
+      - name: Setup XDG_RUNTIME_DIR for the `runtime` tests
+        if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }}
+        run: |
+          XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700))
+          echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> $GITHUB_ENV
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  build-checks-depending-on-kvm:
+    runs-on: garm-ubuntu-2004-smaller
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - runtime-rs
+        include:
+          - component: runtime-rs
+            command: "sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test"
+          - component: runtime-rs
+            component-path: src/dragonball
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install system deps
+        run: |
+          sudo apt-get install -y build-essential musl-tools
+      - name: Install yq
+        run: |
+          sudo -E ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install rust
+        run: |
+          export PATH="$PATH:/usr/local/bin"
+          ./tests/install_rust.sh
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          export PATH="$PATH:${HOME}/.cargo/bin"
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  static-checks:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        cmd:
+          - "make static-checks"
     env:
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
       GOPATH: ${{ github.workspace }}
     steps:
-    - name: Free disk space
-      run: |
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-    - name: Checkout code
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Install Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19.3
-    - name: Check kernel config version
-      run: |
-        cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
-        kernel_dir="tools/packaging/kernel/"
-        kernel_version_file="${kernel_dir}kata_config_version"
-        modified_files=$(git diff --name-only origin/main..HEAD)
-        if git diff --name-only origin/main..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
-          echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
-          if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
-            echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
-          else
-            echo "Readme file changed, no need for kernel config version update."
-          fi
-          echo "Check passed"
-        fi
-    - name: Set PATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    - name: Run check
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: ./src/github.com/${{ github.repository }}
+      - name: Install yq
+        run: |
+          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        run: |
+          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+      - name: Install system dependencies
+        run: |
+          sudo apt-get -y install moreutils hunspell pandoc
+      - name: Run check
+        run: |
+          export PATH=${PATH}:${GOPATH}/bin
+          cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}

VERSION

@@ -1 +1 @@
-3.2.0-rc0
+3.2.0

ci/install_libseccomp.sh

@@ -7,12 +7,10 @@
 
 set -o errexit
 
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+script_name="$(basename "${BASH_SOURCE[0]}")"
 
-clone_tests_repo
-
-source "${tests_repo_dir}/.ci/lib.sh"
+source "${script_dir}/../tests/common.bash"
 
 # The following variables if set on the environment will change the behavior
 # of gperf and libseccomp configure scripts, that may lead this script to
@@ -25,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 # Variables for libseccomp
 libseccomp_version="${LIBSECCOMP_VERSION:-""}"
 if [ -z "${libseccomp_version}" ]; then
-    libseccomp_version=$(get_version "externals.libseccomp.version")
+    libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version")
 fi
 libseccomp_url="${LIBSECCOMP_URL:-""}"
 if [ -z "${libseccomp_url}" ]; then
-    libseccomp_url=$(get_version "externals.libseccomp.url")
+    libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url")
 fi
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
@@ -38,11 +36,11 @@ cflags="-O2"
 # Variables for gperf
 gperf_version="${GPERF_VERSION:-""}"
 if [ -z "${gperf_version}" ]; then
-    gperf_version=$(get_version "externals.gperf.version")
+    gperf_version=$(get_from_kata_deps "externals.gperf.version")
 fi
 gperf_url="${GPERF_URL:-""}"
 if [ -z "${gperf_url}" ]; then
-    gperf_url=$(get_version "externals.gperf.url")
+    gperf_url=$(get_from_kata_deps "externals.gperf.url")
 fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"

docs/design/kata-vra.md

@@ -43,7 +43,7 @@ and perform DMA transactions _anywhere_.
 
 The second feature is ACS (Access Control Services), which controls which
 devices are allowed to communicate with one another and thus avoids improper
-routing of packets irrespectively of whether IOMMU is enabled or not.
+routing of packets `irrespectively` of whether IOMMU is enabled or not.
 
 When IOMMU is enabled, ACS is normally configured to force all PCI Express DMA
 to go through the root complex so IOMMU can translate it, impacting performance
@@ -126,7 +126,7 @@ efficient P2P communication.
 ## PCI Express Virtual P2P Approval Capability
 
 Most of the time, the PCI Express topology is flattened and obfuscated to ensure
-easy migration of the VM image between different physical hardware topologies.
+easy migration of the VM image between different physical hardware `topologies`.
 In Kata, we can configure the hypervisor to use PCI Express root ports to
 hotplug the VFIO  devices one is passing through. A user can select how many PCI
 Express root ports to allocate depending on how many devices are passed through.
@@ -220,7 +220,7 @@ containers that he wants to run with Kata. The goal is to make such things as
 transparent as possible, so we also introduced
 [CDI](https://github.com/container-orchestrated-devices/container-device-interface)
 (Container Device Interface) to Kata. CDI is a[
-specification](https://github.com/container-orchestrated-devices/container-device-interface/blob/master/SPEC.md)
+specification](https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md)
 for container runtimes to support third-party devices.
 
 As written before, we can provide a clique ID for the devices that belong
@@ -300,7 +300,7 @@ pcie_switch_port = 8
 ```
 
 Each device that is passed through is attached to a PCI Express downstream port
-as illustrated below. We can even replicate the host’s two DPUs topologies with
+as illustrated below. We can even replicate the host’s two DPUs `topologies` with
 added metadata through the CDI. Most of the time, a container only needs one
 pair of GPU and NIC for GPUDirect RDMA. This is more of a showcase of what we
 can do with the power of Kata and CDI. One could even think of adding groups of
@@ -328,7 +328,7 @@ $ lspci -tv
 ```
 
 The configuration of using either the root port or switch port can be applied on
-a per Container or Pod basis, meaning we can switch PCI Express topologies on
+a per Container or Pod basis, meaning we can switch PCI Express `topologies` on
 each run of an application.
 
 ## Hypervisor Resource Limits

src/agent/Makefile

@@ -54,7 +54,7 @@ endif
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
 ##VAR DESTDIR=<path> is a directory prepended to each installed target file
-DESTDIR :=
+DESTDIR ?=
 ##VAR BINDIR=<path> is a directory for installing executable programs
 BINDIR := /usr/bin
 
@@ -140,7 +140,7 @@ vendor:
 
 
 #TARGET test: run cargo tests
-test:
+test: $(GENERATED_FILES)
 	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test

src/agent/rustjail/src/lib.rs

@@ -423,12 +423,18 @@ fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
         let mut r = Vec::new();
 
         for d in l.Devices.iter() {
+            // if the filemode for the device is 0 (unset), use a default value as runc does
+            let filemode = if d.FileMode != 0 {
+                Some(d.FileMode)
+            } else {
+                Some(0o666)
+            };
             r.push(oci::LinuxDevice {
                 path: d.Path.clone(),
                 r#type: d.Type.clone(),
                 major: d.Major,
                 minor: d.Minor,
-                file_mode: Some(d.FileMode),
+                file_mode: filemode,
                 uid: Some(d.UID),
                 gid: Some(d.GID),
             });

src/dragonball/Cargo.toml

@@ -27,7 +27,7 @@ kvm-bindings = "0.6.0"
 kvm-ioctls = "0.12.0"
 lazy_static = "1.2"
 libc = "0.2.39"
-linux-loader = "0.6.0"
+linux-loader = "0.8.0"
 log = "0.4.14"
 nix = "0.24.2"
 procfs = "0.12.0"
@@ -40,9 +40,10 @@ slog = "2.5.2"
 slog-scope = "4.4.0"
 thiserror = "1"
 vmm-sys-util = "0.11.0"
-virtio-queue = { version = "0.6.0", optional = true }
-vm-memory = { version = "0.9.0", features = ["backend-mmap"] }
+virtio-queue = { version = "0.7.0", optional = true }
+vm-memory = { version = "0.10.0", features = ["backend-mmap"] }
 crossbeam-channel = "0.5.6"
+fuse-backend-rs = "0.10.5"
 
 [dev-dependencies]
 slog-async = "2.7.0"

src/dragonball/src/api/v1/vmm_action.rs

@@ -358,7 +358,8 @@ impl VmmService {
             Some(ref path) => Some(File::open(path).map_err(|e| BootSource(InvalidInitrdPath(e)))?),
         };
 
-        let mut cmdline = linux_loader::cmdline::Cmdline::new(dbs_boot::layout::CMDLINE_MAX_SIZE);
+        let mut cmdline = linux_loader::cmdline::Cmdline::new(dbs_boot::layout::CMDLINE_MAX_SIZE)
+            .map_err(|err| BootSource(InvalidKernelCommandLine(err)))?;
         let boot_args = boot_source_config
             .boot_args
             .unwrap_or_else(|| String::from(DEFAULT_KERNEL_CMDLINE));

src/dragonball/src/dbs_address_space/Cargo.toml

@@ -17,4 +17,4 @@ nix = "0.23.1"
 lazy_static = "1"
 thiserror = "1"
 vmm-sys-util = "0.11.0"
-vm-memory = { version = "0.9", features = ["backend-mmap", "backend-atomic"] }
+vm-memory = { version = "0.10", features = ["backend-mmap", "backend-atomic"] }

src/dragonball/src/dbs_arch/Cargo.toml

@@ -15,12 +15,12 @@ memoffset = "0.6"
 kvm-bindings = { version = "0.6.0", features = ["fam-wrappers"] }
 kvm-ioctls = "0.12.0"
 thiserror = "1"
-vm-memory = { version = "0.9" }
+vm-memory = { version = "0.10" }
 vmm-sys-util = "0.11.0"
 libc = ">=0.2.39"
 
 [dev-dependencies]
-vm-memory = { version = "0.9", features = ["backend-mmap"] }
+vm-memory = { version = "0.10", features = ["backend-mmap"] }
 
 [package.metadata.docs.rs]
 all-features = true

src/dragonball/src/dbs_boot/Cargo.toml

@@ -17,10 +17,10 @@ kvm-ioctls = "0.12.0"
 lazy_static = "1"
 libc = "0.2.39"
 thiserror = "1"
-vm-memory = "0.9.0"
+vm-memory = "0.10.0"
 vm-fdt = "0.2.0"
 
 [dev-dependencies]
-vm-memory = { version = "0.9.0", features = ["backend-mmap"] }
+vm-memory = { version = "0.10.0", features = ["backend-mmap"] }
 device_tree = ">=1.1.0"
 dbs-device = { path = "../dbs_device" }

src/dragonball/src/dbs_virtio_devices/Cargo.toml

@@ -18,28 +18,28 @@ dbs-interrupt = { path = "../dbs_interrupt", features = ["kvm-legacy-irq", "kvm-
 dbs-utils = { path = "../dbs_utils" }
 epoll = ">=4.3.1, <4.3.2"
 io-uring = "0.5.2"
-fuse-backend-rs = { version = "0.10.0", optional = true }
+fuse-backend-rs = { version = "0.10.5", optional = true }
 kvm-bindings = "0.6.0"
 kvm-ioctls = "0.12.0"
 libc = "0.2.119"
 log = "0.4.14"
 nix = "0.24.3"
-nydus-api = "0.3.0"
-nydus-rafs = "0.3.1"
-nydus-storage = "0.6.3"
+nydus-api = "0.3.1"
+nydus-rafs = "0.3.2"
+nydus-storage = "0.6.4"
 rlimit = "0.7.0"
 serde = "1.0.27"
 serde_json = "1.0.9"
 thiserror = "1"
 threadpool = "1"
 virtio-bindings = "0.1.0"
-virtio-queue = "0.6.0"
+virtio-queue = "0.7.0"
 vmm-sys-util = "0.11.0"
-vm-memory = { version = "0.9.0", features = [ "backend-mmap" ] }
+vm-memory = { version = "0.10.0", features = [ "backend-mmap" ] }
 sendfd = "0.4.3"
 
 [dev-dependencies]
-vm-memory = { version = "0.9.0", features = [ "backend-mmap", "backend-atomic" ] }
+vm-memory = { version = "0.10.0", features = [ "backend-mmap", "backend-atomic" ] }
 
 [features]
 virtio-mmio = []

src/dragonball/src/device_manager/mod.rs

@@ -1195,7 +1195,7 @@ mod tests {
         let mut cmdline = crate::vm::KernelConfigInfo::new(
             kernel_file,
             None,
-            linux_loader::cmdline::Cmdline::new(0x1000),
+            linux_loader::cmdline::Cmdline::new(0x1000).unwrap(),
         );
 
         let address_space = vm.vm_address_space().cloned();

src/dragonball/src/test_utils.rs

@@ -17,7 +17,7 @@ pub mod tests {
         let epoll_manager = EpollManager::default();
         let mut vm = Vm::new(None, instance_info, epoll_manager).unwrap();
         let kernel_file = TempFile::new().unwrap();
-        let cmd_line = Cmdline::new(64);
+        let cmd_line = Cmdline::new(64).unwrap();
         vm.set_kernel_config(KernelConfigInfo::new(
             kernel_file.into_file(),
             None,

src/dragonball/src/vm/kernel_config.rs

@@ -62,7 +62,7 @@ mod tests {
     fn test_kernel_config_info() {
         let kernel = TempFile::new().unwrap();
         let initrd = TempFile::new().unwrap();
-        let mut cmdline = linux_loader::cmdline::Cmdline::new(1024);
+        let mut cmdline = linux_loader::cmdline::Cmdline::new(1024).unwrap();
         cmdline.insert_str("ro").unwrap();
         let mut info = KernelConfigInfo::new(kernel.into_file(), Some(initrd.into_file()), cmdline);
 

src/dragonball/src/vm/mod.rs

@@ -1027,7 +1027,7 @@ pub mod tests {
         );
 
         let kernel_file = TempFile::new().unwrap();
-        let cmd_line = Cmdline::new(64);
+        let cmd_line = Cmdline::new(64).unwrap();
 
         vm.set_kernel_config(KernelConfigInfo::new(
             kernel_file.into_file(),

src/runtime-rs/Cargo.lock

@@ -828,6 +828,7 @@ dependencies = [
  "dbs-upcall",
  "dbs-utils",
  "dbs-virtio-devices",
+ "fuse-backend-rs",
  "kvm-bindings",
  "kvm-ioctls",
  "lazy_static",
@@ -995,9 +996,9 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
 
 [[package]]
 name = "fuse-backend-rs"
-version = "0.10.4"
+version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc24820b14267bec37fa87f5c2a32b5f1c5405b8c60cc3aa77afd481bd2628a6"
+checksum = "f85357722be4bf3d0b7548bedf7499686c77628c2c61cb99c6519463f7a9e5f0"
 dependencies = [
  "arc-swap",
  "bitflags 1.3.2",
@@ -1008,10 +1009,9 @@ dependencies = [
  "log",
  "mio",
  "nix 0.24.3",
- "tokio-uring",
  "virtio-queue",
  "vm-memory",
- "vmm-sys-util 0.10.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
@@ -1593,17 +1593,6 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
-[[package]]
-name = "leaky-bucket"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e8b256cabb5f5c7affd490acbb12f951d725385971fa602dedb11e09c896b6d"
-dependencies = [
- "parking_lot 0.12.1",
- "tokio",
- "tracing",
-]
-
 [[package]]
 name = "libc"
 version = "0.2.147"
@@ -1625,9 +1614,9 @@ dependencies = [
 
 [[package]]
 name = "linux-loader"
-version = "0.6.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62a2f912deca034ec34b0a43a390059ea98daac40e440ebe8bea88f3315fe168"
+checksum = "b9259ddbfbb52cc918f6bbc60390004ddd0228cf1d85f402009ff2b3d95de83f"
 dependencies = [
  "vm-memory",
 ]
@@ -1938,11 +1927,10 @@ dependencies = [
 
 [[package]]
 name = "nydus-api"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33a6ca41dd10813e3d29397550fbb0f15ad149381f312e04659d39e0adcf2002"
+checksum = "c64c62d8a36c10b654b87246a39861b2c05f68e96ab3b2f002f5a54f406d5e0e"
 dependencies = [
- "backtrace",
  "libc",
  "log",
  "serde",
@@ -1952,9 +1940,9 @@ dependencies = [
 
 [[package]]
 name = "nydus-rafs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed21e44a99472850d2afc4fb07427ed46d4e6a8b1cce28b42bd689319e45076d"
+checksum = "adde865ef71c91c5f139c4c05ca5aedb6fbd53f530d646b13409ac5220b85467"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -1974,16 +1962,15 @@ dependencies = [
 
 [[package]]
 name = "nydus-storage"
-version = "0.6.3"
+version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9591fbee1875895bf1f765656695d0be6887fe65372fbf4924b8b3959bd61375"
+checksum = "4023f15303dbbda47797d07e9acd2045862ce82c7e28cd66f70b09bda5584cbb"
 dependencies = [
  "arc-swap",
  "bitflags 1.3.2",
  "fuse-backend-rs",
  "hex",
  "lazy_static",
- "leaky-bucket",
  "libc",
  "log",
  "nix 0.24.3",
@@ -1998,9 +1985,9 @@ dependencies = [
 
 [[package]]
 name = "nydus-utils"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe8b9269e3a370682f272a1b2cac4bdaf6d6657f3f6966560c4fedab36548362"
+checksum = "c1f7bcde0f3906cf49101f2d40e485b0155eee97e3358eefd4783448c4f69c96"
 dependencies = [
  "blake3",
  "flate2",
@@ -2926,12 +2913,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "scoped-tls"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
-
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -3505,20 +3486,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tokio-uring"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d5e02bb137e030b3a547c65a3bd2f1836d66a97369fdcc69034002b10e155ef"
-dependencies = [
- "io-uring",
- "libc",
- "scoped-tls",
- "slab",
- "socket2",
- "tokio",
-]
-
 [[package]]
 name = "tokio-util"
 version = "0.7.8"
@@ -3828,14 +3795,14 @@ checksum = "3ff512178285488516ed85f15b5d0113a7cdb89e9e8a760b269ae4f02b84bd6b"
 
 [[package]]
 name = "virtio-queue"
-version = "0.6.1"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "435dd49c7b38419729afd43675850c7b5dc4728f2fabd70c7a9079a331e4f8c6"
+checksum = "3ba81e2bcc21c0d2fc5e6683e79367e26ad219197423a498df801d79d5ba77bd"
 dependencies = [
  "log",
  "virtio-bindings",
  "vm-memory",
- "vmm-sys-util 0.10.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
@@ -3846,9 +3813,9 @@ checksum = "f43fb5a6bd1a7d423ad72802801036719b7546cf847a103f8fe4575f5b0d45a6"
 
 [[package]]
 name = "vm-memory"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "583f213899e8a5eea23d9c507252d4bed5bc88f0ecbe0783262f80034630744b"
+checksum = "688a70366615b45575a424d9c665561c1b5ab2224d494f706b6a6812911a827c"
 dependencies = [
  "arc-swap",
  "libc",

src/runtime-rs/Makefile

@@ -49,7 +49,7 @@ else
 ##TARGET default: build code
 default: runtime show-header
 ##TARGET test: run cargo tests
-test:
+test: static-checks-build
 	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 install: install-runtime install-configs
 endif

src/runtime/Makefile

@@ -192,7 +192,7 @@ DEFMEMSLOTS := 10
 DEFMAXMEMSZ := 0
 #Default number of bridges
 DEFBRIDGES := 1
-DEFENABLEANNOTATIONS := [\"enable_iommu\"]
+DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\"]
 DEFDISABLEGUESTSECCOMP := true
 DEFDISABLEGUESTEMPTYDIR := false
 #Default experimental features enabled
@@ -603,6 +603,7 @@ USER_VARS += DEFENTROPYSOURCE
 USER_VARS += DEFVALIDENTROPYSOURCES
 USER_VARS += DEFSANDBOXCGROUPONLY
 USER_VARS += DEFSTATICRESOURCEMGMT
+USER_VARS += DEFSTATICRESOURCEMGMT_CLH
 USER_VARS += DEFSTATICRESOURCEMGMT_FC
 USER_VARS += DEFSTATICRESOURCEMGMT_TEE
 USER_VARS += DEFBINDMOUNTS

src/runtime/arch/amd64-options.mk

@@ -26,3 +26,5 @@ ACRNCTLCMD := acrnctl
 
 # cloud-hypervisor binary name
 CLHCMD := cloud-hypervisor
+
+DEFSTATICRESOURCEMGMT_CLH := false

src/runtime/arch/arm64-options.mk

@@ -19,3 +19,5 @@ FCJAILERCMD := jailer
 
 # cloud-hypervisor binary name
 CLHCMD := cloud-hypervisor
+
+DEFSTATICRESOURCEMGMT_CLH := true

src/runtime/config/configuration-clh.toml.in

@@ -188,12 +188,22 @@ block_device_driver = "virtio-blk"
 # Disable the 'seccomp' feature from Cloud Hypervisor, default false
 # disable_seccomp = true
 
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: iommu=pt
+#enable_iommu = true
+
 # This option changes the default hypervisor and kernel parameters
 # to enable debug output where available.
 #
 # Default false
 #enable_debug = true
 
+# Enable hot-plugging of VFIO devices to a root-port.
+# The default setting is  "no-port"
+#hot_plug_vfio = "root-port"
+
 # Path to OCI hook binaries in the *guest rootfs*.
 # This does not affect host-side hooks which must instead be added to
 # the OCI spec passed to the runtime.
@@ -390,7 +400,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
 #   does not yet support sandbox sizing annotations.
 # - When running single containers using a tool like ctr, container sizing information will be available.
-static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_CLH@
 
 # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
 # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.

src/runtime/config/configuration-qemu-nvidia-gpu.toml.in

@@ -327,11 +327,26 @@ valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
 pflashes = []
 
 # This option changes the default hypervisor and kernel parameters
-# to enable debug output where available. And Debug also enable the hmp socket.
+# to enable debug output where available.
 #
 # Default false
 #enable_debug = true
 
+# This option allows to add an extra HMP or QMP socket when `enable_debug = true`
+#
+# WARNING: Anyone with access to the extra socket can take full control of
+# Qemu. This is for debugging purpose only and must *NEVER* be used in
+# production.
+#
+# Valid values are :
+# - "hmp"
+# - "qmp"
+# - "qmp-pretty" (same as "qmp" with pretty json formatting)
+#
+# If set to the empty string "", no extra monitor socket is added. This is
+# the default.
+#extra_monitor_socket = hmp
+
 # Disable the customizations done in the runtime when it detects
 # that it is running on top a VMM. This will result in the runtime
 # behaving as it would when running on bare metal.

src/runtime/config/configuration-qemu.toml.in

@@ -327,11 +327,26 @@ valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
 pflashes = []
 
 # This option changes the default hypervisor and kernel parameters
-# to enable debug output where available. And Debug also enable the hmp socket.
+# to enable debug output where available.
 #
 # Default false
 #enable_debug = true
 
+# This option allows to add an extra HMP or QMP socket when `enable_debug = true`
+#
+# WARNING: Anyone with access to the extra socket can take full control of
+# Qemu. This is for debugging purpose only and must *NEVER* be used in
+# production.
+#
+# Valid values are :
+# - "hmp"
+# - "qmp"
+# - "qmp-pretty" (same as "qmp" with pretty json formatting)
+#
+# If set to the empty string "", no extra monitor socket is added. This is
+# the default.
+#extra_monitor_socket = hmp
+
 # Disable the customizations done in the runtime when it detects
 # that it is running on top a VMM. This will result in the runtime
 # behaving as it would when running on bare metal.

src/runtime/pkg/govmm/qemu/qemu.go

@@ -141,9 +141,16 @@ const (
 func isDimmSupported(config *Config) bool {
 	switch runtime.GOARCH {
 	case "amd64", "386", "ppc64le", "arm64":
-		if config != nil && config.Machine.Type == MachineTypeMicrovm {
-			// microvm does not support NUMA
-			return false
+		if config != nil {
+			if config.Machine.Type == MachineTypeMicrovm {
+				// microvm does not support NUMA
+				return false
+			}
+			if config.Knobs.MemFDPrivate {
+				// TDX guests rely on MemFD Private, which
+				// does not have NUMA support yet
+				return false
+			}
 		}
 		return true
 	default:
@@ -2464,14 +2471,28 @@ const (
 	Unix QMPSocketType = "unix"
 )
 
-// QMPSocket represents a qemu QMP socket configuration.
+// MonitorProtocol tells what protocol is used on a QMPSocket
+type MonitorProtocol string
+
+const (
+	// Socket using a human-friendly text-based protocol.
+	Hmp MonitorProtocol = "hmp"
+
+	// Socket using a richer json-based protocol.
+	Qmp MonitorProtocol = "qmp"
+
+	// Same as Qmp with pretty json formatting.
+	QmpPretty MonitorProtocol = "qmp-pretty"
+)
+
+// QMPSocket represents a qemu QMP or HMP socket configuration.
 // nolint: govet
 type QMPSocket struct {
 	// Type is the socket type (e.g. "unix").
 	Type QMPSocketType
 
-	// Human Monitor Interface (HMP) (true for HMP, false for QMP, default false)
-	IsHmp bool
+	// Protocol is the protocol to be used on the socket.
+	Protocol MonitorProtocol
 
 	// QMP listener file descriptor to be passed to qemu
 	FD *os.File
@@ -2497,6 +2518,10 @@ func (qmp QMPSocket) Valid() bool {
 		return false
 	}
 
+	if qmp.Protocol != Hmp && qmp.Protocol != Qmp && qmp.Protocol != QmpPretty {
+		return false
+	}
+
 	return true
 }
 
@@ -2628,6 +2653,9 @@ type Knobs struct {
 	// MemPrealloc will allocate all the RAM upfront
 	MemPrealloc bool
 
+	// Private Memory FD meant for private memory map/unmap.
+	MemFDPrivate bool
+
 	// FileBackedMem requires Memory.Size and Memory.Path of the VM to
 	// be set.
 	FileBackedMem bool
@@ -2845,10 +2873,11 @@ func (config *Config) appendQMPSockets() {
 			}
 		}
 
-		if q.IsHmp {
+		switch q.Protocol {
+		case Hmp:
 			config.qemuParams = append(config.qemuParams, "-monitor")
-		} else {
-			config.qemuParams = append(config.qemuParams, "-qmp")
+		default:
+			config.qemuParams = append(config.qemuParams, fmt.Sprintf("-%s", q.Protocol))
 		}
 
 		config.qemuParams = append(config.qemuParams, strings.Join(qmpParams, ","))
@@ -2992,10 +3021,13 @@ func (config *Config) appendMemoryKnobs() {
 		return
 	}
 	var objMemParam, numaMemParam string
+
 	dimmName := "dimm1"
 	if config.Knobs.HugePages {
 		objMemParam = "memory-backend-file,id=" + dimmName + ",size=" + config.Memory.Size + ",mem-path=/dev/hugepages"
 		numaMemParam = "node,memdev=" + dimmName
+	} else if config.Knobs.MemFDPrivate {
+		objMemParam = "memory-backend-memfd-private,id=" + dimmName + ",size=" + config.Memory.Size
 	} else if config.Knobs.FileBackedMem && config.Memory.Path != "" {
 		objMemParam = "memory-backend-file,id=" + dimmName + ",size=" + config.Memory.Size + ",mem-path=" + config.Memory.Path
 		numaMemParam = "node,memdev=" + dimmName

src/runtime/pkg/govmm/qemu/qemu_test.go

@@ -632,6 +632,29 @@ func TestAppendMemoryFileBackedMemPrealloc(t *testing.T) {
 	testConfigAppend(conf, knobs, memString+" "+knobsString, t)
 }
 
+func TestAppendMemoryBackedMemFdPrivate(t *testing.T) {
+	conf := &Config{
+		Memory: Memory{
+			Size:  "1G",
+			Slots: 8,
+		},
+	}
+	memString := "-m 1G,slots=8"
+	testConfigAppend(conf, conf.Memory, memString, t)
+
+	knobs := Knobs{
+		MemFDPrivate: true,
+		MemShared:    false,
+	}
+	objMemString := "-object memory-backend-memfd-private,id=dimm1,size=1G"
+	memBackendString := "-machine memory-backend=dimm1"
+
+	knobsString := objMemString + " "
+	knobsString += memBackendString
+
+	testConfigAppend(conf, knobs, memString+" "+knobsString, t)
+}
+
 func TestNoRebootKnob(t *testing.T) {
 	conf := &Config{}
 
@@ -703,10 +726,11 @@ var qmpSingleSocketString = "-qmp unix:path=cc-qmp"
 
 func TestAppendSingleQMPSocketServer(t *testing.T) {
 	qmp := QMPSocket{
-		Type:   "unix",
-		Name:   "cc-qmp",
-		Server: true,
-		NoWait: true,
+		Type:     "unix",
+		Name:     "cc-qmp",
+		Server:   true,
+		NoWait:   true,
+		Protocol: Qmp,
 	}
 
 	testAppend(qmp, qmpSingleSocketServerString, t)
@@ -714,9 +738,10 @@ func TestAppendSingleQMPSocketServer(t *testing.T) {
 
 func TestAppendSingleQMPSocket(t *testing.T) {
 	qmp := QMPSocket{
-		Type:   Unix,
-		Name:   "cc-qmp",
-		Server: false,
+		Type:     Unix,
+		Name:     "cc-qmp",
+		Server:   false,
+		Protocol: Qmp,
 	}
 
 	testAppend(qmp, qmpSingleSocketString, t)
@@ -733,10 +758,11 @@ func TestAppendQMPSocketServerFd(t *testing.T) {
 	}()
 
 	qmp := QMPSocket{
-		Type:   "unix",
-		FD:     foo,
-		Server: true,
-		NoWait: true,
+		Type:     "unix",
+		FD:       foo,
+		Server:   true,
+		NoWait:   true,
+		Protocol: Qmp,
 	}
 
 	testAppend(qmp, qmpSocketServerFdString, t)
@@ -747,16 +773,18 @@ var qmpSocketServerString = "-qmp unix:path=cc-qmp-1,server=on,wait=off -qmp uni
 func TestAppendQMPSocketServer(t *testing.T) {
 	qmp := []QMPSocket{
 		{
-			Type:   "unix",
-			Name:   "cc-qmp-1",
-			Server: true,
-			NoWait: true,
+			Type:     "unix",
+			Name:     "cc-qmp-1",
+			Server:   true,
+			NoWait:   true,
+			Protocol: Qmp,
 		},
 		{
-			Type:   "unix",
-			Name:   "cc-qmp-2",
-			Server: true,
-			NoWait: true,
+			Type:     "unix",
+			Name:     "cc-qmp-2",
+			Server:   true,
+			NoWait:   true,
+			Protocol: Qmp,
 		},
 	}
 

src/runtime/pkg/katautils/config-settings.go.in

@@ -11,6 +11,7 @@ package katautils
 
 import (
 	config "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
 )
 
 // name is the name of the runtime
@@ -79,6 +80,7 @@ const defaultEnableIOMMU bool = false
 const defaultEnableIOMMUPlatform bool = false
 const defaultFileBackedMemRootDir string = ""
 const defaultEnableDebug bool = false
+const defaultExtraMonitorSocket govmmQemu.MonitorProtocol = ""
 const defaultDisableNestingChecks bool = false
 const defaultMsize9p uint32 = 8192
 const defaultEntropySource = "/dev/urandom"

src/runtime/pkg/katautils/config.go

@@ -55,6 +55,8 @@ const (
 
 	// the maximum amount of PCI bridges that can be cold plugged in a VM
 	maxPCIBridges uint32 = 5
+
+	errInvalidHypervisorPrefix = "configuration file contains invalid hypervisor section"
 )
 
 type tomlConfig struct {
@@ -78,87 +80,88 @@ type factory struct {
 }
 
 type hypervisor struct {
-	Path                           string          `toml:"path"`
-	JailerPath                     string          `toml:"jailer_path"`
-	Kernel                         string          `toml:"kernel"`
-	CtlPath                        string          `toml:"ctlpath"`
-	Initrd                         string          `toml:"initrd"`
-	Image                          string          `toml:"image"`
-	RootfsType                     string          `toml:"rootfs_type"`
-	Firmware                       string          `toml:"firmware"`
-	FirmwareVolume                 string          `toml:"firmware_volume"`
-	MachineAccelerators            string          `toml:"machine_accelerators"`
-	CPUFeatures                    string          `toml:"cpu_features"`
-	KernelParams                   string          `toml:"kernel_params"`
-	MachineType                    string          `toml:"machine_type"`
-	BlockDeviceDriver              string          `toml:"block_device_driver"`
-	EntropySource                  string          `toml:"entropy_source"`
-	SharedFS                       string          `toml:"shared_fs"`
-	VirtioFSDaemon                 string          `toml:"virtio_fs_daemon"`
-	VirtioFSCache                  string          `toml:"virtio_fs_cache"`
-	VhostUserStorePath             string          `toml:"vhost_user_store_path"`
-	FileBackedMemRootDir           string          `toml:"file_mem_backend"`
-	GuestHookPath                  string          `toml:"guest_hook_path"`
-	GuestMemoryDumpPath            string          `toml:"guest_memory_dump_path"`
-	SeccompSandbox                 string          `toml:"seccompsandbox"`
-	BlockDeviceAIO                 string          `toml:"block_device_aio"`
-	HypervisorPathList             []string        `toml:"valid_hypervisor_paths"`
-	JailerPathList                 []string        `toml:"valid_jailer_paths"`
-	CtlPathList                    []string        `toml:"valid_ctlpaths"`
-	VirtioFSDaemonList             []string        `toml:"valid_virtio_fs_daemon_paths"`
-	VirtioFSExtraArgs              []string        `toml:"virtio_fs_extra_args"`
-	PFlashList                     []string        `toml:"pflashes"`
-	VhostUserStorePathList         []string        `toml:"valid_vhost_user_store_paths"`
-	FileBackedMemRootList          []string        `toml:"valid_file_mem_backends"`
-	EntropySourceList              []string        `toml:"valid_entropy_sources"`
-	EnableAnnotations              []string        `toml:"enable_annotations"`
-	RxRateLimiterMaxRate           uint64          `toml:"rx_rate_limiter_max_rate"`
-	TxRateLimiterMaxRate           uint64          `toml:"tx_rate_limiter_max_rate"`
-	MemOffset                      uint64          `toml:"memory_offset"`
-	DefaultMaxMemorySize           uint64          `toml:"default_maxmemory"`
-	DiskRateLimiterBwMaxRate       int64           `toml:"disk_rate_limiter_bw_max_rate"`
-	DiskRateLimiterBwOneTimeBurst  int64           `toml:"disk_rate_limiter_bw_one_time_burst"`
-	DiskRateLimiterOpsMaxRate      int64           `toml:"disk_rate_limiter_ops_max_rate"`
-	DiskRateLimiterOpsOneTimeBurst int64           `toml:"disk_rate_limiter_ops_one_time_burst"`
-	NetRateLimiterBwMaxRate        int64           `toml:"net_rate_limiter_bw_max_rate"`
-	NetRateLimiterBwOneTimeBurst   int64           `toml:"net_rate_limiter_bw_one_time_burst"`
-	NetRateLimiterOpsMaxRate       int64           `toml:"net_rate_limiter_ops_max_rate"`
-	NetRateLimiterOpsOneTimeBurst  int64           `toml:"net_rate_limiter_ops_one_time_burst"`
-	VirtioFSCacheSize              uint32          `toml:"virtio_fs_cache_size"`
-	VirtioFSQueueSize              uint32          `toml:"virtio_fs_queue_size"`
-	DefaultMaxVCPUs                uint32          `toml:"default_maxvcpus"`
-	MemorySize                     uint32          `toml:"default_memory"`
-	MemSlots                       uint32          `toml:"memory_slots"`
-	DefaultBridges                 uint32          `toml:"default_bridges"`
-	Msize9p                        uint32          `toml:"msize_9p"`
-	NumVCPUs                       int32           `toml:"default_vcpus"`
-	BlockDeviceCacheSet            bool            `toml:"block_device_cache_set"`
-	BlockDeviceCacheDirect         bool            `toml:"block_device_cache_direct"`
-	BlockDeviceCacheNoflush        bool            `toml:"block_device_cache_noflush"`
-	EnableVhostUserStore           bool            `toml:"enable_vhost_user_store"`
-	VhostUserDeviceReconnect       uint32          `toml:"vhost_user_reconnect_timeout_sec"`
-	DisableBlockDeviceUse          bool            `toml:"disable_block_device_use"`
-	MemPrealloc                    bool            `toml:"enable_mem_prealloc"`
-	HugePages                      bool            `toml:"enable_hugepages"`
-	VirtioMem                      bool            `toml:"enable_virtio_mem"`
-	IOMMU                          bool            `toml:"enable_iommu"`
-	IOMMUPlatform                  bool            `toml:"enable_iommu_platform"`
-	Debug                          bool            `toml:"enable_debug"`
-	DisableNestingChecks           bool            `toml:"disable_nesting_checks"`
-	EnableIOThreads                bool            `toml:"enable_iothreads"`
-	DisableImageNvdimm             bool            `toml:"disable_image_nvdimm"`
-	HotPlugVFIO                    config.PCIePort `toml:"hot_plug_vfio"`
-	ColdPlugVFIO                   config.PCIePort `toml:"cold_plug_vfio"`
-	DisableVhostNet                bool            `toml:"disable_vhost_net"`
-	GuestMemoryDumpPaging          bool            `toml:"guest_memory_dump_paging"`
-	ConfidentialGuest              bool            `toml:"confidential_guest"`
-	SevSnpGuest                    bool            `toml:"sev_snp_guest"`
-	GuestSwap                      bool            `toml:"enable_guest_swap"`
-	Rootless                       bool            `toml:"rootless"`
-	DisableSeccomp                 bool            `toml:"disable_seccomp"`
-	DisableSeLinux                 bool            `toml:"disable_selinux"`
-	DisableGuestSeLinux            bool            `toml:"disable_guest_selinux"`
-	LegacySerial                   bool            `toml:"use_legacy_serial"`
+	Path                           string                    `toml:"path"`
+	JailerPath                     string                    `toml:"jailer_path"`
+	Kernel                         string                    `toml:"kernel"`
+	CtlPath                        string                    `toml:"ctlpath"`
+	Initrd                         string                    `toml:"initrd"`
+	Image                          string                    `toml:"image"`
+	RootfsType                     string                    `toml:"rootfs_type"`
+	Firmware                       string                    `toml:"firmware"`
+	FirmwareVolume                 string                    `toml:"firmware_volume"`
+	MachineAccelerators            string                    `toml:"machine_accelerators"`
+	CPUFeatures                    string                    `toml:"cpu_features"`
+	KernelParams                   string                    `toml:"kernel_params"`
+	MachineType                    string                    `toml:"machine_type"`
+	BlockDeviceDriver              string                    `toml:"block_device_driver"`
+	EntropySource                  string                    `toml:"entropy_source"`
+	SharedFS                       string                    `toml:"shared_fs"`
+	VirtioFSDaemon                 string                    `toml:"virtio_fs_daemon"`
+	VirtioFSCache                  string                    `toml:"virtio_fs_cache"`
+	VhostUserStorePath             string                    `toml:"vhost_user_store_path"`
+	FileBackedMemRootDir           string                    `toml:"file_mem_backend"`
+	GuestHookPath                  string                    `toml:"guest_hook_path"`
+	GuestMemoryDumpPath            string                    `toml:"guest_memory_dump_path"`
+	SeccompSandbox                 string                    `toml:"seccompsandbox"`
+	BlockDeviceAIO                 string                    `toml:"block_device_aio"`
+	HypervisorPathList             []string                  `toml:"valid_hypervisor_paths"`
+	JailerPathList                 []string                  `toml:"valid_jailer_paths"`
+	CtlPathList                    []string                  `toml:"valid_ctlpaths"`
+	VirtioFSDaemonList             []string                  `toml:"valid_virtio_fs_daemon_paths"`
+	VirtioFSExtraArgs              []string                  `toml:"virtio_fs_extra_args"`
+	PFlashList                     []string                  `toml:"pflashes"`
+	VhostUserStorePathList         []string                  `toml:"valid_vhost_user_store_paths"`
+	FileBackedMemRootList          []string                  `toml:"valid_file_mem_backends"`
+	EntropySourceList              []string                  `toml:"valid_entropy_sources"`
+	EnableAnnotations              []string                  `toml:"enable_annotations"`
+	RxRateLimiterMaxRate           uint64                    `toml:"rx_rate_limiter_max_rate"`
+	TxRateLimiterMaxRate           uint64                    `toml:"tx_rate_limiter_max_rate"`
+	MemOffset                      uint64                    `toml:"memory_offset"`
+	DefaultMaxMemorySize           uint64                    `toml:"default_maxmemory"`
+	DiskRateLimiterBwMaxRate       int64                     `toml:"disk_rate_limiter_bw_max_rate"`
+	DiskRateLimiterBwOneTimeBurst  int64                     `toml:"disk_rate_limiter_bw_one_time_burst"`
+	DiskRateLimiterOpsMaxRate      int64                     `toml:"disk_rate_limiter_ops_max_rate"`
+	DiskRateLimiterOpsOneTimeBurst int64                     `toml:"disk_rate_limiter_ops_one_time_burst"`
+	NetRateLimiterBwMaxRate        int64                     `toml:"net_rate_limiter_bw_max_rate"`
+	NetRateLimiterBwOneTimeBurst   int64                     `toml:"net_rate_limiter_bw_one_time_burst"`
+	NetRateLimiterOpsMaxRate       int64                     `toml:"net_rate_limiter_ops_max_rate"`
+	NetRateLimiterOpsOneTimeBurst  int64                     `toml:"net_rate_limiter_ops_one_time_burst"`
+	VirtioFSCacheSize              uint32                    `toml:"virtio_fs_cache_size"`
+	VirtioFSQueueSize              uint32                    `toml:"virtio_fs_queue_size"`
+	DefaultMaxVCPUs                uint32                    `toml:"default_maxvcpus"`
+	MemorySize                     uint32                    `toml:"default_memory"`
+	MemSlots                       uint32                    `toml:"memory_slots"`
+	DefaultBridges                 uint32                    `toml:"default_bridges"`
+	Msize9p                        uint32                    `toml:"msize_9p"`
+	NumVCPUs                       int32                     `toml:"default_vcpus"`
+	BlockDeviceCacheSet            bool                      `toml:"block_device_cache_set"`
+	BlockDeviceCacheDirect         bool                      `toml:"block_device_cache_direct"`
+	BlockDeviceCacheNoflush        bool                      `toml:"block_device_cache_noflush"`
+	EnableVhostUserStore           bool                      `toml:"enable_vhost_user_store"`
+	VhostUserDeviceReconnect       uint32                    `toml:"vhost_user_reconnect_timeout_sec"`
+	DisableBlockDeviceUse          bool                      `toml:"disable_block_device_use"`
+	MemPrealloc                    bool                      `toml:"enable_mem_prealloc"`
+	HugePages                      bool                      `toml:"enable_hugepages"`
+	VirtioMem                      bool                      `toml:"enable_virtio_mem"`
+	IOMMU                          bool                      `toml:"enable_iommu"`
+	IOMMUPlatform                  bool                      `toml:"enable_iommu_platform"`
+	Debug                          bool                      `toml:"enable_debug"`
+	DisableNestingChecks           bool                      `toml:"disable_nesting_checks"`
+	EnableIOThreads                bool                      `toml:"enable_iothreads"`
+	DisableImageNvdimm             bool                      `toml:"disable_image_nvdimm"`
+	HotPlugVFIO                    config.PCIePort           `toml:"hot_plug_vfio"`
+	ColdPlugVFIO                   config.PCIePort           `toml:"cold_plug_vfio"`
+	DisableVhostNet                bool                      `toml:"disable_vhost_net"`
+	GuestMemoryDumpPaging          bool                      `toml:"guest_memory_dump_paging"`
+	ConfidentialGuest              bool                      `toml:"confidential_guest"`
+	SevSnpGuest                    bool                      `toml:"sev_snp_guest"`
+	GuestSwap                      bool                      `toml:"enable_guest_swap"`
+	Rootless                       bool                      `toml:"rootless"`
+	DisableSeccomp                 bool                      `toml:"disable_seccomp"`
+	DisableSeLinux                 bool                      `toml:"disable_selinux"`
+	DisableGuestSeLinux            bool                      `toml:"disable_guest_selinux"`
+	LegacySerial                   bool                      `toml:"use_legacy_serial"`
+	ExtraMonitorSocket             govmmQemu.MonitorProtocol `toml:"extra_monitor_socket"`
 }
 
 type runtime struct {
@@ -516,6 +519,22 @@ func (h hypervisor) blockDeviceAIO() (string, error) {
 	return "", fmt.Errorf("Invalid hypervisor block storage I/O mechanism  %v specified (supported AIO: %v)", h.BlockDeviceAIO, supportedBlockAIO)
 }
 
+func (h hypervisor) extraMonitorSocket() (govmmQemu.MonitorProtocol, error) {
+	supportedExtraMonitor := []govmmQemu.MonitorProtocol{govmmQemu.Hmp, govmmQemu.Qmp, govmmQemu.QmpPretty}
+
+	if h.ExtraMonitorSocket == "" {
+		return "", nil
+	}
+
+	for _, extra := range supportedExtraMonitor {
+		if extra == h.ExtraMonitorSocket {
+			return extra, nil
+		}
+	}
+
+	return "", fmt.Errorf("Invalid hypervisor extra monitor socket %v specified (supported values: %v)", h.ExtraMonitorSocket, supportedExtraMonitor)
+}
+
 func (h hypervisor) sharedFS() (string, error) {
 	supportedSharedFS := []string{config.Virtio9P, config.VirtioFS, config.VirtioFSNydus, config.NoSharedFS}
 
@@ -819,6 +838,11 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	rxRateLimiterMaxRate := h.getRxRateLimiterCfg()
 	txRateLimiterMaxRate := h.getTxRateLimiterCfg()
 
+	extraMonitorSocket, err := h.extraMonitorSocket()
+	if err != nil {
+		return vc.HypervisorConfig{}, err
+	}
+
 	return vc.HypervisorConfig{
 		HypervisorPath:          hypervisor,
 		HypervisorPathList:      h.HypervisorPathList,
@@ -887,6 +911,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		LegacySerial:            h.LegacySerial,
 		DisableSeLinux:          h.DisableSeLinux,
 		DisableGuestSeLinux:     h.DisableGuestSeLinux,
+		ExtraMonitorSocket:      extraMonitorSocket,
 	}, nil
 }
 
@@ -1153,6 +1178,8 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi
 		case dragonballHypervisorTableType:
 			config.HypervisorType = vc.DragonballHypervisor
 			hConfig, err = newDragonballHypervisorConfig(hypervisor)
+		default:
+			err = fmt.Errorf("%s: %+q", errInvalidHypervisorPrefix, k)
 		}
 
 		if err != nil {
@@ -1284,6 +1311,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		IOMMUPlatform:            defaultEnableIOMMUPlatform,
 		FileBackedMemRootDir:     defaultFileBackedMemRootDir,
 		Debug:                    defaultEnableDebug,
+		ExtraMonitorSocket:       defaultExtraMonitorSocket,
 		DisableNestingChecks:     defaultDisableNestingChecks,
 		BlockDeviceDriver:        defaultBlockDeviceDriver,
 		BlockDeviceAIO:           defaultBlockDeviceAIO,
@@ -1680,8 +1708,8 @@ func checkConfig(config oci.RuntimeConfig) error {
 // Only allow one of the following settings for cold-plug:
 // no-port, root-port, switch-port
 func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineType string, hypervisorType virtcontainers.HypervisorType) error {
-	if hypervisorType != virtcontainers.QemuHypervisor {
-		kataUtilsLogger.Warn("Advanced PCIe Topology only available for QEMU hypervisor, ignoring hot(cold)_vfio_port setting")
+	if hypervisorType != virtcontainers.QemuHypervisor && hypervisorType != virtcontainers.ClhHypervisor {
+		kataUtilsLogger.Warn("Advanced PCIe Topology only available for QEMU/CLH hypervisor, ignoring hot(cold)_vfio_port setting")
 		return nil
 	}
 
@@ -1696,6 +1724,14 @@ func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineT
 	if machineType != "q35" && machineType != "virt" {
 		return nil
 	}
+	if hypervisorType == virtcontainers.ClhHypervisor {
+		if coldPlug != config.NoPort {
+			return fmt.Errorf("cold-plug not supported on CLH")
+		}
+		if hotPlug != config.RootPort {
+			return fmt.Errorf("only hot-plug=%s supported on CLH", config.RootPort)
+		}
+	}
 
 	var port config.PCIePort
 	if coldPlug != config.NoPort {
@@ -1704,10 +1740,6 @@ func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineT
 	if hotPlug != config.NoPort {
 		port = hotPlug
 	}
-	if port == config.NoPort {
-		return fmt.Errorf("invalid vfio_port=%s setting, use on of %s, %s, %s",
-			port, config.BridgePort, config.RootPort, config.SwitchPort)
-	}
 	if port == config.BridgePort || port == config.RootPort || port == config.SwitchPort {
 		return nil
 	}

src/runtime/pkg/katautils/config_test.go

@@ -1735,3 +1735,59 @@ vfio_mode="vfio"
 	assert.Equal(t, config.Runtime.InterNetworkModel, "macvtap")
 	assert.Equal(t, config.Runtime.VfioMode, "vfio")
 }
+
+func TestUpdateRuntimeConfigHypervisor(t *testing.T) {
+	assert := assert.New(t)
+
+	type tableTypeEntry struct {
+		name  string
+		valid bool
+	}
+
+	configFile := "/some/where/configuration.toml"
+
+	// Note: We cannot test acrnHypervisorTableType since
+	// newAcrnHypervisorConfig() expects ACRN binaries to be
+	// installed.
+	var entries = []tableTypeEntry{
+		{clhHypervisorTableType, true},
+		{dragonballHypervisorTableType, true},
+		{firecrackerHypervisorTableType, true},
+		{qemuHypervisorTableType, true},
+		{"foo", false},
+		{"bar", false},
+		{clhHypervisorTableType + "baz", false},
+	}
+
+	for i, h := range entries {
+		config := oci.RuntimeConfig{}
+
+		tomlConf := tomlConfig{
+			Hypervisor: map[string]hypervisor{
+				h.name: {
+					NumVCPUs:       int32(2),
+					MemorySize:     uint32(2048),
+					Path:           "/",
+					Kernel:         "/",
+					Image:          "/",
+					Firmware:       "/",
+					FirmwareVolume: "/",
+					SharedFS:       "virtio-fs",
+					VirtioFSDaemon: "/usr/libexec/kata-qemu/virtiofsd",
+				},
+			},
+		}
+
+		err := updateRuntimeConfigHypervisor(configFile, tomlConf, &config)
+
+		if h.valid {
+			assert.NoError(err, "test %d (%+v)", i, h)
+		} else {
+			assert.Error(err, "test %d (%+v)", i, h)
+
+			expectedErr := fmt.Errorf("%v: %v: %+q", configFile, errInvalidHypervisorPrefix, h.name)
+
+			assert.Equal(err, expectedErr, "test %d (%+v)", i, h)
+		}
+	}
+}

src/runtime/pkg/katautils/create_test.go

@@ -18,8 +18,10 @@ import (
 	"syscall"
 	"testing"
 
+	config "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
@@ -419,3 +421,32 @@ func TestCreateContainer(t *testing.T) {
 		assert.NoError(err)
 	}
 }
+
+func TestVfioChecksClh(t *testing.T) {
+	assert := assert.New(t)
+
+	// Check valid CLH vfio configs
+	f := func(coldPlug, hotPlug config.PCIePort) error {
+		return checkPCIeConfig(coldPlug, hotPlug, defaultMachineType, virtcontainers.ClhHypervisor)
+	}
+	assert.NoError(f(config.NoPort, config.NoPort))
+	assert.NoError(f(config.NoPort, config.RootPort))
+	assert.Error(f(config.RootPort, config.RootPort))
+	assert.Error(f(config.RootPort, config.NoPort))
+	assert.Error(f(config.NoPort, config.SwitchPort))
+}
+
+func TestVfioCheckQemu(t *testing.T) {
+	assert := assert.New(t)
+
+	// Check valid Qemu vfio configs
+	f := func(coldPlug, hotPlug config.PCIePort) error {
+		return checkPCIeConfig(coldPlug, hotPlug, defaultMachineType, virtcontainers.QemuHypervisor)
+	}
+
+	assert.NoError(f(config.NoPort, config.NoPort))
+	assert.NoError(f(config.RootPort, config.NoPort))
+	assert.NoError(f(config.NoPort, config.RootPort))
+	assert.Error(f(config.RootPort, config.RootPort))
+	assert.Error(f(config.SwitchPort, config.RootPort))
+}

src/runtime/virtcontainers/acrn_arch_base.go

@@ -366,6 +366,7 @@ func (a *acrnArchBase) capabilities(config HypervisorConfig) types.Capabilities
 
 	caps.SetBlockDeviceSupport()
 	caps.SetBlockDeviceHotplugSupport()
+	caps.SetNetworkDeviceHotplugSupported()
 
 	return caps
 }

src/runtime/virtcontainers/acrn_arch_base_test.go

@@ -89,6 +89,7 @@ func TestAcrnArchBaseCapabilities(t *testing.T) {
 	assert.True(c.IsBlockDeviceSupported())
 	assert.True(c.IsBlockDeviceHotplugSupported())
 	assert.False(c.IsFsSharingSupported())
+	assert.True(c.IsNetworkDeviceHotplugSupported())
 }
 
 func TestAcrnArchBaseMemoryTopology(t *testing.T) {

src/runtime/virtcontainers/acrn_test.go

@@ -82,6 +82,7 @@ func TestAcrnCapabilities(t *testing.T) {
 	caps := a.Capabilities(a.ctx)
 	assert.True(caps.IsBlockDeviceSupported())
 	assert.True(caps.IsBlockDeviceHotplugSupported())
+	assert.True(caps.IsNetworkDeviceHotplugSupported())
 }
 
 func testAcrnAddDevice(t *testing.T, devInfo interface{}, devType DeviceType, expected []Device) {

src/runtime/virtcontainers/clh.go

@@ -490,6 +490,13 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 	}
 	clh.vmconfig.Payload.SetKernel(kernelPath)
 
+	clh.vmconfig.Platform = chclient.NewPlatformConfig()
+	platform := clh.vmconfig.Platform
+	platform.SetNumPciSegments(2)
+	if clh.config.IOMMU {
+		platform.SetIommuSegments([]int32{0})
+	}
+
 	if clh.config.ConfidentialGuest {
 		if err := clh.enableProtection(); err != nil {
 			return err
@@ -528,6 +535,9 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 		// start the guest kernel with 'quiet' in non-debug mode
 		params = append(params, Param{"quiet", ""})
 	}
+	if clh.config.IOMMU {
+		params = append(params, Param{"iommu", "pt"})
+	}
 
 	// Followed by extra kernel parameters defined in the configuration file
 	params = append(params, clh.config.KernelParams...)
@@ -536,16 +546,17 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 
 	// set random device generator to hypervisor
 	clh.vmconfig.Rng = chclient.NewRngConfig(clh.config.EntropySource)
+	clh.vmconfig.Rng.SetIommu(clh.config.IOMMU)
 
 	// set the initial root/boot disk of hypervisor
-	imagePath, err := clh.config.ImageAssetPath()
+	assetPath, assetType, err := clh.config.ImageOrInitrdAssetPath()
 	if err != nil {
 		return err
 	}
 
-	if imagePath != "" {
+	if assetType == types.ImageAsset {
 		if clh.config.ConfidentialGuest {
-			disk := chclient.NewDiskConfig(imagePath)
+			disk := chclient.NewDiskConfig(assetPath)
 			disk.SetReadonly(true)
 
 			diskRateLimiterConfig := clh.getDiskRateLimiterConfig()
@@ -559,8 +570,9 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 				clh.vmconfig.Disks = &[]chclient.DiskConfig{*disk}
 			}
 		} else {
-			pmem := chclient.NewPmemConfig(imagePath)
+			pmem := chclient.NewPmemConfig(assetPath)
 			*pmem.DiscardWrites = true
+			pmem.SetIommu(clh.config.IOMMU)
 
 			if clh.vmconfig.Pmem != nil {
 				*clh.vmconfig.Pmem = append(*clh.vmconfig.Pmem, *pmem)
@@ -569,12 +581,8 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 			}
 		}
 	} else {
-		initrdPath, err := clh.config.InitrdAssetPath()
-		if err != nil {
-			return err
-		}
-
-		clh.vmconfig.Payload.SetInitramfs(initrdPath)
+		// assetType == types.InitrdAsset
+		clh.vmconfig.Payload.SetInitramfs(assetPath)
 	}
 
 	if clh.config.ConfidentialGuest {
@@ -598,6 +606,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 
 		clh.vmconfig.Console = chclient.NewConsoleConfig(cctOFF)
 	}
+	clh.vmconfig.Console.SetIommu(clh.config.IOMMU)
 
 	cpu_topology := chclient.NewCpuTopology()
 	cpu_topology.ThreadsPerCore = func(i int32) *int32 { return &i }(1)
@@ -840,6 +849,7 @@ func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) erro
 	queueSize := int32(1024)
 	clhDisk.NumQueues = &queues
 	clhDisk.QueueSize = &queueSize
+	clhDisk.SetIommu(clh.config.IOMMU)
 
 	diskRateLimiterConfig := clh.getDiskRateLimiterConfig()
 	if diskRateLimiterConfig != nil {
@@ -865,6 +875,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error {
 
 	// Create the clh device config via the constructor to ensure default values are properly assigned
 	clhDevice := *chclient.NewDeviceConfig(device.SysfsDev)
+	clhDevice.SetIommu(clh.config.IOMMU)
 	pciInfo, _, err := cl.VmAddDevicePut(ctx, clhDevice)
 	if err != nil {
 		return fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err))
@@ -1210,6 +1221,7 @@ func (clh *cloudHypervisor) Capabilities(ctx context.Context) types.Capabilities
 		caps.SetFsSharingSupport()
 	}
 	caps.SetBlockDeviceHotplugSupport()
+	caps.SetNetworkDeviceHotplugSupported()
 	return caps
 }
 
@@ -1538,6 +1550,7 @@ func (clh *cloudHypervisor) addVSock(cid int64, path string) {
 	}).Info("Adding HybridVSock")
 
 	clh.vmconfig.Vsock = chclient.NewVsockConfig(cid, path)
+	clh.vmconfig.Vsock.SetIommu(clh.config.IOMMU)
 }
 
 func (clh *cloudHypervisor) getRateLimiterConfig(bwSize, bwOneTimeBurst, opsSize, opsOneTimeBurst int64) *chclient.RateLimiterConfig {
@@ -1607,6 +1620,7 @@ func (clh *cloudHypervisor) addNet(e Endpoint) error {
 	if netRateLimiterConfig != nil {
 		net.SetRateLimiterConfig(*netRateLimiterConfig)
 	}
+	net.SetIommu(clh.config.IOMMU)
 
 	if clh.netDevices != nil {
 		*clh.netDevices = append(*clh.netDevices, *net)
@@ -1639,6 +1653,7 @@ func (clh *cloudHypervisor) addVolume(volume types.Volume) error {
 	}
 
 	fs := chclient.NewFsConfig(volume.MountTag, vfsdSockPath, numQueues, queueSize)
+	fs.SetPciSegment(1)
 	clh.vmconfig.Fs = &[]chclient.FsConfig{*fs}
 
 	clh.Logger().Debug("Adding share volume to hypervisor: ", volume.MountTag)

src/runtime/virtcontainers/clh_test.go

@@ -68,6 +68,7 @@ func newClhConfig() (HypervisorConfig, error) {
 		NetRateLimiterBwOneTimeBurst:  int64(0),
 		NetRateLimiterOpsMaxRate:      int64(0),
 		NetRateLimiterOpsOneTimeBurst: int64(0),
+		HotPlugVFIO:                   config.NoPort,
 	}, nil
 }
 
@@ -752,4 +753,7 @@ func TestClhCapabilities(t *testing.T) {
 
 	c = clh.Capabilities(ctx)
 	assert.False(c.IsFsSharingSupported())
+
+	assert.True(c.IsNetworkDeviceHotplugSupported())
+	assert.True(c.IsBlockDeviceHotplugSupported())
 }

src/runtime/virtcontainers/fc.go

@@ -722,19 +722,12 @@ func (fc *firecracker) fcInitConfiguration(ctx context.Context) error {
 		return err
 	}
 
-	image, err := fc.config.InitrdAssetPath()
+	assetPath, _, err := fc.config.ImageOrInitrdAssetPath()
 	if err != nil {
 		return err
 	}
 
-	if image == "" {
-		image, err = fc.config.ImageAssetPath()
-		if err != nil {
-			return err
-		}
-	}
-
-	if err := fc.fcSetVMRootfs(ctx, image); err != nil {
+	if err := fc.fcSetVMRootfs(ctx, assetPath); err != nil {
 		return err
 	}
 

src/runtime/virtcontainers/hypervisor.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
+	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
 	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 
@@ -84,6 +85,7 @@ const (
 var (
 	hvLogger                   = logrus.WithField("source", "virtcontainers/hypervisor")
 	noGuestMemHotplugErr error = errors.New("guest memory hotplug not supported")
+	conflictingAssets    error = errors.New("cannot set both image and initrd at the same time")
 )
 
 // In some architectures the maximum number of vCPUs depends on the number of physical cores.
@@ -574,7 +576,7 @@ type HypervisorConfig struct {
 	EnableIOThreads bool
 
 	// Debug changes the default hypervisor and kernel parameters to
-	// enable debug output where available. And Debug also enable the hmp socket.
+	// enable debug output where available.
 	Debug bool
 
 	// MemPrealloc specifies if the memory should be pre-allocated
@@ -640,6 +642,9 @@ type HypervisorConfig struct {
 
 	// Use legacy serial for the guest console
 	LegacySerial bool
+
+	// ExtraMonitorSocket allows to add an extra HMP or QMP socket when the VMM is Qemu
+	ExtraMonitorSocket govmmQemu.MonitorProtocol
 }
 
 // vcpu mapping from vcpu number to thread number
@@ -698,6 +703,46 @@ func (conf *HypervisorConfig) AddCustomAsset(a *types.Asset) error {
 	return nil
 }
 
+// ImageOrInitrdAssetPath returns an image or an initrd path, along with the corresponding asset type
+// Annotation path is preferred to config path.
+func (conf *HypervisorConfig) ImageOrInitrdAssetPath() (string, types.AssetType, error) {
+	var image, initrd string
+
+	checkAndReturn := func(image string, initrd string) (string, types.AssetType, error) {
+		if image != "" && initrd != "" {
+			return "", types.UnkownAsset, conflictingAssets
+		}
+
+		if image != "" {
+			return image, types.ImageAsset, nil
+		}
+
+		if initrd != "" {
+			return initrd, types.InitrdAsset, nil
+		}
+
+		return "", types.UnkownAsset, fmt.Errorf("one of image and initrd must be set")
+	}
+
+	if a, ok := conf.customAssets[types.ImageAsset]; ok {
+		image = a.Path()
+	}
+
+	if a, ok := conf.customAssets[types.InitrdAsset]; ok {
+		initrd = a.Path()
+	}
+
+	path, assetType, err := checkAndReturn(image, initrd)
+	if assetType != types.UnkownAsset {
+		return path, assetType, nil
+	}
+	if err == conflictingAssets {
+		return "", types.UnkownAsset, errors.Wrapf(err, "conflicting annotations")
+	}
+
+	return checkAndReturn(conf.ImagePath, conf.InitrdPath)
+}
+
 func (conf *HypervisorConfig) assetPath(t types.AssetType) (string, error) {
 	// Custom assets take precedence over the configured ones
 	a, ok := conf.customAssets[t]

src/runtime/virtcontainers/qemu.go

@@ -119,11 +119,11 @@ type qemu struct {
 }
 
 const (
-	consoleSocket = "console.sock"
-	qmpSocket     = "qmp.sock"
-	hmpSocket     = "hmp.sock"
-	vhostFSSocket = "vhost-fs.sock"
-	nydusdAPISock = "nydusd-api.sock"
+	consoleSocket      = "console.sock"
+	qmpSocket          = "qmp.sock"
+	extraMonitorSocket = "extra-monitor.sock"
+	vhostFSSocket      = "vhost-fs.sock"
+	nydusdAPISock      = "nydusd-api.sock"
 
 	// memory dump format will be set to elf
 	memoryDumpFormat = "elf"
@@ -329,8 +329,8 @@ func (q *qemu) qmpSocketPath(id string) (string, error) {
 	return utils.BuildSocketPath(q.config.VMStorePath, id, qmpSocket)
 }
 
-func (q *qemu) hmpSocketPath(id string) (string, error) {
-	return utils.BuildSocketPath(q.config.VMStorePath, id, hmpSocket)
+func (q *qemu) extraMonitorSocketPath(id string) (string, error) {
+	return utils.BuildSocketPath(q.config.VMStorePath, id, extraMonitorSocket)
 }
 
 func (q *qemu) getQemuMachine() (govmmQemu.Machine, error) {
@@ -347,22 +347,6 @@ func (q *qemu) getQemuMachine() (govmmQemu.Machine, error) {
 	return machine, nil
 }
 
-func (q *qemu) appendImage(ctx context.Context, devices []govmmQemu.Device) ([]govmmQemu.Device, error) {
-	imagePath, err := q.config.ImageAssetPath()
-	if err != nil {
-		return nil, err
-	}
-
-	if imagePath != "" {
-		devices, err = q.arch.appendImage(ctx, devices, imagePath)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return devices, nil
-}
-
 func (q *qemu) createQmpSocket() ([]govmmQemu.QMPSocket, error) {
 	monitorSockPath, err := q.qmpSocketPath(q.id)
 	if err != nil {
@@ -377,35 +361,46 @@ func (q *qemu) createQmpSocket() ([]govmmQemu.QMPSocket, error) {
 	var sockets []govmmQemu.QMPSocket
 
 	sockets = append(sockets, govmmQemu.QMPSocket{
-		Type:   "unix",
-		Server: true,
-		NoWait: true,
+		Type:     "unix",
+		Protocol: govmmQemu.Qmp,
+		Server:   true,
+		NoWait:   true,
 	})
 
-	if q.HypervisorConfig().Debug {
-		humanMonitorSockPath, err := q.hmpSocketPath(q.id)
+	// The extra monitor socket allows an external user to take full
+	// control on Qemu and silently break the VM in all possible ways.
+	// It should only ever be used for debugging purposes, hence the
+	// check on Debug.
+	if q.HypervisorConfig().Debug && q.config.ExtraMonitorSocket != "" {
+		extraMonitorSockPath, err := q.extraMonitorSocketPath(q.id)
 		if err != nil {
 			return nil, err
 		}
 
 		sockets = append(sockets, govmmQemu.QMPSocket{
-			Type:   "unix",
-			IsHmp:  true,
-			Name:   humanMonitorSockPath,
-			Server: true,
-			NoWait: true,
+			Type:     "unix",
+			Protocol: q.config.ExtraMonitorSocket,
+			Name:     extraMonitorSockPath,
+			Server:   true,
+			NoWait:   true,
 		})
+
+		q.Logger().Warn("QEMU configured to start with an untrusted monitor")
 	}
 
 	return sockets, nil
 }
 
-func (q *qemu) buildDevices(ctx context.Context, initrdPath string) ([]govmmQemu.Device, *govmmQemu.IOThread, error) {
+func (q *qemu) buildDevices(ctx context.Context, kernelPath string) ([]govmmQemu.Device, *govmmQemu.IOThread, *govmmQemu.Kernel, error) {
 	var devices []govmmQemu.Device
 
+	kernel := &govmmQemu.Kernel{
+		Path: kernelPath,
+	}
+
 	_, console, err := q.GetVMConsole(ctx, q.id)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	// Add bridges before any other devices. This way we make sure that
@@ -414,20 +409,28 @@ func (q *qemu) buildDevices(ctx context.Context, initrdPath string) ([]govmmQemu
 
 	devices, err = q.arch.appendConsole(ctx, devices, console)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
-	if initrdPath == "" {
-		devices, err = q.appendImage(ctx, devices)
+	assetPath, assetType, err := q.config.ImageOrInitrdAssetPath()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	if assetType == types.ImageAsset {
+		devices, err = q.arch.appendImage(ctx, devices, assetPath)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
+	} else {
+		// InitrdAsset, need to set kernel initrd path
+		kernel.InitrdPath = assetPath
 	}
 
 	if q.config.IOMMU {
 		devices, err = q.arch.appendIOMMU(devices)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 	}
 
@@ -438,10 +441,13 @@ func (q *qemu) buildDevices(ctx context.Context, initrdPath string) ([]govmmQemu
 
 	var ioThread *govmmQemu.IOThread
 	if q.config.BlockDeviceDriver == config.VirtioSCSI {
-		return q.arch.appendSCSIController(ctx, devices, q.config.EnableIOThreads)
+		devices, ioThread, err = q.arch.appendSCSIController(ctx, devices, q.config.EnableIOThreads)
+		if err != nil {
+			return nil, nil, nil, err
+		}
 	}
 
-	return devices, ioThread, nil
+	return devices, ioThread, kernel, nil
 }
 
 func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) govmmQemu.Incoming {
@@ -514,7 +520,7 @@ func (q *qemu) createVirtiofsDaemon(sharedPath string) (VirtiofsDaemon, error) {
 	// in virtiofs if SELinux on the guest side is enabled.
 	if !q.config.DisableGuestSeLinux {
 		q.Logger().Info("Set the xattr option for virtiofsd")
-		q.config.VirtioFSExtraArgs = append(q.config.VirtioFSExtraArgs, "-o", "xattr")
+		q.config.VirtioFSExtraArgs = append(q.config.VirtioFSExtraArgs, "--xattr")
 	}
 
 	// default use virtiofsd
@@ -562,16 +568,6 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 		IOMMUPlatform: q.config.IOMMUPlatform,
 	}
 
-	kernelPath, err := q.config.KernelAssetPath()
-	if err != nil {
-		return err
-	}
-
-	initrdPath, err := q.config.InitrdAssetPath()
-	if err != nil {
-		return err
-	}
-
 	incoming := q.setupTemplate(&knobs, &memory)
 
 	// With the current implementations, VM templating will not work with file
@@ -615,7 +611,36 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 		return err
 	}
 
-	devices, ioThread, err := q.buildDevices(ctx, initrdPath)
+	if q.config.ConfidentialGuest {
+		// At this point we're safe to just check for the protection field
+		// on the hypervisor specific code, as availableGuestProtection()
+		// has been called earlier and we know we have the value stored.
+		if q.arch.getProtection() == tdxProtection {
+			knobs.MemFDPrivate = true
+
+			// In case Nydus or VirtioFS is used, which may become a reality
+			// in the future, whenever we get those hardened for TDX, those
+			// knobs below would be automatically set.  Let's make sure we
+			// pre-emptively disable them, and with that we can avoid some
+			// headaches in the future.
+			knobs.FileBackedMem = false
+			knobs.MemShared = false
+
+			// SMP is currently broken with TDX 1.5, and
+			// we must ensure we use something like:
+			// `...,sockets=1,cores=numvcpus,threads=1,...`
+			smp.Sockets = 1
+			smp.Cores = q.config.NumVCPUs
+			smp.Threads = 1
+		}
+	}
+
+	kernelPath, err := q.config.KernelAssetPath()
+	if err != nil {
+		return err
+	}
+
+	devices, ioThread, kernel, err := q.buildDevices(ctx, kernelPath)
 	if err != nil {
 		return err
 	}
@@ -643,13 +668,8 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 		return err
 	}
 
-	// Breaks hypervisor abstraction has Kata Specific logic
-	kernel := govmmQemu.Kernel{
-		Path:       kernelPath,
-		InitrdPath: initrdPath,
-		// some devices configuration may also change kernel params, make sure this is called afterwards
-		Params: q.kernelParameters(),
-	}
+	// some devices configuration may also change kernel params, make sure this is called afterwards
+	kernel.Params = q.kernelParameters()
 	q.checkBpfEnabled()
 
 	qemuConfig := govmmQemu.Config{
@@ -666,7 +686,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 		Devices:        devices,
 		CPUModel:       cpuModel,
 		SeccompSandbox: q.config.SeccompSandbox,
-		Kernel:         kernel,
+		Kernel:         *kernel,
 		RTC:            rtc,
 		QMPSockets:     qmpSockets,
 		Knobs:          knobs,

src/runtime/virtcontainers/qemu_amd64.go

@@ -162,6 +162,7 @@ func (q *qemuAmd64) capabilities(hConfig HypervisorConfig) types.Capabilities {
 	if q.qemuMachine.Type == QemuQ35 ||
 		q.qemuMachine.Type == QemuVirt {
 		caps.SetBlockDeviceHotplugSupport()
+		caps.SetNetworkDeviceHotplugSupported()
 	}
 
 	caps.SetMultiQueueSupport()

src/runtime/virtcontainers/qemu_amd64_test.go

@@ -47,10 +47,12 @@ func TestQemuAmd64Capabilities(t *testing.T) {
 	amd64 := newTestQemu(assert, QemuQ35)
 	caps := amd64.capabilities(config)
 	assert.True(caps.IsBlockDeviceHotplugSupported())
+	assert.True(caps.IsNetworkDeviceHotplugSupported())
 
 	amd64 = newTestQemu(assert, QemuMicrovm)
 	caps = amd64.capabilities(config)
 	assert.False(caps.IsBlockDeviceHotplugSupported())
+	assert.False(caps.IsNetworkDeviceHotplugSupported())
 }
 
 func TestQemuAmd64Bridges(t *testing.T) {

src/runtime/virtcontainers/qemu_arch_base.go

@@ -71,6 +71,9 @@ type qemuArch interface {
 	// memoryTopology returns the memory topology using the given amount of memoryMb and hostMemoryMb
 	memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) govmmQemu.Memory
 
+	// protection returns platform protection
+	getProtection() guestProtection
+
 	// appendConsole appends a console to devices
 	appendConsole(ctx context.Context, devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error)
 
@@ -280,6 +283,10 @@ func (q *qemuArchBase) machine() govmmQemu.Machine {
 	return q.qemuMachine
 }
 
+func (q *qemuArchBase) getProtection() guestProtection {
+	return q.protection
+}
+
 func (q *qemuArchBase) qemuPath() string {
 	return q.qemuExePath
 }
@@ -300,6 +307,7 @@ func (q *qemuArchBase) capabilities(hConfig HypervisorConfig) types.Capabilities
 	var caps types.Capabilities
 	caps.SetBlockDeviceHotplugSupport()
 	caps.SetMultiQueueSupport()
+	caps.SetNetworkDeviceHotplugSupported()
 	if hConfig.SharedFS != config.NoSharedFS {
 		caps.SetFsSharingSupport()
 	}

src/runtime/virtcontainers/qemu_arch_base_test.go

@@ -123,6 +123,7 @@ func TestQemuArchBaseCapabilities(t *testing.T) {
 	c := qemuArchBase.capabilities(hConfig)
 	assert.True(c.IsBlockDeviceHotplugSupported())
 	assert.True(c.IsFsSharingSupported())
+	assert.True(c.IsNetworkDeviceHotplugSupported())
 
 	hConfig.SharedFS = config.NoSharedFS
 	c = qemuArchBase.capabilities(hConfig)

src/runtime/virtcontainers/qemu_ppc64le.go

@@ -104,6 +104,7 @@ func (q *qemuPPC64le) capabilities(hConfig HypervisorConfig) types.Capabilities
 	// pseries machine type supports hotplugging drives
 	if q.qemuMachine.Type == QemuPseries {
 		caps.SetBlockDeviceHotplugSupport()
+		caps.SetNetworkDeviceHotplugSupported()
 	}
 
 	caps.SetMultiQueueSupport()

src/runtime/virtcontainers/qemu_test.go

@@ -475,6 +475,7 @@ func TestQemuCapabilities(t *testing.T) {
 
 	caps := q.Capabilities(q.ctx)
 	assert.True(caps.IsBlockDeviceHotplugSupported())
+	assert.True(caps.IsNetworkDeviceHotplugSupported())
 }
 
 func TestQemuQemuPath(t *testing.T) {

src/runtime/virtcontainers/sandbox.go

@@ -933,6 +933,17 @@ func (s *Sandbox) createNetwork(ctx context.Context) error {
 		return nil
 	}
 
+	// docker container needs the hypervisor process ID to find out the container netns,
+	// which means that the hypervisor has to support network device hotplug so that docker
+	// can use the prestart hooks to set up container netns.
+	caps := s.hypervisor.Capabilities(ctx)
+	if !caps.IsNetworkDeviceHotplugSupported() {
+		spec := s.GetPatchedOCISpec()
+		if utils.IsDockerContainer(spec) {
+			return errors.New("docker container needs network device hotplug but the configured hypervisor does not support it")
+		}
+	}
+
 	span, ctx := katatrace.Trace(ctx, s.Logger(), "createNetwork", sandboxTracingTags, map[string]string{"sandbox_id": s.id})
 	defer span.End()
 	katatrace.AddTags(span, "network", s.network, "NetworkConfig", s.config.NetworkConfig)
@@ -1288,6 +1299,22 @@ func (s *Sandbox) cleanSwap(ctx context.Context) {
 	}
 }
 
+func (s *Sandbox) runPrestartHooks(ctx context.Context, prestartHookFunc func(context.Context) error) error {
+	hid, _ := s.GetHypervisorPid()
+	// Ignore errors here as hypervisor might not have been started yet, likely in FC case.
+	if hid > 0 {
+		s.Logger().Infof("sandbox %s hypervisor pid is %v", s.id, hid)
+		ctx = context.WithValue(ctx, HypervisorPidKey{}, hid)
+	}
+
+	if err := prestartHookFunc(ctx); err != nil {
+		s.Logger().Errorf("fail to run prestartHook for sandbox %s: %s", s.id, err)
+		return err
+	}
+
+	return nil
+}
+
 // startVM starts the VM.
 func (s *Sandbox) startVM(ctx context.Context, prestartHookFunc func(context.Context) error) (err error) {
 	span, ctx := katatrace.Trace(ctx, s.Logger(), "startVM", sandboxTracingTags, map[string]string{"sandbox_id": s.id})
@@ -1310,6 +1337,17 @@ func (s *Sandbox) startVM(ctx context.Context, prestartHookFunc func(context.Con
 		}
 	}()
 
+	caps := s.hypervisor.Capabilities(ctx)
+	// If the hypervisor does not support device hotplug, run prestart hooks
+	// before spawning the VM so that it is possible to let the hooks set up
+	// netns and thus network devices are set up statically.
+	if !caps.IsNetworkDeviceHotplugSupported() && prestartHookFunc != nil {
+		err = s.runPrestartHooks(ctx, prestartHookFunc)
+		if err != nil {
+			return err
+		}
+	}
+
 	if err := s.network.Run(ctx, func() error {
 		if s.factory != nil {
 			vm, err := s.factory.GetVM(ctx, VMConfig{
@@ -1329,24 +1367,22 @@ func (s *Sandbox) startVM(ctx context.Context, prestartHookFunc func(context.Con
 		return err
 	}
 
-	if prestartHookFunc != nil {
-		hid, err := s.GetHypervisorPid()
+	if caps.IsNetworkDeviceHotplugSupported() && prestartHookFunc != nil {
+		err = s.runPrestartHooks(ctx, prestartHookFunc)
 		if err != nil {
 			return err
 		}
-		s.Logger().Infof("hypervisor pid is %v", hid)
-		ctx = context.WithValue(ctx, HypervisorPidKey{}, hid)
-
-		if err := prestartHookFunc(ctx); err != nil {
-			return err
-		}
 	}
 
-	// 1. Do not scan the netns if we want no network for the vmm.
-	// 2. In case of vm factory, scan the netns to hotplug interfaces after vm is started.
-	// 3. In case of prestartHookFunc, network config might have been changed. We need to
+	// 1. Do not scan the netns if we want no network for the vmm
+	// 2. Do not scan the netns if the vmm does not support device hotplug, in which case
+	//    the network is already set up statically
+	// 3. In case of vm factory, scan the netns to hotplug interfaces after vm is started.
+	// 4. In case of prestartHookFunc, network config might have been changed. We need to
 	//    rescan and handle the change.
-	if !s.config.NetworkConfig.DisableNewNetwork && (s.factory != nil || prestartHookFunc != nil) {
+	if !s.config.NetworkConfig.DisableNewNetwork &&
+		caps.IsNetworkDeviceHotplugSupported() &&
+		(s.factory != nil || prestartHookFunc != nil) {
 		if _, err := s.network.AddEndpoints(ctx, s, nil, true); err != nil {
 			return err
 		}
@@ -1684,12 +1720,14 @@ func (s *Sandbox) Stats(ctx context.Context) (SandboxStats, error) {
 
 	// TODO Do we want to aggregate the overhead cgroup stats to the sandbox ones?
 	switch mt := metrics.(type) {
-	case v1.Metrics:
+	case *v1.Metrics:
 		stats.CgroupStats.CPUStats.CPUUsage.TotalUsage = mt.CPU.Usage.Total
 		stats.CgroupStats.MemoryStats.Usage.Usage = mt.Memory.Usage.Usage
-	case v2.Metrics:
+	case *v2.Metrics:
 		stats.CgroupStats.CPUStats.CPUUsage.TotalUsage = mt.CPU.UsageUsec
 		stats.CgroupStats.MemoryStats.Usage.Usage = mt.Memory.Usage
+	default:
+		return SandboxStats{}, fmt.Errorf("unknown metrics type %T", mt)
 	}
 
 	tids, err := s.hypervisor.GetThreadIDs(ctx)

src/runtime/virtcontainers/types/asset.go

@@ -41,6 +41,8 @@ const (
 	FirmwareAsset AssetType = "firmware"
 
 	FirmwareVolumeAsset AssetType = "firmware_volume"
+
+	UnkownAsset AssetType = "unknown"
 )
 
 // AssetTypes returns a list of all known asset types.

src/runtime/virtcontainers/types/capabilities.go

@@ -10,6 +10,7 @@ const (
 	blockDeviceHotplugSupport
 	multiQueueSupport
 	fsSharingSupported
+	networkDeviceHotplugSupport
 )
 
 // Capabilities describe a virtcontainers hypervisor capabilities
@@ -57,3 +58,13 @@ func (caps *Capabilities) IsFsSharingSupported() bool {
 func (caps *Capabilities) SetFsSharingSupport() {
 	caps.flags |= fsSharingSupported
 }
+
+// IsDeviceHotplugSupported tells if an hypervisor supports device hotplug.
+func (caps *Capabilities) IsNetworkDeviceHotplugSupported() bool {
+	return caps.flags&networkDeviceHotplugSupport != 0
+}
+
+// SetDeviceHotplugSupported sets the host filesystem sharing capability to true.
+func (caps *Capabilities) SetNetworkDeviceHotplugSupported() {
+	caps.flags |= networkDeviceHotplugSupport
+}

src/runtime/virtcontainers/utils/utils.go

@@ -12,9 +12,11 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"time"
 
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
@@ -494,3 +496,21 @@ func RevertBytes(num uint64) uint64 {
 	}
 	return 1024*RevertBytes(a) + b
 }
+
+// IsDockerContainer returns if the container is managed by docker
+// This is done by checking the prestart hook for `libnetwork` arguments.
+func IsDockerContainer(spec *specs.Spec) bool {
+	if spec == nil || spec.Hooks == nil {
+		return false
+	}
+
+	for _, hook := range spec.Hooks.Prestart {
+		for _, arg := range hook.Args {
+			if strings.HasPrefix(arg, "libnetwork") {
+				return true
+			}
+		}
+	}
+
+	return false
+}

src/runtime/virtcontainers/utils/utils_test.go

@@ -16,6 +16,7 @@ import (
 	"syscall"
 	"testing"
 
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
@@ -580,3 +581,25 @@ func TestRevertBytes(t *testing.T) {
 	num := RevertBytes(testNum)
 	assert.Equal(expectedNum, num)
 }
+
+func TestIsDockerContainer(t *testing.T) {
+	assert := assert.New(t)
+
+	ociSpec := &specs.Spec{
+		Hooks: &specs.Hooks{
+			Prestart: []specs.Hook{
+				{
+					Args: []string{
+						"haha",
+					},
+				},
+			},
+		},
+	}
+	assert.False(IsDockerContainer(ociSpec))
+
+	ociSpec.Hooks.Prestart = append(ociSpec.Hooks.Prestart, specs.Hook{
+		Args: []string{"libnetwork-xxx"},
+	})
+	assert.True(IsDockerContainer(ociSpec))
+}

src/runtime/virtcontainers/virtiofsd.go

@@ -39,7 +39,6 @@ var (
 
 const (
 	typeVirtioFSCacheModeNever  = "never"
-	typeVirtioFSCacheModeNone   = "none"
 	typeVirtioFSCacheModeAlways = "always"
 	typeVirtioFSCacheModeAuto   = "auto"
 )
@@ -219,9 +218,6 @@ func (v *virtiofsd) valid() error {
 
 	if v.cache == "" {
 		v.cache = typeVirtioFSCacheModeAuto
-	} else if v.cache == typeVirtioFSCacheModeNone {
-		v.Logger().Warn("virtio-fs cache mode `none` is deprecated since Kata Containers 2.5.0 and will be removed in the future release, please use `never` instead. For more details please refer to https://github.com/kata-containers/kata-containers/issues/4234.")
-		v.cache = typeVirtioFSCacheModeNever
 	} else if v.cache != typeVirtioFSCacheModeAuto && v.cache != typeVirtioFSCacheModeAlways && v.cache != typeVirtioFSCacheModeNever {
 		return errVirtiofsdInvalidVirtiofsCacheMode(v.cache)
 	}

src/runtime/virtcontainers/virtiofsd_test.go

@@ -76,15 +76,15 @@ func TestVirtiofsdArgs(t *testing.T) {
 	v := &virtiofsd{
 		path:       "/usr/bin/virtiofsd",
 		sourcePath: "/run/kata-shared/foo",
-		cache:      "none",
+		cache:      "never",
 	}
 
-	expected := "--syslog --cache=none --shared-dir=/run/kata-shared/foo --fd=123"
+	expected := "--syslog --cache=never --shared-dir=/run/kata-shared/foo --fd=123"
 	args, err := v.args(123)
 	assert.NoError(err)
 	assert.Equal(expected, strings.Join(args, " "))
 
-	expected = "--syslog --cache=none --shared-dir=/run/kata-shared/foo --fd=456"
+	expected = "--syslog --cache=never --shared-dir=/run/kata-shared/foo --fd=456"
 	args, err = v.args(456)
 	assert.NoError(err)
 	assert.Equal(expected, strings.Join(args, " "))
@@ -130,12 +130,7 @@ func TestValid(t *testing.T) {
 		{"source is not available", func(v *virtiofsd) {
 			v.sourcePath = "/foo/bar"
 		}, errVirtiofsdSourceNotAvailable, nil},
-		{"replace cache mode none by never", func(v *virtiofsd) {
-			v.cache = "none"
-		}, nil, func(name string, a *assert.Assertions, v *virtiofsd) {
-			a.Equal("never", v.cache, "test case %+s, cache mode none should be replaced by never", name)
-		}},
-		{"invald cache mode: replace none by never", func(v *virtiofsd) {
+		{"invalid cache mode", func(v *virtiofsd) {
 			v.cache = "foo"
 		}, errVirtiofsdInvalidVirtiofsCacheMode("foo"), nil},
 	}

src/tools/kata-ctl/Cargo.lock

@@ -1946,6 +1946,7 @@ dependencies = [
  "anyhow",
  "hyper",
  "hyperlocal",
+ "kata-types",
  "tokio",
 ]
 

src/tools/kata-ctl/Makefile

@@ -52,13 +52,13 @@ clean:
 vendor:
 	cargo vendor
 
-test:
+test: $(GENERATED_CODE)
 	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo test --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 install:
 	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path . --root $(INSTALL_PATH)
 
-check: standard_rust_check
+check: $(GENERATED_CODE) standard_rust_check
 
 .PHONY: \
 	build \

src/tools/kata-ctl/src/check.rs

@@ -539,10 +539,10 @@ mod tests {
             },
             // Success scenarios
             TestData {
-                module_name: "kvm",
+                module_name: "loop",
                 param_name: "",
                 kernel_module: &KernelModule {
-                    name: "kvm",
+                    name: "loop",
                     params: &[KernelParam {
                         name: "nonexistantparam",
                         value: KernelParamType::Simple("Y"),
@@ -552,16 +552,16 @@ mod tests {
                 result: Ok(()),
             },
             TestData {
-                module_name: "kvm",
-                param_name: "kvmclock_periodic_sync",
+                module_name: "loop",
+                param_name: "hw_queue_depth",
                 kernel_module: &KernelModule {
-                    name: "kvm",
+                    name: "loop",
                     params: &[KernelParam {
-                        name: "kvmclock_periodic_sync",
-                        value: KernelParamType::Simple("Y"),
+                        name: "hw_queue_depth",
+                        value: KernelParamType::Simple("128"),
                     }],
                 },
-                param_value: "Y",
+                param_value: "128",
                 result: Ok(()),
             },
         ];

tests/common.bash

@@ -158,7 +158,7 @@ function clean_env_ctr()
 	info "Wait until the containers gets removed"
 
 	for task_id in "${running_tasks[@]}"; do
-		sudo ctr t kill -a -s SIGTERM ${task_id} >/dev/null 2>&1
+		sudo timeout -s SIGKILL 30s ctr t kill -a -s SIGTERM ${task_id} >/dev/null 2>&1 || true
 		sleep 0.5
 	done
 
@@ -186,31 +186,19 @@ function clean_env_ctr()
 	if (( count_tasks > 0 )); then
 		die "Can't remove running containers."
 	fi
-
-	kill_kata_components
 }
 
 # Kills running shim and hypervisor components
 function kill_kata_components() {
-	local kata_bin_dir="/opt/kata/bin"
-	local shim_path="${kata_bin_dir}/containerd-shim-kata-v2"
-	local hypervisor_path="${kata_bin_dir}/qemu-system-x86_64"
-	local pid_shim_count="$(pgrep -fc ${shim_path} || exit 0)"
+	local TIMEOUT="30s"
+	local PID_NAMES=( "containerd-shim-kata-v2" "qemu-system-x86_64" "cloud-hypervisor" )
 
-	[ ${pid_shim_count} -gt "0" ] && sudo kill -SIGKILL "$(pgrep -f ${shim_path})" > /dev/null 2>&1
-
-        if [ "${KATA_HYPERVISOR}" = 'clh' ]; then
-		hypervisor_path="${kata_bin_dir}/cloud-hypervisor"
-	elif [ "${KATA_HYPERVISOR}" != 'qemu' ]; then
-                echo "Failed to stop the hypervisor: '${KATA_HYPERVISOR}' as it is not recognized"
-		return
-        fi
-
-	local pid_hypervisor_count="$(pgrep -fc ${hypervisor_path} || exit 0)"
-
-	if [ ${pid_hypervisor_count} -gt "0" ]; then
-		sudo kill -SIGKILL "$(pgrep -f ${hypervisor_path})" > /dev/null 2>&1
-	fi
+	sudo systemctl stop containerd
+	# iterate over the list of kata components and stop them
+	for PID_NAME in "${PID_NAMES[@]}"; do
+		[[ ! -z "$(pidof ${PID_NAME})" ]] && sudo killall "${PID_NAME}" > /dev/null 2>&1 || true
+	done
+	sudo timeout -s SIGKILL "${TIMEOUT}" systemctl start containerd
 }
 
 # Restarts a systemd service while ensuring the start-limit-burst is set to 0.
@@ -269,25 +257,71 @@ function restart_containerd_service() {
 	return 0
 }
 
+function restart_crio_service() {
+	sudo systemctl restart crio
+}
+
 # Configures containerd
 function overwrite_containerd_config() {
 	containerd_config="/etc/containerd/config.toml"
 	sudo rm -f "${containerd_config}"
 	sudo tee "${containerd_config}" << EOF
 version = 2
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
-  SystemdCgroup = true
 
 [plugins]
   [plugins."io.containerd.grpc.v1.cri"]
     [plugins."io.containerd.grpc.v1.cri".containerd]
-      default_runtime_name = "kata"
       [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          base_runtime_spec = ""
+          cni_conf_dir = ""
+          cni_max_conf_num = 0
+          container_annotations = []
+          pod_annotations = []
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_path = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = ""
+            CriuImagePath = ""
+            CriuPath = ""
+            CriuWorkPath = ""
+            IoGid = 0
+            IoUid = 0
+            NoNewKeyring = false
+            NoPivotRoot = false
+            Root = ""
+            ShimCgroup = ""
+            SystemdCgroup = false
         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
           runtime_type = "io.containerd.kata.v2"
 EOF
 }
 
+# Configures CRI-O
+function overwrite_crio_config() {
+	crio_conf_d="/etc/crio/crio.conf.d"
+	sudo mkdir -p ${crio_conf_d}
+
+	kata_config="${crio_conf_d}/99-kata-containers"
+	sudo tee "${kata_config}" << EOF
+[crio.runtime.runtimes.kata]
+runtime_path = "/usr/local/bin/containerd-shim-kata-v2"
+runtime_type = "vm"
+runtime_root = "/run/vc"
+runtime_config_path = "/opt/kata/share/defaults/kata-containers/configuration.toml"
+privileged_without_host_devices = true
+EOF
+
+	debug_config="${crio_conf_d}/100-debug"
+	sudo tee "${debug_config}" << EOF
+[crio]
+log_level = "debug"
+EOF
+}
+
 function install_kata() {
 	local kata_tarball="kata-static.tar.xz"
 	declare -r katadir="/opt/kata"
@@ -306,18 +340,35 @@ function install_kata() {
 		sudo ln -sf "${b}" "${local_bin_dir}/$(basename $b)"
 	done
 
-	if [[ ${KATA_HYPERVISOR} == "dragonball" ]]; then
-		sudo ln -sf "${katadir}/runtime-rs/bin/containerd-shim-kata-v2" "${local_bin_dir}/containerd-shim-kata-${KATA_HYPERVISOR}-v2"
+	if [ "${CONTAINER_ENGINE:=containerd}" = "containerd" ]; then
+		check_containerd_config_for_kata
+		restart_containerd_service
 	else
-		sudo ln -sf "${katadir}/bin/containerd-shim-kata-v2" "${local_bin_dir}/containerd-shim-kata-${KATA_HYPERVISOR}-v2"
+		overwrite_crio_config
+		restart_crio_service
 	fi
 
-	sudo ln -sf ${katadir}/share/defaults/kata-containers/configuration-${KATA_HYPERVISOR}.toml ${katadir}/share/defaults/kata-containers/configuration.toml 
-
-	check_containerd_config_for_kata
-	restart_containerd_service
 }
 
+# creates a new kata configuration.toml hard link that
+# points to the hypervisor passed by KATA_HYPERVISOR env var.
+function enabling_hypervisor() {
+	declare -r KATA_DIR="/opt/kata"
+	declare -r CONFIG_DIR="${KATA_DIR}/share/defaults/kata-containers"
+	declare -r SRC_HYPERVISOR_CONFIG="${CONFIG_DIR}/configuration-${KATA_HYPERVISOR}.toml"
+	declare -r DEST_KATA_CONFIG="${CONFIG_DIR}/configuration.toml"
+	declare -r CONTAINERD_SHIM_KATA="/usr/local/bin/containerd-shim-kata-${KATA_HYPERVISOR}-v2"
+
+	if [[ ${KATA_HYPERVISOR} == "dragonball" ]]; then
+		sudo ln -sf "${KATA_DIR}/runtime-rs/bin/containerd-shim-kata-v2" "${CONTAINERD_SHIM_KATA}"
+	else
+		sudo ln -sf "${KATA_DIR}/bin/containerd-shim-kata-v2" "${CONTAINERD_SHIM_KATA}"
+	fi
+
+	sudo ln -sf "${SRC_HYPERVISOR_CONFIG}" "${DEST_KATA_CONFIG}"
+}
+
+
 function check_containerd_config_for_kata() {
 	# check containerd config
 	declare -r line1="default_runtime_name = \"kata\""
@@ -339,6 +390,7 @@ function ensure_yq() {
     export GOPATH
     export PATH="${GOPATH}/bin:${PATH}"
     INSTALL_IN_GOPATH=true "${repo_root_dir}/ci/install_yq.sh"
+    hash -d yq 2> /dev/null || true # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache.
 }
 
 # dependency: What we want to get the version from the versions.yaml file
@@ -383,6 +435,19 @@ function download_github_project_tarball() {
 	wget https://github.com/${project}/releases/download/${version}/${tarball_name}
 }
 
+# version: The version to be intalled
+function install_cni_plugins() {
+	version="${1}"
+
+	project="containernetworking/plugins"
+	tarball_name="cni-plugins-linux-$(${repo_root_dir}/tests/kata-arch.sh -g)-${version}.tgz"
+
+	download_github_project_tarball "${project}" "${version}" "${tarball_name}"
+	sudo mkdir -p /opt/cni/bin
+	sudo tar -xvf "${tarball_name}" -C /opt/cni/bin
+	rm -f "${tarball_name}"
+}
+
 # base_version: The version to be intalled in the ${major}.${minor} format
 function install_cri_containerd() {
 	base_version="${1}"
@@ -413,3 +478,110 @@ function install_cri_tools() {
 	sudo tar -xvf "${tarball_name}" -C /usr/local/bin
 	rm -f "${tarball_name}"
 }
+
+function install_nydus() {
+	version="${1}"
+
+	project="dragonflyoss/image-service"
+	tarball_name="nydus-static-${version}-linux-$(${repo_root_dir}/tests/kata-arch.sh -g).tgz"
+
+	download_github_project_tarball "${project}" "${version}" "${tarball_name}"
+	sudo tar xfz "${tarball_name}" -C /usr/local/bin --strip-components=1
+	rm -f "${tarball_name}"
+}
+
+function install_nydus_snapshotter() {
+	version="${1}"
+
+	project="containerd/nydus-snapshotter"
+	tarball_name="nydus-snapshotter-${version}-$(${repo_root_dir}/tests/kata-arch.sh).tgz"
+
+	download_github_project_tarball "${project}" "${version}" "${tarball_name}"
+	sudo tar xfz "${tarball_name}" -C /usr/local/bin --strip-components=1
+	rm -f "${tarball_name}"
+}
+
+function _get_os_for_crio() {
+	source /etc/os-release
+
+	if [ "${NAME}" != "Ubuntu" ]; then
+		echo "Only Ubuntu is supported for now"
+		exit 2
+	fi
+
+	echo "x${NAME}_${VERSION_ID}"
+}
+
+# version: the CRI-O version to be installe
+function install_crio() {
+	local version=${1}
+
+	os=$(_get_os_for_crio)
+
+	echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/${os}/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+	echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/${version}/${os}/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:${version}.list
+	curl -L https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:${version}/${os}/Release.key | sudo apt-key add -
+	curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/${os}/Release.key | sudo apt-key add -
+	sudo apt update
+	sudo apt install -y cri-o cri-o-runc
+
+	# We need to set the default capabilities to ensure our tests will pass
+	# See: https://github.com/kata-containers/kata-containers/issues/8034
+	sudo mkdir -p /etc/crio/crio.conf.d/
+	cat <<EOF | sudo tee /etc/crio/crio.conf.d/00-default-capabilities
+[crio.runtime]
+default_capabilities = [
+       "CHOWN",
+       "DAC_OVERRIDE",
+       "FSETID",
+       "FOWNER",
+       "SETGID",
+       "SETUID",
+       "SETPCAP",
+       "NET_BIND_SERVICE",
+       "KILL",
+       "SYS_CHROOT",
+]
+EOF
+
+	sudo systemctl enable --now crio
+}
+
+# Convert architecture to the name used by golang
+function arch_to_golang() {
+	local arch="$(uname -m)"
+
+	case "${arch}" in
+		aarch64) echo "arm64";;
+		ppc64le) echo "${arch}";;
+		x86_64) echo "amd64";;
+		s390x) echo "s390x";;
+		*) die "unsupported architecture: ${arch}";;
+	esac
+}
+
+# Convert architecture to the name used by rust
+function arch_to_rust() {
+	local -r arch="$(uname -m)"
+
+	case "${arch}" in
+		aarch64) echo "${arch}";;
+		ppc64le) echo "powerpc64le";;
+		x86_64) echo "${arch}";;
+		s390x) echo "${arch}";;
+		*) die "unsupported architecture: ${arch}";;
+	esac
+}
+
+# Convert architecture to the name used by the Linux kernel build system
+function arch_to_kernel() {
+	local -r arch="$(uname -m)"
+
+	case "${arch}" in
+		aarch64) echo "arm64";;
+		ppc64le) echo "powerpc";;
+		x86_64) echo "${arch}";;
+		s390x) echo "s390x";;
+		*) die "unsupported architecture: ${arch}";;
+	esac
+}

tests/functional/kata-deploy/gha-run.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2023 Microsoft Corporation
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+kata_deploy_dir="$(dirname "$(readlink -f "$0")")"
+source "${kata_deploy_dir}/../../gha-run-k8s-common.sh"
+
+function run_tests() {
+	cleanup_runtimeclasses || true
+
+	pushd "${kata_deploy_dir}"
+	bash run-kata-deploy-tests.sh
+	popd
+}
+
+function cleanup_runtimeclasses() {
+	# Cleanup any runtime class that was left behind in the cluster, in
+	# case of a test failure, apart from the default one that comes from
+	# AKS
+	for rc in `kubectl get runtimeclass -o name | grep -v "kata-mshv-vm-isolation" | sed 's|runtimeclass.node.k8s.io/||'`
+	do
+		kubectl delete runtimeclass $rc;
+	done
+}
+
+function cleanup() {
+	platform="${1}"
+	test_type="${2:-k8s}"
+
+	cleanup_runtimeclasses || true
+
+	if [ "${platform}" = "aks" ]; then
+		delete_cluster ${test_type}
+	fi
+}
+
+function main() {
+    export KATA_HOST_OS="${KATA_HOST_OS:-}"
+
+    platform="aks"
+    if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then
+	    platform="tdx"
+    fi
+    export platform
+
+    action="${1:-}"
+
+    case "${action}" in
+        install-azure-cli) install_azure_cli ;;
+        login-azure) login_azure ;;
+        create-cluster) create_cluster "kata-deploy" ;;
+        deploy-k8s) deploy_k8s ;;
+        install-bats) install_bats ;;
+        install-kubectl) install_kubectl ;;
+        get-cluster-credentials) get_cluster_credentials "kata-deploy" ;;
+        run-tests) run_tests ;;
+        delete-cluster) cleanup "aks" "kata-deploy" ;;
+        *) >&2 echo "Invalid argument"; exit 2 ;;
+    esac
+}
+
+main "$@"

tests/functional/kata-deploy/kata-deploy.bats

@@ -0,0 +1,116 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/../../common.bash"
+
+setup() {
+	repo_root_dir="${BATS_TEST_DIRNAME}/../../../"
+	ensure_yq
+
+	# We expect 2 runtime classes because:
+	# * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`.
+	# * `kata-${KATA_HYPERVISOR}` is the other one
+	#    * As part of the tests we're only deploying the specific runtimeclass that will be used, instead of all of them.
+	expected_runtime_classes=2
+
+	# We expect both runtime classes to have the same handler: kata-${KATA_HYPERVISOR}
+	expected_handlers_re=( \
+		"kata\s+kata-${KATA_HYPERVISOR}" \
+		"kata-${KATA_HYPERVISOR}\s+kata-${KATA_HYPERVISOR}" \
+	)
+
+	# Set the latest image, the one generated as part of the PR, to be used as part of the tests
+	sed -i -e "s|quay.io/kata-containers/kata-deploy:3.2.0|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+
+	# Enable debug for Kata Containers
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[1].value' --tag '!!str' "true"
+	# Create the runtime class only for the shim that's being tested
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[2].value' "${KATA_HYPERVISOR}"
+	# Set the tested hypervisor as the default `kata` shim
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}"
+	# Let the `kata-deploy` script take care of the runtime class creation / removal
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
+	# Let the `kata-deploy` create the default `kata` runtime class
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true"
+	
+	if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
+		yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
+		yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
+	fi
+	
+	echo "::group::Final kata-deploy.yaml that is used in the test"
+	cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
+	echo "::endgroup::"
+	
+	kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
+	if [ "${KUBERNETES}" = "k0s" ]; then
+		kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s"
+	elif [ "${KUBERNETES}" = "k3s" ]; then
+		kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"
+	elif [ "${KUBERNETES}" = "rke2" ]; then
+		kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2"
+	else
+		kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	fi
+	kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
+
+	# Give some time for the pod to finish what's doing and have the
+	# runtimeclasses properly created
+	sleep 30s
+}
+
+@test "Test runtimeclasses are being properly created" {
+	# We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy
+	current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l)
+	[[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]]
+
+	for handler_re in ${expected_handlers_re[@]}
+	do
+		kubectl get runtimeclass | grep -E "${handler_re}"
+	done
+}
+
+teardown() {
+	kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation"
+
+	if [ "${KUBERNETES}" = "k0s" ]; then
+		deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s\""
+		cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\""
+	elif [ "${KUBERNETES}" = "k3s" ]; then
+		deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s\""
+		cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\""
+	elif [ "${KUBERNETES}" = "rke2" ]; then
+		deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2\""
+		cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\""
+	else
+		deploy_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\""
+		cleanup_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\""
+	fi
+
+	kubectl delete ${deploy_spec}
+	kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
+	
+	# Let the `kata-deploy` script take care of the runtime class creation / removal
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
+	# Create the runtime class only for the shim that's being tested
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[2].value' "${KATA_HYPERVISOR}"
+	# Set the tested hypervisor as the default `kata` shim
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}"
+	# Let the `kata-deploy` create the default `kata` runtime class
+	yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true"
+	
+	sed -i -e "s|quay.io/kata-containers/kata-deploy:3.2.0|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
+	cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
+	grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"
+
+	kubectl apply ${cleanup_spec}
+	sleep 30s
+	
+	kubectl delete ${cleanup_spec}
+	kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
+}

tests/functional/kata-deploy/run-kata-deploy-tests.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+kata_deploy_dir=$(dirname "$(readlink -f "$0")")
+source "${kata_deploy_dir}/../../common.bash"
+
+if [ -n "${KATA_DEPLOY_TEST_UNION:-}" ]; then
+	KATA_DEPLOY_TEST_UNION=($KATA_DEPLOY_TEST_UNION)
+else
+	KATA_DEPLOY_TEST_UNION=( \
+		"kata-deploy.bats" \
+	)
+fi
+
+info "Run tests"
+for KATA_DEPLOY_TEST_ENTRY in ${KATA_DEPLOY_TEST_UNION[@]}
+do
+	bats "${KATA_DEPLOY_TEST_ENTRY}"
+done