Merge pull request #10620 from kata-containers/topic/fix-remove-artifact-ordering

workflows: Remove potential timing issues with artifacts

.github/actionlint.yaml

@@ -21,4 +21,5 @@ self-hosted-runner:
     - sev-snp
     - s390x
     - s390x-large
-    - tdx
+    - tdx-no-attestation
+    - tdx-attestation

.github/workflows/actionlint.yaml

@@ -0,0 +1,33 @@
+name: Lint GHA workflows
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths:
+      - '.github/workflows/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-actionlint:
+    env:
+      GH_TOKEN: ${{ github.token }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install actionlint gh extension
+        run: gh extension install https://github.com/cschleiden/gh-actionlint
+
+      - name: Run actionlint
+        run:  gh actionlint

.github/workflows/add-issues-to-project.yaml

@@ -33,7 +33,7 @@ jobs:
         run: |
           # Clone into a temporary directory to avoid overwriting
           # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
+          pushd "$(mktemp -d)" &>/dev/null
           git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
           sudo install hub-util.sh /usr/local/bin
           popd &>/dev/null

.github/workflows/add-pr-sizing-label.yaml

@@ -36,7 +36,7 @@ jobs:
         run: |
           # Clone into a temporary directory to avoid overwriting
           # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
+          pushd "$(mktemp -d)" &>/dev/null
           git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
           sudo install pr-add-size-label.sh /usr/local/bin
           popd &>/dev/null

.github/workflows/basic-ci-amd64.yaml

@@ -138,6 +138,8 @@ jobs:
         run: bash tests/integration/nydus/gha-run.sh run
 
   run-runk:
+    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
+    if: false
     runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: lts

.github/workflows/build-checks.yaml

@@ -19,7 +19,6 @@ jobs:
           - runtime-rs
           - agent-ctl
           - kata-ctl
-          - runk
           - trace-forwarder
           - genpolicy
         command:
@@ -40,22 +39,18 @@ jobs:
             component-path: src/tools/agent-ctl
           - component: kata-ctl
             component-path: src/tools/kata-ctl
-          - component: runk
-            component-path: src/tools/runk
           - component: trace-forwarder
             component-path: src/tools/trace-forwarder
           - install-libseccomp: no
           - component: agent
             install-libseccomp: yes
-          - component: runk
-            install-libseccomp: yes
           - component: genpolicy
             component-path: src/tools/genpolicy
     steps:
       - name: Adjust a permission for repo
         run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE $HOME
-          sudo rm -rf $GITHUB_WORKSPACE/* && echo "GITHUB_WORKSPACE removed" || { sleep 10 && sudo rm -rf $GITHUB_WORKSPACE/*; }
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE" "$HOME"
+          sudo rm -rf "$GITHUB_WORKSPACE"/* || { sleep 10 && sudo rm -rf "$GITHUB_WORKSPACE"/*; }
           sudo rm -f /tmp/kata_hybrid*  # Sometime we got leftover from test_setup_hvsock_failed()
 
       - name: Checkout the code
@@ -72,12 +67,12 @@ jobs:
         if: ${{ matrix.component == 'runtime' }}
         run: |
           ./tests/install_go.sh -f -p
-          echo "/usr/local/go/bin" >> $GITHUB_PATH
+          echo "/usr/local/go/bin" >> "$GITHUB_PATH"
       - name: Install rust
         if: ${{ matrix.component != 'runtime' }}
         run: |
           ./tests/install_rust.sh
-          echo "${HOME}/.cargo/bin" >> $GITHUB_PATH
+          echo "${HOME}/.cargo/bin" >> "$GITHUB_PATH"
       - name: Install musl-tools
         if: ${{ matrix.component != 'runtime' }}
         run: sudo apt-get -y install musl-tools
@@ -91,10 +86,10 @@ jobs:
           gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
           ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
           echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-          echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-          echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
+          echo "LIBSECCOMP_LINK_TYPE=static" >> "$GITHUB_ENV"
+          echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> "$GITHUB_ENV"
       - name: Install protobuf-compiler
-        if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'runk' || matrix.component == 'genpolicy' || matrix.component == 'agent-ctl') }}
+        if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'genpolicy' || matrix.component == 'agent-ctl') }}
         run: sudo apt-get -y install protobuf-compiler
       - name: Install clang
         if: ${{ matrix.command == 'make check' && (matrix.component == 'agent' || matrix.component == 'agent-ctl') }}
@@ -102,8 +97,8 @@ jobs:
       - name: Setup XDG_RUNTIME_DIR for the `runtime` tests
         if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }}
         run: |
-          XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700))
-          echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> $GITHUB_ENV
+          XDG_RUNTIME_DIR=$(mktemp -d "/tmp/kata-tests-$USER.XXX" | tee >(xargs chmod 0700))
+          echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> "$GITHUB_ENV"
       - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
         run: |
           cd ${{ matrix.component-path }}

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -37,6 +37,7 @@ jobs:
           - cloud-hypervisor
           - cloud-hypervisor-glibc
           - coco-guest-components
+          - csi-kata-directvolume
           - firecracker
           - genpolicy
           - kata-ctl
@@ -53,12 +54,6 @@ jobs:
           - qemu
           - qemu-snp-experimental
           - stratovirt
-          - rootfs-image
-          - rootfs-image-confidential
-          - rootfs-image-mariner
-          - rootfs-initrd
-          - rootfs-initrd-confidential
-          - runk
           - trace-forwarder
           - virtiofsd
         stage:
@@ -94,7 +89,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -134,7 +129,6 @@ jobs:
           push-to-registry: true
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ matrix.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -142,9 +136,17 @@ jobs:
           retention-days: 15
           if-no-files-found: error
 
-  build-asset-shim-v2:
+  build-asset-rootfs:
     runs-on: ubuntu-22.04
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-image-confidential
+          - rootfs-image-mariner
+          - rootfs-initrd
+          - rootfs-initrd-confidential
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -165,13 +167,93 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
-      - name: Build shim-v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
         id: build
         run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - agent
+          - coco-guest-components
+          - pause-image
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: ubuntu-22.04
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: shim-v2
           TAR_OUTPUT: shim-v2.tar.gz
@@ -181,6 +263,7 @@ jobs:
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
 
       - name: store-artifact shim-v2
         uses: actions/upload-artifact@v4
@@ -192,7 +275,7 @@ jobs:
 
   create-kata-tarball:
     runs-on: ubuntu-22.04
-    needs: [build-asset, build-asset-shim-v2]
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -35,8 +35,6 @@ jobs:
           - nydus
           - qemu
           - stratovirt
-          - rootfs-image
-          - rootfs-initrd
           - virtiofsd
     steps:
       - name: Login to Kata Containers quay.io
@@ -63,7 +61,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -75,7 +73,6 @@ jobs:
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -83,9 +80,14 @@ jobs:
           retention-days: 15
           if-no-files-found: error
 
-  build-asset-shim-v2:
+  build-asset-rootfs:
     runs-on: arm64-builder
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-initrd
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -106,12 +108,89 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
-      - name: Build shim-v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
         run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - agent
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: arm64-builder
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: shim-v2
           TAR_OUTPUT: shim-v2.tar.gz
@@ -132,11 +211,11 @@ jobs:
 
   create-kata-tarball:
     runs-on: arm64-builder
-    needs: [build-asset, build-asset-shim-v2]
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - name: Adjust a permission for repo
         run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
 
       - uses: actions/checkout@v4
         with:

.github/workflows/build-kata-static-tarball-ppc64le.yaml

@@ -30,15 +30,14 @@ jobs:
           - agent
           - kernel
           - qemu
-          - rootfs-initrd
           - virtiofsd
         stage:
           - ${{ inputs.stage }}
     steps:
       - name: Prepare the self-hosted runner
         run: |
-            ${HOME}/scripts/prepare_runner.sh
-            sudo rm -rf $GITHUB_WORKSPACE/*
+            "${HOME}/scripts/prepare_runner.sh"
+            sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -64,7 +63,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -76,7 +75,6 @@ jobs:
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -84,14 +82,20 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
-  build-asset-shim-v2:
+  build-asset-rootfs:
     runs-on: ppc64le
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-initrd
+        stage:
+          - ${{ inputs.stage }}
     steps:
       - name: Prepare the self-hosted runner
         run: |
-            ${HOME}/scripts/prepare_runner.sh
-            sudo rm -rf $GITHUB_WORKSPACE/*
+            "${HOME}/scripts/prepare_runner.sh"
+            sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -112,12 +116,94 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
-      - name: Build shim-v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
         run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - agent
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-ppc64le-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: ppc64le
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts]
+    steps:
+      - name: Prepare the self-hosted runner
+        run: |
+            "${HOME}/scripts/prepare_runner.sh"
+            sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: shim-v2
           TAR_OUTPUT: shim-v2.tar.gz
@@ -138,11 +224,11 @@ jobs:
 
   create-kata-tarball:
     runs-on: ppc64le
-    needs: [build-asset, build-asset-shim-v2]
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - name: Adjust a permission for repo
         run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
 
       - uses: actions/checkout@v4
         with:

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -38,10 +38,6 @@ jobs:
           - kernel-confidential
           - pause-image
           - qemu
-          - rootfs-image
-          - rootfs-image-confidential
-          - rootfs-initrd
-          - rootfs-initrd-confidential
           - virtiofsd
     env:
       PERFORM_ATTESTATION: ${{ matrix.asset == 'agent' && inputs.push-to-registry == 'yes' && 'yes' || 'no' }}
@@ -71,7 +67,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -106,7 +102,69 @@ jobs:
           push-to-registry: true
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-rootfs:
+    runs-on: s390x
+    needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-image-confidential
+          - rootfs-initrd
+          - rootfs-initrd-confidential
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -116,7 +174,7 @@ jobs:
 
   build-asset-boot-image-se:
     runs-on: s390x
-    needs: build-asset
+    needs: [build-asset, build-asset-rootfs]
     steps:
       - uses: actions/checkout@v4
 
@@ -142,15 +200,11 @@ jobs:
 
       - name: Build boot-image-se
         run: |
-          base_dir=tools/packaging/kata-deploy/local-build/
-          cp -r kata-artifacts ${base_dir}/build
-          # Skip building dependant artifacts of boot-image-se-tarball
-          # because we already have them from the previous build
-          sed -i 's/\(^boot-image-se-tarball:\).*/\1/g' ${base_dir}/Makefile
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "boot-image-se"
           make boot-image-se-tarball
           build_dir=$(readlink -f build)
           sudo cp -r "${build_dir}" "kata-build"
-          sudo chown -R $(id -u):$(id -g) "kata-build"
+          sudo chown -R "$(id -u)":"$(id -g)" "kata-build"
         env:
           HKD_PATH: "host-key-document"
 
@@ -162,9 +216,25 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: [build-asset-rootfs, build-asset-boot-image-se]
+    strategy:
+      matrix:
+        asset:
+          - agent
+          - coco-guest-components
+          - pause-image
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-s390x-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
   build-asset-shim-v2:
     runs-on: s390x
-    needs: build-asset
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts]
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -185,13 +255,21 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
       - name: Build shim-v2
         id: build
         run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: shim-v2
           TAR_OUTPUT: shim-v2.tar.gz
@@ -201,6 +279,7 @@ jobs:
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
 
       - name: store-artifact shim-v2
         uses: actions/upload-artifact@v4
@@ -212,7 +291,11 @@ jobs:
 
   create-kata-tarball:
     runs-on: s390x
-    needs: [build-asset, build-asset-boot-image-se, build-asset-shim-v2]
+    needs:
+      - build-asset
+      - build-asset-rootfs
+      - build-asset-boot-image-se
+      - build-asset-shim-v2
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/cargo-deny-runner.yaml

@@ -24,7 +24,7 @@ jobs:
         run: bash cargo-deny-generator.sh
         working-directory: ./.github/cargo-deny-composite-action/
         env:
-          GOPATH: ${{ runner.workspace }}/kata-containers
+          GOPATH: ${{ github.workspace }}/kata-containers
       - name: Run Action
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         uses: ./.github/cargo-deny-composite-action

.github/workflows/ci-nightly-s390x.yaml

@@ -16,6 +16,6 @@ jobs:
     - name: Fetch a test result for {{ matrix.test_title }}
       run: |
         file_name="${TEST_TITLE}-$(date +%Y-%m-%d).log"
-        /home/${USER}/script/handle_test_log.sh download $file_name
+        "/home/${USER}/script/handle_test_log.sh" download "$file_name"
       env:
         TEST_TITLE: ${{ matrix.test_title }}

.github/workflows/ci-weekly.yaml

@@ -37,7 +37,7 @@ jobs:
     secrets: inherit
 
   build-and-publish-tee-confidential-unencrypted-image:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -83,4 +83,5 @@ jobs:
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
+      tarball-suffix: -${{ inputs.tag }}
     secrets: inherit

.github/workflows/ci.yaml

@@ -135,6 +135,56 @@ jobs:
           platforms: linux/amd64, linux/s390x
           file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
 
+  publish-csi-driver-amd64:
+    needs: build-kata-static-tarball-amd64
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64-${{ inputs.tag }}
+          path: kata-artifacts
+
+      - name: Install tools
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
+      - name: Copy binary into Docker context
+        run: |
+          # Copy to the location where the Dockerfile expects the binary.
+          mkdir -p src/tools/csi-kata-directvolume/bin/
+          cp /opt/kata/bin/csi-kata-directvolume src/tools/csi-kata-directvolume/bin/directvolplugin
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Kata Containers ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker build and push
+        uses: docker/build-push-action@v5
+        with:
+          tags: ghcr.io/kata-containers/csi-kata-directvolume:${{ inputs.pr-number }}
+          push: true
+          context: src/tools/csi-kata-directvolume/
+          platforms: linux/amd64
+          file: src/tools/csi-kata-directvolume/Dockerfile
+
   run-kata-monitor-tests:
     if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-amd64
@@ -173,9 +223,13 @@ jobs:
 
   run-kata-coco-tests:
     if: ${{ inputs.skip-test != 'yes' }}
-    needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
+    needs:
+     - publish-kata-deploy-payload-amd64
+     - build-and-publish-tee-confidential-unencrypted-image
+     - publish-csi-driver-amd64
     uses: ./.github/workflows/run-kata-coco-tests.yaml
     with:
+      tarball-suffix: -${{ inputs.tag }}
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64

.github/workflows/darwin-tests.yaml

@@ -16,7 +16,7 @@ jobs:
     runs-on: macos-latest
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.22.2
     - name: Checkout code

.github/workflows/docs-url-alive-check.yaml

@@ -12,15 +12,15 @@ jobs:
       target_branch: ${{ github.base_ref }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.22.2
       env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
+        GOPATH: ${{ github.workspace }}/kata-containers
     - name: Set env
       run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+        echo "GOPATH=${{ github.workspace }}" >> "$GITHUB_ENV"
+        echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Checkout code
       uses: actions/checkout@v4
       with:
@@ -29,4 +29,4 @@ jobs:
     # docs url alive check
     - name: Docs URL Alive Check
       run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make docs-url-alive-check
+        cd "${GOPATH}/src/github.com/${{ github.repository }}" && make docs-url-alive-check

.github/workflows/gatekeeper-skipper.yaml

@@ -34,7 +34,7 @@ on:
 
 jobs:
   skipper:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     outputs:
       skip_build: ${{ steps.skipper.outputs.skip_build }}
       skip_test: ${{ steps.skipper.outputs.skip_test }}

.github/workflows/gatekeeper.yaml

@@ -18,7 +18,7 @@ concurrency:
 
 jobs:
   gatekeeper:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/kata-runtime-classes-sync.yaml

@@ -20,9 +20,9 @@ jobs:
       run: |
         pushd tools/packaging/kata-deploy/runtimeclasses/
         echo "::group::Combine runtime classes"
-        for runtimeClass in `find . -type f \( -name "*.yaml" -and -not -name "kata-runtimeClasses.yaml" \) | sort`; do
+        for runtimeClass in $(find . -type f \( -name "*.yaml" -and -not -name "kata-runtimeClasses.yaml" \) | sort); do
             echo "Adding ${runtimeClass} to the resultingRuntimeClasses.yaml"
-            cat ${runtimeClass} >> resultingRuntimeClasses.yaml;
+            cat "${runtimeClass}" >> resultingRuntimeClasses.yaml;
         done
         echo "::endgroup::"
         echo "::group::Displaying the content of resultingRuntimeClasses.yaml"

.github/workflows/move-issues-to-in-progress.yaml

@@ -31,7 +31,7 @@ jobs:
         run: |
           # Clone into a temporary directory to avoid overwriting
           # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
+          pushd "$(mktemp -d)" &>/dev/null
           git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
           sudo install hub-util.sh /usr/local/bin
           popd &>/dev/null
@@ -72,9 +72,9 @@ jobs:
           project_type="org"
           project_column="In progress"
 
-          for issue_url in $(echo "$linked_issue_urls")
+          for issue_url in $linked_issue_urls
           do
-            issue=$(echo "$issue_url"| awk -F\/ '{print $NF}' || true)
+            issue=$(echo "$issue_url"| awk -F/ '{print $NF}' || true)
 
             [ -z "$issue" ] && {
               echo "::error::Cannot determine issue number from $issue_url for PR $pr"

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -62,5 +62,5 @@ jobs:
         id: build-and-push-kata-payload
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
+          "$(pwd)"/kata-static.tar.xz \
           ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -28,7 +28,7 @@ jobs:
     steps:
       - name: Adjust a permission for repo
         run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
 
       - uses: actions/checkout@v4
         with:
@@ -66,6 +66,5 @@ jobs:
         id: build-and-push-kata-payload
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
+          "$(pwd)"/kata-static.tar.xz \
           ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}
-

.github/workflows/publish-kata-deploy-payload-ppc64le.yaml

@@ -28,12 +28,12 @@ jobs:
     steps:
       - name: Prepare the self-hosted runner
         run: |
-          ${HOME}/scripts/prepare_runner.sh
-          sudo rm -rf $GITHUB_WORKSPACE/*
+          "${HOME}/scripts/prepare_runner.sh"
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - name: Adjust a permission for repo
         run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
 
       - uses: actions/checkout@v4
         with:
@@ -71,5 +71,5 @@ jobs:
         id: build-and-push-kata-payload
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
+          "$(pwd)"/kata-static.tar.xz \
           ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -62,5 +62,5 @@ jobs:
         id: build-and-push-kata-payload
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
+          "$(pwd)"/kata-static.tar.xz \
           ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/release-amd64.yaml

@@ -42,18 +42,18 @@ jobs:
         run: |
           # We need to do such trick here as the format of the $GITHUB_REF
           # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
           if [ "${tag}" = "main" ]; then
               tag=$(./tools/packaging/release/release.sh release-version)
-              tags=(${tag} "latest")
+              tags=("${tag}" "latest")
           else
-              tags=(${tag})
+              tags=("${tag}")
           fi
-          for tag in ${tags[@]}; do
+          for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
           done

.github/workflows/release-arm64.yaml

@@ -42,18 +42,18 @@ jobs:
         run: |
           # We need to do such trick here as the format of the $GITHUB_REF
           # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
           if [ "${tag}" = "main" ]; then
               tag=$(./tools/packaging/release/release.sh release-version)
-              tags=(${tag} "latest")
+              tags=("${tag}" "latest")
           else
-              tags=(${tag})
+              tags=("${tag}")
           fi
-          for tag in ${tags[@]}; do
+          for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
           done

.github/workflows/release-ppc64le.yaml

@@ -20,8 +20,8 @@ jobs:
     steps:
       - name: Prepare the self-hosted runner
         run: |
-          bash ${HOME}/scripts/prepare_runner.sh
-          sudo rm -rf $GITHUB_WORKSPACE/*
+          bash "${HOME}/scripts/prepare_runner.sh"
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v3
@@ -47,18 +47,18 @@ jobs:
         run: |
           # We need to do such trick here as the format of the $GITHUB_REF
           # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
           if [ "${tag}" = "main" ]; then
               tag=$(./tools/packaging/release/release.sh release-version)
-              tags=(${tag} "latest")
+              tags=("${tag}" "latest")
           else
-              tags=(${tag})
+              tags=("${tag}")
           fi
-          for tag in ${tags[@]}; do
+          for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
           done

.github/workflows/release-s390x.yaml

@@ -42,18 +42,18 @@ jobs:
         run: |
           # We need to do such trick here as the format of the $GITHUB_REF
           # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
           if [ "${tag}" = "main" ]; then
               tag=$(./tools/packaging/release/release.sh release-version)
-              tags=(${tag} "latest")
+              tags=("${tag}" "latest")
           else
-              tags=(${tag})
+              tags=("${tag}")
           fi
-          for tag in ${tags[@]}; do
+          for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
           done

.github/workflows/release.yaml

@@ -175,6 +175,23 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
 
+  upload-helm-chart-tarball:
+    needs: release
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install helm
+        uses: azure/setup-helm@v4.2.0
+        id: install
+
+      - name: Generate and upload helm chart tarball
+        run: |
+          ./tools/packaging/release/release.sh upload-helm-chart-tarball
+        env:
+          GH_TOKEN: ${{ github.token }}
+
   publish-release:
     needs: [ build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x, build-and-push-assets-ppc64le, publish-multi-arch-images, upload-multi-arch-static-tarball, upload-versions-yaml, upload-cargo-vendored-tarball, upload-libseccomp-tarball ]
     runs-on: ubuntu-22.04

.github/workflows/run-cri-containerd-tests-ppc64le.yaml

@@ -30,12 +30,12 @@ jobs:
       KATA_HYPERVISOR: ${{ matrix.vmm }}
     steps:
       - name: Adjust a permission for repo
-        run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-  
+        run: sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
+
       - name: Prepare the self-hosted runner
         run: |
-          bash ${HOME}/scripts/prepare_runner.sh cri-containerd
-          sudo rm -rf $GITHUB_WORKSPACE/*
+          bash "${HOME}/scripts/prepare_runner.sh" cri-containerd
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/checkout@v4
         with:
@@ -62,6 +62,6 @@ jobs:
 
       - name: Run cri-containerd tests
         run: bash tests/integration/cri-containerd/gha-run.sh run
-      
+
       - name: Cleanup actions for the self hosted runner
-        run: ${HOME}/scripts/cleanup_runner.sh
+        run: bash "${HOME}/scripts/cleanup_runner.sh"

.github/workflows/run-k8s-tests-on-amd64.yaml

@@ -56,6 +56,7 @@ jobs:
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: "false"
       K8S_TEST_HOST_TYPE: all
+      CONTAINER_RUNTIME: ${{ matrix.container_runtime }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -85,11 +86,11 @@ jobs:
 
       - name: Install `bats`
         run: bash tests/integration/kubernetes/gha-run.sh install-bats
-  
+
       - name: Run tests
         timeout-minutes: 30
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
-  
+
       - name: Collect artifacts ${{ matrix.vmm }}
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh collect-artifacts
@@ -98,7 +99,7 @@ jobs:
       - name: Archive artifacts ${{ matrix.vmm }}
         uses: actions/upload-artifact@v4
         with:
-          name: k8s-tests-${{ matrix.vmm }}-${{ matrix.snapshotter }}-${{ matrix.k8s }}-${{ matrix.instance }}-${{ inputs.tag }}
+          name: k8s-tests-${{ matrix.vmm }}-${{ matrix.snapshotter }}-${{ matrix.k8s }}-${{ inputs.tag }}
           path: /tmp/artifacts
           retention-days: 1
 

.github/workflows/run-k8s-tests-on-ppc64le.yaml

@@ -44,9 +44,9 @@ jobs:
       TARGET_ARCH: "ppc64le"
     steps:
       - name: Prepare the self-hosted runner
-        run: | 
-          bash ${HOME}/scripts/prepare_runner.sh kubernetes
-          sudo rm -rf $GITHUB_WORKSPACE/*
+        run: |
+          bash "${HOME}/scripts/prepare_runner.sh" kubernetes
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/checkout@v4
         with:
@@ -62,13 +62,13 @@ jobs:
       - name: Install golang
         run: |
           ./tests/install_go.sh -f -p
-          echo "/usr/local/go/bin" >> $GITHUB_PATH
+          echo "/usr/local/go/bin" >> "$GITHUB_PATH"
 
       - name: Prepare the runner for k8s cluster creation
-        run: bash ${HOME}/scripts/k8s_cluster_cleanup.sh
+        run: bash "${HOME}/scripts/k8s_cluster_cleanup.sh"
 
       - name: Create k8s cluster using kubeadm
-        run: bash ${HOME}/scripts/k8s_cluster_create.sh
+        run: bash "${HOME}/scripts/k8s_cluster_create.sh"
 
       - name: Deploy Kata
         timeout-minutes: 10
@@ -79,4 +79,4 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete cluster and post cleanup actions
-        run: bash ${HOME}/scripts/k8s_cluster_cleanup.sh
+        run: bash "${HOME}/scripts/k8s_cluster_cleanup.sh"

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -36,7 +36,7 @@ jobs:
           - qemu-runtime-rs
           - qemu-coco-dev
         k8s:
-          - k3s
+          - kubeadm
         include:
           - snapshotter: devmapper
             pull-type: default
@@ -88,18 +88,15 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Set SNAPSHOTTER to empty if overlayfs
-        run: echo "SNAPSHOTTER=" >> $GITHUB_ENV
+        run: echo "SNAPSHOTTER=" >> "$GITHUB_ENV"
         if: ${{ matrix.snapshotter == 'overlayfs' }}
 
       - name: Set KBS and KBS_INGRESS if qemu-coco-dev
         run: |
-          echo "KBS=true" >> $GITHUB_ENV
-          echo "KBS_INGRESS=nodeport" >> $GITHUB_ENV
+          echo "KBS=true" >> "$GITHUB_ENV"
+          echo "KBS_INGRESS=nodeport" >> "$GITHUB_ENV"
         if: ${{ matrix.vmm == 'qemu-coco-dev' }}
 
-      - name: Deploy ${{ matrix.k8s }}
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
-
       # qemu-runtime-rs only works with overlayfs
       # See: https://github.com/kata-containers/kata-containers/issues/10066
       - name: Configure the ${{ matrix.snapshotter }} snapshotter

.github/workflows/run-kata-coco-stability-tests.yaml

@@ -21,6 +21,9 @@ on:
         required: false
         type: string
         default: ""
+      tarball-suffix:
+        required: false
+        type: string
 
 jobs:
   # Generate jobs for testing CoCo on non-TEE environments
@@ -34,13 +37,12 @@ jobs:
           - nydus
         pull-type:
           - guest-pull
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
       DOCKER_TAG: ${{ inputs.tag }}
       GH_PR_NUMBER: ${{ inputs.pr-number }}
-      KATA_HOST_OS: ${{ matrix.host_os }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
       # Some tests rely on that variable to run (or not)
       KBS: "true"
@@ -64,6 +66,15 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
 

.github/workflows/run-kata-coco-tests.yaml

@@ -2,6 +2,9 @@ name: CI | Run kata coco tests
 on:
   workflow_call:
     inputs:
+      tarball-suffix:
+        required: false
+        type: string
       registry:
         required: true
         type: string
@@ -33,7 +36,15 @@ jobs:
           - nydus
         pull-type:
           - guest-pull
-    runs-on: tdx
+        k8s-test-host-type:
+          - baremetal-attestation
+          - baremetal-no-attestation
+        include:
+          - k8s-test-host-type: baremetal-attestation
+            machine: tdx-attestation
+          - k8s-test-host-type: baremetal-no-attestation
+            machine: tdx-no-attestation
+    runs-on: ${{ matrix.machine }}
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
@@ -43,7 +54,7 @@ jobs:
       KUBERNETES: "vanilla"
       USING_NFD: "true"
       KBS: "true"
-      K8S_TEST_HOST_TYPE: "baremetal"
+      K8S_TEST_HOST_TYPE: ${{ matrix.k8s-test-host-type }}
       KBS_INGRESS: "nodeport"
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       PULL_TYPE: ${{ matrix.pull-type }}
@@ -72,17 +83,24 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
 
       - name: Uninstall previous `kbs-client`
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
 
       - name: Deploy CoCo KBS
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
 
       - name: Install `kbs-client`
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
       - name: Run tests
         timeout-minutes: 100
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
@@ -96,9 +114,13 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
       - name: Delete CoCo KBS
-        if: always()
+        if: ${{ always() && matrix.machine != 'tdx-no-attestation' }}
         run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
 
+      - name: Delete CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
   run-k8s-tests-on-sev:
     strategy:
       fail-fast: false
@@ -145,10 +167,18 @@ jobs:
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
 
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
       - name: Run tests
         timeout-minutes: 50
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
+      - name: Delete CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
       - name: Delete kata-deploy
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-sev
@@ -217,6 +247,10 @@ jobs:
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
       - name: Run tests
         timeout-minutes: 50
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
@@ -233,6 +267,10 @@ jobs:
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
 
+      - name: Delete CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
   # Generate jobs for testing CoCo on non-TEE environments
   run-k8s-tests-coco-nontee:
     strategy:
@@ -250,7 +288,6 @@ jobs:
       DOCKER_REPO: ${{ inputs.repo }}
       DOCKER_TAG: ${{ inputs.tag }}
       GH_PR_NUMBER: ${{ inputs.pr-number }}
-      KATA_HOST_OS: ${{ matrix.host_os }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
       # Some tests rely on that variable to run (or not)
       KBS: "true"
@@ -262,6 +299,7 @@ jobs:
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: "false"
+      AUTO_GENERATE_POLICY: "yes"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -274,6 +312,15 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
 
@@ -314,8 +361,12 @@ jobs:
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
       - name: Run tests
-        timeout-minutes: 60
+        timeout-minutes: 80
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete AKS cluster

.github/workflows/run-metrics.yaml

@@ -48,7 +48,7 @@ jobs:
       # all the tests due to a single flaky instance.
       fail-fast: false
       matrix:
-        vmm: ['clh', 'qemu', 'stratovirt']
+        vmm: ['clh', 'qemu']
       max-parallel: 1
     runs-on: metrics
     env:

.github/workflows/run-runk-tests.yaml

@@ -15,6 +15,8 @@ on:
 
 jobs:
   run-runk:
+    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
+    if: false
     runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: lts

.github/workflows/static-checks.yaml

@@ -31,8 +31,8 @@ jobs:
         run: |
           kernel_dir="tools/packaging/kernel/"
           kernel_version_file="${kernel_dir}kata_config_version"
-          modified_files=$(git diff --name-only origin/$GITHUB_BASE_REF..HEAD)
-          if git diff --name-only origin/$GITHUB_BASE_REF..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+          modified_files=$(git diff --name-only origin/"$GITHUB_BASE_REF"..HEAD)
+          if git diff --name-only origin/"$GITHUB_BASE_REF"..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
             echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
             if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
               echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
@@ -107,19 +107,19 @@ jobs:
           path: ./src/github.com/${{ github.repository }}
       - name: Install yq
         run: |
-          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          cd "${GOPATH}/src/github.com/${{ github.repository }}"
           ./ci/install_yq.sh
         env:
           INSTALL_IN_GOPATH: false
       - name: Install golang
         run: |
-          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          cd "${GOPATH}/src/github.com/${{ github.repository }}"
           ./tests/install_go.sh -f -p
-          echo "/usr/local/go/bin" >> $GITHUB_PATH
+          echo "/usr/local/go/bin" >> "$GITHUB_PATH"
       - name: Install system dependencies
         run: |
           sudo apt-get -y install moreutils hunspell hunspell-en-gb hunspell-en-us pandoc
       - name: Run check
         run: |
-          export PATH=${PATH}:${GOPATH}/bin
-          cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}
+          export PATH="${PATH}:${GOPATH}/bin"
+          cd "${GOPATH}/src/github.com/${{ github.repository }}" && ${{ matrix.cmd }}

VERSION

@@ -1 +1 @@
-3.10.1
+3.12.0

ci/openshift-ci/smoke/http-server.yaml.in

@@ -13,7 +13,7 @@ metadata:
 spec:
   containers:
     - name: http-server
-      image: registry.fedoraproject.org/fedora
+      image: docker.io/library/python:3
       ports:
         - containerPort: 8080
       command: ["python3"]

docs/Developer-Guide.md

@@ -627,7 +627,7 @@ the following steps (using rootfs or initrd image).
 >
 > Look for `INIT_PROCESS=systemd` in the `config.sh` osbuilder rootfs config file
 > to verify an osbuilder distro supports systemd for the distro you want to build rootfs for.
-> For an example, see the [Clear Linux config.sh file](../tools/osbuilder/rootfs-builder/clearlinux/config.sh).
+> For an example, see the [Ubuntu config.sh file](../tools/osbuilder/rootfs-builder/ubuntu/config.sh).
 >
 > For a non-systemd-based distro, create an equivalent system
 > service using that distro’s init system syntax. Alternatively, you can build a distro

docs/design/architecture/guest-assets.md

@@ -135,7 +135,7 @@ See also the [process overview](README.md#process-overview).
 
 | Image type | Default distro | Init daemon | Reason | Notes |
 |-|-|-|-|-|
-| [image](background.md#root-filesystem-image) | [Clear Linux](https://clearlinux.org) (for x86_64 systems)| systemd | Minimal and highly optimized | systemd offers flexibility |
+| [image](background.md#root-filesystem-image) | [Ubuntu](https://ubuntu.com) (for x86_64 systems) | systemd | Fully tested in our CI |  systemd offers flexibility |
 | [initrd](#initrd-image) | [Alpine Linux](https://alpinelinux.org) | Kata [agent](README.md#agent) (as no systemd support) | Security hardened and tiny C library |
 
 See also:

docs/design/virtualization.md

@@ -98,8 +98,7 @@ of Kata Containers, the Cloud Hypervisor configuration supports both CPU
 and memory resize, device hotplug (disk and VFIO), file-system sharing through virtio-fs,
 block-based volumes, booting from VM images backed by pmem device, and
 fine-grained seccomp filters for each VMM threads (e.g. all virtio
-device worker threads). Please check [this GitHub Project](https://github.com/orgs/kata-containers/projects/21)
-for details of ongoing integration efforts.
+device worker threads).
 
 Devices and features used:
 - virtio VSOCK or virtio serial

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -29,16 +29,16 @@ __SNP-specific steps:__
 - Build the SNP-specific kernel as shown below (see this [guide](../../tools/packaging/kernel/README.md#build-kata-containers-kernel) for more information)
 ```bash
 $ pushd kata-containers/tools/packaging/
-$ ./kernel/build-kernel.sh -a x86_64 -x snp setup
-$ ./kernel/build-kernel.sh -a x86_64 -x snp build
-$ sudo -E PATH="${PATH}" ./kernel/build-kernel.sh -x snp install
+$ ./kernel/build-kernel.sh -a x86_64 -x setup
+$ ./kernel/build-kernel.sh -a x86_64 -x build
+$ sudo -E PATH="${PATH}" ./kernel/build-kernel.sh -x install
 $ popd
 ```
 - Build a current OVMF capable of SEV-SNP:
 ```bash
 $ pushd kata-containers/tools/packaging/static-build/ovmf
-$ ./build.sh
-$ tar -xvf edk2-x86_64.tar.gz
+$ ovmf_build=sev ./build.sh
+$ tar -xvf edk2-sev.tar.gz
 $ popd
 ```
 - Build a custom QEMU
@@ -106,7 +106,7 @@ sev_snp_guest = true
 ```
   - Configure an OVMF (add path)
 ```toml
-firmware = "/path/to/kata-containers/tools/packaging/static-build/ovmf/opt/kata/share/ovmf/OVMF.fd"
+firmware = "/path/to/kata-containers/tools/packaging/static-build/ovmf/opt/kata/share/ovmf/AMDSEV.fd"
 ```
   - SNP attestation (add cert-chain to default path or add the path with cert-chain)
 ```toml

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -94,6 +94,8 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
 | `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
+| `io.katacontainers.config.hypervisor.default_gpus` | uint32 | the minimum number of GPUs required for the VM. Only used by remote hypervisor to help with instance selection |
+| `io.katacontainers.config.hypervisor.default_gpu_model` | string | the GPU model required for the VM. Only used by remote hypervisor to help with instance selection |
 
 ## Container Options
 | Key | Value Type | Comments |

src/agent/Cargo.lock

@@ -64,6 +64,20 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "ahash"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
+dependencies = [
+ "cfg-if 1.0.0",
+ "getrandom",
+ "once_cell",
+ "serde",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -97,6 +111,55 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "anstream"
+version = "0.6.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.86"
@@ -712,6 +775,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "bytecount"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -814,6 +883,30 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cdi"
+version = "0.1.0"
+source = "git+https://github.com/cncf-tags/container-device-interface-rs?rev=fba5677a8e7cc962fc6e495fcec98d7d765e332a#fba5677a8e7cc962fc6e495fcec98d7d765e332a"
+dependencies = [
+ "anyhow",
+ "clap 4.5.13",
+ "const_format",
+ "jsonschema",
+ "lazy_static",
+ "libc",
+ "nix 0.24.3",
+ "notify",
+ "oci-spec",
+ "once_cell",
+ "path-clean",
+ "regex",
+ "semver",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "serde_yaml",
+]
+
 [[package]]
 name = "cesu8"
 version = "1.1.0"
@@ -914,8 +1007,8 @@ checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
 dependencies = [
  "atty",
  "bitflags 1.3.2",
- "clap_derive",
- "clap_lex",
+ "clap_derive 3.2.25",
+ "clap_lex 0.2.4",
  "indexmap 1.9.3",
  "once_cell",
  "strsim 0.10.0",
@@ -923,6 +1016,28 @@ dependencies = [
  "textwrap",
 ]
 
+[[package]]
+name = "clap"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
+dependencies = [
+ "clap_builder",
+ "clap_derive 4.5.13",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex 0.7.2",
+ "strsim 0.11.1",
+]
+
 [[package]]
 name = "clap_derive"
 version = "3.2.25"
@@ -936,6 +1051,18 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "clap_derive"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.71",
+]
+
 [[package]]
 name = "clap_lex"
 version = "0.2.4"
@@ -945,6 +1072,12 @@ dependencies = [
  "os_str_bytes",
 ]
 
+[[package]]
+name = "clap_lex"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
+
 [[package]]
 name = "cmac"
 version = "0.7.2"
@@ -967,6 +1100,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "colorchoice"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -1741,6 +1880,17 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "fancy-regex"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
+dependencies = [
+ "bit-set",
+ "regex-automata 0.4.7",
+ "regex-syntax 0.8.4",
+]
+
 [[package]]
 name = "fastrand"
 version = "1.9.0"
@@ -1812,6 +1962,15 @@ dependencies = [
  "miniz_oxide",
 ]
 
+[[package]]
+name = "fluent-uri"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17c704e9dbe1ddd863da1e6ff3567795087b1eb201ce80d8fa81162e1516500d"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1827,6 +1986,25 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fraction"
+version = "0.15.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7"
+dependencies = [
+ "lazy_static",
+ "num",
+]
+
+[[package]]
+name = "fsevent-sys"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "funty"
 version = "2.0.0"
@@ -2052,7 +2230,7 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 dependencies = [
- "ahash",
+ "ahash 0.7.8",
 ]
 
 [[package]]
@@ -2605,6 +2783,21 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
+[[package]]
+name = "iso8601"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -2623,15 +2816,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -2690,6 +2874,18 @@ dependencies = [
  "smallvec",
 ]
 
+[[package]]
+name = "json-patch"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b1fb8864823fad91877e6caea0baca82e49e8db50f8e5c9f9a453e27d3330fc"
+dependencies = [
+ "jsonptr",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
 [[package]]
 name = "json-syntax"
 version = "0.12.5"
@@ -2709,6 +2905,47 @@ dependencies = [
  "utf8-decode",
 ]
 
+[[package]]
+name = "jsonptr"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627"
+dependencies = [
+ "fluent-uri",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "jsonschema"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0afd06142c9bcb03f4a8787c77897a87b6be9c4918f1946c33caa714c27578"
+dependencies = [
+ "ahash 0.8.11",
+ "anyhow",
+ "base64 0.22.1",
+ "bytecount",
+ "clap 4.5.13",
+ "fancy-regex",
+ "fraction",
+ "getrandom",
+ "iso8601",
+ "itoa",
+ "memchr",
+ "num-cmp",
+ "once_cell",
+ "parking_lot 0.12.3",
+ "percent-encoding",
+ "regex",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "time",
+ "url",
+ "uuid",
+]
+
 [[package]]
 name = "jwt"
 version = "0.16.0"
@@ -2773,14 +3010,16 @@ dependencies = [
  "async-std",
  "async-trait",
  "capctl",
+ "cdi",
  "cfg-if 1.0.0",
  "cgroups-rs",
- "clap",
+ "clap 3.2.25",
  "const_format",
  "derivative",
  "futures",
  "image-rs",
  "ipnetwork",
+ "json-patch",
  "kata-sys-util",
  "kata-types",
  "lazy_static",
@@ -2835,7 +3074,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -2934,6 +3172,26 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "kqueue"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c"
+dependencies = [
+ "kqueue-sys",
+ "libc",
+]
+
+[[package]]
+name = "kqueue-sys"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+]
+
 [[package]]
 name = "krata-tokio-tar"
 version = "0.4.2"
@@ -3285,6 +3543,18 @@ dependencies = [
  "adler",
 ]
 
+[[package]]
+name = "mio"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+dependencies = [
+ "libc",
+ "log",
+ "wasi",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "mio"
 version = "1.0.2"
@@ -3465,6 +3735,25 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "notify"
+version = "6.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
+dependencies = [
+ "bitflags 2.6.0",
+ "crossbeam-channel",
+ "filetime",
+ "fsevent-sys",
+ "inotify",
+ "kqueue",
+ "libc",
+ "log",
+ "mio 0.8.11",
+ "walkdir",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "ntapi"
 version = "0.4.1"
@@ -3515,6 +3804,12 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "num-cmp"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa"
+
 [[package]]
 name = "num-complex"
 version = "0.4.6"
@@ -3894,6 +4189,12 @@ dependencies = [
  "slash-formatter",
 ]
 
+[[package]]
+name = "path-clean"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
+
 [[package]]
 name = "path-dedot"
 version = "1.2.4"
@@ -4376,7 +4677,6 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.5.1",
  "serde",
@@ -4626,14 +4926,12 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
 
 [[package]]
 name = "regorus"
-version = "0.1.5"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77dd872918e5c172bd42ac49716f89a15e35be513bba3d902e355a531529a87f"
+checksum = "843c3d97f07e3b5ac0955d53ad0af4c91fe4a4f8525843ece5bf014f27829b73"
 dependencies = [
  "anyhow",
- "itertools 0.12.1",
  "lazy_static",
- "num",
  "rand",
  "regex",
  "scientific",
@@ -4666,6 +4964,7 @@ dependencies = [
  "bytes 1.6.1",
  "cookie",
  "cookie_store",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "http",
@@ -5919,7 +6218,7 @@ dependencies = [
  "backtrace",
  "bytes 1.6.1",
  "libc",
- "mio",
+ "mio 1.0.2",
  "parking_lot 0.12.3",
  "pin-project-lite",
  "signal-hook-registry",
@@ -6346,6 +6645,12 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
 [[package]]
 name = "uuid"
 version = "1.10.0"
@@ -7021,6 +7326,26 @@ dependencies = [
  "zvariant",
 ]
 
+[[package]]
+name = "zerocopy"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.71",
+]
+
 [[package]]
 name = "zerofrom"
 version = "0.1.4"

src/agent/Cargo.toml

@@ -80,10 +80,13 @@ strum_macros = "0.26.2"
 image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "v0.10.0", default-features = false, optional = true }
 
 # Agent Policy
-regorus = { version = "0.1.4", default-features = false, features = [
+regorus = { version = "0.2.6", default-features = false, features = [
     "arc",
     "regex",
+    "std",
 ], optional = true }
+cdi = { git = "https://github.com/cncf-tags/container-device-interface-rs", rev = "fba5677a8e7cc962fc6e495fcec98d7d765e332a" }
+json-patch = "2.0.0"
 
 [dev-dependencies]
 tempfile = "3.1.0"

src/agent/rustjail/src/mount.rs

@@ -233,7 +233,7 @@ pub fn init_rootfs(
         // bind may be only specified in the oci spec options -> flags update r#type
         let m = &{
             let mut mbind = m.clone();
-            if mbind.typ().is_none() && flags & MsFlags::MS_BIND == MsFlags::MS_BIND {
+            if is_none_mount_type(mbind.typ()) && flags & MsFlags::MS_BIND == MsFlags::MS_BIND {
                 mbind.set_typ(Some("bind".to_string()));
             }
             mbind
@@ -397,6 +397,13 @@ fn mount_cgroups_v2(cfd_log: RawFd, m: &Mount, rootfs: &str, flags: MsFlags) ->
     Ok(())
 }
 
+fn is_none_mount_type(typ: &Option<String>) -> bool {
+    match typ {
+        Some(t) => t == "none",
+        None => true,
+    }
+}
+
 fn mount_cgroups(
     cfd_log: RawFd,
     m: &Mount,

src/agent/src/device/mod.rs

@@ -11,6 +11,9 @@ use self::vfio_device_handler::{VfioApDeviceHandler, VfioPciDeviceHandler};
 use crate::pci;
 use crate::sandbox::Sandbox;
 use anyhow::{anyhow, Context, Result};
+use cdi::annotations::parse_annotations;
+use cdi::cache::{new_cache, with_auto_refresh, CdiOption};
+use cdi::spec_dirs::with_spec_dirs;
 use kata_types::device::DeviceHandlerManager;
 use nix::sys::stat;
 use oci::{LinuxDeviceCgroup, Spec};
@@ -25,6 +28,8 @@ use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
+use tokio::time;
+use tokio::time::Duration;
 use tracing::instrument;
 
 pub mod block_device_handler;
@@ -238,6 +243,69 @@ pub async fn add_devices(
     update_spec_devices(logger, spec, dev_updates)
 }
 
+#[instrument]
+pub async fn handle_cdi_devices(
+    logger: &Logger,
+    spec: &mut Spec,
+    spec_dir: &str,
+    cdi_timeout: u64,
+) -> Result<()> {
+    if let Some(container_type) = spec
+        .annotations()
+        .as_ref()
+        .and_then(|a| a.get("io.katacontainers.pkg.oci.container_type"))
+    {
+        if container_type == "pod_sandbox" {
+            return Ok(());
+        }
+    }
+
+    let (_, devices) = parse_annotations(spec.annotations().as_ref().unwrap())?;
+
+    if devices.is_empty() {
+        info!(logger, "no CDI annotations, no devices to inject");
+        return Ok(());
+    }
+    // Explicitly set the cache options to disable auto-refresh and
+    // to use the single spec dir "/var/run/cdi" for tests it can be overridden
+    let options: Vec<CdiOption> = vec![with_auto_refresh(false), with_spec_dirs(&[spec_dir])];
+    let cache: Arc<std::sync::Mutex<cdi::cache::Cache>> = new_cache(options);
+
+    for _ in 0..=cdi_timeout {
+        let inject_result = {
+            // Lock cache within this scope, std::sync::Mutex has no Send
+            // and await will not work with time::sleep
+            let mut cache = cache.lock().unwrap();
+            match cache.refresh() {
+                Ok(_) => {}
+                Err(e) => {
+                    return Err(anyhow!("error refreshing cache: {:?}", e));
+                }
+            }
+            cache.inject_devices(Some(spec), devices.clone())
+        };
+
+        match inject_result {
+            Ok(_) => {
+                info!(
+                    logger,
+                    "all devices injected successfully, modified CDI container spec: {:?}", &spec
+                );
+                return Ok(());
+            }
+            Err(e) => {
+                info!(logger, "error injecting devices: {:?}", e);
+                println!("error injecting devices: {:?}", e);
+            }
+        }
+        time::sleep(Duration::from_millis(1000)).await;
+    }
+    Err(anyhow!(
+        "failed to inject devices after CDI timeout of {} seconds",
+        cdi_timeout
+    ))
+}
+
 #[instrument]
 async fn validate_device(
     logger: &Logger,
@@ -1110,4 +1178,94 @@ mod tests {
         assert!(name.is_ok(), "{}", name.unwrap_err());
         assert_eq!(name.unwrap(), devname);
     }
+
+    #[tokio::test]
+    async fn test_handle_cdi_devices() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let mut spec = Spec::default();
+
+        let mut annotations = HashMap::new();
+        // cdi.k8s.io/vendor1_devices: vendor1.com/device=foo
+        annotations.insert(
+            "cdi.k8s.io/vfio17".to_string(),
+            "kata.com/gpu=0".to_string(),
+        );
+        spec.set_annotations(Some(annotations));
+
+        let temp_dir = tempdir().expect("Failed to create temporary directory");
+        let cdi_file = temp_dir.path().join("kata.json");
+
+        let cdi_version = "0.6.0";
+        let kind = "kata.com/gpu";
+        let device_name = "0";
+        let annotation_whatever = "false";
+        let annotation_whenever = "true";
+        let inner_env = "TEST_INNER_ENV=TEST_INNER_ENV_VALUE";
+        let outer_env = "TEST_OUTER_ENV=TEST_OUTER_ENV_VALUE";
+        let inner_device = "/dev/zero";
+        let outer_device = "/dev/null";
+
+        let cdi_content = format!(
+            r#"{{
+            "cdiVersion": "{cdi_version}",
+            "kind": "{kind}",
+            "devices": [
+                {{
+                    "name": "{device_name}",
+                    "annotations": {{
+                        "whatever": "{annotation_whatever}",
+                        "whenever": "{annotation_whenever}"
+                    }},
+                    "containerEdits": {{
+                        "env": [
+                            "{inner_env}"
+                        ],
+                        "deviceNodes": [
+                            {{
+                                "path": "{inner_device}"
+                            }}
+                        ]
+                    }}
+                }}
+            ],
+            "containerEdits": {{
+                "env": [
+                    "{outer_env}"
+                ],
+                "deviceNodes": [
+                    {{
+                        "path": "{outer_device}"
+                    }}
+                ]
+            }}
+        }}"#
+        );
+
+        fs::write(&cdi_file, cdi_content).expect("Failed to write CDI file");
+
+        let res =
+            handle_cdi_devices(&logger, &mut spec, temp_dir.path().to_str().unwrap(), 0).await;
+        println!("modfied spec {:?}", spec);
+        assert!(res.is_ok(), "{}", res.err().unwrap());
+
+        let linux = spec.linux().as_ref().unwrap();
+        let devices = linux
+            .resources()
+            .as_ref()
+            .unwrap()
+            .devices()
+            .as_ref()
+            .unwrap();
+        assert_eq!(devices.len(), 2);
+
+        let env = spec.process().as_ref().unwrap().env().as_ref().unwrap();
+
+        // find string TEST_OUTER_ENV in env
+        let outer_env = env.iter().find(|e| e.starts_with("TEST_OUTER_ENV"));
+        assert!(outer_env.is_some(), "TEST_OUTER_ENV not found in env");
+
+        // find TEST_INNER_ENV in env
+        let inner_env = env.iter().find(|e| e.starts_with("TEST_INNER_ENV"));
+        assert!(inner_env.is_some(), "TEST_INNER_ENV not found in env");
+    }
 }

src/agent/src/image.rs

@@ -21,6 +21,9 @@ use tokio::sync::Mutex;
 use crate::rpc::CONTAINER_BASE;
 use crate::AGENT_CONFIG;
 
+use kata_types::mount::KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL;
+use protocols::agent::Storage;
+
 pub const KATA_IMAGE_WORK_DIR: &str = "/run/kata-containers/image/";
 const CONFIG_JSON: &str = "config.json";
 const KATA_PAUSE_BUNDLE: &str = "/pause_bundle";
@@ -81,6 +84,28 @@ impl ImageService {
         Self { image_client }
     }
 
+    /// get guest pause image process specification
+    fn get_pause_image_process() -> Result<oci::Process> {
+        let guest_pause_bundle = Path::new(KATA_PAUSE_BUNDLE);
+        if !guest_pause_bundle.exists() {
+            bail!("Pause image not present in rootfs");
+        }
+        let guest_pause_config = scoped_join(guest_pause_bundle, CONFIG_JSON)?;
+
+        let image_oci = oci::Spec::load(guest_pause_config.to_str().ok_or_else(|| {
+            anyhow!(
+                "Failed to load the guest pause image config from {:?}",
+                guest_pause_config
+            )
+        })?)
+        .context("load image config file")?;
+
+        let image_oci_process = image_oci.process().as_ref().ok_or_else(|| {
+            anyhow!("The guest pause image config does not contain a process specification. Please check the pause image.")
+        })?;
+        Ok(image_oci_process.clone())
+    }
+
     /// pause image is packaged in rootfs
     fn unpack_pause_image(cid: &str) -> Result<String> {
         verify_id(cid).context("The guest pause image cid contains invalid characters.")?;
@@ -132,6 +157,20 @@ impl ImageService {
         Ok(pause_rootfs.display().to_string())
     }
 
+    /// check whether the image is for sandbox or for container.
+    fn is_sandbox(image_metadata: &HashMap<String, String>) -> bool {
+        let mut is_sandbox = false;
+        for key in K8S_CONTAINER_TYPE_KEYS.iter() {
+            if let Some(value) = image_metadata.get(key as &str) {
+                if value == "sandbox" {
+                    is_sandbox = true;
+                    break;
+                }
+            }
+        }
+        is_sandbox
+    }
+
     /// pull_image is used for call image-rs to pull image in the guest.
     /// # Parameters
     /// - `image`: Image name (exp: quay.io/prometheus/busybox:latest)
@@ -147,18 +186,7 @@ impl ImageService {
     ) -> Result<String> {
         info!(sl(), "image metadata: {image_metadata:?}");
 
-        //Check whether the image is for sandbox or for container.
-        let mut is_sandbox = false;
-        for key in K8S_CONTAINER_TYPE_KEYS.iter() {
-            if let Some(value) = image_metadata.get(key as &str) {
-                if value == "sandbox" {
-                    is_sandbox = true;
-                    break;
-                }
-            }
-        }
-
-        if is_sandbox {
+        if Self::is_sandbox(image_metadata) {
             let mount_path = Self::unpack_pause_image(cid)?;
             return Ok(mount_path);
         }
@@ -194,6 +222,32 @@ impl ImageService {
     }
 }
 
+/// get_process overrides the OCI process spec with pause image process spec if needed
+pub fn get_process(
+    ocip: &oci::Process,
+    oci: &oci::Spec,
+    storages: Vec<Storage>,
+) -> Result<oci::Process> {
+    let mut guest_pull = false;
+    for storage in storages {
+        if storage.driver == KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL {
+            guest_pull = true;
+            break;
+        }
+    }
+    if guest_pull {
+        match oci.annotations() {
+            Some(a) => {
+                if ImageService::is_sandbox(a) {
+                    return ImageService::get_pause_image_process();
+                }
+            }
+            None => {}
+        }
+    }
+    Ok(ocip.clone())
+}
+
 /// Set proxy environment from AGENT_CONFIG
 pub async fn set_proxy_env_vars() {
     if env::var("HTTPS_PROXY").is_err() {

src/agent/src/main.rs

@@ -21,7 +21,7 @@ extern crate slog;
 use anyhow::{anyhow, Context, Result};
 use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
-use const_format::concatcp;
+use const_format::{concatcp, formatcp};
 use nix::fcntl::OFlag;
 use nix::sys::reboot::{reboot, RebootMode};
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
@@ -29,7 +29,7 @@ use nix::unistd::{self, dup, sync, Pid};
 use std::env;
 use std::ffi::OsStr;
 use std::fs::{self, File};
-use std::os::unix::fs as unixfs;
+use std::os::unix::fs::{self as unixfs, FileTypeExt};
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
 use std::process::exit;
@@ -109,7 +109,18 @@ const CDH_SOCKET_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, CDH_SOCKET);
 const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest";
 
 /// Path of ocicrypt config file. This is used by image-rs when decrypting image.
-const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+const OCICRYPT_CONFIG_PATH: &str = "/run/confidential-containers/ocicrypt_config.json";
+
+const OCICRYPT_CONFIG: &str = formatcp!(
+    r#"{{
+    "key-providers": {{
+        "attestation-agent": {{
+            "ttrpc": "{}"
+        }}
+    }}
+}}"#,
+    CDH_SOCKET_URI
+);
 
 const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6;
 
@@ -408,15 +419,13 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     let gc_procs = config.guest_components_procs;
-    if gc_procs != GuestComponentsProcs::None {
-        if !attestation_binaries_available(logger, &gc_procs) {
-            warn!(
-                logger,
-                "attestation binaries requested for launch not available"
-            );
-        } else {
-            init_attestation_components(logger, config).await?;
-        }
+    if !attestation_binaries_available(logger, &gc_procs) {
+        warn!(
+            logger,
+            "attestation binaries requested for launch not available"
+        );
+    } else {
+        init_attestation_components(logger, config).await?;
     }
 
     // vsock:///dev/vsock, port
@@ -447,12 +456,7 @@ fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs)
     true
 }
 
-// Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
-// and the corresponding procs are enabled in the agent configuration. the process will be
-// launched in the background and the function will return immediately.
-// If the CDH is started, a CDH client will be instantiated and returned.
-async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> {
-    // skip launch of any guest-component
+async fn launch_guest_component_procs(logger: &Logger, config: &AgentConfig) -> Result<()> {
     if config.guest_components_procs == GuestComponentsProcs::None {
         return Ok(());
     }
@@ -472,17 +476,6 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
         return Ok(());
     }
 
-    let ocicrypt_config = serde_json::json!({
-        "key-providers": {
-            "attestation-agent":{
-                "ttrpc":CDH_SOCKET_URI
-            }
-        }
-    });
-
-    fs::write(OCICRYPT_CONFIG_PATH, ocicrypt_config.to_string().as_bytes())?;
-    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
-
     debug!(
         logger,
         "spawning confidential-data-hub process {}", CDH_PATH
@@ -497,9 +490,6 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
     )
     .map_err(|e| anyhow!("launch_process {} failed: {:?}", CDH_PATH, e))?;
 
-    // initialize cdh client
-    cdh::init_cdh_client(CDH_SOCKET_URI).await?;
-
     // skip launch of api-server-rest
     if config.guest_components_procs == GuestComponentsProcs::ConfidentialDataHub {
         return Ok(());
@@ -522,6 +512,33 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
     Ok(())
 }
 
+// Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
+// and the corresponding procs are enabled in the agent configuration. the process will be
+// launched in the background and the function will return immediately.
+// If the CDH is started, a CDH client will be instantiated and returned.
+async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> {
+    launch_guest_component_procs(logger, config).await?;
+
+    // If a CDH socket exists, initialize the CDH client and enable ocicrypt
+    match tokio::fs::metadata(CDH_SOCKET).await {
+        Ok(md) => {
+            if md.file_type().is_socket() {
+                cdh::init_cdh_client(CDH_SOCKET_URI).await?;
+                fs::write(OCICRYPT_CONFIG_PATH, OCICRYPT_CONFIG.as_bytes())?;
+                env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
+            } else {
+                debug!(logger, "File {} is not a socket", CDH_SOCKET);
+            }
+        }
+        Err(err) => warn!(
+            logger,
+            "Failed to probe CDH socket file {}: {:?}", CDH_SOCKET, err
+        ),
+    }
+
+    Ok(())
+}
+
 fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> {
     let p = Path::new(path);
     let mut attempts = 0;

src/agent/src/policy.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::Result;
+use anyhow::{bail, Result};
 use protobuf::MessageDyn;
 use tokio::io::AsyncWriteExt;
 
@@ -68,6 +68,12 @@ pub struct AgentPolicy {
     engine: regorus::Engine,
 }
 
+#[derive(serde::Deserialize, Debug)]
+struct MetadataResponse {
+    allowed: bool,
+    ops: Option<json_patch::Patch>,
+}
+
 impl AgentPolicy {
     /// Create AgentPolicy object.
     pub fn new() -> Self {
@@ -82,6 +88,17 @@ impl AgentPolicy {
         let mut engine = regorus::Engine::new();
         engine.set_strict_builtin_errors(false);
         engine.set_gather_prints(true);
+        // assign a slice of the engine data "pstate" to be used as policy state
+        engine
+            .add_data(
+                regorus::Value::from_json_str(
+                    r#"{
+                        "pstate": {}
+                    }"#,
+                )
+                .unwrap(),
+            )
+            .unwrap();
         engine
     }
 
@@ -112,6 +129,23 @@ impl AgentPolicy {
         Ok(())
     }
 
+    async fn apply_patch_to_state(&mut self, patch: json_patch::Patch) -> Result<()> {
+        // Convert the current engine data to a JSON value
+        let mut state = serde_json::to_value(self.engine.get_data())?;
+
+        // Apply the patch to the state
+        json_patch::patch(&mut state, &patch)?;
+
+        // Clear the existing data in the engine
+        self.engine.clear_data();
+
+        // Add the patched state back to the engine
+        self.engine
+            .add_data(regorus::Value::from_json_str(&state.to_string())?)?;
+
+        Ok(())
+    }
+
     /// Ask regorus if an API call should be allowed or not.
     async fn allow_request(&mut self, ep: &str, ep_input: &str) -> Result<(bool, String)> {
         debug!(sl!(), "policy check: {ep}");
@@ -120,13 +154,56 @@ impl AgentPolicy {
         let query = format!("data.agent_policy.{ep}");
         self.engine.set_input_json(ep_input)?;
 
-        let mut allow = match self.engine.eval_bool_query(query, false) {
-            Ok(a) => a,
-            Err(e) => {
-                if !self.allow_failures {
-                    return Err(e);
+        let results = self.engine.eval_query(query, false)?;
+
+        let prints = match self.engine.take_prints() {
+            Ok(p) => p.join(" "),
+            Err(e) => format!("Failed to get policy log: {e}"),
+        };
+
+        if results.result.len() != 1 {
+            // Results are empty when AllowRequestsFailingPolicy is used to allow a Request that hasn't been defined in the policy
+            if self.allow_failures {
+                return Ok((true, prints));
+            }
+            bail!(
+                "policy check: unexpected eval_query result len {:?}",
+                results
+            );
+        }
+
+        if results.result[0].expressions.len() != 1 {
+            bail!(
+                "policy check: unexpected eval_query result expressions {:?}",
+                results
+            );
+        }
+
+        let mut allow = match &results.result[0].expressions[0].value {
+            regorus::Value::Bool(b) => *b,
+
+            // Match against a specific variant that could be interpreted as MetadataResponse
+            regorus::Value::Object(obj) => {
+                let json_str = serde_json::to_string(obj)?;
+
+                self.log_eval_input(ep, &json_str).await;
+
+                let metadata_response: MetadataResponse = serde_json::from_str(&json_str)?;
+
+                if metadata_response.allowed {
+                    if let Some(ops) = metadata_response.ops {
+                        self.apply_patch_to_state(ops).await?;
+                    }
                 }
-                false
+                metadata_response.allowed
+            }
+
+            _ => {
+                error!(sl!(), "allow_request: unexpected eval_query result type");
+                bail!(
+                    "policy check: unexpected eval_query result type {:?}",
+                    results
+                );
             }
         };
 
@@ -135,11 +212,6 @@ impl AgentPolicy {
             allow = true;
         }
 
-        let prints = match self.engine.take_prints() {
-            Ok(p) => p.join(" "),
-            Err(e) => format!("Failed to get policy log: {e}"),
-        };
-
         Ok((allow, prints))
     }
 

src/agent/src/rpc.rs

@@ -58,7 +58,7 @@ use rustjail::process::ProcessOperations;
 use crate::cdh;
 use crate::device::block_device_handler::get_virtio_blk_pci_device_name;
 use crate::device::network_device_handler::wait_for_net_interface;
-use crate::device::{add_devices, update_env_pci};
+use crate::device::{add_devices, handle_cdi_devices, update_env_pci};
 use crate::features::get_build_features;
 use crate::image::KATA_IMAGE_WORK_DIR;
 use crate::linux_abi::*;
@@ -130,6 +130,8 @@ const ERR_NO_SANDBOX_PIDNS: &str = "Sandbox does not have sandbox_pidns";
 // not available.
 const IPTABLES_RESTORE_WAIT_SEC: u64 = 5;
 
+const CDI_TIMEOUT_LIMIT: u64 = 100;
+
 // Convenience function to obtain the scope logger.
 fn sl() -> slog::Logger {
     slog_scope::logger()
@@ -224,6 +226,15 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&sl(), &req.devices, &mut oci, &self.sandbox).await?;
 
+        // In guest-kernel mode some devices need extra handling. Taking the
+        // GPU as an example the shim will inject CDI annotations that will
+        // be used by the kata-agent to do containerEdits according to the
+        // CDI spec coming from a registry that is created on the fly by UDEV
+        // or other entities for a specifc device.
+        // In Kata we only consider the directory "/var/run/cdi", "/etc" may be
+        // readonly
+        handle_cdi_devices(&sl(), &mut oci, "/var/run/cdi", CDI_TIMEOUT_LIMIT).await?;
+
         cdh_handler(&mut oci).await?;
 
         // Both rootfs and volumes (invoked with --volume for instance) will
@@ -233,7 +244,13 @@ impl AgentService {
         // After all those storages have been processed, no matter the order
         // here, the agent will rely on rustjail (using the oci.Mounts
         // list) to bind mount all of them inside the container.
-        let m = add_storages(sl(), req.storages, &self.sandbox, Some(req.container_id)).await?;
+        let m = add_storages(
+            sl(),
+            req.storages.clone(),
+            &self.sandbox,
+            Some(req.container_id),
+        )
+        .await?;
 
         let mut s = self.sandbox.lock().await;
         s.container_mounts.insert(cid.clone(), m);
@@ -288,6 +305,13 @@ impl AgentService {
         let pipe_size = AGENT_CONFIG.container_pipe_size;
 
         let p = if let Some(p) = oci.process() {
+            #[cfg(feature = "guest-pull")]
+            {
+                let new_p = image::get_process(p, &oci, req.storages.clone())?;
+                Process::new(&sl(), &new_p, cid.as_str(), true, pipe_size, proc_io)?
+            }
+
+            #[cfg(not(feature = "guest-pull"))]
             Process::new(&sl(), p, cid.as_str(), true, pipe_size, proc_io)?
         } else {
             info!(sl(), "no process configurations!");

src/agent/src/storage/ephemeral_handler.rs

@@ -170,7 +170,7 @@ impl EphemeralHandler {
         let size = size_str
             .unwrap()
             .parse::<u64>()
-            .context(format!("parse size: {:?}", &pagesize_str))?;
+            .context(format!("parse size: {:?}", &size_str))?;
 
         Ok((pagesize, size))
     }

src/libs/Cargo.lock

@@ -240,19 +240,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "cgroups-rs"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
-dependencies = [
- "libc",
- "log",
- "nix 0.25.1",
- "regex",
- "thiserror",
-]
-
 [[package]]
 name = "chrono"
 version = "0.4.20"
@@ -814,7 +801,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -975,18 +961,6 @@ dependencies = [
  "memoffset 0.6.5",
 ]
 
-[[package]]
-name = "nix"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
-dependencies = [
- "autocfg",
- "bitflags",
- "cfg-if",
- "libc",
-]
-
 [[package]]
 name = "nix"
 version = "0.26.4"
@@ -1316,7 +1290,6 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.2.0",
  "serde",

src/libs/kata-sys-util/Cargo.toml

@@ -13,7 +13,6 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0.31"
 byteorder = "1.4.3"
-cgroups = { package = "cgroups-rs", version = "0.3.2" }
 chrono = "0.4.0"
 common-path = "=1.0.0"
 fail = "0.5.0"

src/libs/kata-sys-util/README.md

@@ -1,10 +1,9 @@
-# kata-sys-util
+# `kata-sys-util`
 
 This crate is a collection of utilities and helpers for 
 [Kata Containers](https://github.com/kata-containers/kata-containers/) components to access system services.
 
 It provides safe wrappers over system services, such as:
-- cgroups
 - file systems
 - mount
 - NUMA

src/libs/kata-sys-util/src/mount.rs

@@ -53,6 +53,7 @@ use std::time::Instant;
 use lazy_static::lazy_static;
 use nix::mount::{mount, MntFlags, MsFlags};
 use nix::{unistd, NixPath};
+use oci_spec::runtime as oci;
 
 use crate::fs::is_symlink;
 use crate::sl;
@@ -799,8 +800,20 @@ pub fn get_mount_options(options: &Option<Vec<String>>) -> Vec<String> {
     }
 }
 
-pub fn get_mount_type(typ: &Option<String>) -> String {
-    typ.clone().unwrap_or("bind".to_string())
+pub fn get_mount_type(m: &oci::Mount) -> String {
+    m.typ()
+        .clone()
+        .map(|typ| {
+            if typ.as_str() == "none" {
+                if let Some(opts) = m.options() {
+                    if opts.iter().any(|opt| opt == "bind" || opt == "rbind") {
+                        return "bind".to_string();
+                    }
+                }
+            }
+            typ
+        })
+        .unwrap_or("bind".to_string())
 }
 
 #[cfg(test)]

src/libs/kata-sys-util/src/spec.rs

@@ -97,11 +97,3 @@ pub fn load_oci_spec() -> Result<oci::Spec, OciSpecError> {
 
     oci::Spec::load(spec_file.to_str().unwrap_or_default())
 }
-
-/// handle string parsing for input possibly be JSON string.
-pub fn parse_json_string(input: &str) -> &str {
-    let json_str: &str = serde_json::from_str(input).unwrap_or(input);
-    let stripped_str = json_str.strip_prefix("CAP_").unwrap_or(json_str);
-
-    stripped_str
-}

src/libs/kata-types/src/annotations/cri_containerd.rs

@@ -11,7 +11,12 @@ pub const CONTAINER_NAME_LABEL_KEY: &str = "io.kubernetes.cri.container-name";
 pub const SANDBOX: &str = "sandbox";
 pub const CONTAINER: &str = "container";
 
+// SandboxID is the sandbox ID annotation
 pub const SANDBOX_ID_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-id";
+// SandboxName is the name of the sandbox (pod)
+pub const SANDBOX_NAME_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-name";
+// SandboxNamespace is the name of the namespace of the sandbox (pod)
+pub const SANDBOX_NAMESPACE_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-namespace";
 
 // Ref: https://pkg.go.dev/github.com/containerd/containerd@v1.6.7/pkg/cri/annotations
 // SandboxCPU annotations are based on the initial CPU configuration for the sandbox. This is calculated as the

src/libs/kata-types/src/config/default.rs

@@ -98,3 +98,11 @@ pub const DEFAULT_FIRECRACKER_GUEST_KERNEL_IMAGE: &str = "vmlinux";
 pub const DEFAULT_FIRECRACKER_GUEST_KERNEL_PARAMS: &str = "";
 pub const MAX_FIRECRACKER_VCPUS: u32 = 32;
 pub const MIN_FIRECRACKER_MEMORY_SIZE_MB: u32 = 128;
+
+// Default configuration for remote
+pub const DEFAULT_REMOTE_HYPERVISOR_SOCKET: &str = "/run/peerpod/hypervisor.sock";
+pub const DEFAULT_REMOTE_HYPERVISOR_TIMEOUT: i32 = 600; // 600 Seconds
+pub const MAX_REMOTE_VCPUS: u32 = 32;
+pub const MIN_REMOTE_MEMORY_SIZE_MB: u32 = 64;
+pub const DEFAULT_REMOTE_MEMORY_SIZE_MB: u32 = 128;
+pub const DEFAULT_REMOTE_MEMORY_SLOTS: u32 = 128;

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -44,6 +44,9 @@ pub use self::qemu::{QemuConfig, HYPERVISOR_NAME_QEMU};
 mod ch;
 pub use self::ch::{CloudHypervisorConfig, HYPERVISOR_NAME_CH};
 
+mod remote;
+pub use self::remote::{RemoteConfig, HYPERVISOR_NAME_REMOTE};
+
 /// Virtual PCI block device driver.
 pub const VIRTIO_BLK_PCI: &str = "virtio-blk-pci";
 
@@ -540,6 +543,7 @@ impl TopologyConfigInfo {
             HYPERVISOR_NAME_CH,
             HYPERVISOR_NAME_DRAGONBALL,
             HYPERVISOR_NAME_FIRECRACKER,
+            HYPERVISOR_NAME_REMOTE,
         ];
         let hypervisor_name = toml_config.runtime.hypervisor_name.as_str();
         if !hypervisor_names.contains(&hypervisor_name) {
@@ -1040,6 +1044,18 @@ impl SharedFsInfo {
     }
 }
 
+/// Configuration information for remote hypervisor type.
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub struct RemoteInfo {
+    /// Remote hypervisor socket path
+    #[serde(default)]
+    pub hypervisor_socket: String,
+
+    /// Remote hyperisor timeout of creating (in seconds)
+    #[serde(default)]
+    pub hypervisor_timeout: i32,
+}
+
 /// Common configuration information for hypervisors.
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct Hypervisor {
@@ -1123,6 +1139,10 @@ pub struct Hypervisor {
     #[serde(default, flatten)]
     pub shared_fs: SharedFsInfo,
 
+    /// Remote hypervisor configuration information.
+    #[serde(default, flatten)]
+    pub remote_info: RemoteInfo,
+
     /// A sandbox annotation used to specify prefetch_files.list host path container image
     /// being used, and runtime will pass it to Hypervisor to  search for corresponding
     /// prefetch list file:
@@ -1164,6 +1184,10 @@ impl ConfigOps for Hypervisor {
     fn adjust_config(conf: &mut TomlConfig) -> Result<()> {
         HypervisorVendor::adjust_config(conf)?;
         let hypervisors: Vec<String> = conf.hypervisor.keys().cloned().collect();
+        info!(
+            sl!(),
+            "Adjusting hypervisor configuration {:?}", hypervisors
+        );
         for hypervisor in hypervisors.iter() {
             if let Some(plugin) = get_hypervisor_plugin(hypervisor) {
                 plugin.adjust_config(conf)?;

src/libs/kata-types/src/config/hypervisor/remote.rs

@@ -0,0 +1,116 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use byte_unit::{Byte, Unit};
+use std::io::Result;
+use std::path::Path;
+use std::sync::Arc;
+use sysinfo::System;
+
+use crate::{
+    config::{
+        default::{self, MAX_REMOTE_VCPUS, MIN_REMOTE_MEMORY_SIZE_MB},
+        ConfigPlugin,
+    }, device::DRIVER_NVDIMM_TYPE, eother, resolve_path
+};
+
+use super::register_hypervisor_plugin;
+
+/// Hypervisor name for remote, used to index `TomlConfig::hypervisor`.
+pub const HYPERVISOR_NAME_REMOTE: &str = "remote";
+
+/// Configuration information for remote.
+#[derive(Default, Debug)]
+pub struct RemoteConfig {}
+
+impl RemoteConfig {
+    /// Create a new instance of `RemoteConfig`
+    pub fn new() -> Self {
+        RemoteConfig {}
+    }
+
+    /// Register the remote plugin.
+    pub fn register(self) {
+        let plugin = Arc::new(self);
+        register_hypervisor_plugin(HYPERVISOR_NAME_REMOTE, plugin);
+    }
+}
+
+impl ConfigPlugin for RemoteConfig {
+    fn name(&self) -> &str {
+        HYPERVISOR_NAME_REMOTE
+    }
+
+    /// Adjust the configuration information after loading from configuration file.
+    fn adjust_config(&self, conf: &mut crate::config::TomlConfig) -> Result<()> {
+        if let Some(remote) = conf.hypervisor.get_mut(HYPERVISOR_NAME_REMOTE) {
+            if remote.remote_info.hypervisor_socket.is_empty() {
+                remote.remote_info.hypervisor_socket =
+                    default::DEFAULT_REMOTE_HYPERVISOR_SOCKET.to_string();
+            }
+            resolve_path!(
+                remote.remote_info.hypervisor_socket,
+                "Remote hypervisor socket `{}` is invalid: {}"
+            )?;
+            if remote.remote_info.hypervisor_timeout == 0 {
+                remote.remote_info.hypervisor_timeout = default::DEFAULT_REMOTE_HYPERVISOR_TIMEOUT;
+            }
+            if remote.memory_info.default_memory == 0 {
+                remote.memory_info.default_memory = default::MIN_REMOTE_MEMORY_SIZE_MB;
+            }
+            if remote.memory_info.memory_slots == 0 {
+                remote.memory_info.memory_slots = default::DEFAULT_REMOTE_MEMORY_SLOTS
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Validate the configuration information.
+    fn validate(&self, conf: &crate::config::TomlConfig) -> Result<()> {
+        if let Some(remote) = conf.hypervisor.get(HYPERVISOR_NAME_REMOTE) {
+            let s = System::new_all();
+            let total_memory = Byte::from_u64(s.total_memory())
+                .get_adjusted_unit(Unit::MiB)
+                .get_value() as u32;
+            if remote.memory_info.default_maxmemory != total_memory {
+                return Err(eother!(
+                    "Remote hypervisor does not support memory hotplug, default_maxmemory must be equal to the total system memory",
+                ));
+            }
+            let cpus = num_cpus::get() as u32;
+            if remote.cpu_info.default_maxvcpus != cpus {
+                return Err(eother!(
+                    "Remote hypervisor does not support CPU hotplug, default_maxvcpus must be equal to the total system CPUs",
+                ));
+            }
+            if !remote.boot_info.initrd.is_empty() {
+                return Err(eother!("Remote hypervisor does not support initrd"));
+            }
+            if !remote.boot_info.rootfs_type.is_empty() {
+                return Err(eother!("Remote hypervisor does not support rootfs_type"));
+            }
+            if remote.blockdev_info.block_device_driver.as_str() == DRIVER_NVDIMM_TYPE {
+                return Err(eother!("Remote hypervisor does not support nvdimm"));
+            }
+            if remote.memory_info.default_memory < MIN_REMOTE_MEMORY_SIZE_MB {
+                return Err(eother!(
+                    "Remote hypervisor has minimal memory limitation {}",
+                    MIN_REMOTE_MEMORY_SIZE_MB
+                ));
+            }
+        }
+
+        Ok(())
+    }
+
+    fn get_min_memory(&self) -> u32 {
+        MIN_REMOTE_MEMORY_SIZE_MB
+    }
+
+    fn get_max_cpus(&self) -> u32 {
+        MAX_REMOTE_VCPUS
+    }
+}

src/libs/kata-types/src/config/mod.rs

@@ -26,7 +26,7 @@ pub use self::agent::Agent;
 use self::default::DEFAULT_AGENT_DBG_CONSOLE_PORT;
 pub use self::hypervisor::{
     BootInfo, CloudHypervisorConfig, DragonballConfig, FirecrackerConfig, Hypervisor, QemuConfig,
-    HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_FIRECRACKER, HYPERVISOR_NAME_QEMU,
+    RemoteConfig, HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_FIRECRACKER, HYPERVISOR_NAME_QEMU,
 };
 
 mod runtime;

src/libs/protocols/Cargo.toml

@@ -7,19 +7,17 @@ license = "Apache-2.0"
 
 [features]
 default = []
-with-serde = [ "serde", "serde_json" ]
+with-serde = []
 async = ["ttrpc/async", "async-trait"]
 
 [dependencies]
 ttrpc = "0.8"
 async-trait = { version = "0.1.42", optional = true }
 protobuf = { version = "3.2.0" }
-serde = { version = "1.0.130", features = ["derive"], optional = true }
-serde_json = { version = "1.0.68", optional = true }
+serde = { version = "1.0.130", features = ["derive"] }
+serde_json = "1.0.68"
 oci-spec = { version = "0.6.8", features = ["runtime"] }
 
-kata-sys-util = { path = "../kata-sys-util" }
-
 [build-dependencies]
 ttrpc-codegen = "0.4.2"
 protobuf = { version = "3.2.0" }

src/libs/protocols/build.rs

@@ -204,6 +204,7 @@ fn real_main() -> Result<(), std::io::Error> {
                 "protos/agent.proto",
                 "protos/health.proto",
                 "protos/confidential_data_hub.proto",
+                "protos/remote.proto",
             ],
             true,
         )?;
@@ -214,6 +215,7 @@ fn real_main() -> Result<(), std::io::Error> {
             "src/confidential_data_hub_ttrpc.rs",
             "src/confidential_data_hub_ttrpc_async.rs",
         )?;
+        fs::rename("src/remote_ttrpc.rs", "src/remote_ttrpc_async.rs")?;
     }
 
     codegen(
@@ -222,6 +224,7 @@ fn real_main() -> Result<(), std::io::Error> {
             "protos/agent.proto",
             "protos/health.proto",
             "protos/confidential_data_hub.proto",
+            "protos/remote.proto",
         ],
         false,
     )?;

src/libs/protocols/protos/remote.proto

@@ -0,0 +1,47 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+syntax = "proto3";
+
+package remote;
+
+service Hypervisor {
+	rpc CreateVM(CreateVMRequest) returns (CreateVMResponse) {}
+	rpc StartVM(StartVMRequest) returns (StartVMResponse) {}
+	rpc StopVM(StopVMRequest) returns (StopVMResponse) {}
+	rpc Version(VersionRequest) returns (VersionResponse) {}
+}
+
+message VersionRequest {
+	string version = 1;
+}
+
+message VersionResponse {
+	string version = 1;
+}
+
+message CreateVMRequest {
+	string id = 1;
+	map<string, string> annotations = 2;
+	string networkNamespacePath = 3;
+}
+
+message CreateVMResponse {
+	string agentSocketPath = 1;
+}
+
+message StartVMRequest {
+	string id = 1;
+}
+
+message StartVMResponse {
+}
+
+message StopVMRequest {
+	string id = 1;
+}
+
+message StopVMResponse {
+}

src/libs/protocols/src/lib.rs

@@ -21,6 +21,10 @@ pub mod oci;
 mod serde_config;
 pub mod trans;
 pub mod types;
+pub mod remote;
+pub mod remote_ttrpc;
+#[cfg(feature = "async")]
+pub mod remote_ttrpc_async;
 
 #[cfg(feature = "with-serde")]
 pub use serde_config::{

src/libs/protocols/src/trans.rs

@@ -10,7 +10,6 @@ use std::convert::TryFrom;
 use std::path::PathBuf;
 
 use crate::oci as grpc;
-use kata_sys_util::spec::parse_json_string;
 use oci_spec::runtime as oci;
 
 // translate from interface to ttprc tools
@@ -41,8 +40,9 @@ fn cap_hashset2vec(hash_set: &Option<HashSet<oci::Capability>>) -> Vec<String> {
 fn cap_vec2hashset(caps: Vec<String>) -> HashSet<oci::Capability> {
     caps.iter()
         .map(|cap: &String| {
-            let cap_str = parse_json_string(cap);
-            cap_str
+            // cap might be JSON-encoded
+            let decoded: &str = serde_json::from_str(cap).unwrap_or(cap);
+            decoded.strip_prefix("CAP_").unwrap_or(decoded)
                 .parse::<oci::Capability>()
                 .unwrap_or_else(|_| panic!("Failed to parse {:?} to Enum Capability", cap))
         })
@@ -97,6 +97,8 @@ impl From<oci::LinuxCapabilities> for grpc::LinuxCapabilities {
     }
 }
 
+// TODO(burgerdev): remove condition here and below after upgrading to oci_spec > 0.7.
+#[cfg(target_os = "linux")]
 impl From<oci::PosixRlimit> for grpc::POSIXRlimit {
     fn from(from: oci::PosixRlimit) -> Self {
         grpc::POSIXRlimit {
@@ -118,6 +120,7 @@ impl From<oci::Process> for grpc::Process {
             Env: option_vec_to_vec(from.env()),
             Cwd: from.cwd().display().to_string(),
             Capabilities: from_option(from.capabilities().clone()),
+            #[cfg(target_os = "linux")]
             Rlimits: from_option_vec(from.rlimits().clone()),
             NoNewPrivileges: from.no_new_privileges().unwrap_or_default(),
             ApparmorProfile: from
@@ -993,6 +996,7 @@ impl From<grpc::Linux> for oci::Linux {
     }
 }
 
+#[cfg(target_os = "linux")]
 impl From<grpc::POSIXRlimit> for oci::PosixRlimit {
     fn from(proto: grpc::POSIXRlimit) -> Self {
         oci::PosixRlimitBuilder::default()
@@ -1078,6 +1082,8 @@ impl From<grpc::Process> for oci::Process {
         } else {
             process.set_capabilities(None);
         }
+
+        #[cfg(target_os = "linux")]
         if !from.Rlimits().is_empty() {
             process.set_rlimits(Some(
                 from.Rlimits().iter().cloned().map(|r| r.into()).collect(),
@@ -1238,6 +1244,11 @@ impl From<grpc::LinuxIntelRdt> for oci::LinuxIntelRdt {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashSet;
+
+    use super::cap_vec2hashset;
+    use super::oci;
+
     fn from_vec<F: Sized, T: From<F>>(from: Vec<F>) -> Vec<T> {
         let mut to: Vec<T> = vec![];
         for data in from {
@@ -1289,4 +1300,26 @@ mod tests {
         assert_eq!(from.len(), to.len());
         assert_eq!(from[0].from, to[0].to);
     }
+
+    #[test]
+    fn test_cap_vec2hashset_good() {
+        let expected: HashSet<oci::Capability> =
+            vec![oci::Capability::NetAdmin, oci::Capability::Mknod]
+                .into_iter()
+                .collect();
+        let actual = cap_vec2hashset(vec![
+            "CAP_NET_ADMIN".to_string(),
+            "\"CAP_MKNOD\"".to_string(),
+        ]);
+
+        assert_eq!(expected, actual);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_cap_vec2hashset_bad() {
+        cap_vec2hashset(vec![
+            "CAP_DOES_NOT_EXIST".to_string(),
+        ]);
+    }
 }

src/runtime-rs/Cargo.lock

@@ -1685,8 +1685,11 @@ dependencies = [
  "libc",
  "logging",
  "nix 0.24.3",
+ "oci-spec",
  "path-clean",
  "persist",
+ "protobuf 3.2.0",
+ "protocols",
  "qapi",
  "qapi-qmp",
  "qapi-spec",
@@ -1706,6 +1709,8 @@ dependencies = [
  "thiserror",
  "tokio",
  "tracing",
+ "ttrpc",
+ "ttrpc-codegen",
  "vmm-sys-util 0.11.1",
 ]
 
@@ -1839,7 +1844,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -2994,9 +2998,10 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.2.0",
+ "serde",
+ "serde_json",
  "ttrpc",
  "ttrpc-codegen",
 ]

src/runtime-rs/Makefile

@@ -86,6 +86,7 @@ HYPERVISOR_DB = dragonball
 HYPERVISOR_FC = firecracker
 HYPERVISOR_QEMU = qemu
 HYPERVISOR_CLH = cloud-hypervisor
+HYPERVISOR_REMOTE = remote
 
 # When set to true, builds the built-in Dragonball hypervisor
 USE_BUILDIN_DB := true
@@ -93,7 +94,7 @@ USE_BUILDIN_DB := true
 HYPERVISOR ?= $(HYPERVISOR_DB)
 
 ##VAR HYPERVISOR=<hypervisor_name> List of hypervisors this build system can generate configuration for.
-HYPERVISORS := $(HYPERVISOR_DB) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
+HYPERVISORS := $(HYPERVISOR_DB) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_REMOTE)
 
 CLHPATH := $(CLHBINDIR)/$(CLHCMD)
 CLHVALIDHYPERVISORPATHS := [\"$(CLHPATH)\"]
@@ -317,6 +318,20 @@ ifneq (,$(FCCMD))
     DEFSTATICRESOURCEMGMT_FC := true
 endif
 
+ifneq (,$(REMOTE))
+    KNOWN_HYPERVISORS += $(HYPERVISOR_REMOTE)
+    CONFIG_FILE_REMOTE = configuration-remote.toml
+    CONFIG_REMOTE = config/$(CONFIG_FILE_REMOTE)
+    CONFIG_REMOTE_IN = $(CONFIG_REMOTE).in
+    CONFIG_PATH_REMOTE = $(abspath $(CONFDIR)/$(CONFIG_FILE_REMOTE))
+    CONFIG_PATHS += $(CONFIG_PATH_REMOTE)
+    SYSCONFDIR_REMOTE = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_REMOTE))
+    SYSCONFIG_PATHS += $(SYSCONFDIR_REMOTE)
+    CONFIGS += $(CONFIG_REMOTE)
+    # remote-specific options (all should be suffixed by "_REMOTE")
+    DEFSANDBOXCGROUPONLY_REMOTE := false
+endif
+
 ifeq ($(HYPERVISOR),$(HYPERVISOR_DB))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_DB)
 endif
@@ -327,6 +342,9 @@ endif
 ifeq ($(HYPERVISOR),$(HYPERVISOR_FC))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_FC)
 endif
+ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_REMOTE))
+    DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_REMOTE)
+endif
 # list of variables the user may wish to override
 USER_VARS += ARCH
 USER_VARS += BINDIR
@@ -334,6 +352,7 @@ USER_VARS += CONFIG_DB_IN
 USER_VARS += CONFIG_FC_IN
 USER_VARS += CONFIG_PATH
 USER_VARS += CONFIG_QEMU_IN
+USER_VARS += CONFIG_REMOTE_IN
 USER_VARS += DESTDIR
 USER_VARS += HYPERVISOR
 USER_VARS += USE_BUILDIN_DB
@@ -443,6 +462,7 @@ USER_VARS += DEFSANDBOXCGROUPONLY_QEMU
 USER_VARS += DEFSANDBOXCGROUPONLY_DB
 USER_VARS += DEFSANDBOXCGROUPONLY_FC
 USER_VARS += DEFSANDBOXCGROUPONLY_CLH
+USER_VARS += DEFSANDBOXCGROUPONLY_REMOTE
 USER_VARS += DEFSTATICRESOURCEMGMT_DB
 USER_VARS += DEFSTATICRESOURCEMGMT_FC
 USER_VARS += DEFSTATICRESOURCEMGMT_CLH
@@ -560,7 +580,7 @@ static-checks-build: $(GENERATED_FILES)
 $(TARGET): $(GENERATED_FILES) $(TARGET_PATH)
 
 $(TARGET_PATH): $(SOURCES) | show-summary
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed \

src/runtime-rs/arch/x86_64-options.mk

@@ -20,3 +20,5 @@ CLHCMD := cloud-hypervisor
 # firecracker binary (vmm and jailer)
 FCCMD := firecracker
 FCJAILERCMD := jailer
+
+REMOTE := remote

src/runtime-rs/config/configuration-remote.toml.in

@@ -0,0 +1,297 @@
+# Copyright 2024 Kata Contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_REMOTE_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+
+[hypervisor.remote]
+# Default VM information query service unix domain socket, created by cloud-api-adaptor
+# Ref: https://github.com/confidential-containers/cloud-api-adaptor/blob/main/src/cloud-api-adaptor/docs/vminfo.md
+remote_hypervisor_socket = "/run/peerpod/hypervisor.sock"
+# Timeout in seconds for creating a remote hypervisor, 600s(10min) by default
+remote_hypervisor_timeout = 600
+
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+# confidential_guest = true
+
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+# Note: Remote hypervisor is only handling the following annotations
+enable_annotations = ["machine_type", "default_memory", "default_vcpus"]
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+# NOTE: kernel_params are not currently passed over in remote hypervisor
+# kernel_params = ""
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWAREPATH@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+# default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+# default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+# default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+#memory_slots = @DEFMEMSLOTS@
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. And Debug also enable the hmp socket.
+#
+# Default false
+# enable_debug = true
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+# Note: The remote hypervisor has a different guest, so currently requires this to be disabled
+disable_guest_selinux = true
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+# enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+# enable_tracing = true
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+# enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+# Note: The remote hypervisor, uses it's own network, so "none" is required
+internetworking_model="none"
+
+name="virt_container"
+hypervisor_name="remote"
+agent_name="kata"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+# Note: The remote hypervisor has a different guest, so currently requires this to be set to true
+disable_guest_seccomp=true
+
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+# Note: The remote hypervisor has a different networking model, which requires true 
+disable_new_netns = false
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY_REMOTE@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+# Note: the remote hypervisor uses the peer pod config to determine the sandbox size, so requires this to be set to true
+static_sandbox_resource_mgmt=true
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+# Note: remote hypervisor has no sharing of emptydir mounts from host to guest
+disable_guest_empty_dir=false
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true

src/runtime-rs/crates/agent/src/sock/mod.rs

@@ -8,6 +8,8 @@ mod hybrid_vsock;
 pub use hybrid_vsock::HybridVsock;
 mod vsock;
 pub use vsock::Vsock;
+mod remote;
+pub use remote::Remote;
 
 use std::{
     pin::Pin,
@@ -28,6 +30,7 @@ use url::Url;
 
 const VSOCK_SCHEME: &str = "vsock";
 const HYBRID_VSOCK_SCHEME: &str = "hvsock";
+const REMOTE_SCHEME: &str = "remote";
 
 /// Socket stream
 pub enum Stream {
@@ -98,6 +101,7 @@ impl ConnectConfig {
 enum SockType {
     Vsock(Vsock),
     HybridVsock(HybridVsock),
+    Remote(Remote),
 }
 
 #[async_trait]
@@ -114,6 +118,7 @@ pub fn new(address: &str, port: u32) -> Result<Arc<dyn Sock>> {
     match parse(address, port).context("parse url")? {
         SockType::Vsock(sock) => Ok(Arc::new(sock)),
         SockType::HybridVsock(sock) => Ok(Arc::new(sock)),
+        SockType::Remote(sock) => Ok(Arc::new(sock)),
     }
 }
 
@@ -136,6 +141,13 @@ fn parse(address: &str, port: u32) -> Result<SockType> {
             let uds = path[0];
             Ok(SockType::HybridVsock(HybridVsock::new(uds, port)))
         }
+        REMOTE_SCHEME => {
+            let path: Vec<&str> = url.path().split(':').collect();
+            if path.len() != 1 {
+                return Err(anyhow!("invalid path {:?}", path));
+            }
+            Ok(SockType::Remote(Remote::new(path[0].to_string())))
+        }
         _ => Err(anyhow!("Unsupported scheme")),
     }
 }

src/runtime-rs/crates/agent/src/sock/remote.rs

@@ -0,0 +1,61 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::{os::unix::prelude::AsRawFd, path::Path};
+
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use tokio::{io::Interest, net::UnixStream};
+
+use super::{ConnectConfig, Sock, Stream};
+
+#[derive(Debug, PartialEq)]
+pub struct Remote {
+    path: String,
+}
+
+impl Remote {
+    pub fn new(path: String) -> Self {
+        Self { path }
+    }
+}
+
+#[async_trait]
+impl Sock for Remote {
+    async fn connect(&self, config: &ConnectConfig) -> Result<Stream> {
+        let retry_times = config.reconnect_timeout_ms / config.dial_timeout_ms;
+        for i in 0..retry_times {
+            match connect_helper(&self.path).await {
+                Ok(stream) => {
+                    info!(
+                        sl!(),
+                        "remote connect success on {} current client fd {}",
+                        i,
+                        stream.as_raw_fd()
+                    );
+                    return Ok(Stream::Unix(stream));
+                }
+                Err(err) => {
+                    debug!(sl!(), "remote connect on {} err : {:?}", i, err);
+                    tokio::time::sleep(std::time::Duration::from_millis(config.dial_timeout_ms))
+                        .await;
+                    continue;
+                }
+            }
+        }
+        Err(anyhow!("cannot connect to agent ttrpc server {:?}", config))
+    }
+}
+
+async fn connect_helper(address: &str) -> Result<UnixStream> {
+    let stream = UnixStream::connect(Path::new(&address))
+        .await
+        .context("failed to create UnixAddr")?;
+    stream
+        .ready(Interest::READABLE | Interest::WRITABLE)
+        .await?;
+    Ok(stream)
+}

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -28,11 +28,15 @@ rand = "0.8.4"
 path-clean = "1.0.1"
 lazy_static = "1.4"
 tracing = "0.1.36"
+ttrpc = {version = "0.8.1", features = ["async"] }
+protobuf = "3.1.0"
 
 kata-sys-util = { path = "../../../libs/kata-sys-util" }
 kata-types = { path = "../../../libs/kata-types" }
 logging = { path = "../../../libs/logging" }
+protocols = { path = "../../../libs/protocols", features = ["async"] }
 shim-interface = { path = "../../../libs/shim-interface" }
+oci-spec = { version = "0.6.8", features = ["runtime"] }
 
 ch-config = { path = "ch-config", optional = true }
 tests_utils = { path = "../../tests/utils" }
@@ -68,3 +72,7 @@ hypervisor = { path = ".", features = ["cloud-hypervisor"] }
 test-utils = { path = "../../../libs/test-utils" }
 
 serial_test = "2.0.0"
+
+
+[build-dependencies]
+ttrpc-codegen = "0.4.2"

src/runtime-rs/crates/hypervisor/src/ch/mod.rs

@@ -11,6 +11,7 @@ use async_trait::async_trait;
 use kata_types::capabilities::{Capabilities, CapabilityBits};
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 use persist::sandbox_persist::Persist;
+use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::{mpsc, Mutex, RwLock};
 
@@ -59,7 +60,12 @@ impl Default for CloudHypervisor {
 
 #[async_trait]
 impl Hypervisor for CloudHypervisor {
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }

src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs

@@ -438,7 +438,7 @@ impl VfioDevice {
             let mut hostdev: HostDevice = self
                 .set_vfio_config(iommu_devs_path.clone(), device)
                 .context("set vfio config failed")?;
-            let dev_prefix = self.get_vfio_prefix();
+            let dev_prefix = format!("{}_{}", self.get_vfio_prefix(), &vfio_group);
             hostdev.hostdev_id = make_device_nameid(&dev_prefix, index, MAX_DEV_ID_SIZE);
 
             self.devices.push(hostdev);

src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs

@@ -12,6 +12,7 @@ use inner::DragonballInner;
 use persist::sandbox_persist::Persist;
 pub mod vmm_instance;
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use anyhow::{Context, Result};
@@ -69,7 +70,12 @@ impl Dragonball {
 #[async_trait]
 impl Hypervisor for Dragonball {
     #[instrument]
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }

src/runtime-rs/crates/hypervisor/src/firecracker/mod.rs

@@ -18,6 +18,7 @@ use inner::FcInner;
 use kata_types::capabilities::Capabilities;
 use kata_types::capabilities::CapabilityBits;
 use persist::sandbox_persist::Persist;
+use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use tokio::sync::Mutex;
@@ -58,7 +59,12 @@ impl Firecracker {
 
 #[async_trait]
 impl Hypervisor for Firecracker {
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -19,6 +19,7 @@ pub mod dragonball;
 pub mod firecracker;
 mod kernel_param;
 pub mod qemu;
+pub mod remote;
 pub use kernel_param::Param;
 pub mod utils;
 use std::collections::HashMap;
@@ -66,6 +67,7 @@ const HUGE_SHMEM: &str = "hugeshmem";
 pub const HYPERVISOR_DRAGONBALL: &str = "dragonball";
 pub const HYPERVISOR_QEMU: &str = "qemu";
 pub const HYPERVISOR_FIRECRACKER: &str = "firecracker";
+pub const HYPERVISOR_REMOTE: &str = "remote";
 
 pub const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock";
 pub const JAILER_ROOT: &str = "root";
@@ -95,7 +97,12 @@ pub struct MemoryConfig {
 #[async_trait]
 pub trait Hypervisor: std::fmt::Debug + Send + Sync {
     // vm manager
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()>;
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        annotations: &HashMap<String, String>,
+    ) -> Result<()>;
     async fn start_vm(&self, timeout: i32) -> Result<()>;
     async fn stop_vm(&self) -> Result<()>;
     async fn wait_vm(&self) -> Result<i32>;

src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs

@@ -369,6 +369,88 @@ impl ToQemuParams for Cpu {
     }
 }
 
+/// Error type for CCW Subchannel operations
+#[derive(Debug)]
+enum CcwError {
+    DeviceAlreadyExists(String), // Error when trying to add an existing device
+    #[allow(dead_code)]
+    DeviceNotFound(String), // Error when trying to remove a nonexistent device
+}
+
+/// Represents a CCW subchannel for managing devices
+#[derive(Debug)]
+struct CcwSubChannel {
+    devices: HashMap<String, u32>, // Maps device IDs to slot indices
+    addr: u32,                     // Subchannel address
+    next_slot: u32,                // Next available slot index
+}
+
+impl CcwSubChannel {
+    fn new() -> Self {
+        Self {
+            devices: HashMap::new(),
+            addr: 0,
+            next_slot: 0,
+        }
+    }
+
+    /// Adds a device to the subchannel.
+    ///
+    /// # Arguments
+    /// - `dev_id`: device ID to add
+    ///
+    /// # Returns
+    /// - `Result<u32, CcwError>`: slot index of the added device
+    ///   or an error if the device already exists
+    fn add_device(&mut self, dev_id: &str) -> Result<u32, CcwError> {
+        if self.devices.contains_key(dev_id) {
+            Err(CcwError::DeviceAlreadyExists(dev_id.to_owned()))
+        } else {
+            let slot = self.next_slot;
+            self.devices.insert(dev_id.to_owned(), slot);
+            self.next_slot += 1;
+            Ok(slot)
+        }
+    }
+
+    /// Removes a device from the subchannel by its ID.
+    ///
+    /// # Arguments
+    /// - `dev_id`: device ID to remove
+    ///
+    /// # Returns
+    /// - `Result<(), CcwError>`: Ok(()) if the device was removed
+    ///  or an error if the device was not found
+    #[allow(dead_code)]
+    fn remove_device(&mut self, dev_id: &str) -> Result<(), CcwError> {
+        if self.devices.remove(dev_id).is_some() {
+            Ok(())
+        } else {
+            Err(CcwError::DeviceNotFound(dev_id.to_owned()))
+        }
+    }
+
+    /// Formats the CCW address for a given slot
+    ///
+    /// # Arguments
+    /// - `slot`: slot index
+    ///
+    /// # Returns
+    /// - `String`: formatted CCW address (e.g. `fe.0.0000`)
+    fn address_format_ccw(&self, slot: u32) -> String {
+        format!("fe.{:x}.{:04x}", self.addr, slot)
+    }
+
+    /// Sets the address of the subchannel.
+    /// # Arguments
+    /// - `addr`: subchannel address to set
+    #[allow(dead_code)]
+    fn set_addr(&mut self, addr: u32) -> &mut Self {
+        self.addr = addr;
+        self
+    }
+}
+
 #[derive(Debug)]
 struct Machine {
     r#type: String,
@@ -657,10 +739,16 @@ struct DeviceVhostUserFs {
     queue_size: u64,
     romfile: String,
     iommu_platform: bool,
+    devno: Option<String>,
 }
 
 impl DeviceVhostUserFs {
-    fn new(chardev: &str, tag: &str, bus_type: VirtioBusType) -> DeviceVhostUserFs {
+    fn new(
+        chardev: &str,
+        tag: &str,
+        bus_type: VirtioBusType,
+        devno: Option<String>,
+    ) -> DeviceVhostUserFs {
         DeviceVhostUserFs {
             bus_type,
             chardev: chardev.to_owned(),
@@ -668,6 +756,7 @@ impl DeviceVhostUserFs {
             queue_size: 0,
             romfile: String::new(),
             iommu_platform: false,
+            devno,
         }
     }
 
@@ -715,6 +804,9 @@ impl ToQemuParams for DeviceVhostUserFs {
         if self.iommu_platform {
             params.push("iommu_platform=on".to_owned());
         }
+        if let Some(devno) = &self.devno {
+            params.push(format!("devno={}", devno));
+        }
         Ok(vec!["-device".to_owned(), params.join(",")])
     }
 }
@@ -835,15 +927,17 @@ struct DeviceVirtioBlk {
     id: String,
     config_wce: bool,
     share_rw: bool,
+    devno: Option<String>,
 }
 
 impl DeviceVirtioBlk {
-    fn new(id: &str, bus_type: VirtioBusType) -> DeviceVirtioBlk {
+    fn new(id: &str, bus_type: VirtioBusType, devno: Option<String>) -> DeviceVirtioBlk {
         DeviceVirtioBlk {
             bus_type,
             id: id.to_owned(),
             config_wce: false,
             share_rw: true,
+            devno,
         }
     }
 
@@ -877,7 +971,9 @@ impl ToQemuParams for DeviceVirtioBlk {
             params.push("share-rw=off".to_owned());
         }
         params.push(format!("serial=image-{}", self.id));
-
+        if let Some(devno) = &self.devno {
+            params.push(format!("devno={}", devno));
+        }
         Ok(vec!["-device".to_owned(), params.join(",")])
     }
 }
@@ -888,16 +984,23 @@ struct VhostVsock {
     guest_cid: u32,
     disable_modern: bool,
     iommu_platform: bool,
+    devno: Option<String>,
 }
 
 impl VhostVsock {
-    fn new(vhostfd: tokio::fs::File, guest_cid: u32, bus_type: VirtioBusType) -> VhostVsock {
+    fn new(
+        vhostfd: tokio::fs::File,
+        guest_cid: u32,
+        bus_type: VirtioBusType,
+        devno: Option<String>,
+    ) -> VhostVsock {
         VhostVsock {
             bus_type,
             vhostfd,
             guest_cid,
             disable_modern: false,
             iommu_platform: false,
+            devno,
         }
     }
 
@@ -923,6 +1026,9 @@ impl ToQemuParams for VhostVsock {
         if self.iommu_platform {
             params.push("iommu_platform=on".to_owned());
         }
+        if let Some(devno) = &self.devno {
+            params.push(format!("devno={}", devno));
+        }
         params.push(format!("vhostfd={}", self.vhostfd.as_raw_fd()));
         params.push(format!("guest-cid={}", self.guest_cid));
 
@@ -1154,14 +1260,16 @@ struct DeviceVirtioSerial {
     id: String,
     bus_type: VirtioBusType,
     iommu_platform: bool,
+    devno: Option<String>,
 }
 
 impl DeviceVirtioSerial {
-    fn new(id: &str, bus_type: VirtioBusType) -> DeviceVirtioSerial {
+    fn new(id: &str, bus_type: VirtioBusType, devno: Option<String>) -> DeviceVirtioSerial {
         DeviceVirtioSerial {
             id: id.to_owned(),
             bus_type,
             iommu_platform: false,
+            devno,
         }
     }
 
@@ -1180,6 +1288,9 @@ impl ToQemuParams for DeviceVirtioSerial {
         if self.iommu_platform {
             params.push("iommu_platform=on".to_owned());
         }
+        if let Some(devno) = &self.devno {
+            params.push(format!("devno={}", devno));
+        }
         Ok(vec!["-device".to_owned(), params.join(",")])
     }
 }
@@ -1520,21 +1631,30 @@ struct DeviceVirtioScsi {
     id: String,
     disable_modern: bool,
     iothread: String,
+    iommu_platform: bool,
+    devno: Option<String>,
 }
 
 impl DeviceVirtioScsi {
-    fn new(id: &str, disable_modern: bool, bus_type: VirtioBusType) -> Self {
+    fn new(id: &str, disable_modern: bool, bus_type: VirtioBusType, devno: Option<String>) -> Self {
         DeviceVirtioScsi {
             bus_type,
             id: id.to_owned(),
             disable_modern,
             iothread: "".to_owned(),
+            iommu_platform: false,
+            devno,
         }
     }
 
     fn set_iothread(&mut self, iothread: &str) {
         self.iothread = iothread.to_owned();
     }
+
+    fn set_iommu_platform(&mut self, iommu_platform: bool) -> &mut Self {
+        self.iommu_platform = iommu_platform;
+        self
+    }
 }
 
 #[async_trait]
@@ -1549,6 +1669,12 @@ impl ToQemuParams for DeviceVirtioScsi {
         if !self.iothread.is_empty() {
             params.push(format!("iothread={}", self.iothread));
         }
+        if self.iommu_platform {
+            params.push("iommu_platform=on".to_owned());
+        }
+        if let Some(devno) = &self.devno {
+            params.push(format!("devno={}", devno));
+        }
         Ok(vec!["-device".to_owned(), params.join(",")])
     }
 }
@@ -1622,10 +1748,15 @@ pub struct QemuCmdLine<'a> {
     knobs: Knobs,
 
     devices: Vec<Box<dyn ToQemuParams>>,
+    ccw_subchannel: Option<CcwSubChannel>,
 }
 
 impl<'a> QemuCmdLine<'a> {
     pub fn new(id: &str, config: &'a HypervisorConfig) -> Result<QemuCmdLine<'a>> {
+        let ccw_subchannel = match bus_type(config) {
+            VirtioBusType::Ccw => Some(CcwSubChannel::new()),
+            _ => None,
+        };
         let mut qemu_cmd_line = QemuCmdLine {
             id: id.to_string(),
             config,
@@ -1637,6 +1768,7 @@ impl<'a> QemuCmdLine<'a> {
             qmp_socket: QmpSocket::new(MonitorProtocol::Qmp)?,
             knobs: Knobs::new(config),
             devices: Vec::new(),
+            ccw_subchannel,
         };
 
         if config.device_info.enable_iommu {
@@ -1703,8 +1835,20 @@ impl<'a> QemuCmdLine<'a> {
     }
 
     fn add_scsi_controller(&mut self) {
-        let mut virtio_scsi =
-            DeviceVirtioScsi::new("scsi0", should_disable_modern(), bus_type(self.config));
+        let devno = get_devno_ccw(&mut self.ccw_subchannel, "scsi0");
+        let mut virtio_scsi = DeviceVirtioScsi::new(
+            "scsi0",
+            should_disable_modern(),
+            bus_type(self.config),
+            devno,
+        );
+
+        if self.config.device_info.enable_iommu_platform
+            && bus_type(self.config) == VirtioBusType::Ccw
+        {
+            virtio_scsi.set_iommu_platform(true);
+        }
+
         if self.config.enable_iothreads {
             let iothread_id = "scsi-io-thread";
             let iothread = ObjectIoThread::new(iothread_id);
@@ -1731,8 +1875,8 @@ impl<'a> QemuCmdLine<'a> {
         self.devices.push(Box::new(virtiofsd_socket_chardev));
 
         let bus_type = bus_type(self.config);
-
-        let mut virtiofs_device = DeviceVhostUserFs::new(chardev_name, mount_tag, bus_type);
+        let devno = get_devno_ccw(&mut self.ccw_subchannel, chardev_name);
+        let mut virtiofs_device = DeviceVhostUserFs::new(chardev_name, mount_tag, bus_type, devno);
         virtiofs_device.set_queue_size(queue_size);
         if self.config.device_info.enable_iommu_platform && bus_type == VirtioBusType::Ccw {
             virtiofs_device.set_iommu_platform(true);
@@ -1762,13 +1906,16 @@ impl<'a> QemuCmdLine<'a> {
     pub fn add_vsock(&mut self, vhostfd: tokio::fs::File, guest_cid: u32) -> Result<()> {
         clear_cloexec(vhostfd.as_raw_fd()).context("clearing O_CLOEXEC failed on vsock fd")?;
 
-        let mut vhost_vsock_pci = VhostVsock::new(vhostfd, guest_cid, bus_type(self.config));
+        let devno = get_devno_ccw(&mut self.ccw_subchannel, "vsock-0");
+        let mut vhost_vsock_pci = VhostVsock::new(vhostfd, guest_cid, bus_type(self.config), devno);
 
         if !self.config.disable_nesting_checks && should_disable_modern() {
             vhost_vsock_pci.set_disable_modern(true);
         }
 
-        if self.config.device_info.enable_iommu_platform {
+        if self.config.device_info.enable_iommu_platform
+            && bus_type(self.config) == VirtioBusType::Ccw
+        {
             vhost_vsock_pci.set_iommu_platform(true);
         }
 
@@ -1809,9 +1956,11 @@ impl<'a> QemuCmdLine<'a> {
     pub fn add_block_device(&mut self, device_id: &str, path: &str) -> Result<()> {
         self.devices
             .push(Box::new(BlockBackend::new(device_id, path)));
+        let devno = get_devno_ccw(&mut self.ccw_subchannel, device_id);
         self.devices.push(Box::new(DeviceVirtioBlk::new(
             device_id,
             bus_type(self.config),
+            devno,
         )));
         Ok(())
     }
@@ -1836,7 +1985,8 @@ impl<'a> QemuCmdLine<'a> {
     }
 
     pub fn add_console(&mut self, console_socket_path: &str) {
-        let mut serial_dev = DeviceVirtioSerial::new("serial0", bus_type(self.config));
+        let devno = get_devno_ccw(&mut self.ccw_subchannel, "serial0");
+        let mut serial_dev = DeviceVirtioSerial::new("serial0", bus_type(self.config), devno);
         if self.config.device_info.enable_iommu_platform
             && bus_type(self.config) == VirtioBusType::Ccw
         {
@@ -1909,3 +2059,15 @@ pub fn get_network_device(
 
     Ok((netdev, virtio_net_device))
 }
+
+fn get_devno_ccw(ccw_subchannel: &mut Option<CcwSubChannel>, device_name: &str) -> Option<String> {
+    ccw_subchannel.as_mut().and_then(|subchannel| {
+        subchannel.add_device(device_name).map_or_else(
+            |err| {
+                info!(sl!(), "failed to add device to subchannel: {:?}", err);
+                None
+            },
+            |slot| Some(subchannel.address_format_ccw(slot)),
+        )
+    })
+}

src/runtime-rs/crates/hypervisor/src/qemu/mod.rs

@@ -18,6 +18,7 @@ use persist::sandbox_persist::Persist;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 
+use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 use tokio::sync::{mpsc, Mutex};
@@ -52,7 +53,12 @@ impl Qemu {
 
 #[async_trait]
 impl Hypervisor for Qemu {
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }

src/runtime-rs/crates/hypervisor/src/remote/inner.rs

@@ -0,0 +1,387 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::{
+    device::DeviceType, hypervisor_persist::HypervisorState, HypervisorConfig, HYPERVISOR_REMOTE,
+};
+use crate::{MemoryConfig, VcpuThreadIds};
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+use kata_types::{
+    annotations::{
+        cri_containerd::{SANDBOX_NAMESPACE_LABEL_KEY, SANDBOX_NAME_LABEL_KEY},
+        KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS,
+        KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH, KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE,
+    },
+    capabilities::{Capabilities, CapabilityBits},
+};
+use persist::sandbox_persist::Persist;
+use protocols::{
+    remote::{CreateVMRequest, StartVMRequest, StopVMRequest},
+    remote_ttrpc_async::HypervisorClient,
+};
+use std::{collections::HashMap, time};
+use tokio::sync::{mpsc, Mutex};
+use ttrpc::context::{self};
+use ttrpc::r#async::Client;
+
+const REMOTE_SCHEME: &str = "remote";
+const DEFAULT_MIN_TIMEOUT: i32 = time::Duration::from_secs(60).as_millis() as i32;
+
+pub struct RemoteInner {
+    /// sandbox id
+    pub(crate) id: String,
+    /// hypervisor config
+    pub(crate) config: HypervisorConfig,
+    /// agent socket path
+    pub(crate) agent_socket_path: String,
+    /// netns path
+    pub(crate) netns: Option<String>,
+    /// hypervisor unix client
+    pub(crate) client: Option<Client>,
+
+    exit_notify: Option<mpsc::Sender<i32>>,
+    exit_waiter: Mutex<(mpsc::Receiver<i32>, i32)>,
+}
+
+impl std::fmt::Debug for RemoteInner {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RemoteInner")
+            .field("id", &self.id)
+            .field("config", &self.config)
+            .field("agent_socket_path", &self.agent_socket_path)
+            .field("netns", &self.netns)
+            .finish()
+    }
+}
+
+impl RemoteInner {
+    pub fn new() -> Self {
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
+        Self {
+            id: "".to_string(),
+            config: HypervisorConfig::default(),
+            agent_socket_path: "".to_string(),
+            netns: None,
+            client: None,
+
+            exit_notify: Some(exit_notify),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
+        }
+    }
+
+    fn get_ttrpc_client(&mut self) -> Result<HypervisorClient> {
+        match self.client {
+            Some(ref c) => Ok(HypervisorClient::new(c.clone())),
+            None => {
+                let c = Client::connect(&format!(
+                    "unix://{}",
+                    &self.config.remote_info.hypervisor_socket
+                ))
+                .context("connect to ")?;
+                self.client = Some(c.clone());
+                Ok(HypervisorClient::new(c))
+            }
+        }
+    }
+
+    fn prepare_annotations(
+        &self,
+        oci_annotations: &HashMap<String, String>,
+    ) -> HashMap<String, String> {
+        let mut annotations: HashMap<String, String> = HashMap::new();
+        let config = &self.config;
+        annotations.insert(
+            SANDBOX_NAME_LABEL_KEY.to_string(),
+            oci_annotations
+                .get(SANDBOX_NAME_LABEL_KEY)
+                .cloned()
+                .unwrap_or_default(),
+        );
+        annotations.insert(
+            SANDBOX_NAMESPACE_LABEL_KEY.to_string(),
+            oci_annotations
+                .get(SANDBOX_NAMESPACE_LABEL_KEY)
+                .cloned()
+                .unwrap_or_default(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE.to_string(),
+            config.machine_info.machine_type.to_string(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS.to_string(),
+            config.cpu_info.default_vcpus.to_string(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY.to_string(),
+            config.memory_info.default_memory.to_string(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH.to_string(),
+            config.boot_info.image.to_string(),
+        );
+        annotations
+    }
+
+    pub(crate) async fn prepare_vm(
+        &mut self,
+        id: &str,
+        netns: Option<String>,
+        annotations: &HashMap<String, String>,
+    ) -> Result<()> {
+        info!(sl!(), "Preparing REMOTE VM");
+        self.id = id.to_string();
+
+        if let Some(netns_path) = &netns {
+            debug!(sl!(), "set netns for vmm master {:?}", &netns_path);
+            std::fs::metadata(netns_path).context("check netns path")?;
+        }
+
+        let client = self.get_ttrpc_client()?;
+
+        let ctx = context::Context::default();
+        let req = CreateVMRequest {
+            id: id.to_string(),
+            annotations: self.prepare_annotations(annotations),
+            networkNamespacePath: netns.clone().unwrap_or_default(),
+            ..Default::default()
+        };
+        info!(sl!(), "Preparing REMOTE VM req: {:?}", req.clone());
+        let resp = client.create_vm(ctx, &req).await?;
+        info!(sl!(), "Preparing REMOTE VM resp: {:?}", resp.clone());
+        self.agent_socket_path = resp.agentSocketPath;
+        self.netns = netns;
+        Ok(())
+    }
+
+    pub(crate) async fn start_vm(&mut self, timeout: i32) -> Result<()> {
+        info!(sl!(), "Starting REMOTE VM");
+
+        let mut min_timeout = DEFAULT_MIN_TIMEOUT;
+        if self.config.remote_info.hypervisor_timeout > 0 {
+            min_timeout = self.config.remote_info.hypervisor_timeout.min(timeout);
+        }
+        let timeout = min_timeout;
+
+        let client = self.get_ttrpc_client()?;
+
+        let req = StartVMRequest {
+            id: self.id.clone(),
+            ..Default::default()
+        };
+        let ctx =
+            context::with_timeout(time::Duration::from_secs(timeout as u64).as_nanos() as i64);
+        let _resp = client.start_vm(ctx, &req).await?;
+
+        Ok(())
+    }
+
+    pub(crate) async fn stop_vm(&mut self) -> Result<()> {
+        info!(sl!(), "Stopping REMOTE VM");
+
+        let client = self.get_ttrpc_client()?;
+
+        let ctx = context::with_timeout(time::Duration::from_secs(1).as_nanos() as i64);
+        let req = StopVMRequest {
+            id: self.id.clone(),
+            ..Default::default()
+        };
+        let _resp = client.stop_vm(ctx, &req).await?;
+
+        self.exit_notify.take().unwrap().send(1).await?;
+        Ok(())
+    }
+
+    pub(crate) async fn pause_vm(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::pause_vm(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn wait_vm(&self) -> Result<i32> {
+        info!(sl!(), "Wait Remote VM");
+        let mut waiter = self.exit_waiter.lock().await;
+        if let Some(exitcode) = waiter.0.recv().await {
+            waiter.1 = exitcode;
+        }
+
+        Ok(waiter.1)
+    }
+
+    pub(crate) async fn resume_vm(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::resume_vm(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn save_vm(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::save_vm(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn add_device(&self, device: DeviceType) -> Result<DeviceType> {
+        warn!(sl!(), "RemoteInner::add_device(): NOT YET IMPLEMENTED");
+        Ok(device)
+    }
+
+    pub(crate) async fn remove_device(&self, _device: DeviceType) -> Result<()> {
+        warn!(sl!(), "RemoteInner::remove_device(): NOT YET IMPLEMENTED");
+        Ok(())
+    }
+
+    pub(crate) async fn update_device(&self, _device: DeviceType) -> Result<()> {
+        warn!(sl!(), "RemoteInner::update_device(): NOT YET IMPLEMENTED");
+        Ok(())
+    }
+
+    pub(crate) async fn get_agent_socket(&self) -> Result<String> {
+        Ok(format!("{}://{}", REMOTE_SCHEME, &self.agent_socket_path))
+    }
+
+    pub(crate) async fn disconnect(&mut self) {
+        warn!(sl!(), "RemoteInner::disconnect(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub fn hypervisor_config(&self) -> HypervisorConfig {
+        info!(
+            sl!(),
+            "RemoteInner::hypervisor_config(): {:?}",
+            self.config.clone()
+        );
+        self.config.clone()
+    }
+
+    pub(crate) async fn get_thread_ids(&self) -> Result<VcpuThreadIds> {
+        warn!(sl!(), "RemoteInner::get_thread_ids(): NOT YET IMPLEMENTED");
+        let vcpu_thread_ids: VcpuThreadIds = VcpuThreadIds {
+            vcpus: HashMap::new(),
+        };
+        Ok(vcpu_thread_ids)
+    }
+
+    pub(crate) async fn get_vmm_master_tid(&self) -> Result<u32> {
+        warn!(sl!(), "RemoteInner::get_vmm_master_tid()");
+        let tid = nix::unistd::gettid().as_raw();
+        Ok(tid as u32)
+    }
+
+    pub(crate) async fn get_ns_path(&self) -> Result<String> {
+        info!(sl!(), "RemoteInner::get_ns_path()");
+        Ok(self.netns.clone().unwrap_or_default())
+    }
+
+    pub(crate) async fn cleanup(&self) -> Result<()> {
+        info!(sl!(), "RemoteInner::cleanup(): NOT YET IMPLEMENTED");
+        Ok(())
+    }
+
+    pub(crate) async fn resize_vcpu(
+        &mut self,
+        _old_vcpus: u32,
+        _new_vcpus: u32,
+    ) -> Result<(u32, u32)> {
+        info!(sl!(), "RemoteInner::resize_vcpu(): NOT YET IMPLEMENTED");
+        Ok((_old_vcpus, _new_vcpus))
+    }
+
+    pub(crate) async fn get_pids(&self) -> Result<Vec<u32>> {
+        warn!(sl!(), "RemoteInner::get_pids(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn check(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::check(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn get_jailer_root(&self) -> Result<String> {
+        warn!(sl!(), "RemoteInner::get_jailer_root(): NOT YET IMPLEMENTED");
+        Ok("".into())
+    }
+
+    pub(crate) async fn capabilities(&self) -> Result<Capabilities> {
+        Ok(Capabilities::default())
+    }
+
+    pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) {
+        self.config = config;
+    }
+
+    pub(crate) async fn get_hypervisor_metrics(&self) -> Result<String> {
+        warn!(
+            sl!(),
+            "RemoteInner::get_hypervisor_metrics(): NOT YET IMPLEMENTED"
+        );
+        todo!()
+    }
+
+    pub(crate) fn set_capabilities(&mut self, _flag: CapabilityBits) {
+        warn!(
+            sl!(),
+            "RemoteInner::set_capabilities(): NOT YET IMPLEMENTED"
+        );
+        todo!()
+    }
+
+    pub(crate) fn set_guest_memory_block_size(&mut self, _size: u32) {
+        info!(
+            sl!(),
+            "RemoteInner::set_guest_memory_block_size(): NOT YET IMPLEMENTED"
+        )
+    }
+
+    pub(crate) fn guest_memory_block_size_mb(&self) -> u32 {
+        warn!(
+            sl!(),
+            "RemoteInner::guest_memory_block_size_mb(): NOT YET IMPLEMENTED"
+        );
+        0
+    }
+
+    pub(crate) fn resize_memory(&self, _new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
+        Ok((
+            _new_mem_mb,
+            MemoryConfig {
+                ..Default::default()
+            },
+        ))
+    }
+}
+
+#[async_trait]
+impl Persist for RemoteInner {
+    type State = HypervisorState;
+    type ConstructorArgs = ();
+
+    /// Save a state of hypervisor
+    async fn save(&self) -> Result<Self::State> {
+        Ok(HypervisorState {
+            hypervisor_type: HYPERVISOR_REMOTE.to_string(),
+            id: self.id.clone(),
+            config: self.config.clone(),
+            netns: self.netns.clone(),
+            ..Default::default()
+        })
+    }
+
+    /// Restore hypervisor
+    async fn restore(
+        _hypervisor_args: Self::ConstructorArgs,
+        hypervisor_state: Self::State,
+    ) -> Result<Self> {
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
+        Ok(RemoteInner {
+            id: hypervisor_state.id,
+            config: hypervisor_state.config,
+            agent_socket_path: "".to_string(),
+            netns: hypervisor_state.netns,
+            client: None,
+            exit_notify: Some(exit_notify),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
+        })
+    }
+}

src/runtime-rs/crates/hypervisor/src/remote/mod.rs

@@ -0,0 +1,216 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::HypervisorState;
+use crate::{device::DeviceType, Hypervisor, HypervisorConfig, MemoryConfig, VcpuThreadIds};
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+use inner::RemoteInner;
+use kata_types::capabilities::{Capabilities, CapabilityBits};
+use persist::sandbox_persist::Persist;
+use std::collections::HashMap;
+
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+mod inner;
+
+#[derive(Debug)]
+pub struct Remote {
+    inner: Arc<RwLock<RemoteInner>>,
+}
+
+impl Default for Remote {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Remote {
+    pub fn new() -> Self {
+        Self {
+            inner: Arc::new(RwLock::new(RemoteInner::new())),
+        }
+    }
+
+    pub async fn set_hypervisor_config(&mut self, config: HypervisorConfig) {
+        let mut inner = self.inner.write().await;
+        inner.set_hypervisor_config(config)
+    }
+}
+
+#[async_trait]
+impl Hypervisor for Remote {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        annotations: &HashMap<String, String>,
+    ) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.prepare_vm(id, netns, annotations).await
+    }
+
+    async fn start_vm(&self, timeout: i32) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.start_vm(timeout).await
+    }
+
+    async fn stop_vm(&self) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.stop_vm().await
+    }
+
+    async fn wait_vm(&self) -> Result<i32> {
+        let inner = self.inner.read().await;
+        inner.wait_vm().await
+    }
+
+    async fn pause_vm(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.pause_vm().await
+    }
+
+    async fn resume_vm(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.resume_vm().await
+    }
+
+    async fn save_vm(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.save_vm().await
+    }
+
+    async fn add_device(&self, device: DeviceType) -> Result<DeviceType> {
+        let inner = self.inner.write().await;
+        inner.add_device(device).await
+    }
+
+    async fn remove_device(&self, device: DeviceType) -> Result<()> {
+        let inner = self.inner.write().await;
+        inner.remove_device(device).await
+    }
+
+    async fn update_device(&self, device: DeviceType) -> Result<()> {
+        let inner = self.inner.write().await;
+        inner.update_device(device).await
+    }
+
+    async fn get_agent_socket(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_agent_socket().await
+    }
+
+    async fn disconnect(&self) {
+        let mut inner = self.inner.write().await;
+        inner.disconnect().await
+    }
+
+    async fn hypervisor_config(&self) -> HypervisorConfig {
+        let inner = self.inner.read().await;
+        inner.hypervisor_config()
+    }
+
+    async fn get_thread_ids(&self) -> Result<VcpuThreadIds> {
+        let inner = self.inner.read().await;
+        inner.get_thread_ids().await
+    }
+
+    async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let inner = self.inner.read().await;
+        inner.get_vmm_master_tid().await
+    }
+
+    async fn get_ns_path(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_ns_path().await
+    }
+
+    async fn cleanup(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.cleanup().await
+    }
+
+    async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> {
+        let mut inner = self.inner.write().await;
+        inner.resize_vcpu(old_vcpus, new_vcpus).await
+    }
+
+    async fn get_pids(&self) -> Result<Vec<u32>> {
+        let inner = self.inner.read().await;
+        inner.get_pids().await
+    }
+
+    async fn check(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.check().await
+    }
+
+    async fn get_jailer_root(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_jailer_root().await
+    }
+
+    async fn save_state(&self) -> Result<HypervisorState> {
+        self.save().await
+    }
+
+    async fn capabilities(&self) -> Result<Capabilities> {
+        let inner = self.inner.read().await;
+        inner.capabilities().await
+    }
+
+    async fn get_hypervisor_metrics(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_hypervisor_metrics().await
+    }
+
+    async fn set_capabilities(&self, flag: CapabilityBits) {
+        let mut inner = self.inner.write().await;
+        inner.set_capabilities(flag)
+    }
+
+    async fn set_guest_memory_block_size(&self, size: u32) {
+        let mut inner = self.inner.write().await;
+        inner.set_guest_memory_block_size(size);
+    }
+
+    async fn guest_memory_block_size(&self) -> u32 {
+        let inner = self.inner.read().await;
+        inner.guest_memory_block_size_mb()
+    }
+
+    async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
+        let inner = self.inner.read().await;
+        inner.resize_memory(new_mem_mb)
+    }
+
+    async fn get_passfd_listener_addr(&self) -> Result<(String, u32)> {
+        Err(anyhow::anyhow!("Not yet supported"))
+    }
+}
+
+#[async_trait]
+impl Persist for Remote {
+    type State = HypervisorState;
+    type ConstructorArgs = ();
+
+    /// Save a state of the component.
+    async fn save(&self) -> Result<Self::State> {
+        let inner = self.inner.read().await;
+        inner.save().await.context("save remote hypervisor state")
+    }
+
+    /// Restore a component from a specified state.
+    async fn restore(
+        hypervisor_args: Self::ConstructorArgs,
+        hypervisor_state: Self::State,
+    ) -> Result<Self> {
+        let inner = RemoteInner::restore(hypervisor_args, hypervisor_state).await?;
+        Ok(Self {
+            inner: Arc::new(RwLock::new(inner)),
+        })
+    }
+}

src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs

@@ -120,7 +120,7 @@ impl ShareFsMount for VirtiofsShareMount {
                 guest_path,
                 storages,
             });
-        } else if get_mount_type(config.mount.typ()).as_str() == mount::KATA_EPHEMERAL_VOLUME_TYPE {
+        } else if get_mount_type(&config.mount).as_str() == mount::KATA_EPHEMERAL_VOLUME_TYPE {
             // refer to the golang `handleEphemeralStorage` code at
             // https://github.com/kata-containers/kata-containers/blob/9516286f6dd5cfd6b138810e5d7c9e01cf6fc043/src/runtime/virtcontainers/kata_agent.go#L1354
 

src/runtime-rs/crates/resource/src/volume/direct_volume.rs

@@ -98,7 +98,7 @@ pub(crate) async fn handle_direct_volume(
 }
 
 pub(crate) fn is_direct_volume(m: &oci::Mount) -> Result<bool> {
-    let mnt_type = get_mount_type(m.typ());
+    let mnt_type = get_mount_type(m);
     let mount_type = mnt_type.as_str();
 
     // Filter the non-bind volume and non-direct-vol volume

src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs

@@ -125,7 +125,7 @@ impl SPDKVolume {
             .context("generate host-guest shared path failed")?;
         storage.mount_point = guest_path.clone();
 
-        if get_mount_type(m.typ()).as_str() != "bind" {
+        if get_mount_type(m).as_str() != "bind" {
             storage.fs_type = mount_info.fs_type.clone();
         } else {
             storage.fs_type = DEFAULT_VOLUME_FS_TYPE.to_string();

src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs

@@ -80,7 +80,7 @@ impl VfioVolume {
             .context("generate host-guest shared path failed")?;
         storage.mount_point = guest_path.clone();
 
-        if get_mount_type(m.typ()).as_str() != "bind" {
+        if get_mount_type(m).as_str() != "bind" {
             storage.fs_type = mount_info.fs_type.clone();
         } else {
             storage.fs_type = DEFAULT_VOLUME_FS_TYPE.to_string();

src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs

@@ -308,8 +308,8 @@ impl Volume for ShareFsVolume {
 }
 
 pub(crate) fn is_share_fs_volume(m: &oci::Mount) -> bool {
-    (get_mount_type(m.typ()).as_str() == "bind"
-        || get_mount_type(m.typ()).as_str() == mount::KATA_EPHEMERAL_VOLUME_TYPE)
+    let mount_type = get_mount_type(m);
+    (mount_type == "bind" || mount_type == mount::KATA_EPHEMERAL_VOLUME_TYPE)
         && !is_host_device(&get_mount_path(&Some(m.destination().clone())))
         && !is_system_mount(&get_mount_path(m.source()))
 }

src/runtime-rs/crates/resource/src/volume/shm_volume.rs

@@ -115,5 +115,5 @@ impl Volume for ShmVolume {
 
 pub(crate) fn is_shm_volume(m: &oci::Mount) -> bool {
     get_mount_path(&Some(m.destination().clone())).as_str() == "/dev/shm"
-        && get_mount_type(m.typ()).as_str() != KATA_EPHEMERAL_DEV_TYPE
+        && get_mount_type(m).as_str() != KATA_EPHEMERAL_DEV_TYPE
 }

src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs

@@ -128,11 +128,13 @@ impl Process {
 
     pub fn pre_fifos_open(&mut self) -> Result<()> {
         if let Some(ref stdout) = self.stdout {
-            self.stdout_r = Some(open_fifo_read(stdout)?);
+            self.stdout_r = Some(open_fifo_read(stdout).context("open stdout")?);
         }
 
-        if let Some(ref stderr) = self.stderr {
-            self.stderr_r = Some(open_fifo_read(stderr)?);
+        if !self.terminal {
+            if let Some(ref stderr) = self.stderr {
+                self.stderr_r = Some(open_fifo_read(stderr).context("open stderr")?);
+            }
         }
 
         Ok(())

src/runtime-rs/crates/runtimes/virt_container/src/lib.rs

@@ -26,10 +26,12 @@ use hypervisor::{dragonball::Dragonball, HYPERVISOR_DRAGONBALL};
 #[cfg(not(target_arch = "s390x"))]
 use hypervisor::{firecracker::Firecracker, HYPERVISOR_FIRECRACKER};
 use hypervisor::{qemu::Qemu, HYPERVISOR_QEMU};
+use hypervisor::{remote::Remote, HYPERVISOR_REMOTE};
 #[cfg(all(feature = "dragonball", not(target_arch = "s390x")))]
 use kata_types::config::DragonballConfig;
 #[cfg(not(target_arch = "s390x"))]
 use kata_types::config::FirecrackerConfig;
+use kata_types::config::RemoteConfig;
 use kata_types::config::{hypervisor::register_hypervisor_plugin, QemuConfig, TomlConfig};
 
 #[cfg(all(feature = "cloud-hypervisor", not(target_arch = "s390x")))]
@@ -75,6 +77,9 @@ impl RuntimeHandler for VirtContainer {
             register_hypervisor_plugin(HYPERVISOR_NAME_CH, ch_config);
         }
 
+        let remote_config = Arc::new(RemoteConfig::new());
+        register_hypervisor_plugin("remote", remote_config);
+
         Ok(())
     }
 
@@ -179,7 +184,6 @@ async fn new_hypervisor(toml_config: &TomlConfig) -> Result<Arc<dyn Hypervisor>>
                 .await;
             Ok(Arc::new(hypervisor))
         }
-
         #[cfg(all(feature = "cloud-hypervisor", not(target_arch = "s390x")))]
         HYPERVISOR_NAME_CH => {
             let mut hypervisor = CloudHypervisor::new();
@@ -190,6 +194,13 @@ async fn new_hypervisor(toml_config: &TomlConfig) -> Result<Arc<dyn Hypervisor>>
 
             Ok(Arc::new(hypervisor))
         }
+        HYPERVISOR_REMOTE => {
+            let mut hypervisor = Remote::new();
+            hypervisor
+                .set_hypervisor_config(hypervisor_config.clone())
+                .await;
+            Ok(Arc::new(hypervisor))
+        }
         _ => Err(anyhow!("Unsupported hypervisor {}", &hypervisor_name)),
     }
 }

src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs

@@ -19,6 +19,7 @@ use containerd_shim_protos::events::task::{TaskExit, TaskOOM};
 use hypervisor::VsockConfig;
 #[cfg(not(target_arch = "s390x"))]
 use hypervisor::HYPERVISOR_FIRECRACKER;
+use hypervisor::HYPERVISOR_REMOTE;
 #[cfg(all(feature = "dragonball", not(target_arch = "s390x")))]
 use hypervisor::{dragonball::Dragonball, HYPERVISOR_DRAGONBALL};
 use hypervisor::{qemu::Qemu, HYPERVISOR_QEMU};
@@ -326,7 +327,11 @@ impl Sandbox for VirtSandbox {
         }
 
         self.hypervisor
-            .prepare_vm(id, sandbox_config.network_env.netns.clone())
+            .prepare_vm(
+                id,
+                sandbox_config.network_env.netns.clone(),
+                &sandbox_config.annotations,
+            )
             .await
             .context("prepare vm")?;
 
@@ -649,6 +654,7 @@ impl Persist for VirtSandbox {
                 #[cfg(not(target_arch = "s390x"))]
                 HYPERVISOR_FIRECRACKER => Ok(Some(hypervisor_state)),
                 HYPERVISOR_QEMU => Ok(Some(hypervisor_state)),
+                HYPERVISOR_REMOTE => Ok(Some(hypervisor_state)),
                 _ => Err(anyhow!(
                     "Unsupported hypervisor {}",
                     hypervisor_state.hypervisor_type

src/runtime/Makefile

@@ -151,8 +151,8 @@ FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd
 SNPCERTSPATH := /opt/snp/cert_chain.cert
 
 ROOTMEASURECONFIG ?= ""
-KERNELPARAMS += $(ROOTMEASURECONFIG)
 KERNELTDXPARAMS += $(ROOTMEASURECONFIG)
+KERNELQEMUCOCODEVPARAMS += $(ROOTMEASURECONFIG)
 
 # Name of default configuration file the runtime will use.
 CONFIG_FILE = configuration.toml
@@ -646,6 +646,7 @@ USER_VARS += DEFMACHINETYPE_CLH
 USER_VARS += DEFMACHINETYPE_STRATOVIRT
 USER_VARS += KERNELPARAMS
 USER_VARS += KERNELTDXPARAMS
+USER_VARS += KERNELQEMUCOCODEVPARAMS
 USER_VARS += LIBEXECDIR
 USER_VARS += LOCALSTATEDIR
 USER_VARS += PKGDATADIR

src/runtime/config/configuration-qemu-coco-dev.toml.in

@@ -73,7 +73,7 @@ valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
 # may stop the virtual machine from booting.
 # To see the list of default parameters, enable hypervisor debug, create a
 # container and look for 'default-kernel-parameters' log entries.
-kernel_params = "@KERNELPARAMS@"
+kernel_params = "@KERNELQEMUCOCODEVPARAMS@"
 
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
@@ -653,7 +653,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
 #   does not yet support sandbox sizing annotations.
 # - When running single containers using a tool like ctr, container sizing information will be available.
-static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_TEE@
 
 # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
 # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.

src/runtime/config/configuration-remote.toml.in

@@ -38,7 +38,7 @@ remote_hypervisor_timeout = 600
 # Each member of the list is a regular expression, which is the base name
 # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
 # Note: Remote hypervisor is only handling the following annotations
-enable_annotations = ["machine_type", "default_memory", "default_vcpus", "image"]
+enable_annotations = ["machine_type", "default_memory", "default_vcpus", "image", "default_gpus", "gpu_model"]
 
 # Optional space-separated list of options to pass to the guest kernel.
 # For example, use `kernel_params = "vsyscall=emulate"` if you are having

src/runtime/pkg/govmm/.github/workflows/main.yml

@@ -5,17 +5,17 @@ jobs:
     strategy:
       matrix:
         go-version: [1.15.x, 1.16.x]
-        os: [ubuntu-latest]
+        os: [ubuntu-22.04]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v6
         with:
           version: latest
           args: -c .golangci.yml -v

src/runtime/pkg/oci/utils.go

@@ -560,6 +560,10 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig,
 		config.HypervisorConfig.Initdata = initdata
 	}
 
+	if err := addHypervisorGPUOverrides(ocispec, config); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -754,6 +758,26 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
 	})
 }
 
+func addHypervisorGPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
+	if sbConfig.HypervisorType != vc.RemoteHypervisor {
+		return nil
+	}
+
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.DefaultGPUs).setUint(func(gpus uint64) {
+		sbConfig.HypervisorConfig.DefaultGPUs = uint32(gpus)
+	}); err != nil {
+		return err
+	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.DefaultGPUModel]; ok {
+		if value != "" {
+			sbConfig.HypervisorConfig.DefaultGPUModel = value
+		}
+	}
+
+	return nil
+}
+
 func addHypervisorBlockOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
 	if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceDriver]; ok {
 		supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW}
@@ -1076,6 +1100,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
 		sandboxConfig.HypervisorConfig.NumVCPUsF += sandboxConfig.SandboxResources.WorkloadCPUs
 		sandboxConfig.HypervisorConfig.MemorySize += sandboxConfig.SandboxResources.WorkloadMemMB
 
+		sandboxConfig.HypervisorConfig.DefaultMaxVCPUs = sandboxConfig.HypervisorConfig.NumVCPUs()
+
 		ociLog.WithFields(logrus.Fields{
 			"workload cpu":       sandboxConfig.SandboxResources.WorkloadCPUs,
 			"default cpu":        sandboxConfig.SandboxResources.BaseCPUs,

src/runtime/pkg/oci/utils_test.go

@@ -775,6 +775,23 @@ func TestAddRemoteHypervisorAnnotations(t *testing.T) {
 	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
 	assert.NoError(err)
 	assert.Equal(sbConfig.HypervisorConfig.Initdata, "initdata")
+
+	// When GPU annotations are specified, remote hypervisor annotations have the annotation added
+	ocispec.Annotations[vcAnnotations.DefaultGPUs] = "-1"
+	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
+	assert.Error(err)
+
+	ocispec.Annotations[vcAnnotations.DefaultGPUs] = "1"
+	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
+	assert.NoError(err)
+	assert.Equal(sbConfig.HypervisorConfig.DefaultGPUs, uint32(1))
+
+	// When GPU annotations are specified, remote hypervisor annotations have the annotation added
+	ocispec.Annotations[vcAnnotations.DefaultGPUModel] = "tesla"
+	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
+	assert.NoError(err)
+	assert.Equal(sbConfig.HypervisorConfig.DefaultGPUModel, "tesla")
+
 }
 
 func TestAddProtectedHypervisorAnnotations(t *testing.T) {