Merge pull request #10562 from stevenhorsman/remove-release-artifactor-skips

workflows: Remove skipping of artifact uploads

.github/workflows/basic-ci-amd64.yaml

@@ -138,6 +138,8 @@ jobs:
         run: bash tests/integration/nydus/gha-run.sh run
 
   run-runk:
+    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
+    if: false
     runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: lts

.github/workflows/build-checks.yaml

@@ -19,7 +19,6 @@ jobs:
           - runtime-rs
           - agent-ctl
           - kata-ctl
-          - runk
           - trace-forwarder
           - genpolicy
         command:
@@ -40,15 +39,11 @@ jobs:
             component-path: src/tools/agent-ctl
           - component: kata-ctl
             component-path: src/tools/kata-ctl
-          - component: runk
-            component-path: src/tools/runk
           - component: trace-forwarder
             component-path: src/tools/trace-forwarder
           - install-libseccomp: no
           - component: agent
             install-libseccomp: yes
-          - component: runk
-            install-libseccomp: yes
           - component: genpolicy
             component-path: src/tools/genpolicy
     steps:
@@ -94,7 +89,7 @@ jobs:
           echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
           echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
       - name: Install protobuf-compiler
-        if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'runk' || matrix.component == 'genpolicy' || matrix.component == 'agent-ctl') }}
+        if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'genpolicy' || matrix.component == 'agent-ctl') }}
         run: sudo apt-get -y install protobuf-compiler
       - name: Install clang
         if: ${{ matrix.command == 'make check' && (matrix.component == 'agent' || matrix.component == 'agent-ctl') }}

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -53,12 +53,6 @@ jobs:
           - qemu
           - qemu-snp-experimental
           - stratovirt
-          - rootfs-image
-          - rootfs-image-confidential
-          - rootfs-image-mariner
-          - rootfs-initrd
-          - rootfs-initrd-confidential
-          - runk
           - trace-forwarder
           - virtiofsd
         stage:
@@ -134,7 +128,6 @@ jobs:
           push-to-registry: true
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ matrix.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -142,9 +135,17 @@ jobs:
           retention-days: 15
           if-no-files-found: error
 
-  build-asset-shim-v2:
+  build-asset-rootfs:
     runs-on: ubuntu-22.04
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-image-confidential
+          - rootfs-image-mariner
+          - rootfs-initrd
+          - rootfs-initrd-confidential
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -165,9 +166,73 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-shim-v2:
+    runs-on: ubuntu-22.04
+    needs: [build-asset, build-asset-rootfs]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
       - name: Build shim-v2
         id: build
         run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
@@ -181,6 +246,7 @@ jobs:
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
 
       - name: store-artifact shim-v2
         uses: actions/upload-artifact@v4
@@ -192,7 +258,7 @@ jobs:
 
   create-kata-tarball:
     runs-on: ubuntu-22.04
-    needs: [build-asset, build-asset-shim-v2]
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -35,8 +35,6 @@ jobs:
           - nydus
           - qemu
           - stratovirt
-          - rootfs-image
-          - rootfs-initrd
           - virtiofsd
     steps:
       - name: Login to Kata Containers quay.io
@@ -75,7 +73,6 @@ jobs:
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -83,9 +80,14 @@ jobs:
           retention-days: 15
           if-no-files-found: error
 
-  build-asset-shim-v2:
+  build-asset-rootfs:
     runs-on: arm64-builder
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-initrd
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -106,8 +108,71 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-shim-v2:
+    runs-on: arm64-builder
+    needs: [build-asset, build-asset-rootfs]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
       - name: Build shim-v2
         run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
@@ -132,7 +197,7 @@ jobs:
 
   create-kata-tarball:
     runs-on: arm64-builder
-    needs: [build-asset, build-asset-shim-v2]
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - name: Adjust a permission for repo
         run: |

.github/workflows/build-kata-static-tarball-ppc64le.yaml

@@ -30,7 +30,6 @@ jobs:
           - agent
           - kernel
           - qemu
-          - rootfs-initrd
           - virtiofsd
         stage:
           - ${{ inputs.stage }}
@@ -76,7 +75,6 @@ jobs:
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -84,9 +82,15 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
-  build-asset-shim-v2:
+  build-asset-rootfs:
     runs-on: ppc64le
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-initrd
+        stage:
+          - ${{ inputs.stage }}
     steps:
       - name: Prepare the self-hosted runner
         run: |
@@ -112,8 +116,76 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  build-asset-shim-v2:
+    runs-on: ppc64le
+    needs: [build-asset, build-asset-rootfs]
+    steps:
+      - name: Prepare the self-hosted runner
+        run: |
+            ${HOME}/scripts/prepare_runner.sh
+            sudo rm -rf $GITHUB_WORKSPACE/*
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
       - name: Build shim-v2
         run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
@@ -138,7 +210,7 @@ jobs:
 
   create-kata-tarball:
     runs-on: ppc64le
-    needs: [build-asset, build-asset-shim-v2]
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - name: Adjust a permission for repo
         run: |

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -38,10 +38,6 @@ jobs:
           - kernel-confidential
           - pause-image
           - qemu
-          - rootfs-image
-          - rootfs-image-confidential
-          - rootfs-initrd
-          - rootfs-initrd-confidential
           - virtiofsd
     env:
       PERFORM_ATTESTATION: ${{ matrix.asset == 'agent' && inputs.push-to-registry == 'yes' && 'yes' || 'no' }}
@@ -106,7 +102,69 @@ jobs:
           push-to-registry: true
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-rootfs:
+    runs-on: s390x
+    needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-image-confidential
+          - rootfs-initrd
+          - rootfs-initrd-confidential
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -116,7 +174,7 @@ jobs:
 
   build-asset-boot-image-se:
     runs-on: s390x
-    needs: build-asset
+    needs: [build-asset, build-asset-rootfs]
     steps:
       - uses: actions/checkout@v4
 
@@ -142,11 +200,7 @@ jobs:
 
       - name: Build boot-image-se
         run: |
-          base_dir=tools/packaging/kata-deploy/local-build/
-          cp -r kata-artifacts ${base_dir}/build
-          # Skip building dependant artifacts of boot-image-se-tarball
-          # because we already have them from the previous build
-          sed -i 's/\(^boot-image-se-tarball:\).*/\1/g' ${base_dir}/Makefile
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "boot-image-se"
           make boot-image-se-tarball
           build_dir=$(readlink -f build)
           sudo cp -r "${build_dir}" "kata-build"
@@ -164,7 +218,7 @@ jobs:
 
   build-asset-shim-v2:
     runs-on: s390x
-    needs: build-asset
+    needs: [build-asset, build-asset-rootfs]
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -185,9 +239,17 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
       - name: Build shim-v2
         id: build
         run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
@@ -201,6 +263,7 @@ jobs:
           ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
           TARGET_BRANCH: ${{ inputs.target-branch }}
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
 
       - name: store-artifact shim-v2
         uses: actions/upload-artifact@v4
@@ -212,7 +275,7 @@ jobs:
 
   create-kata-tarball:
     runs-on: s390x
-    needs: [build-asset, build-asset-boot-image-se, build-asset-shim-v2]
+    needs: [build-asset, build-asset-rootfs, build-asset-boot-image-se, build-asset-shim-v2]
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/ci-weekly.yaml

@@ -37,7 +37,7 @@ jobs:
     secrets: inherit
 
   build-and-publish-tee-confidential-unencrypted-image:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/ci.yaml

@@ -176,6 +176,7 @@ jobs:
     needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
     uses: ./.github/workflows/run-kata-coco-tests.yaml
     with:
+      tarball-suffix: -${{ inputs.tag }}
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64

.github/workflows/gatekeeper-skipper.yaml

@@ -34,7 +34,7 @@ on:
 
 jobs:
   skipper:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     outputs:
       skip_build: ${{ steps.skipper.outputs.skip_build }}
       skip_test: ${{ steps.skipper.outputs.skip_test }}

.github/workflows/gatekeeper.yaml

@@ -18,7 +18,7 @@ concurrency:
 
 jobs:
   gatekeeper:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/run-k8s-tests-on-amd64.yaml

@@ -56,6 +56,7 @@ jobs:
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: "false"
       K8S_TEST_HOST_TYPE: all
+      CONTAINER_RUNTIME: ${{ matrix.container_runtime }}
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -36,7 +36,7 @@ jobs:
           - qemu-runtime-rs
           - qemu-coco-dev
         k8s:
-          - k3s
+          - kubeadm
         include:
           - snapshotter: devmapper
             pull-type: default
@@ -97,9 +97,6 @@ jobs:
           echo "KBS_INGRESS=nodeport" >> $GITHUB_ENV
         if: ${{ matrix.vmm == 'qemu-coco-dev' }}
 
-      - name: Deploy ${{ matrix.k8s }}
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
-
       # qemu-runtime-rs only works with overlayfs
       # See: https://github.com/kata-containers/kata-containers/issues/10066
       - name: Configure the ${{ matrix.snapshotter }} snapshotter

.github/workflows/run-kata-coco-stability-tests.yaml

@@ -34,7 +34,7 @@ jobs:
           - nydus
         pull-type:
           - guest-pull
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
@@ -64,6 +64,15 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
 

.github/workflows/run-kata-coco-tests.yaml

@@ -2,6 +2,9 @@ name: CI | Run kata coco tests
 on:
   workflow_call:
     inputs:
+      tarball-suffix:
+        required: false
+        type: string
       registry:
         required: true
         type: string
@@ -33,7 +36,15 @@ jobs:
           - nydus
         pull-type:
           - guest-pull
-    runs-on: tdx
+        k8s-test-host-type:
+          - baremetal-attestation
+          - baremetal-no-attestation
+        include:
+          - k8s-test-host-type: baremetal-attestation
+            machine: tdx-attestation
+          - k8s-test-host-type: baremetal-no-attestation
+            machine: tdx-no-attestation
+    runs-on: ${{ matrix.machine }}
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
@@ -43,7 +54,7 @@ jobs:
       KUBERNETES: "vanilla"
       USING_NFD: "true"
       KBS: "true"
-      K8S_TEST_HOST_TYPE: "baremetal"
+      K8S_TEST_HOST_TYPE: ${{ matrix.k8s-test-host-type }}
       KBS_INGRESS: "nodeport"
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       PULL_TYPE: ${{ matrix.pull-type }}
@@ -72,14 +83,17 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
 
       - name: Uninstall previous `kbs-client`
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
 
       - name: Deploy CoCo KBS
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
 
       - name: Install `kbs-client`
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
@@ -96,7 +110,7 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
       - name: Delete CoCo KBS
-        if: always()
+        if: ${{ always() && matrix.machine != 'tdx-no-attestation' }}
         run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
 
   run-k8s-tests-on-sev:
@@ -158,6 +172,9 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
   run-k8s-tests-sev-snp:
+    # Skipping SNP tests to unblock the CI. 
+    # Will revert after issue is fixed.
+    if: false
     strategy:
       fail-fast: false
       matrix:
@@ -262,6 +279,7 @@ jobs:
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: "false"
+      AUTO_GENERATE_POLICY: "yes"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -274,6 +292,15 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
 
@@ -315,7 +342,7 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
       - name: Run tests
-        timeout-minutes: 60
+        timeout-minutes: 80
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete AKS cluster

.github/workflows/run-metrics.yaml

@@ -48,7 +48,7 @@ jobs:
       # all the tests due to a single flaky instance.
       fail-fast: false
       matrix:
-        vmm: ['clh', 'qemu', 'stratovirt']
+        vmm: ['clh', 'qemu']
       max-parallel: 1
     runs-on: metrics
     env:

.github/workflows/run-runk-tests.yaml

@@ -15,6 +15,8 @@ on:
 
 jobs:
   run-runk:
+    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
+    if: false
     runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: lts

VERSION

@@ -1 +1 @@
-3.10.1
+3.11.0

ci/openshift-ci/smoke/http-server.yaml.in

@@ -13,7 +13,7 @@ metadata:
 spec:
   containers:
     - name: http-server
-      image: registry.fedoraproject.org/fedora
+      image: docker.io/library/python:3
       ports:
         - containerPort: 8080
       command: ["python3"]

docs/design/virtualization.md

@@ -98,8 +98,7 @@ of Kata Containers, the Cloud Hypervisor configuration supports both CPU
 and memory resize, device hotplug (disk and VFIO), file-system sharing through virtio-fs,
 block-based volumes, booting from VM images backed by pmem device, and
 fine-grained seccomp filters for each VMM threads (e.g. all virtio
-device worker threads). Please check [this GitHub Project](https://github.com/orgs/kata-containers/projects/21)
-for details of ongoing integration efforts.
+device worker threads).
 
 Devices and features used:
 - virtio VSOCK or virtio serial

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -94,6 +94,8 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
 | `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
+| `io.katacontainers.config.hypervisor.default_gpus` | uint32 | the minimum number of GPUs required for the VM. Only used by remote hypervisor to help with instance selection |
+| `io.katacontainers.config.hypervisor.default_gpu_model` | string | the GPU model required for the VM. Only used by remote hypervisor to help with instance selection |
 
 ## Container Options
 | Key | Value Type | Comments |

src/agent/Cargo.lock

@@ -64,6 +64,20 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "ahash"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
+dependencies = [
+ "cfg-if 1.0.0",
+ "getrandom",
+ "once_cell",
+ "serde",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -97,6 +111,55 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "anstream"
+version = "0.6.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.86"
@@ -712,6 +775,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "bytecount"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -814,6 +883,30 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cdi"
+version = "0.1.0"
+source = "git+https://github.com/cncf-tags/container-device-interface-rs?rev=fba5677a8e7cc962fc6e495fcec98d7d765e332a#fba5677a8e7cc962fc6e495fcec98d7d765e332a"
+dependencies = [
+ "anyhow",
+ "clap 4.5.13",
+ "const_format",
+ "jsonschema",
+ "lazy_static",
+ "libc",
+ "nix 0.24.3",
+ "notify",
+ "oci-spec",
+ "once_cell",
+ "path-clean",
+ "regex",
+ "semver",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "serde_yaml",
+]
+
 [[package]]
 name = "cesu8"
 version = "1.1.0"
@@ -914,8 +1007,8 @@ checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
 dependencies = [
  "atty",
  "bitflags 1.3.2",
- "clap_derive",
- "clap_lex",
+ "clap_derive 3.2.25",
+ "clap_lex 0.2.4",
  "indexmap 1.9.3",
  "once_cell",
  "strsim 0.10.0",
@@ -923,6 +1016,28 @@ dependencies = [
  "textwrap",
 ]
 
+[[package]]
+name = "clap"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
+dependencies = [
+ "clap_builder",
+ "clap_derive 4.5.13",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex 0.7.2",
+ "strsim 0.11.1",
+]
+
 [[package]]
 name = "clap_derive"
 version = "3.2.25"
@@ -936,6 +1051,18 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "clap_derive"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.71",
+]
+
 [[package]]
 name = "clap_lex"
 version = "0.2.4"
@@ -945,6 +1072,12 @@ dependencies = [
  "os_str_bytes",
 ]
 
+[[package]]
+name = "clap_lex"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
+
 [[package]]
 name = "cmac"
 version = "0.7.2"
@@ -967,6 +1100,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "colorchoice"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -1741,6 +1880,17 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "fancy-regex"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
+dependencies = [
+ "bit-set",
+ "regex-automata 0.4.7",
+ "regex-syntax 0.8.4",
+]
+
 [[package]]
 name = "fastrand"
 version = "1.9.0"
@@ -1812,6 +1962,15 @@ dependencies = [
  "miniz_oxide",
 ]
 
+[[package]]
+name = "fluent-uri"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17c704e9dbe1ddd863da1e6ff3567795087b1eb201ce80d8fa81162e1516500d"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1827,6 +1986,25 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fraction"
+version = "0.15.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7"
+dependencies = [
+ "lazy_static",
+ "num",
+]
+
+[[package]]
+name = "fsevent-sys"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "funty"
 version = "2.0.0"
@@ -2052,7 +2230,7 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 dependencies = [
- "ahash",
+ "ahash 0.7.8",
 ]
 
 [[package]]
@@ -2605,6 +2783,21 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
+[[package]]
+name = "iso8601"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -2623,15 +2816,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -2690,6 +2874,18 @@ dependencies = [
  "smallvec",
 ]
 
+[[package]]
+name = "json-patch"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b1fb8864823fad91877e6caea0baca82e49e8db50f8e5c9f9a453e27d3330fc"
+dependencies = [
+ "jsonptr",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
 [[package]]
 name = "json-syntax"
 version = "0.12.5"
@@ -2709,6 +2905,47 @@ dependencies = [
  "utf8-decode",
 ]
 
+[[package]]
+name = "jsonptr"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627"
+dependencies = [
+ "fluent-uri",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "jsonschema"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0afd06142c9bcb03f4a8787c77897a87b6be9c4918f1946c33caa714c27578"
+dependencies = [
+ "ahash 0.8.11",
+ "anyhow",
+ "base64 0.22.1",
+ "bytecount",
+ "clap 4.5.13",
+ "fancy-regex",
+ "fraction",
+ "getrandom",
+ "iso8601",
+ "itoa",
+ "memchr",
+ "num-cmp",
+ "once_cell",
+ "parking_lot 0.12.3",
+ "percent-encoding",
+ "regex",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "time",
+ "url",
+ "uuid",
+]
+
 [[package]]
 name = "jwt"
 version = "0.16.0"
@@ -2773,14 +3010,16 @@ dependencies = [
  "async-std",
  "async-trait",
  "capctl",
+ "cdi",
  "cfg-if 1.0.0",
  "cgroups-rs",
- "clap",
+ "clap 3.2.25",
  "const_format",
  "derivative",
  "futures",
  "image-rs",
  "ipnetwork",
+ "json-patch",
  "kata-sys-util",
  "kata-types",
  "lazy_static",
@@ -2835,7 +3074,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -2934,6 +3172,26 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "kqueue"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c"
+dependencies = [
+ "kqueue-sys",
+ "libc",
+]
+
+[[package]]
+name = "kqueue-sys"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+]
+
 [[package]]
 name = "krata-tokio-tar"
 version = "0.4.2"
@@ -3285,6 +3543,18 @@ dependencies = [
  "adler",
 ]
 
+[[package]]
+name = "mio"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+dependencies = [
+ "libc",
+ "log",
+ "wasi",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "mio"
 version = "1.0.2"
@@ -3465,6 +3735,25 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "notify"
+version = "6.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
+dependencies = [
+ "bitflags 2.6.0",
+ "crossbeam-channel",
+ "filetime",
+ "fsevent-sys",
+ "inotify",
+ "kqueue",
+ "libc",
+ "log",
+ "mio 0.8.11",
+ "walkdir",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "ntapi"
 version = "0.4.1"
@@ -3515,6 +3804,12 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "num-cmp"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa"
+
 [[package]]
 name = "num-complex"
 version = "0.4.6"
@@ -3894,6 +4189,12 @@ dependencies = [
  "slash-formatter",
 ]
 
+[[package]]
+name = "path-clean"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
+
 [[package]]
 name = "path-dedot"
 version = "1.2.4"
@@ -4376,7 +4677,6 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.5.1",
  "serde",
@@ -4626,14 +4926,12 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
 
 [[package]]
 name = "regorus"
-version = "0.1.5"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77dd872918e5c172bd42ac49716f89a15e35be513bba3d902e355a531529a87f"
+checksum = "843c3d97f07e3b5ac0955d53ad0af4c91fe4a4f8525843ece5bf014f27829b73"
 dependencies = [
  "anyhow",
- "itertools 0.12.1",
  "lazy_static",
- "num",
  "rand",
  "regex",
  "scientific",
@@ -4666,6 +4964,7 @@ dependencies = [
  "bytes 1.6.1",
  "cookie",
  "cookie_store",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "http",
@@ -5919,7 +6218,7 @@ dependencies = [
  "backtrace",
  "bytes 1.6.1",
  "libc",
- "mio",
+ "mio 1.0.2",
  "parking_lot 0.12.3",
  "pin-project-lite",
  "signal-hook-registry",
@@ -6346,6 +6645,12 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
 [[package]]
 name = "uuid"
 version = "1.10.0"
@@ -7021,6 +7326,26 @@ dependencies = [
  "zvariant",
 ]
 
+[[package]]
+name = "zerocopy"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.71",
+]
+
 [[package]]
 name = "zerofrom"
 version = "0.1.4"

src/agent/Cargo.toml

@@ -80,10 +80,13 @@ strum_macros = "0.26.2"
 image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "v0.10.0", default-features = false, optional = true }
 
 # Agent Policy
-regorus = { version = "0.1.4", default-features = false, features = [
+regorus = { version = "0.2.6", default-features = false, features = [
     "arc",
     "regex",
+    "std",
 ], optional = true }
+cdi = { git = "https://github.com/cncf-tags/container-device-interface-rs", rev = "fba5677a8e7cc962fc6e495fcec98d7d765e332a" }
+json-patch = "2.0.0"
 
 [dev-dependencies]
 tempfile = "3.1.0"

src/agent/src/device/mod.rs

@@ -11,6 +11,9 @@ use self::vfio_device_handler::{VfioApDeviceHandler, VfioPciDeviceHandler};
 use crate::pci;
 use crate::sandbox::Sandbox;
 use anyhow::{anyhow, Context, Result};
+use cdi::annotations::parse_annotations;
+use cdi::cache::{new_cache, with_auto_refresh, CdiOption};
+use cdi::spec_dirs::with_spec_dirs;
 use kata_types::device::DeviceHandlerManager;
 use nix::sys::stat;
 use oci::{LinuxDeviceCgroup, Spec};
@@ -25,6 +28,8 @@ use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
+use tokio::time;
+use tokio::time::Duration;
 use tracing::instrument;
 
 pub mod block_device_handler;
@@ -238,6 +243,69 @@ pub async fn add_devices(
     update_spec_devices(logger, spec, dev_updates)
 }
 
+#[instrument]
+pub async fn handle_cdi_devices(
+    logger: &Logger,
+    spec: &mut Spec,
+    spec_dir: &str,
+    cdi_timeout: u64,
+) -> Result<()> {
+    if let Some(container_type) = spec
+        .annotations()
+        .as_ref()
+        .and_then(|a| a.get("io.katacontainers.pkg.oci.container_type"))
+    {
+        if container_type == "pod_sandbox" {
+            return Ok(());
+        }
+    }
+
+    let (_, devices) = parse_annotations(spec.annotations().as_ref().unwrap())?;
+
+    if devices.is_empty() {
+        info!(logger, "no CDI annotations, no devices to inject");
+        return Ok(());
+    }
+    // Explicitly set the cache options to disable auto-refresh and
+    // to use the single spec dir "/var/run/cdi" for tests it can be overridden
+    let options: Vec<CdiOption> = vec![with_auto_refresh(false), with_spec_dirs(&[spec_dir])];
+    let cache: Arc<std::sync::Mutex<cdi::cache::Cache>> = new_cache(options);
+
+    for _ in 0..=cdi_timeout {
+        let inject_result = {
+            // Lock cache within this scope, std::sync::Mutex has no Send
+            // and await will not work with time::sleep
+            let mut cache = cache.lock().unwrap();
+            match cache.refresh() {
+                Ok(_) => {}
+                Err(e) => {
+                    return Err(anyhow!("error refreshing cache: {:?}", e));
+                }
+            }
+            cache.inject_devices(Some(spec), devices.clone())
+        };
+
+        match inject_result {
+            Ok(_) => {
+                info!(
+                    logger,
+                    "all devices injected successfully, modified CDI container spec: {:?}", &spec
+                );
+                return Ok(());
+            }
+            Err(e) => {
+                info!(logger, "error injecting devices: {:?}", e);
+                println!("error injecting devices: {:?}", e);
+            }
+        }
+        time::sleep(Duration::from_millis(1000)).await;
+    }
+    Err(anyhow!(
+        "failed to inject devices after CDI timeout of {} seconds",
+        cdi_timeout
+    ))
+}
+
 #[instrument]
 async fn validate_device(
     logger: &Logger,
@@ -1110,4 +1178,95 @@ mod tests {
         assert!(name.is_ok(), "{}", name.unwrap_err());
         assert_eq!(name.unwrap(), devname);
     }
+
+    #[tokio::test]
+    async fn test_handle_cdi_devices() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let mut spec = Spec::default();
+
+        let mut annotations = HashMap::new();
+        // cdi.k8s.io/vendor1_devices: vendor1.com/device=foo
+        annotations.insert(
+            "cdi.k8s.io/vfio17".to_string(),
+            "kata.com/gpu=0".to_string(),
+        );
+        spec.set_annotations(Some(annotations));
+
+        // create a file in /tmp/cdi with nvidia.json content
+        let cdi_dir = PathBuf::from("/tmp/cdi");
+        let cdi_file = cdi_dir.join("kata.json");
+
+        let cdi_version = "0.6.0";
+        let kind = "kata.com/gpu";
+        let device_name = "0";
+        let annotation_whatever = "false";
+        let annotation_whenever = "true";
+        let inner_env = "TEST_INNER_ENV=TEST_INNER_ENV_VALUE";
+        let outer_env = "TEST_OUTER_ENV=TEST_OUTER_ENV_VALUE";
+        let inner_device = "/dev/zero";
+        let outer_device = "/dev/null";
+
+        let cdi_content = format!(
+            r#"{{
+            "cdiVersion": "{cdi_version}",
+            "kind": "{kind}",
+            "devices": [
+                {{
+                    "name": "{device_name}",
+                    "annotations": {{
+                        "whatever": "{annotation_whatever}",
+                        "whenever": "{annotation_whenever}"
+                    }},
+                    "containerEdits": {{
+                        "env": [
+                            "{inner_env}"
+                        ],
+                        "deviceNodes": [
+                            {{
+                                "path": "{inner_device}"
+                            }}
+                        ]
+                    }}
+                }}
+            ],
+            "containerEdits": {{
+                "env": [
+                    "{outer_env}"
+                ],
+                "deviceNodes": [
+                    {{
+                        "path": "{outer_device}"
+                    }}
+                ]
+            }}
+        }}"#
+        );
+
+        fs::create_dir_all(&cdi_dir).unwrap();
+        fs::write(&cdi_file, cdi_content).unwrap();
+
+        let res = handle_cdi_devices(&logger, &mut spec, "/tmp/cdi", 0).await;
+        println!("modfied spec {:?}", spec);
+        assert!(res.is_ok(), "{}", res.err().unwrap());
+
+        let linux = spec.linux().as_ref().unwrap();
+        let devices = linux
+            .resources()
+            .as_ref()
+            .unwrap()
+            .devices()
+            .as_ref()
+            .unwrap();
+        assert_eq!(devices.len(), 2);
+
+        let env = spec.process().as_ref().unwrap().env().as_ref().unwrap();
+
+        // find string TEST_OUTER_ENV in evn
+        let outer_env = env.iter().find(|e| e.starts_with("TEST_OUTER_ENV"));
+        assert!(outer_env.is_some(), "TEST_OUTER_ENV not found in env");
+
+        // find TEST_INNER_ENV in env
+        let inner_env = env.iter().find(|e| e.starts_with("TEST_INNER_ENV"));
+        assert!(inner_env.is_some(), "TEST_INNER_ENV not found in env");
+    }
 }

src/agent/src/image.rs

@@ -21,6 +21,9 @@ use tokio::sync::Mutex;
 use crate::rpc::CONTAINER_BASE;
 use crate::AGENT_CONFIG;
 
+use kata_types::mount::KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL;
+use protocols::agent::Storage;
+
 pub const KATA_IMAGE_WORK_DIR: &str = "/run/kata-containers/image/";
 const CONFIG_JSON: &str = "config.json";
 const KATA_PAUSE_BUNDLE: &str = "/pause_bundle";
@@ -81,6 +84,28 @@ impl ImageService {
         Self { image_client }
     }
 
+    /// get guest pause image process specification
+    fn get_pause_image_process() -> Result<oci::Process> {
+        let guest_pause_bundle = Path::new(KATA_PAUSE_BUNDLE);
+        if !guest_pause_bundle.exists() {
+            bail!("Pause image not present in rootfs");
+        }
+        let guest_pause_config = scoped_join(guest_pause_bundle, CONFIG_JSON)?;
+
+        let image_oci = oci::Spec::load(guest_pause_config.to_str().ok_or_else(|| {
+            anyhow!(
+                "Failed to load the guest pause image config from {:?}",
+                guest_pause_config
+            )
+        })?)
+        .context("load image config file")?;
+
+        let image_oci_process = image_oci.process().as_ref().ok_or_else(|| {
+            anyhow!("The guest pause image config does not contain a process specification. Please check the pause image.")
+        })?;
+        Ok(image_oci_process.clone())
+    }
+
     /// pause image is packaged in rootfs
     fn unpack_pause_image(cid: &str) -> Result<String> {
         verify_id(cid).context("The guest pause image cid contains invalid characters.")?;
@@ -132,6 +157,20 @@ impl ImageService {
         Ok(pause_rootfs.display().to_string())
     }
 
+    /// check whether the image is for sandbox or for container.
+    fn is_sandbox(image_metadata: &HashMap<String, String>) -> bool {
+        let mut is_sandbox = false;
+        for key in K8S_CONTAINER_TYPE_KEYS.iter() {
+            if let Some(value) = image_metadata.get(key as &str) {
+                if value == "sandbox" {
+                    is_sandbox = true;
+                    break;
+                }
+            }
+        }
+        is_sandbox
+    }
+
     /// pull_image is used for call image-rs to pull image in the guest.
     /// # Parameters
     /// - `image`: Image name (exp: quay.io/prometheus/busybox:latest)
@@ -147,18 +186,7 @@ impl ImageService {
     ) -> Result<String> {
         info!(sl(), "image metadata: {image_metadata:?}");
 
-        //Check whether the image is for sandbox or for container.
-        let mut is_sandbox = false;
-        for key in K8S_CONTAINER_TYPE_KEYS.iter() {
-            if let Some(value) = image_metadata.get(key as &str) {
-                if value == "sandbox" {
-                    is_sandbox = true;
-                    break;
-                }
-            }
-        }
-
-        if is_sandbox {
+        if Self::is_sandbox(image_metadata) {
             let mount_path = Self::unpack_pause_image(cid)?;
             return Ok(mount_path);
         }
@@ -194,6 +222,32 @@ impl ImageService {
     }
 }
 
+/// get_process overrides the OCI process spec with pause image process spec if needed
+pub fn get_process(
+    ocip: &oci::Process,
+    oci: &oci::Spec,
+    storages: Vec<Storage>,
+) -> Result<oci::Process> {
+    let mut guest_pull = false;
+    for storage in storages {
+        if storage.driver == KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL {
+            guest_pull = true;
+            break;
+        }
+    }
+    if guest_pull {
+        match oci.annotations() {
+            Some(a) => {
+                if ImageService::is_sandbox(a) {
+                    return ImageService::get_pause_image_process();
+                }
+            }
+            None => {}
+        }
+    }
+    Ok(ocip.clone())
+}
+
 /// Set proxy environment from AGENT_CONFIG
 pub async fn set_proxy_env_vars() {
     if env::var("HTTPS_PROXY").is_err() {

src/agent/src/main.rs

@@ -21,7 +21,7 @@ extern crate slog;
 use anyhow::{anyhow, Context, Result};
 use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
-use const_format::concatcp;
+use const_format::{concatcp, formatcp};
 use nix::fcntl::OFlag;
 use nix::sys::reboot::{reboot, RebootMode};
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
@@ -29,7 +29,7 @@ use nix::unistd::{self, dup, sync, Pid};
 use std::env;
 use std::ffi::OsStr;
 use std::fs::{self, File};
-use std::os::unix::fs as unixfs;
+use std::os::unix::fs::{self as unixfs, FileTypeExt};
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
 use std::process::exit;
@@ -109,7 +109,18 @@ const CDH_SOCKET_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, CDH_SOCKET);
 const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest";
 
 /// Path of ocicrypt config file. This is used by image-rs when decrypting image.
-const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+const OCICRYPT_CONFIG_PATH: &str = "/run/confidential-containers/ocicrypt_config.json";
+
+const OCICRYPT_CONFIG: &str = formatcp!(
+    r#"{{
+    "key-providers": {{
+        "attestation-agent": {{
+            "ttrpc": "{}"
+        }}
+    }}
+}}"#,
+    CDH_SOCKET_URI
+);
 
 const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6;
 
@@ -408,15 +419,13 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     let gc_procs = config.guest_components_procs;
-    if gc_procs != GuestComponentsProcs::None {
-        if !attestation_binaries_available(logger, &gc_procs) {
-            warn!(
-                logger,
-                "attestation binaries requested for launch not available"
-            );
-        } else {
-            init_attestation_components(logger, config).await?;
-        }
+    if !attestation_binaries_available(logger, &gc_procs) {
+        warn!(
+            logger,
+            "attestation binaries requested for launch not available"
+        );
+    } else {
+        init_attestation_components(logger, config).await?;
     }
 
     // vsock:///dev/vsock, port
@@ -447,12 +456,7 @@ fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs)
     true
 }
 
-// Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
-// and the corresponding procs are enabled in the agent configuration. the process will be
-// launched in the background and the function will return immediately.
-// If the CDH is started, a CDH client will be instantiated and returned.
-async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> {
-    // skip launch of any guest-component
+async fn launch_guest_component_procs(logger: &Logger, config: &AgentConfig) -> Result<()> {
     if config.guest_components_procs == GuestComponentsProcs::None {
         return Ok(());
     }
@@ -472,17 +476,6 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
         return Ok(());
     }
 
-    let ocicrypt_config = serde_json::json!({
-        "key-providers": {
-            "attestation-agent":{
-                "ttrpc":CDH_SOCKET_URI
-            }
-        }
-    });
-
-    fs::write(OCICRYPT_CONFIG_PATH, ocicrypt_config.to_string().as_bytes())?;
-    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
-
     debug!(
         logger,
         "spawning confidential-data-hub process {}", CDH_PATH
@@ -497,9 +490,6 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
     )
     .map_err(|e| anyhow!("launch_process {} failed: {:?}", CDH_PATH, e))?;
 
-    // initialize cdh client
-    cdh::init_cdh_client(CDH_SOCKET_URI).await?;
-
     // skip launch of api-server-rest
     if config.guest_components_procs == GuestComponentsProcs::ConfidentialDataHub {
         return Ok(());
@@ -522,6 +512,34 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
     Ok(())
 }
 
+// Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
+// and the corresponding procs are enabled in the agent configuration. the process will be
+// launched in the background and the function will return immediately.
+// If the CDH is started, a CDH client will be instantiated and returned.
+async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> {
+    launch_guest_component_procs(logger, config).await?;
+
+    fs::write(OCICRYPT_CONFIG_PATH, OCICRYPT_CONFIG.as_bytes())?;
+    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
+
+    // If a CDH socket exists, initialize the CDH client
+    match tokio::fs::metadata(CDH_SOCKET).await {
+        Ok(md) => {
+            if md.file_type().is_socket() {
+                cdh::init_cdh_client(CDH_SOCKET_URI).await?;
+            } else {
+                debug!(logger, "File {} is not a socket", CDH_SOCKET);
+            }
+        }
+        Err(err) => warn!(
+            logger,
+            "Failed to probe CDH socket file {}: {:?}", CDH_SOCKET, err
+        ),
+    }
+
+    Ok(())
+}
+
 fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> {
     let p = Path::new(path);
     let mut attempts = 0;

src/agent/src/policy.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::Result;
+use anyhow::{bail, Result};
 use protobuf::MessageDyn;
 use tokio::io::AsyncWriteExt;
 
@@ -68,6 +68,12 @@ pub struct AgentPolicy {
     engine: regorus::Engine,
 }
 
+#[derive(serde::Deserialize, Debug)]
+struct MetadataResponse {
+    allowed: bool,
+    ops: Option<json_patch::Patch>,
+}
+
 impl AgentPolicy {
     /// Create AgentPolicy object.
     pub fn new() -> Self {
@@ -82,6 +88,17 @@ impl AgentPolicy {
         let mut engine = regorus::Engine::new();
         engine.set_strict_builtin_errors(false);
         engine.set_gather_prints(true);
+        // assign a slice of the engine data "pstate" to be used as policy state
+        engine
+            .add_data(
+                regorus::Value::from_json_str(
+                    r#"{
+                        "pstate": {}
+                    }"#,
+                )
+                .unwrap(),
+            )
+            .unwrap();
         engine
     }
 
@@ -112,6 +129,23 @@ impl AgentPolicy {
         Ok(())
     }
 
+    async fn apply_patch_to_state(&mut self, patch: json_patch::Patch) -> Result<()> {
+        // Convert the current engine data to a JSON value
+        let mut state = serde_json::to_value(self.engine.get_data())?;
+
+        // Apply the patch to the state
+        json_patch::patch(&mut state, &patch)?;
+
+        // Clear the existing data in the engine
+        self.engine.clear_data();
+
+        // Add the patched state back to the engine
+        self.engine
+            .add_data(regorus::Value::from_json_str(&state.to_string())?)?;
+
+        Ok(())
+    }
+
     /// Ask regorus if an API call should be allowed or not.
     async fn allow_request(&mut self, ep: &str, ep_input: &str) -> Result<(bool, String)> {
         debug!(sl!(), "policy check: {ep}");
@@ -120,13 +154,56 @@ impl AgentPolicy {
         let query = format!("data.agent_policy.{ep}");
         self.engine.set_input_json(ep_input)?;
 
-        let mut allow = match self.engine.eval_bool_query(query, false) {
-            Ok(a) => a,
-            Err(e) => {
-                if !self.allow_failures {
-                    return Err(e);
+        let results = self.engine.eval_query(query, false)?;
+
+        let prints = match self.engine.take_prints() {
+            Ok(p) => p.join(" "),
+            Err(e) => format!("Failed to get policy log: {e}"),
+        };
+
+        if results.result.len() != 1 {
+            // Results are empty when AllowRequestsFailingPolicy is used to allow a Request that hasn't been defined in the policy
+            if self.allow_failures {
+                return Ok((true, prints));
+            }
+            bail!(
+                "policy check: unexpected eval_query result len {:?}",
+                results
+            );
+        }
+
+        if results.result[0].expressions.len() != 1 {
+            bail!(
+                "policy check: unexpected eval_query result expressions {:?}",
+                results
+            );
+        }
+
+        let mut allow = match &results.result[0].expressions[0].value {
+            regorus::Value::Bool(b) => *b,
+
+            // Match against a specific variant that could be interpreted as MetadataResponse
+            regorus::Value::Object(obj) => {
+                let json_str = serde_json::to_string(obj)?;
+
+                self.log_eval_input(ep, &json_str).await;
+
+                let metadata_response: MetadataResponse = serde_json::from_str(&json_str)?;
+
+                if metadata_response.allowed {
+                    if let Some(ops) = metadata_response.ops {
+                        self.apply_patch_to_state(ops).await?;
+                    }
                 }
-                false
+                metadata_response.allowed
+            }
+
+            _ => {
+                error!(sl!(), "allow_request: unexpected eval_query result type");
+                bail!(
+                    "policy check: unexpected eval_query result type {:?}",
+                    results
+                );
             }
         };
 
@@ -135,11 +212,6 @@ impl AgentPolicy {
             allow = true;
         }
 
-        let prints = match self.engine.take_prints() {
-            Ok(p) => p.join(" "),
-            Err(e) => format!("Failed to get policy log: {e}"),
-        };
-
         Ok((allow, prints))
     }
 

src/agent/src/rpc.rs

@@ -58,7 +58,7 @@ use rustjail::process::ProcessOperations;
 use crate::cdh;
 use crate::device::block_device_handler::get_virtio_blk_pci_device_name;
 use crate::device::network_device_handler::wait_for_net_interface;
-use crate::device::{add_devices, update_env_pci};
+use crate::device::{add_devices, handle_cdi_devices, update_env_pci};
 use crate::features::get_build_features;
 use crate::image::KATA_IMAGE_WORK_DIR;
 use crate::linux_abi::*;
@@ -130,6 +130,8 @@ const ERR_NO_SANDBOX_PIDNS: &str = "Sandbox does not have sandbox_pidns";
 // not available.
 const IPTABLES_RESTORE_WAIT_SEC: u64 = 5;
 
+const CDI_TIMEOUT_LIMIT: u64 = 100;
+
 // Convenience function to obtain the scope logger.
 fn sl() -> slog::Logger {
     slog_scope::logger()
@@ -224,6 +226,15 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&sl(), &req.devices, &mut oci, &self.sandbox).await?;
 
+        // In guest-kernel mode some devices need extra handling. Taking the
+        // GPU as an example the shim will inject CDI annotations that will
+        // be used by the kata-agent to do containerEdits according to the
+        // CDI spec coming from a registry that is created on the fly by UDEV
+        // or other entities for a specifc device.
+        // In Kata we only consider the directory "/var/run/cdi", "/etc" may be
+        // readonly
+        handle_cdi_devices(&sl(), &mut oci, "/var/run/cdi", CDI_TIMEOUT_LIMIT).await?;
+
         cdh_handler(&mut oci).await?;
 
         // Both rootfs and volumes (invoked with --volume for instance) will
@@ -233,7 +244,13 @@ impl AgentService {
         // After all those storages have been processed, no matter the order
         // here, the agent will rely on rustjail (using the oci.Mounts
         // list) to bind mount all of them inside the container.
-        let m = add_storages(sl(), req.storages, &self.sandbox, Some(req.container_id)).await?;
+        let m = add_storages(
+            sl(),
+            req.storages.clone(),
+            &self.sandbox,
+            Some(req.container_id),
+        )
+        .await?;
 
         let mut s = self.sandbox.lock().await;
         s.container_mounts.insert(cid.clone(), m);
@@ -288,6 +305,13 @@ impl AgentService {
         let pipe_size = AGENT_CONFIG.container_pipe_size;
 
         let p = if let Some(p) = oci.process() {
+            #[cfg(feature = "guest-pull")]
+            {
+                let new_p = image::get_process(p, &oci, req.storages.clone())?;
+                Process::new(&sl(), &new_p, cid.as_str(), true, pipe_size, proc_io)?
+            }
+
+            #[cfg(not(feature = "guest-pull"))]
             Process::new(&sl(), p, cid.as_str(), true, pipe_size, proc_io)?
         } else {
             info!(sl(), "no process configurations!");

src/agent/src/storage/ephemeral_handler.rs

@@ -170,7 +170,7 @@ impl EphemeralHandler {
         let size = size_str
             .unwrap()
             .parse::<u64>()
-            .context(format!("parse size: {:?}", &pagesize_str))?;
+            .context(format!("parse size: {:?}", &size_str))?;
 
         Ok((pagesize, size))
     }

src/libs/Cargo.lock

@@ -240,19 +240,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "cgroups-rs"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
-dependencies = [
- "libc",
- "log",
- "nix 0.25.1",
- "regex",
- "thiserror",
-]
-
 [[package]]
 name = "chrono"
 version = "0.4.20"
@@ -814,7 +801,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -975,18 +961,6 @@ dependencies = [
  "memoffset 0.6.5",
 ]
 
-[[package]]
-name = "nix"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
-dependencies = [
- "autocfg",
- "bitflags",
- "cfg-if",
- "libc",
-]
-
 [[package]]
 name = "nix"
 version = "0.26.4"
@@ -1316,7 +1290,6 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.2.0",
  "serde",

src/libs/kata-sys-util/Cargo.toml

@@ -13,7 +13,6 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0.31"
 byteorder = "1.4.3"
-cgroups = { package = "cgroups-rs", version = "0.3.2" }
 chrono = "0.4.0"
 common-path = "=1.0.0"
 fail = "0.5.0"

src/libs/kata-sys-util/README.md

@@ -1,10 +1,9 @@
-# kata-sys-util
+# `kata-sys-util`
 
 This crate is a collection of utilities and helpers for 
 [Kata Containers](https://github.com/kata-containers/kata-containers/) components to access system services.
 
 It provides safe wrappers over system services, such as:
-- cgroups
 - file systems
 - mount
 - NUMA

src/libs/kata-sys-util/src/spec.rs

@@ -97,11 +97,3 @@ pub fn load_oci_spec() -> Result<oci::Spec, OciSpecError> {
 
     oci::Spec::load(spec_file.to_str().unwrap_or_default())
 }
-
-/// handle string parsing for input possibly be JSON string.
-pub fn parse_json_string(input: &str) -> &str {
-    let json_str: &str = serde_json::from_str(input).unwrap_or(input);
-    let stripped_str = json_str.strip_prefix("CAP_").unwrap_or(json_str);
-
-    stripped_str
-}

src/libs/kata-types/src/annotations/cri_containerd.rs

@@ -11,7 +11,12 @@ pub const CONTAINER_NAME_LABEL_KEY: &str = "io.kubernetes.cri.container-name";
 pub const SANDBOX: &str = "sandbox";
 pub const CONTAINER: &str = "container";
 
+// SandboxID is the sandbox ID annotation
 pub const SANDBOX_ID_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-id";
+// SandboxName is the name of the sandbox (pod)
+pub const SANDBOX_NAME_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-name";
+// SandboxNamespace is the name of the namespace of the sandbox (pod)
+pub const SANDBOX_NAMESPACE_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-namespace";
 
 // Ref: https://pkg.go.dev/github.com/containerd/containerd@v1.6.7/pkg/cri/annotations
 // SandboxCPU annotations are based on the initial CPU configuration for the sandbox. This is calculated as the

src/libs/kata-types/src/config/default.rs

@@ -98,3 +98,11 @@ pub const DEFAULT_FIRECRACKER_GUEST_KERNEL_IMAGE: &str = "vmlinux";
 pub const DEFAULT_FIRECRACKER_GUEST_KERNEL_PARAMS: &str = "";
 pub const MAX_FIRECRACKER_VCPUS: u32 = 32;
 pub const MIN_FIRECRACKER_MEMORY_SIZE_MB: u32 = 128;
+
+// Default configuration for remote
+pub const DEFAULT_REMOTE_HYPERVISOR_SOCKET: &str = "/run/peerpod/hypervisor.sock";
+pub const DEFAULT_REMOTE_HYPERVISOR_TIMEOUT: i32 = 600; // 600 Seconds
+pub const MAX_REMOTE_VCPUS: u32 = 32;
+pub const MIN_REMOTE_MEMORY_SIZE_MB: u32 = 64;
+pub const DEFAULT_REMOTE_MEMORY_SIZE_MB: u32 = 128;
+pub const DEFAULT_REMOTE_MEMORY_SLOTS: u32 = 128;

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -44,6 +44,9 @@ pub use self::qemu::{QemuConfig, HYPERVISOR_NAME_QEMU};
 mod ch;
 pub use self::ch::{CloudHypervisorConfig, HYPERVISOR_NAME_CH};
 
+mod remote;
+pub use self::remote::{RemoteConfig, HYPERVISOR_NAME_REMOTE};
+
 /// Virtual PCI block device driver.
 pub const VIRTIO_BLK_PCI: &str = "virtio-blk-pci";
 
@@ -540,6 +543,7 @@ impl TopologyConfigInfo {
             HYPERVISOR_NAME_CH,
             HYPERVISOR_NAME_DRAGONBALL,
             HYPERVISOR_NAME_FIRECRACKER,
+            HYPERVISOR_NAME_REMOTE,
         ];
         let hypervisor_name = toml_config.runtime.hypervisor_name.as_str();
         if !hypervisor_names.contains(&hypervisor_name) {
@@ -1040,6 +1044,18 @@ impl SharedFsInfo {
     }
 }
 
+/// Configuration information for remote hypervisor type.
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub struct RemoteInfo {
+    /// Remote hypervisor socket path
+    #[serde(default)]
+    pub hypervisor_socket: String,
+
+    /// Remote hyperisor timeout of creating (in seconds)
+    #[serde(default)]
+    pub hypervisor_timeout: i32,
+}
+
 /// Common configuration information for hypervisors.
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct Hypervisor {
@@ -1123,6 +1139,10 @@ pub struct Hypervisor {
     #[serde(default, flatten)]
     pub shared_fs: SharedFsInfo,
 
+    /// Remote hypervisor configuration information.
+    #[serde(default, flatten)]
+    pub remote_info: RemoteInfo,
+
     /// A sandbox annotation used to specify prefetch_files.list host path container image
     /// being used, and runtime will pass it to Hypervisor to  search for corresponding
     /// prefetch list file:
@@ -1164,6 +1184,10 @@ impl ConfigOps for Hypervisor {
     fn adjust_config(conf: &mut TomlConfig) -> Result<()> {
         HypervisorVendor::adjust_config(conf)?;
         let hypervisors: Vec<String> = conf.hypervisor.keys().cloned().collect();
+        info!(
+            sl!(),
+            "Adjusting hypervisor configuration {:?}", hypervisors
+        );
         for hypervisor in hypervisors.iter() {
             if let Some(plugin) = get_hypervisor_plugin(hypervisor) {
                 plugin.adjust_config(conf)?;

src/libs/kata-types/src/config/hypervisor/remote.rs

@@ -0,0 +1,116 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use byte_unit::{Byte, Unit};
+use std::io::Result;
+use std::path::Path;
+use std::sync::Arc;
+use sysinfo::System;
+
+use crate::{
+    config::{
+        default::{self, MAX_REMOTE_VCPUS, MIN_REMOTE_MEMORY_SIZE_MB},
+        ConfigPlugin,
+    }, device::DRIVER_NVDIMM_TYPE, eother, resolve_path
+};
+
+use super::register_hypervisor_plugin;
+
+/// Hypervisor name for remote, used to index `TomlConfig::hypervisor`.
+pub const HYPERVISOR_NAME_REMOTE: &str = "remote";
+
+/// Configuration information for remote.
+#[derive(Default, Debug)]
+pub struct RemoteConfig {}
+
+impl RemoteConfig {
+    /// Create a new instance of `RemoteConfig`
+    pub fn new() -> Self {
+        RemoteConfig {}
+    }
+
+    /// Register the remote plugin.
+    pub fn register(self) {
+        let plugin = Arc::new(self);
+        register_hypervisor_plugin(HYPERVISOR_NAME_REMOTE, plugin);
+    }
+}
+
+impl ConfigPlugin for RemoteConfig {
+    fn name(&self) -> &str {
+        HYPERVISOR_NAME_REMOTE
+    }
+
+    /// Adjust the configuration information after loading from configuration file.
+    fn adjust_config(&self, conf: &mut crate::config::TomlConfig) -> Result<()> {
+        if let Some(remote) = conf.hypervisor.get_mut(HYPERVISOR_NAME_REMOTE) {
+            if remote.remote_info.hypervisor_socket.is_empty() {
+                remote.remote_info.hypervisor_socket =
+                    default::DEFAULT_REMOTE_HYPERVISOR_SOCKET.to_string();
+            }
+            resolve_path!(
+                remote.remote_info.hypervisor_socket,
+                "Remote hypervisor socket `{}` is invalid: {}"
+            )?;
+            if remote.remote_info.hypervisor_timeout == 0 {
+                remote.remote_info.hypervisor_timeout = default::DEFAULT_REMOTE_HYPERVISOR_TIMEOUT;
+            }
+            if remote.memory_info.default_memory == 0 {
+                remote.memory_info.default_memory = default::MIN_REMOTE_MEMORY_SIZE_MB;
+            }
+            if remote.memory_info.memory_slots == 0 {
+                remote.memory_info.memory_slots = default::DEFAULT_REMOTE_MEMORY_SLOTS
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Validate the configuration information.
+    fn validate(&self, conf: &crate::config::TomlConfig) -> Result<()> {
+        if let Some(remote) = conf.hypervisor.get(HYPERVISOR_NAME_REMOTE) {
+            let s = System::new_all();
+            let total_memory = Byte::from_u64(s.total_memory())
+                .get_adjusted_unit(Unit::MiB)
+                .get_value() as u32;
+            if remote.memory_info.default_maxmemory != total_memory {
+                return Err(eother!(
+                    "Remote hypervisor does not support memory hotplug, default_maxmemory must be equal to the total system memory",
+                ));
+            }
+            let cpus = num_cpus::get() as u32;
+            if remote.cpu_info.default_maxvcpus != cpus {
+                return Err(eother!(
+                    "Remote hypervisor does not support CPU hotplug, default_maxvcpus must be equal to the total system CPUs",
+                ));
+            }
+            if !remote.boot_info.initrd.is_empty() {
+                return Err(eother!("Remote hypervisor does not support initrd"));
+            }
+            if !remote.boot_info.rootfs_type.is_empty() {
+                return Err(eother!("Remote hypervisor does not support rootfs_type"));
+            }
+            if remote.blockdev_info.block_device_driver.as_str() == DRIVER_NVDIMM_TYPE {
+                return Err(eother!("Remote hypervisor does not support nvdimm"));
+            }
+            if remote.memory_info.default_memory < MIN_REMOTE_MEMORY_SIZE_MB {
+                return Err(eother!(
+                    "Remote hypervisor has minimal memory limitation {}",
+                    MIN_REMOTE_MEMORY_SIZE_MB
+                ));
+            }
+        }
+
+        Ok(())
+    }
+
+    fn get_min_memory(&self) -> u32 {
+        MIN_REMOTE_MEMORY_SIZE_MB
+    }
+
+    fn get_max_cpus(&self) -> u32 {
+        MAX_REMOTE_VCPUS
+    }
+}

src/libs/kata-types/src/config/mod.rs

@@ -26,7 +26,7 @@ pub use self::agent::Agent;
 use self::default::DEFAULT_AGENT_DBG_CONSOLE_PORT;
 pub use self::hypervisor::{
     BootInfo, CloudHypervisorConfig, DragonballConfig, FirecrackerConfig, Hypervisor, QemuConfig,
-    HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_FIRECRACKER, HYPERVISOR_NAME_QEMU,
+    RemoteConfig, HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_FIRECRACKER, HYPERVISOR_NAME_QEMU,
 };
 
 mod runtime;

src/libs/protocols/Cargo.toml

@@ -7,19 +7,17 @@ license = "Apache-2.0"
 
 [features]
 default = []
-with-serde = [ "serde", "serde_json" ]
+with-serde = []
 async = ["ttrpc/async", "async-trait"]
 
 [dependencies]
 ttrpc = "0.8"
 async-trait = { version = "0.1.42", optional = true }
 protobuf = { version = "3.2.0" }
-serde = { version = "1.0.130", features = ["derive"], optional = true }
-serde_json = { version = "1.0.68", optional = true }
+serde = { version = "1.0.130", features = ["derive"] }
+serde_json = "1.0.68"
 oci-spec = { version = "0.6.8", features = ["runtime"] }
 
-kata-sys-util = { path = "../kata-sys-util" }
-
 [build-dependencies]
 ttrpc-codegen = "0.4.2"
 protobuf = { version = "3.2.0" }

src/libs/protocols/build.rs

@@ -204,6 +204,7 @@ fn real_main() -> Result<(), std::io::Error> {
                 "protos/agent.proto",
                 "protos/health.proto",
                 "protos/confidential_data_hub.proto",
+                "protos/remote.proto",
             ],
             true,
         )?;
@@ -214,6 +215,7 @@ fn real_main() -> Result<(), std::io::Error> {
             "src/confidential_data_hub_ttrpc.rs",
             "src/confidential_data_hub_ttrpc_async.rs",
         )?;
+        fs::rename("src/remote_ttrpc.rs", "src/remote_ttrpc_async.rs")?;
     }
 
     codegen(
@@ -222,6 +224,7 @@ fn real_main() -> Result<(), std::io::Error> {
             "protos/agent.proto",
             "protos/health.proto",
             "protos/confidential_data_hub.proto",
+            "protos/remote.proto",
         ],
         false,
     )?;

src/libs/protocols/protos/remote.proto

@@ -0,0 +1,47 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+syntax = "proto3";
+
+package remote;
+
+service Hypervisor {
+	rpc CreateVM(CreateVMRequest) returns (CreateVMResponse) {}
+	rpc StartVM(StartVMRequest) returns (StartVMResponse) {}
+	rpc StopVM(StopVMRequest) returns (StopVMResponse) {}
+	rpc Version(VersionRequest) returns (VersionResponse) {}
+}
+
+message VersionRequest {
+	string version = 1;
+}
+
+message VersionResponse {
+	string version = 1;
+}
+
+message CreateVMRequest {
+	string id = 1;
+	map<string, string> annotations = 2;
+	string networkNamespacePath = 3;
+}
+
+message CreateVMResponse {
+	string agentSocketPath = 1;
+}
+
+message StartVMRequest {
+	string id = 1;
+}
+
+message StartVMResponse {
+}
+
+message StopVMRequest {
+	string id = 1;
+}
+
+message StopVMResponse {
+}

src/libs/protocols/src/lib.rs

@@ -21,6 +21,10 @@ pub mod oci;
 mod serde_config;
 pub mod trans;
 pub mod types;
+pub mod remote;
+pub mod remote_ttrpc;
+#[cfg(feature = "async")]
+pub mod remote_ttrpc_async;
 
 #[cfg(feature = "with-serde")]
 pub use serde_config::{

src/libs/protocols/src/trans.rs

@@ -10,7 +10,6 @@ use std::convert::TryFrom;
 use std::path::PathBuf;
 
 use crate::oci as grpc;
-use kata_sys_util::spec::parse_json_string;
 use oci_spec::runtime as oci;
 
 // translate from interface to ttprc tools
@@ -41,8 +40,9 @@ fn cap_hashset2vec(hash_set: &Option<HashSet<oci::Capability>>) -> Vec<String> {
 fn cap_vec2hashset(caps: Vec<String>) -> HashSet<oci::Capability> {
     caps.iter()
         .map(|cap: &String| {
-            let cap_str = parse_json_string(cap);
-            cap_str
+            // cap might be JSON-encoded
+            let decoded: &str = serde_json::from_str(cap).unwrap_or(cap);
+            decoded.strip_prefix("CAP_").unwrap_or(decoded)
                 .parse::<oci::Capability>()
                 .unwrap_or_else(|_| panic!("Failed to parse {:?} to Enum Capability", cap))
         })
@@ -97,6 +97,8 @@ impl From<oci::LinuxCapabilities> for grpc::LinuxCapabilities {
     }
 }
 
+// TODO(burgerdev): remove condition here and below after upgrading to oci_spec > 0.7.
+#[cfg(target_os = "linux")]
 impl From<oci::PosixRlimit> for grpc::POSIXRlimit {
     fn from(from: oci::PosixRlimit) -> Self {
         grpc::POSIXRlimit {
@@ -118,6 +120,7 @@ impl From<oci::Process> for grpc::Process {
             Env: option_vec_to_vec(from.env()),
             Cwd: from.cwd().display().to_string(),
             Capabilities: from_option(from.capabilities().clone()),
+            #[cfg(target_os = "linux")]
             Rlimits: from_option_vec(from.rlimits().clone()),
             NoNewPrivileges: from.no_new_privileges().unwrap_or_default(),
             ApparmorProfile: from
@@ -993,6 +996,7 @@ impl From<grpc::Linux> for oci::Linux {
     }
 }
 
+#[cfg(target_os = "linux")]
 impl From<grpc::POSIXRlimit> for oci::PosixRlimit {
     fn from(proto: grpc::POSIXRlimit) -> Self {
         oci::PosixRlimitBuilder::default()
@@ -1078,6 +1082,8 @@ impl From<grpc::Process> for oci::Process {
         } else {
             process.set_capabilities(None);
         }
+
+        #[cfg(target_os = "linux")]
         if !from.Rlimits().is_empty() {
             process.set_rlimits(Some(
                 from.Rlimits().iter().cloned().map(|r| r.into()).collect(),
@@ -1238,6 +1244,11 @@ impl From<grpc::LinuxIntelRdt> for oci::LinuxIntelRdt {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashSet;
+
+    use super::cap_vec2hashset;
+    use super::oci;
+
     fn from_vec<F: Sized, T: From<F>>(from: Vec<F>) -> Vec<T> {
         let mut to: Vec<T> = vec![];
         for data in from {
@@ -1289,4 +1300,26 @@ mod tests {
         assert_eq!(from.len(), to.len());
         assert_eq!(from[0].from, to[0].to);
     }
+
+    #[test]
+    fn test_cap_vec2hashset_good() {
+        let expected: HashSet<oci::Capability> =
+            vec![oci::Capability::NetAdmin, oci::Capability::Mknod]
+                .into_iter()
+                .collect();
+        let actual = cap_vec2hashset(vec![
+            "CAP_NET_ADMIN".to_string(),
+            "\"CAP_MKNOD\"".to_string(),
+        ]);
+
+        assert_eq!(expected, actual);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_cap_vec2hashset_bad() {
+        cap_vec2hashset(vec![
+            "CAP_DOES_NOT_EXIST".to_string(),
+        ]);
+    }
 }

src/runtime-rs/Cargo.lock

@@ -1685,8 +1685,11 @@ dependencies = [
  "libc",
  "logging",
  "nix 0.24.3",
+ "oci-spec",
  "path-clean",
  "persist",
+ "protobuf 3.2.0",
+ "protocols",
  "qapi",
  "qapi-qmp",
  "qapi-spec",
@@ -1706,6 +1709,8 @@ dependencies = [
  "thiserror",
  "tokio",
  "tracing",
+ "ttrpc",
+ "ttrpc-codegen",
  "vmm-sys-util 0.11.1",
 ]
 
@@ -1839,7 +1844,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -2994,9 +2998,10 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.2.0",
+ "serde",
+ "serde_json",
  "ttrpc",
  "ttrpc-codegen",
 ]

src/runtime-rs/Makefile

@@ -86,6 +86,7 @@ HYPERVISOR_DB = dragonball
 HYPERVISOR_FC = firecracker
 HYPERVISOR_QEMU = qemu
 HYPERVISOR_CLH = cloud-hypervisor
+HYPERVISOR_REMOTE = remote
 
 # When set to true, builds the built-in Dragonball hypervisor
 USE_BUILDIN_DB := true
@@ -93,7 +94,7 @@ USE_BUILDIN_DB := true
 HYPERVISOR ?= $(HYPERVISOR_DB)
 
 ##VAR HYPERVISOR=<hypervisor_name> List of hypervisors this build system can generate configuration for.
-HYPERVISORS := $(HYPERVISOR_DB) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
+HYPERVISORS := $(HYPERVISOR_DB) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_REMOTE)
 
 CLHPATH := $(CLHBINDIR)/$(CLHCMD)
 CLHVALIDHYPERVISORPATHS := [\"$(CLHPATH)\"]
@@ -317,6 +318,20 @@ ifneq (,$(FCCMD))
     DEFSTATICRESOURCEMGMT_FC := true
 endif
 
+ifneq (,$(REMOTE))
+    KNOWN_HYPERVISORS += $(HYPERVISOR_REMOTE)
+    CONFIG_FILE_REMOTE = configuration-remote.toml
+    CONFIG_REMOTE = config/$(CONFIG_FILE_REMOTE)
+    CONFIG_REMOTE_IN = $(CONFIG_REMOTE).in
+    CONFIG_PATH_REMOTE = $(abspath $(CONFDIR)/$(CONFIG_FILE_REMOTE))
+    CONFIG_PATHS += $(CONFIG_PATH_REMOTE)
+    SYSCONFDIR_REMOTE = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_REMOTE))
+    SYSCONFIG_PATHS += $(SYSCONFDIR_REMOTE)
+    CONFIGS += $(CONFIG_REMOTE)
+    # remote-specific options (all should be suffixed by "_REMOTE")
+    DEFSANDBOXCGROUPONLY_REMOTE := false
+endif
+
 ifeq ($(HYPERVISOR),$(HYPERVISOR_DB))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_DB)
 endif
@@ -327,6 +342,9 @@ endif
 ifeq ($(HYPERVISOR),$(HYPERVISOR_FC))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_FC)
 endif
+ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_REMOTE))
+    DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_REMOTE)
+endif
 # list of variables the user may wish to override
 USER_VARS += ARCH
 USER_VARS += BINDIR
@@ -334,6 +352,7 @@ USER_VARS += CONFIG_DB_IN
 USER_VARS += CONFIG_FC_IN
 USER_VARS += CONFIG_PATH
 USER_VARS += CONFIG_QEMU_IN
+USER_VARS += CONFIG_REMOTE_IN
 USER_VARS += DESTDIR
 USER_VARS += HYPERVISOR
 USER_VARS += USE_BUILDIN_DB
@@ -443,6 +462,7 @@ USER_VARS += DEFSANDBOXCGROUPONLY_QEMU
 USER_VARS += DEFSANDBOXCGROUPONLY_DB
 USER_VARS += DEFSANDBOXCGROUPONLY_FC
 USER_VARS += DEFSANDBOXCGROUPONLY_CLH
+USER_VARS += DEFSANDBOXCGROUPONLY_REMOTE
 USER_VARS += DEFSTATICRESOURCEMGMT_DB
 USER_VARS += DEFSTATICRESOURCEMGMT_FC
 USER_VARS += DEFSTATICRESOURCEMGMT_CLH

src/runtime-rs/arch/x86_64-options.mk

@@ -20,3 +20,5 @@ CLHCMD := cloud-hypervisor
 # firecracker binary (vmm and jailer)
 FCCMD := firecracker
 FCJAILERCMD := jailer
+
+REMOTE := remote

src/runtime-rs/config/configuration-remote.toml.in

@@ -0,0 +1,297 @@
+# Copyright 2024 Kata Contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_REMOTE_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+
+[hypervisor.remote]
+# Default VM information query service unix domain socket, created by cloud-api-adaptor
+# Ref: https://github.com/confidential-containers/cloud-api-adaptor/blob/main/src/cloud-api-adaptor/docs/vminfo.md
+remote_hypervisor_socket = "/run/peerpod/hypervisor.sock"
+# Timeout in seconds for creating a remote hypervisor, 600s(10min) by default
+remote_hypervisor_timeout = 600
+
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+# confidential_guest = true
+
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+# Note: Remote hypervisor is only handling the following annotations
+enable_annotations = ["machine_type", "default_memory", "default_vcpus"]
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+# NOTE: kernel_params are not currently passed over in remote hypervisor
+# kernel_params = ""
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWAREPATH@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+# default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+# default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+# default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+#memory_slots = @DEFMEMSLOTS@
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. And Debug also enable the hmp socket.
+#
+# Default false
+# enable_debug = true
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+# Note: The remote hypervisor has a different guest, so currently requires this to be disabled
+disable_guest_selinux = true
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+# enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+# enable_tracing = true
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+# enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+# Note: The remote hypervisor, uses it's own network, so "none" is required
+internetworking_model="none"
+
+name="virt_container"
+hypervisor_name="remote"
+agent_name="kata"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+# Note: The remote hypervisor has a different guest, so currently requires this to be set to true
+disable_guest_seccomp=true
+
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+# Note: The remote hypervisor has a different networking model, which requires true 
+disable_new_netns = false
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY_REMOTE@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+# Note: the remote hypervisor uses the peer pod config to determine the sandbox size, so requires this to be set to true
+static_sandbox_resource_mgmt=true
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+# Note: remote hypervisor has no sharing of emptydir mounts from host to guest
+disable_guest_empty_dir=false
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true

src/runtime-rs/crates/agent/src/sock/mod.rs

@@ -8,6 +8,8 @@ mod hybrid_vsock;
 pub use hybrid_vsock::HybridVsock;
 mod vsock;
 pub use vsock::Vsock;
+mod remote;
+pub use remote::Remote;
 
 use std::{
     pin::Pin,
@@ -28,6 +30,7 @@ use url::Url;
 
 const VSOCK_SCHEME: &str = "vsock";
 const HYBRID_VSOCK_SCHEME: &str = "hvsock";
+const REMOTE_SCHEME: &str = "remote";
 
 /// Socket stream
 pub enum Stream {
@@ -98,6 +101,7 @@ impl ConnectConfig {
 enum SockType {
     Vsock(Vsock),
     HybridVsock(HybridVsock),
+    Remote(Remote),
 }
 
 #[async_trait]
@@ -114,6 +118,7 @@ pub fn new(address: &str, port: u32) -> Result<Arc<dyn Sock>> {
     match parse(address, port).context("parse url")? {
         SockType::Vsock(sock) => Ok(Arc::new(sock)),
         SockType::HybridVsock(sock) => Ok(Arc::new(sock)),
+        SockType::Remote(sock) => Ok(Arc::new(sock)),
     }
 }
 
@@ -136,6 +141,13 @@ fn parse(address: &str, port: u32) -> Result<SockType> {
             let uds = path[0];
             Ok(SockType::HybridVsock(HybridVsock::new(uds, port)))
         }
+        REMOTE_SCHEME => {
+            let path: Vec<&str> = url.path().split(':').collect();
+            if path.len() != 1 {
+                return Err(anyhow!("invalid path {:?}", path));
+            }
+            Ok(SockType::Remote(Remote::new(path[0].to_string())))
+        }
         _ => Err(anyhow!("Unsupported scheme")),
     }
 }

src/runtime-rs/crates/agent/src/sock/remote.rs

@@ -0,0 +1,61 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::{os::unix::prelude::AsRawFd, path::Path};
+
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use tokio::{io::Interest, net::UnixStream};
+
+use super::{ConnectConfig, Sock, Stream};
+
+#[derive(Debug, PartialEq)]
+pub struct Remote {
+    path: String,
+}
+
+impl Remote {
+    pub fn new(path: String) -> Self {
+        Self { path }
+    }
+}
+
+#[async_trait]
+impl Sock for Remote {
+    async fn connect(&self, config: &ConnectConfig) -> Result<Stream> {
+        let retry_times = config.reconnect_timeout_ms / config.dial_timeout_ms;
+        for i in 0..retry_times {
+            match connect_helper(&self.path).await {
+                Ok(stream) => {
+                    info!(
+                        sl!(),
+                        "remote connect success on {} current client fd {}",
+                        i,
+                        stream.as_raw_fd()
+                    );
+                    return Ok(Stream::Unix(stream));
+                }
+                Err(err) => {
+                    debug!(sl!(), "remote connect on {} err : {:?}", i, err);
+                    tokio::time::sleep(std::time::Duration::from_millis(config.dial_timeout_ms))
+                        .await;
+                    continue;
+                }
+            }
+        }
+        Err(anyhow!("cannot connect to agent ttrpc server {:?}", config))
+    }
+}
+
+async fn connect_helper(address: &str) -> Result<UnixStream> {
+    let stream = UnixStream::connect(Path::new(&address))
+        .await
+        .context("failed to create UnixAddr")?;
+    stream
+        .ready(Interest::READABLE | Interest::WRITABLE)
+        .await?;
+    Ok(stream)
+}

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -28,11 +28,15 @@ rand = "0.8.4"
 path-clean = "1.0.1"
 lazy_static = "1.4"
 tracing = "0.1.36"
+ttrpc = {version = "0.8.1", features = ["async"] }
+protobuf = "3.1.0"
 
 kata-sys-util = { path = "../../../libs/kata-sys-util" }
 kata-types = { path = "../../../libs/kata-types" }
 logging = { path = "../../../libs/logging" }
+protocols = { path = "../../../libs/protocols", features = ["async"] }
 shim-interface = { path = "../../../libs/shim-interface" }
+oci-spec = { version = "0.6.8", features = ["runtime"] }
 
 ch-config = { path = "ch-config", optional = true }
 tests_utils = { path = "../../tests/utils" }
@@ -68,3 +72,7 @@ hypervisor = { path = ".", features = ["cloud-hypervisor"] }
 test-utils = { path = "../../../libs/test-utils" }
 
 serial_test = "2.0.0"
+
+
+[build-dependencies]
+ttrpc-codegen = "0.4.2"

src/runtime-rs/crates/hypervisor/src/ch/mod.rs

@@ -11,6 +11,7 @@ use async_trait::async_trait;
 use kata_types::capabilities::{Capabilities, CapabilityBits};
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 use persist::sandbox_persist::Persist;
+use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::{mpsc, Mutex, RwLock};
 
@@ -59,7 +60,12 @@ impl Default for CloudHypervisor {
 
 #[async_trait]
 impl Hypervisor for CloudHypervisor {
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }

src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs

@@ -12,6 +12,7 @@ use inner::DragonballInner;
 use persist::sandbox_persist::Persist;
 pub mod vmm_instance;
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use anyhow::{Context, Result};
@@ -69,7 +70,12 @@ impl Dragonball {
 #[async_trait]
 impl Hypervisor for Dragonball {
     #[instrument]
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }

src/runtime-rs/crates/hypervisor/src/firecracker/mod.rs

@@ -18,6 +18,7 @@ use inner::FcInner;
 use kata_types::capabilities::Capabilities;
 use kata_types::capabilities::CapabilityBits;
 use persist::sandbox_persist::Persist;
+use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use tokio::sync::Mutex;
@@ -58,7 +59,12 @@ impl Firecracker {
 
 #[async_trait]
 impl Hypervisor for Firecracker {
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -19,6 +19,7 @@ pub mod dragonball;
 pub mod firecracker;
 mod kernel_param;
 pub mod qemu;
+pub mod remote;
 pub use kernel_param::Param;
 pub mod utils;
 use std::collections::HashMap;
@@ -66,6 +67,7 @@ const HUGE_SHMEM: &str = "hugeshmem";
 pub const HYPERVISOR_DRAGONBALL: &str = "dragonball";
 pub const HYPERVISOR_QEMU: &str = "qemu";
 pub const HYPERVISOR_FIRECRACKER: &str = "firecracker";
+pub const HYPERVISOR_REMOTE: &str = "remote";
 
 pub const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock";
 pub const JAILER_ROOT: &str = "root";
@@ -95,7 +97,12 @@ pub struct MemoryConfig {
 #[async_trait]
 pub trait Hypervisor: std::fmt::Debug + Send + Sync {
     // vm manager
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()>;
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        annotations: &HashMap<String, String>,
+    ) -> Result<()>;
     async fn start_vm(&self, timeout: i32) -> Result<()>;
     async fn stop_vm(&self) -> Result<()>;
     async fn wait_vm(&self) -> Result<i32>;

src/runtime-rs/crates/hypervisor/src/qemu/mod.rs

@@ -18,6 +18,7 @@ use persist::sandbox_persist::Persist;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 
+use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 use tokio::sync::{mpsc, Mutex};
@@ -52,7 +53,12 @@ impl Qemu {
 
 #[async_trait]
 impl Hypervisor for Qemu {
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }

src/runtime-rs/crates/hypervisor/src/remote/inner.rs

@@ -0,0 +1,387 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::{
+    device::DeviceType, hypervisor_persist::HypervisorState, HypervisorConfig, HYPERVISOR_REMOTE,
+};
+use crate::{MemoryConfig, VcpuThreadIds};
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+use kata_types::{
+    annotations::{
+        cri_containerd::{SANDBOX_NAMESPACE_LABEL_KEY, SANDBOX_NAME_LABEL_KEY},
+        KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS,
+        KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH, KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE,
+    },
+    capabilities::{Capabilities, CapabilityBits},
+};
+use persist::sandbox_persist::Persist;
+use protocols::{
+    remote::{CreateVMRequest, StartVMRequest, StopVMRequest},
+    remote_ttrpc_async::HypervisorClient,
+};
+use std::{collections::HashMap, time};
+use tokio::sync::{mpsc, Mutex};
+use ttrpc::context::{self};
+use ttrpc::r#async::Client;
+
+const REMOTE_SCHEME: &str = "remote";
+const DEFAULT_MIN_TIMEOUT: i32 = time::Duration::from_secs(60).as_millis() as i32;
+
+pub struct RemoteInner {
+    /// sandbox id
+    pub(crate) id: String,
+    /// hypervisor config
+    pub(crate) config: HypervisorConfig,
+    /// agent socket path
+    pub(crate) agent_socket_path: String,
+    /// netns path
+    pub(crate) netns: Option<String>,
+    /// hypervisor unix client
+    pub(crate) client: Option<Client>,
+
+    exit_notify: Option<mpsc::Sender<i32>>,
+    exit_waiter: Mutex<(mpsc::Receiver<i32>, i32)>,
+}
+
+impl std::fmt::Debug for RemoteInner {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RemoteInner")
+            .field("id", &self.id)
+            .field("config", &self.config)
+            .field("agent_socket_path", &self.agent_socket_path)
+            .field("netns", &self.netns)
+            .finish()
+    }
+}
+
+impl RemoteInner {
+    pub fn new() -> Self {
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
+        Self {
+            id: "".to_string(),
+            config: HypervisorConfig::default(),
+            agent_socket_path: "".to_string(),
+            netns: None,
+            client: None,
+
+            exit_notify: Some(exit_notify),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
+        }
+    }
+
+    fn get_ttrpc_client(&mut self) -> Result<HypervisorClient> {
+        match self.client {
+            Some(ref c) => Ok(HypervisorClient::new(c.clone())),
+            None => {
+                let c = Client::connect(&format!(
+                    "unix://{}",
+                    &self.config.remote_info.hypervisor_socket
+                ))
+                .context("connect to ")?;
+                self.client = Some(c.clone());
+                Ok(HypervisorClient::new(c))
+            }
+        }
+    }
+
+    fn prepare_annotations(
+        &self,
+        oci_annotations: &HashMap<String, String>,
+    ) -> HashMap<String, String> {
+        let mut annotations: HashMap<String, String> = HashMap::new();
+        let config = &self.config;
+        annotations.insert(
+            SANDBOX_NAME_LABEL_KEY.to_string(),
+            oci_annotations
+                .get(SANDBOX_NAME_LABEL_KEY)
+                .cloned()
+                .unwrap_or_default(),
+        );
+        annotations.insert(
+            SANDBOX_NAMESPACE_LABEL_KEY.to_string(),
+            oci_annotations
+                .get(SANDBOX_NAMESPACE_LABEL_KEY)
+                .cloned()
+                .unwrap_or_default(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE.to_string(),
+            config.machine_info.machine_type.to_string(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS.to_string(),
+            config.cpu_info.default_vcpus.to_string(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY.to_string(),
+            config.memory_info.default_memory.to_string(),
+        );
+        annotations.insert(
+            KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH.to_string(),
+            config.boot_info.image.to_string(),
+        );
+        annotations
+    }
+
+    pub(crate) async fn prepare_vm(
+        &mut self,
+        id: &str,
+        netns: Option<String>,
+        annotations: &HashMap<String, String>,
+    ) -> Result<()> {
+        info!(sl!(), "Preparing REMOTE VM");
+        self.id = id.to_string();
+
+        if let Some(netns_path) = &netns {
+            debug!(sl!(), "set netns for vmm master {:?}", &netns_path);
+            std::fs::metadata(netns_path).context("check netns path")?;
+        }
+
+        let client = self.get_ttrpc_client()?;
+
+        let ctx = context::Context::default();
+        let req = CreateVMRequest {
+            id: id.to_string(),
+            annotations: self.prepare_annotations(annotations),
+            networkNamespacePath: netns.clone().unwrap_or_default(),
+            ..Default::default()
+        };
+        info!(sl!(), "Preparing REMOTE VM req: {:?}", req.clone());
+        let resp = client.create_vm(ctx, &req).await?;
+        info!(sl!(), "Preparing REMOTE VM resp: {:?}", resp.clone());
+        self.agent_socket_path = resp.agentSocketPath;
+        self.netns = netns;
+        Ok(())
+    }
+
+    pub(crate) async fn start_vm(&mut self, timeout: i32) -> Result<()> {
+        info!(sl!(), "Starting REMOTE VM");
+
+        let mut min_timeout = DEFAULT_MIN_TIMEOUT;
+        if self.config.remote_info.hypervisor_timeout > 0 {
+            min_timeout = self.config.remote_info.hypervisor_timeout.min(timeout);
+        }
+        let timeout = min_timeout;
+
+        let client = self.get_ttrpc_client()?;
+
+        let req = StartVMRequest {
+            id: self.id.clone(),
+            ..Default::default()
+        };
+        let ctx =
+            context::with_timeout(time::Duration::from_secs(timeout as u64).as_nanos() as i64);
+        let _resp = client.start_vm(ctx, &req).await?;
+
+        Ok(())
+    }
+
+    pub(crate) async fn stop_vm(&mut self) -> Result<()> {
+        info!(sl!(), "Stopping REMOTE VM");
+
+        let client = self.get_ttrpc_client()?;
+
+        let ctx = context::with_timeout(time::Duration::from_secs(1).as_nanos() as i64);
+        let req = StopVMRequest {
+            id: self.id.clone(),
+            ..Default::default()
+        };
+        let _resp = client.stop_vm(ctx, &req).await?;
+
+        self.exit_notify.take().unwrap().send(1).await?;
+        Ok(())
+    }
+
+    pub(crate) async fn pause_vm(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::pause_vm(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn wait_vm(&self) -> Result<i32> {
+        info!(sl!(), "Wait Remote VM");
+        let mut waiter = self.exit_waiter.lock().await;
+        if let Some(exitcode) = waiter.0.recv().await {
+            waiter.1 = exitcode;
+        }
+
+        Ok(waiter.1)
+    }
+
+    pub(crate) async fn resume_vm(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::resume_vm(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn save_vm(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::save_vm(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn add_device(&self, device: DeviceType) -> Result<DeviceType> {
+        warn!(sl!(), "RemoteInner::add_device(): NOT YET IMPLEMENTED");
+        Ok(device)
+    }
+
+    pub(crate) async fn remove_device(&self, _device: DeviceType) -> Result<()> {
+        warn!(sl!(), "RemoteInner::remove_device(): NOT YET IMPLEMENTED");
+        Ok(())
+    }
+
+    pub(crate) async fn update_device(&self, _device: DeviceType) -> Result<()> {
+        warn!(sl!(), "RemoteInner::update_device(): NOT YET IMPLEMENTED");
+        Ok(())
+    }
+
+    pub(crate) async fn get_agent_socket(&self) -> Result<String> {
+        Ok(format!("{}://{}", REMOTE_SCHEME, &self.agent_socket_path))
+    }
+
+    pub(crate) async fn disconnect(&mut self) {
+        warn!(sl!(), "RemoteInner::disconnect(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub fn hypervisor_config(&self) -> HypervisorConfig {
+        info!(
+            sl!(),
+            "RemoteInner::hypervisor_config(): {:?}",
+            self.config.clone()
+        );
+        self.config.clone()
+    }
+
+    pub(crate) async fn get_thread_ids(&self) -> Result<VcpuThreadIds> {
+        warn!(sl!(), "RemoteInner::get_thread_ids(): NOT YET IMPLEMENTED");
+        let vcpu_thread_ids: VcpuThreadIds = VcpuThreadIds {
+            vcpus: HashMap::new(),
+        };
+        Ok(vcpu_thread_ids)
+    }
+
+    pub(crate) async fn get_vmm_master_tid(&self) -> Result<u32> {
+        warn!(sl!(), "RemoteInner::get_vmm_master_tid()");
+        let tid = nix::unistd::gettid().as_raw();
+        Ok(tid as u32)
+    }
+
+    pub(crate) async fn get_ns_path(&self) -> Result<String> {
+        info!(sl!(), "RemoteInner::get_ns_path()");
+        Ok(self.netns.clone().unwrap_or_default())
+    }
+
+    pub(crate) async fn cleanup(&self) -> Result<()> {
+        info!(sl!(), "RemoteInner::cleanup(): NOT YET IMPLEMENTED");
+        Ok(())
+    }
+
+    pub(crate) async fn resize_vcpu(
+        &mut self,
+        _old_vcpus: u32,
+        _new_vcpus: u32,
+    ) -> Result<(u32, u32)> {
+        info!(sl!(), "RemoteInner::resize_vcpu(): NOT YET IMPLEMENTED");
+        Ok((_old_vcpus, _new_vcpus))
+    }
+
+    pub(crate) async fn get_pids(&self) -> Result<Vec<u32>> {
+        warn!(sl!(), "RemoteInner::get_pids(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn check(&self) -> Result<()> {
+        warn!(sl!(), "RemoteInner::check(): NOT YET IMPLEMENTED");
+        todo!()
+    }
+
+    pub(crate) async fn get_jailer_root(&self) -> Result<String> {
+        warn!(sl!(), "RemoteInner::get_jailer_root(): NOT YET IMPLEMENTED");
+        Ok("".into())
+    }
+
+    pub(crate) async fn capabilities(&self) -> Result<Capabilities> {
+        Ok(Capabilities::default())
+    }
+
+    pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) {
+        self.config = config;
+    }
+
+    pub(crate) async fn get_hypervisor_metrics(&self) -> Result<String> {
+        warn!(
+            sl!(),
+            "RemoteInner::get_hypervisor_metrics(): NOT YET IMPLEMENTED"
+        );
+        todo!()
+    }
+
+    pub(crate) fn set_capabilities(&mut self, _flag: CapabilityBits) {
+        warn!(
+            sl!(),
+            "RemoteInner::set_capabilities(): NOT YET IMPLEMENTED"
+        );
+        todo!()
+    }
+
+    pub(crate) fn set_guest_memory_block_size(&mut self, _size: u32) {
+        info!(
+            sl!(),
+            "RemoteInner::set_guest_memory_block_size(): NOT YET IMPLEMENTED"
+        )
+    }
+
+    pub(crate) fn guest_memory_block_size_mb(&self) -> u32 {
+        warn!(
+            sl!(),
+            "RemoteInner::guest_memory_block_size_mb(): NOT YET IMPLEMENTED"
+        );
+        0
+    }
+
+    pub(crate) fn resize_memory(&self, _new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
+        Ok((
+            _new_mem_mb,
+            MemoryConfig {
+                ..Default::default()
+            },
+        ))
+    }
+}
+
+#[async_trait]
+impl Persist for RemoteInner {
+    type State = HypervisorState;
+    type ConstructorArgs = ();
+
+    /// Save a state of hypervisor
+    async fn save(&self) -> Result<Self::State> {
+        Ok(HypervisorState {
+            hypervisor_type: HYPERVISOR_REMOTE.to_string(),
+            id: self.id.clone(),
+            config: self.config.clone(),
+            netns: self.netns.clone(),
+            ..Default::default()
+        })
+    }
+
+    /// Restore hypervisor
+    async fn restore(
+        _hypervisor_args: Self::ConstructorArgs,
+        hypervisor_state: Self::State,
+    ) -> Result<Self> {
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
+        Ok(RemoteInner {
+            id: hypervisor_state.id,
+            config: hypervisor_state.config,
+            agent_socket_path: "".to_string(),
+            netns: hypervisor_state.netns,
+            client: None,
+            exit_notify: Some(exit_notify),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
+        })
+    }
+}

src/runtime-rs/crates/hypervisor/src/remote/mod.rs

@@ -0,0 +1,216 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::HypervisorState;
+use crate::{device::DeviceType, Hypervisor, HypervisorConfig, MemoryConfig, VcpuThreadIds};
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+use inner::RemoteInner;
+use kata_types::capabilities::{Capabilities, CapabilityBits};
+use persist::sandbox_persist::Persist;
+use std::collections::HashMap;
+
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+mod inner;
+
+#[derive(Debug)]
+pub struct Remote {
+    inner: Arc<RwLock<RemoteInner>>,
+}
+
+impl Default for Remote {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Remote {
+    pub fn new() -> Self {
+        Self {
+            inner: Arc::new(RwLock::new(RemoteInner::new())),
+        }
+    }
+
+    pub async fn set_hypervisor_config(&mut self, config: HypervisorConfig) {
+        let mut inner = self.inner.write().await;
+        inner.set_hypervisor_config(config)
+    }
+}
+
+#[async_trait]
+impl Hypervisor for Remote {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        annotations: &HashMap<String, String>,
+    ) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.prepare_vm(id, netns, annotations).await
+    }
+
+    async fn start_vm(&self, timeout: i32) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.start_vm(timeout).await
+    }
+
+    async fn stop_vm(&self) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.stop_vm().await
+    }
+
+    async fn wait_vm(&self) -> Result<i32> {
+        let inner = self.inner.read().await;
+        inner.wait_vm().await
+    }
+
+    async fn pause_vm(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.pause_vm().await
+    }
+
+    async fn resume_vm(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.resume_vm().await
+    }
+
+    async fn save_vm(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.save_vm().await
+    }
+
+    async fn add_device(&self, device: DeviceType) -> Result<DeviceType> {
+        let inner = self.inner.write().await;
+        inner.add_device(device).await
+    }
+
+    async fn remove_device(&self, device: DeviceType) -> Result<()> {
+        let inner = self.inner.write().await;
+        inner.remove_device(device).await
+    }
+
+    async fn update_device(&self, device: DeviceType) -> Result<()> {
+        let inner = self.inner.write().await;
+        inner.update_device(device).await
+    }
+
+    async fn get_agent_socket(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_agent_socket().await
+    }
+
+    async fn disconnect(&self) {
+        let mut inner = self.inner.write().await;
+        inner.disconnect().await
+    }
+
+    async fn hypervisor_config(&self) -> HypervisorConfig {
+        let inner = self.inner.read().await;
+        inner.hypervisor_config()
+    }
+
+    async fn get_thread_ids(&self) -> Result<VcpuThreadIds> {
+        let inner = self.inner.read().await;
+        inner.get_thread_ids().await
+    }
+
+    async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let inner = self.inner.read().await;
+        inner.get_vmm_master_tid().await
+    }
+
+    async fn get_ns_path(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_ns_path().await
+    }
+
+    async fn cleanup(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.cleanup().await
+    }
+
+    async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> {
+        let mut inner = self.inner.write().await;
+        inner.resize_vcpu(old_vcpus, new_vcpus).await
+    }
+
+    async fn get_pids(&self) -> Result<Vec<u32>> {
+        let inner = self.inner.read().await;
+        inner.get_pids().await
+    }
+
+    async fn check(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.check().await
+    }
+
+    async fn get_jailer_root(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_jailer_root().await
+    }
+
+    async fn save_state(&self) -> Result<HypervisorState> {
+        self.save().await
+    }
+
+    async fn capabilities(&self) -> Result<Capabilities> {
+        let inner = self.inner.read().await;
+        inner.capabilities().await
+    }
+
+    async fn get_hypervisor_metrics(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_hypervisor_metrics().await
+    }
+
+    async fn set_capabilities(&self, flag: CapabilityBits) {
+        let mut inner = self.inner.write().await;
+        inner.set_capabilities(flag)
+    }
+
+    async fn set_guest_memory_block_size(&self, size: u32) {
+        let mut inner = self.inner.write().await;
+        inner.set_guest_memory_block_size(size);
+    }
+
+    async fn guest_memory_block_size(&self) -> u32 {
+        let inner = self.inner.read().await;
+        inner.guest_memory_block_size_mb()
+    }
+
+    async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
+        let inner = self.inner.read().await;
+        inner.resize_memory(new_mem_mb)
+    }
+
+    async fn get_passfd_listener_addr(&self) -> Result<(String, u32)> {
+        Err(anyhow::anyhow!("Not yet supported"))
+    }
+}
+
+#[async_trait]
+impl Persist for Remote {
+    type State = HypervisorState;
+    type ConstructorArgs = ();
+
+    /// Save a state of the component.
+    async fn save(&self) -> Result<Self::State> {
+        let inner = self.inner.read().await;
+        inner.save().await.context("save remote hypervisor state")
+    }
+
+    /// Restore a component from a specified state.
+    async fn restore(
+        hypervisor_args: Self::ConstructorArgs,
+        hypervisor_state: Self::State,
+    ) -> Result<Self> {
+        let inner = RemoteInner::restore(hypervisor_args, hypervisor_state).await?;
+        Ok(Self {
+            inner: Arc::new(RwLock::new(inner)),
+        })
+    }
+}

src/runtime-rs/crates/runtimes/virt_container/src/lib.rs

@@ -26,10 +26,12 @@ use hypervisor::{dragonball::Dragonball, HYPERVISOR_DRAGONBALL};
 #[cfg(not(target_arch = "s390x"))]
 use hypervisor::{firecracker::Firecracker, HYPERVISOR_FIRECRACKER};
 use hypervisor::{qemu::Qemu, HYPERVISOR_QEMU};
+use hypervisor::{remote::Remote, HYPERVISOR_REMOTE};
 #[cfg(all(feature = "dragonball", not(target_arch = "s390x")))]
 use kata_types::config::DragonballConfig;
 #[cfg(not(target_arch = "s390x"))]
 use kata_types::config::FirecrackerConfig;
+use kata_types::config::RemoteConfig;
 use kata_types::config::{hypervisor::register_hypervisor_plugin, QemuConfig, TomlConfig};
 
 #[cfg(all(feature = "cloud-hypervisor", not(target_arch = "s390x")))]
@@ -75,6 +77,9 @@ impl RuntimeHandler for VirtContainer {
             register_hypervisor_plugin(HYPERVISOR_NAME_CH, ch_config);
         }
 
+        let remote_config = Arc::new(RemoteConfig::new());
+        register_hypervisor_plugin("remote", remote_config);
+
         Ok(())
     }
 
@@ -179,7 +184,6 @@ async fn new_hypervisor(toml_config: &TomlConfig) -> Result<Arc<dyn Hypervisor>>
                 .await;
             Ok(Arc::new(hypervisor))
         }
-
         #[cfg(all(feature = "cloud-hypervisor", not(target_arch = "s390x")))]
         HYPERVISOR_NAME_CH => {
             let mut hypervisor = CloudHypervisor::new();
@@ -190,6 +194,13 @@ async fn new_hypervisor(toml_config: &TomlConfig) -> Result<Arc<dyn Hypervisor>>
 
             Ok(Arc::new(hypervisor))
         }
+        HYPERVISOR_REMOTE => {
+            let mut hypervisor = Remote::new();
+            hypervisor
+                .set_hypervisor_config(hypervisor_config.clone())
+                .await;
+            Ok(Arc::new(hypervisor))
+        }
         _ => Err(anyhow!("Unsupported hypervisor {}", &hypervisor_name)),
     }
 }

src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs

@@ -19,6 +19,7 @@ use containerd_shim_protos::events::task::{TaskExit, TaskOOM};
 use hypervisor::VsockConfig;
 #[cfg(not(target_arch = "s390x"))]
 use hypervisor::HYPERVISOR_FIRECRACKER;
+use hypervisor::HYPERVISOR_REMOTE;
 #[cfg(all(feature = "dragonball", not(target_arch = "s390x")))]
 use hypervisor::{dragonball::Dragonball, HYPERVISOR_DRAGONBALL};
 use hypervisor::{qemu::Qemu, HYPERVISOR_QEMU};
@@ -326,7 +327,11 @@ impl Sandbox for VirtSandbox {
         }
 
         self.hypervisor
-            .prepare_vm(id, sandbox_config.network_env.netns.clone())
+            .prepare_vm(
+                id,
+                sandbox_config.network_env.netns.clone(),
+                &sandbox_config.annotations,
+            )
             .await
             .context("prepare vm")?;
 
@@ -649,6 +654,7 @@ impl Persist for VirtSandbox {
                 #[cfg(not(target_arch = "s390x"))]
                 HYPERVISOR_FIRECRACKER => Ok(Some(hypervisor_state)),
                 HYPERVISOR_QEMU => Ok(Some(hypervisor_state)),
+                HYPERVISOR_REMOTE => Ok(Some(hypervisor_state)),
                 _ => Err(anyhow!(
                     "Unsupported hypervisor {}",
                     hypervisor_state.hypervisor_type

src/runtime/Makefile

@@ -151,8 +151,8 @@ FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd
 SNPCERTSPATH := /opt/snp/cert_chain.cert
 
 ROOTMEASURECONFIG ?= ""
-KERNELPARAMS += $(ROOTMEASURECONFIG)
 KERNELTDXPARAMS += $(ROOTMEASURECONFIG)
+KERNELQEMUCOCODEVPARAMS += $(ROOTMEASURECONFIG)
 
 # Name of default configuration file the runtime will use.
 CONFIG_FILE = configuration.toml
@@ -646,6 +646,7 @@ USER_VARS += DEFMACHINETYPE_CLH
 USER_VARS += DEFMACHINETYPE_STRATOVIRT
 USER_VARS += KERNELPARAMS
 USER_VARS += KERNELTDXPARAMS
+USER_VARS += KERNELQEMUCOCODEVPARAMS
 USER_VARS += LIBEXECDIR
 USER_VARS += LOCALSTATEDIR
 USER_VARS += PKGDATADIR

src/runtime/config/configuration-qemu-coco-dev.toml.in

@@ -73,7 +73,7 @@ valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
 # may stop the virtual machine from booting.
 # To see the list of default parameters, enable hypervisor debug, create a
 # container and look for 'default-kernel-parameters' log entries.
-kernel_params = "@KERNELPARAMS@"
+kernel_params = "@KERNELQEMUCOCODEVPARAMS@"
 
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
@@ -653,7 +653,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
 #   does not yet support sandbox sizing annotations.
 # - When running single containers using a tool like ctr, container sizing information will be available.
-static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_TEE@
 
 # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
 # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.

src/runtime/config/configuration-remote.toml.in

@@ -38,7 +38,7 @@ remote_hypervisor_timeout = 600
 # Each member of the list is a regular expression, which is the base name
 # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
 # Note: Remote hypervisor is only handling the following annotations
-enable_annotations = ["machine_type", "default_memory", "default_vcpus", "image"]
+enable_annotations = ["machine_type", "default_memory", "default_vcpus", "image", "default_gpus", "gpu_model"]
 
 # Optional space-separated list of options to pass to the guest kernel.
 # For example, use `kernel_params = "vsyscall=emulate"` if you are having

src/runtime/pkg/govmm/.github/workflows/main.yml

@@ -5,7 +5,7 @@ jobs:
     strategy:
       matrix:
         go-version: [1.15.x, 1.16.x]
-        os: [ubuntu-latest]
+        os: [ubuntu-22.04]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Install Go

src/runtime/pkg/oci/utils.go

@@ -560,6 +560,10 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig,
 		config.HypervisorConfig.Initdata = initdata
 	}
 
+	if err := addHypervisorGPUOverrides(ocispec, config); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -754,6 +758,26 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
 	})
 }
 
+func addHypervisorGPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
+	if sbConfig.HypervisorType != vc.RemoteHypervisor {
+		return nil
+	}
+
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.DefaultGPUs).setUint(func(gpus uint64) {
+		sbConfig.HypervisorConfig.DefaultGPUs = uint32(gpus)
+	}); err != nil {
+		return err
+	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.DefaultGPUModel]; ok {
+		if value != "" {
+			sbConfig.HypervisorConfig.DefaultGPUModel = value
+		}
+	}
+
+	return nil
+}
+
 func addHypervisorBlockOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
 	if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceDriver]; ok {
 		supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW}
@@ -1076,6 +1100,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
 		sandboxConfig.HypervisorConfig.NumVCPUsF += sandboxConfig.SandboxResources.WorkloadCPUs
 		sandboxConfig.HypervisorConfig.MemorySize += sandboxConfig.SandboxResources.WorkloadMemMB
 
+		sandboxConfig.HypervisorConfig.DefaultMaxVCPUs = sandboxConfig.HypervisorConfig.NumVCPUs()
+
 		ociLog.WithFields(logrus.Fields{
 			"workload cpu":       sandboxConfig.SandboxResources.WorkloadCPUs,
 			"default cpu":        sandboxConfig.SandboxResources.BaseCPUs,

src/runtime/pkg/oci/utils_test.go

@@ -775,6 +775,23 @@ func TestAddRemoteHypervisorAnnotations(t *testing.T) {
 	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
 	assert.NoError(err)
 	assert.Equal(sbConfig.HypervisorConfig.Initdata, "initdata")
+
+	// When GPU annotations are specified, remote hypervisor annotations have the annotation added
+	ocispec.Annotations[vcAnnotations.DefaultGPUs] = "-1"
+	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
+	assert.Error(err)
+
+	ocispec.Annotations[vcAnnotations.DefaultGPUs] = "1"
+	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
+	assert.NoError(err)
+	assert.Equal(sbConfig.HypervisorConfig.DefaultGPUs, uint32(1))
+
+	// When GPU annotations are specified, remote hypervisor annotations have the annotation added
+	ocispec.Annotations[vcAnnotations.DefaultGPUModel] = "tesla"
+	err = addAnnotations(ocispec, &sbConfig, runtimeConfig)
+	assert.NoError(err)
+	assert.Equal(sbConfig.HypervisorConfig.DefaultGPUModel, "tesla")
+
 }
 
 func TestAddProtectedHypervisorAnnotations(t *testing.T) {

src/runtime/virtcontainers/clh.go

@@ -1359,11 +1359,10 @@ func (clh *cloudHypervisor) launchClh() error {
 	if clh.config.Debug {
 		// Cloud hypervisor log levels
 		// 'v' occurrences increase the level
-		//0 =>  Error
-		//1 =>  Warn
-		//2 =>  Info
-		//3 =>  Debug
-		//4+ => Trace
+		//0 =>  Warn
+		//1 =>  Info
+		//2 =>  Debug
+		//3+ => Trace
 		// Use Info, the CI runs with debug enabled
 		// a high level of logging increases the boot time
 		// and in a nested environment this could increase

src/runtime/virtcontainers/fs_share_linux.go

@@ -64,6 +64,14 @@ func resolveRootDir() string {
 		// Use the default root dir in case of any errors resolving the root dir symlink
 		return defaultKubernetesRootDir
 	}
+	// Make root dir an absolute path if needed
+	if !filepath.IsAbs(rootDir) {
+		rootDir, err = filepath.Abs(filepath.Join(filepath.Dir(defaultKubernetesRootDir), rootDir))
+		if err != nil {
+			// Use the default root dir in case of any errors resolving the root dir symlink
+			return defaultKubernetesRootDir
+		}
+	}
 	return rootDir
 }
 

src/runtime/virtcontainers/hypervisor.go

@@ -673,6 +673,12 @@ type HypervisorConfig struct {
 
 	// Initdata defines the initdata passed into guest when CreateVM
 	Initdata string
+
+	// GPU specific annotations (currently only applicable for Remote Hypervisor)
+	//DefaultGPUs specifies the number of GPUs required for the Kata VM
+	DefaultGPUs uint32
+	// DefaultGPUModel specifies GPU model like tesla, h100, readeon etc.
+	DefaultGPUModel string
 }
 
 // vcpu mapping from vcpu number to thread number

src/runtime/virtcontainers/hypervisor_config_linux.go

@@ -58,11 +58,6 @@ func validateHypervisorConfig(conf *HypervisorConfig) error {
 		conf.DefaultMaxVCPUs = defaultMaxVCPUs
 	}
 
-	if conf.ConfidentialGuest && conf.NumVCPUs() != conf.DefaultMaxVCPUs {
-		hvLogger.Warnf("Confidential guests do not support hotplugging of vCPUs. Setting DefaultMaxVCPUs to NumVCPUs (%d)", conf.NumVCPUs())
-		conf.DefaultMaxVCPUs = conf.NumVCPUs()
-	}
-
 	if conf.Msize9p == 0 && conf.SharedFS != config.VirtioFS {
 		conf.Msize9p = defaultMsize9p
 	}

src/runtime/virtcontainers/pkg/annotations/annotations.go

@@ -132,6 +132,12 @@ const (
 	// UseLegacySerial sets legacy serial device for guest console if available and implemented for architecture
 	UseLegacySerial = kataAnnotHypervisorPrefix + "use_legacy_serial"
 
+	// GPU specific annotations used by remote hypervisor for instance selection
+	// Number of GPUs required in the Kata VM
+	DefaultGPUs = kataAnnotHypervisorPrefix + "default_gpus"
+	// GPU model - tesla, h100, radeon etc..
+	DefaultGPUModel = kataAnnotHypervisorPrefix + "default_gpu_model"
+
 	//
 	// CPU Annotations
 	//

src/runtime/virtcontainers/remote.go

@@ -81,6 +81,8 @@ func (rh *remoteHypervisor) CreateVM(ctx context.Context, id string, network Net
 	annotations[hypannotations.DefaultVCPUs] = strconv.FormatUint(uint64(hypervisorConfig.NumVCPUs()), 10)
 	annotations[hypannotations.DefaultMemory] = strconv.FormatUint(uint64(hypervisorConfig.MemorySize), 10)
 	annotations[hypannotations.Initdata] = hypervisorConfig.Initdata
+	annotations[hypannotations.DefaultGPUs] = strconv.FormatUint(uint64(hypervisorConfig.DefaultGPUs), 10)
+	annotations[hypannotations.DefaultGPUModel] = hypervisorConfig.DefaultGPUModel
 
 	req := &pb.CreateVMRequest{
 		Id:                   id,

src/runtime/virtcontainers/sandbox.go

@@ -1458,6 +1458,8 @@ func (s *Sandbox) startVM(ctx context.Context, prestartHookFunc func(context.Con
 
 	defer func() {
 		if err != nil {
+			// Log error, otherwise nobody might see it - StopVM could kill this process.
+			s.Logger().WithError(err).Error("Cannot start VM")
 			s.hypervisor.StopVM(ctx, false)
 		}
 	}()

src/tools/agent-ctl/Cargo.toml

@@ -28,6 +28,7 @@ slog = "2.7.0"
 slog-scope = "4.4.0"
 rand = "0.8.4"
 protobuf = "3.2.0"
+log = "0.4.22"
 
 nix = "0.23.0"
 libc = "0.2.112"
@@ -41,4 +42,11 @@ humantime = "2.1.0"
 serde = { version = "1.0.131", features = ["derive"] }
 serde_json = "1.0.73"
 
+# Image pull/unpack
+image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "v0.10.0", features = ["snapshot-overlayfs", "oci-client-rustls", "signature-cosign-rustls"] }
+
+safe-path = { path = "../../libs/safe-path" }
+
+tokio = { version = "1.28.1", features = ["signal"] }
+
 [workspace]

src/tools/agent-ctl/src/client.rs

@@ -5,7 +5,7 @@
 
 // Description: Client side of ttRPC comms
 
-use crate::types::{Config, CopyFileInput, Options, SetPolicyInput};
+use crate::types::*;
 use crate::utils;
 use anyhow::{anyhow, Result};
 use byteorder::ByteOrder;
@@ -34,7 +34,7 @@ macro_rules! run_if_auto_values {
         let cfg = $ctx.metadata.get(METADATA_CFG_NS);
 
         if let Some(v) = cfg {
-            if v.contains(&NO_AUTO_VALUES_CFG_NAME.to_string()) {
+            if v.contains(&AUTO_VALUES_CFG_NAME.to_string()) {
                 debug!(sl!(), "Running closure to generate values");
 
                 $closure()?;
@@ -103,9 +103,9 @@ const ERR_API_FAILED: &str = "API failed";
 // Value used as a "namespace" in the ttRPC Context's metadata.
 const METADATA_CFG_NS: &str = "agent-ctl-cfg";
 
-// Special value which if found means do not generate any values
+// Special value which if found means generate any values
 // automatically.
-const NO_AUTO_VALUES_CFG_NAME: &str = "no-auto-values";
+const AUTO_VALUES_CFG_NAME: &str = "auto-values";
 
 static AGENT_CMDS: &[AgentCmd] = &[
     AgentCmd {
@@ -640,7 +640,7 @@ pub fn client(cfg: &Config, commands: Vec<&str>) -> Result<()> {
     // of this option.
 
     if !cfg.no_auto_values {
-        ttrpc_ctx.add(METADATA_CFG_NS.into(), NO_AUTO_VALUES_CFG_NAME.to_string());
+        ttrpc_ctx.add(METADATA_CFG_NS.into(), AUTO_VALUES_CFG_NAME.to_string());
 
         debug!(sl!(), "Automatic value generation disabled");
     }
@@ -921,20 +921,18 @@ fn agent_cmd_sandbox_create(
     ctx: &Context,
     client: &AgentServiceClient,
     _health: &HealthClient,
-    options: &mut Options,
+    _options: &mut Options,
     args: &str,
 ) -> Result<()> {
     let mut req: CreateSandboxRequest = utils::make_request(args)?;
 
+    // Generate sandbox_id if it is empty
+    if req.sandbox_id.is_empty() {
+        req.set_sandbox_id(utils::random_sandbox_id());
+    }
+
     let ctx = clone_context(ctx);
 
-    run_if_auto_values!(ctx, || -> Result<()> {
-        let sid = utils::get_option("sid", options, args)?;
-        req.set_sandbox_id(sid);
-
-        Ok(())
-    });
-
     debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
 
     let reply = client
@@ -974,26 +972,19 @@ fn agent_cmd_container_create(
     ctx: &Context,
     client: &AgentServiceClient,
     _health: &HealthClient,
-    options: &mut Options,
+    _options: &mut Options,
     args: &str,
 ) -> Result<()> {
-    let mut req: CreateContainerRequest = utils::make_request(args)?;
+    let input: CreateContainerInput = utils::make_request(args)?;
+
+    if input.image.is_empty() {
+        info!(sl!(), "create container: error image is empty");
+        return Err(anyhow!("CreateContainer needs image reference"));
+    }
 
     let ctx = clone_context(ctx);
 
-    // FIXME: container create: add back "spec=file:///" support
-
-    run_if_auto_values!(ctx, || -> Result<()> {
-        let cid = utils::get_option("cid", options, args)?;
-        let exec_id = utils::get_option("exec_id", options, args)?;
-        let ttrpc_spec = utils::get_ttrpc_spec(options, &cid).map_err(|e| anyhow!(e))?;
-
-        req.set_container_id(cid);
-        req.set_exec_id(exec_id);
-        req.set_OCI(ttrpc_spec);
-
-        Ok(())
-    });
+    let req = utils::make_create_container_request(input)?;
 
     debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
 
@@ -1011,19 +1002,13 @@ fn agent_cmd_container_remove(
     ctx: &Context,
     client: &AgentServiceClient,
     _health: &HealthClient,
-    options: &mut Options,
+    _options: &mut Options,
     args: &str,
 ) -> Result<()> {
-    let mut req: RemoveContainerRequest = utils::make_request(args)?;
+    let req: RemoveContainerRequest = utils::make_request(args)?;
 
     let ctx = clone_context(ctx);
 
-    run_if_auto_values!(ctx, || -> Result<()> {
-        let cid = utils::get_option("cid", options, args)?;
-        req.set_container_id(cid);
-        Ok(())
-    });
-
     debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
 
     let reply = client
@@ -1033,6 +1018,9 @@ fn agent_cmd_container_remove(
     info!(sl!(), "response received";
         "response" => format!("{:?}", reply));
 
+    // Un-mount the rootfs mount point.
+    utils::remove_container_image_mount(req.container_id())?;
+
     Ok(())
 }
 
@@ -1180,20 +1168,13 @@ fn agent_cmd_container_start(
     ctx: &Context,
     client: &AgentServiceClient,
     _health: &HealthClient,
-    options: &mut Options,
+    _options: &mut Options,
     args: &str,
 ) -> Result<()> {
-    let mut req: StartContainerRequest = utils::make_request(args)?;
+    let req: StartContainerRequest = utils::make_request(args)?;
 
     let ctx = clone_context(ctx);
 
-    run_if_auto_values!(ctx, || -> Result<()> {
-        let cid = utils::get_option("cid", options, args)?;
-
-        req.set_container_id(cid);
-        Ok(())
-    });
-
     debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
 
     let reply = client

src/tools/agent-ctl/src/image.rs

@@ -0,0 +1,63 @@
+// Copyright (c) 2024 Microsoft Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Description: Image client to manage container images for testing container creation
+
+use anyhow::{anyhow, Context, Result};
+use image_rs::image::ImageClient;
+use nix::mount::umount;
+use safe_path::scoped_join;
+use slog::{debug, warn};
+use std::fs;
+use std::path::PathBuf;
+
+const IMAGE_WORK_DIR: &str = "/run/kata-containers/test_image/";
+const CONTAINER_BASE_TEST: &str = "/run/kata-containers/testing/";
+
+// Pulls the container image referenced in `image` using image-rs
+// and returns the bundle path containing the rootfs (mounted by
+// the underlying snapshotter, overlayfs in this case) & config.json
+// Uses anonymous image registry authentication.
+pub fn pull_image(image: &str, cid: &str) -> Result<String> {
+    if image.is_empty() || cid.is_empty() {
+        warn!(sl!(), "pull_image: invalid inputs");
+        return Err(anyhow!(
+            "Invalid image reference or container id to pull image"
+        ));
+    }
+
+    debug!(sl!(), "pull_image: creating image client");
+    let mut image_client = ImageClient::new(PathBuf::from(IMAGE_WORK_DIR));
+    image_client.config.auth = false;
+    image_client.config.security_validate = false;
+
+    // setup the container test base path
+    fs::create_dir_all(CONTAINER_BASE_TEST)?;
+
+    // setup the container bundle path
+    let bundle_dir = scoped_join(CONTAINER_BASE_TEST, cid)?;
+    fs::create_dir_all(bundle_dir.clone())?;
+
+    // pull the image
+    let image_id = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?
+        .block_on(image_client.pull_image(image, &bundle_dir, &None, &None))
+        .context("pull and unpack container image")?;
+
+    debug!(
+        sl!(),
+        "pull_image: image pull for {:?} successfull", image_id
+    );
+
+    // return the bundle path created by unpacking the images
+    Ok(bundle_dir.as_path().display().to_string())
+}
+
+pub fn remove_image_mount(cid: &str) -> Result<()> {
+    let bundle_path = scoped_join(CONTAINER_BASE_TEST, cid)?;
+    let rootfs_path = scoped_join(bundle_path, "rootfs")?;
+    umount(&rootfs_path)?;
+    Ok(())
+}

src/tools/agent-ctl/src/main.rs

@@ -21,6 +21,7 @@ macro_rules! sl {
 }
 
 mod client;
+mod image;
 mod rpc;
 mod types;
 mod utils;

src/tools/agent-ctl/src/types.rs

@@ -33,3 +33,10 @@ pub struct CopyFileInput {
 pub struct SetPolicyInput {
     pub policy_file: String,
 }
+
+// CreateContainer input
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct CreateContainerInput {
+    pub image: String,
+    pub id: String,
+}

src/tools/agent-ctl/src/utils.rs

@@ -3,20 +3,24 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::types::{Config, CopyFileInput, Options, SetPolicyInput};
+use crate::image;
+use crate::types::*;
 use anyhow::{anyhow, Result};
 use oci::{Root as ociRoot, Spec as ociSpec};
 use oci_spec::runtime as oci;
-use protocols::agent::{CopyFileRequest, SetPolicyRequest};
-use protocols::oci::{Mount as ttrpcMount, Root as ttrpcRoot, Spec as ttrpcSpec};
+use protocols::agent::{CopyFileRequest, CreateContainerRequest, SetPolicyRequest};
+use protocols::oci::{
+    Mount as ttrpcMount, Process as ttrpcProcess, Root as ttrpcRoot, Spec as ttrpcSpec,
+};
 use rand::Rng;
+use safe_path::scoped_join;
 use serde::de::DeserializeOwned;
 use slog::{debug, warn};
 use std::collections::HashMap;
 use std::fs::{self, File};
 use std::io::Read;
 use std::os::unix::fs::MetadataExt;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
 // Length of a sandbox identifier
@@ -30,6 +34,10 @@ const MIN_HOSTNAME_LEN: u8 = 8;
 // Name of the OCI configuration file found at the root of an OCI bundle.
 const CONFIG_FILE: &str = "config.json";
 
+// Path to OCI configuration template
+const OCI_CONFIG_TEMPLATE: &str =
+    "/opt/kata/share/defaults/kata-containers/agent-ctl/oci_config.json";
+
 lazy_static! {
     // Create a mutable hash map statically
     static ref SIGNALS: Arc<Mutex<HashMap<&'static str, u8>>> = {
@@ -492,3 +500,67 @@ pub fn make_set_policy_request(input: &SetPolicyInput) -> Result<SetPolicyReques
     req.set_policy(policy_data);
     Ok(req)
 }
+
+fn fix_oci_process_args(spec: &mut ttrpcSpec, bundle: &str) -> Result<()> {
+    let config_path = scoped_join(bundle, CONFIG_FILE)?;
+
+    let file = File::open(config_path)?;
+    let oci_from_config: ociSpec = serde_json::from_reader(file)?;
+
+    let mut process: ttrpcProcess = match &oci_from_config.process() {
+        Some(p) => p.clone().into(),
+        None => {
+            return Err(anyhow!("Failed to set container process args"));
+        }
+    };
+
+    spec.take_Process().set_Args(process.take_Args());
+    Ok(())
+}
+
+// Helper function to generate create container request
+pub fn make_create_container_request(
+    input: CreateContainerInput,
+) -> Result<CreateContainerRequest> {
+    // read in the oci configuration template
+    if !Path::new(OCI_CONFIG_TEMPLATE).exists() {
+        warn!(sl!(), "make_create_container_request: Missig template file");
+        return Err(anyhow!("Missing OCI Config template file"));
+    }
+
+    let file = File::open(OCI_CONFIG_TEMPLATE)?;
+    let spec: ociSpec = serde_json::from_reader(file)?;
+
+    let mut req = CreateContainerRequest::default();
+
+    let c_id = if !input.id.is_empty() {
+        input.id
+    } else {
+        random_container_id()
+    };
+
+    debug!(
+        sl!(),
+        "make_create_container_request: pulling container image"
+    );
+
+    // Pull and unpack the container image
+    let bundle = image::pull_image(&input.image, &c_id)?;
+
+    let mut ttrpc_spec = oci_to_ttrpc(&bundle, &c_id, &spec)?;
+
+    // Rootfs has been handled with bundle after pulling image
+    // Fix the container process argument.
+    fix_oci_process_args(&mut ttrpc_spec, &bundle)?;
+
+    req.set_container_id(c_id);
+    req.set_OCI(ttrpc_spec);
+
+    debug!(sl!(), "CreateContainer request generated successfully");
+
+    Ok(req)
+}
+
+pub fn remove_container_image_mount(c_id: &str) -> Result<()> {
+    image::remove_image_mount(c_id)
+}

src/tools/agent-ctl/template/oci_config.json

@@ -0,0 +1,170 @@
+{
+	"ociVersion": "1.1.0-rc.1-test",
+	"process": {
+		"user": {
+		},
+		"args": [
+			""
+		],
+		"env": [
+			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+			"TERM=xterm"
+		],
+		"cwd": "/",
+		"capabilities": {
+			"bounding": [
+				"CAP_CHOWN",
+				"CAP_DAC_OVERRIDE",
+				"CAP_FSETID",
+				"CAP_FOWNER",
+				"CAP_MKNOD",
+				"CAP_NET_RAW",
+				"CAP_SETGID",
+				"CAP_SETUID",
+				"CAP_SETFCAP",
+				"CAP_SETPCAP",
+				"CAP_NET_BIND_SERVICE",
+				"CAP_SYS_CHROOT",
+				"CAP_KILL",
+				"CAP_AUDIT_WRITE"
+			],
+			"effective": [
+				"CAP_CHOWN",
+				"CAP_DAC_OVERRIDE",
+				"CAP_FSETID",
+				"CAP_FOWNER",
+				"CAP_MKNOD",
+				"CAP_NET_RAW",
+				"CAP_SETGID",
+				"CAP_SETUID",
+				"CAP_SETFCAP",
+				"CAP_SETPCAP",
+				"CAP_NET_BIND_SERVICE",
+				"CAP_SYS_CHROOT",
+				"CAP_KILL",
+				"CAP_AUDIT_WRITE"
+			],
+			"permitted": [
+				"CAP_CHOWN",
+				"CAP_DAC_OVERRIDE",
+				"CAP_FSETID",
+				"CAP_FOWNER",
+				"CAP_MKNOD",
+				"CAP_NET_RAW",
+				"CAP_SETGID",
+				"CAP_SETUID",
+				"CAP_SETFCAP",
+				"CAP_SETPCAP",
+				"CAP_NET_BIND_SERVICE",
+				"CAP_SYS_CHROOT",
+				"CAP_KILL",
+				"CAP_AUDIT_WRITE"
+			]
+		},
+		"noNewPrivileges": true
+	},
+	"root": {
+		"path": "rootfs",
+		"readonly": true
+
+	},
+	"mounts": [
+		{
+			"destination": "/proc",
+			"type": "proc",
+			"source": "proc",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev"
+			]
+		},
+		{
+			"destination": "/dev",
+			"type": "tmpfs",
+			"source": "tmpfs",
+			"options": [
+				"nosuid",
+				"strictatime",
+				"mode=755",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/pts",
+			"type": "devpts",
+			"source": "devpts",
+			"options": [
+				"nosuid",
+				"noexec",
+				"newinstance",
+				"ptmxmode=0666",
+				"mode=0620",
+				"gid=5"
+			]
+		},
+		{
+			"destination": "/dev/shm",
+			"type": "tmpfs",
+			"source": "shm",
+			"options": [
+                		"nosuid",
+                		"noexec",
+                		"nodev",
+                		"mode=1777",
+                		"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/mqueue",
+			"type": "mqueue",
+			"source": "mqueue",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev"
+			]
+		},
+		{
+			"destination": "/sys",
+			"type": "sysfs",
+			"source": "sysfs",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"ro"
+			]
+		}
+	],
+	"annotations": {
+			"io.katacontainers.pkg.oci.container_type": "pod_container",
+			"io.kubernetes.cri.sandbox-id": "",
+			"io.kubernetes.cri.image-name": "" ,
+			"io.kubernetes.cri.container-type": "container",
+			"io.kubernetes.cri.container-name": "",
+			"io.kubernetes.cri.sandbox-namespace": "default",
+			"io.kubernetes.cri.sandbox-name": ""
+	},
+	"linux": {
+		"resources": {
+		},
+		"namespaces": [
+			{
+				"type": "pid"
+			},
+			{
+				"type": "network"
+			},
+			{
+				"type": "ipc"
+			},
+			{
+				"type": "uts"
+			},
+			{
+				"type": "mount"
+			}
+		]
+	}
+}

src/tools/genpolicy/Cargo.lock

@@ -17,17 +17,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
-[[package]]
-name = "ahash"
-version = "0.7.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
-dependencies = [
- "getrandom",
- "once_cell",
- "version_check",
-]
-
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -58,18 +47,6 @@ version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"
 
-[[package]]
-name = "arc-swap"
-version = "1.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
-
-[[package]]
-name = "arrayvec"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
-
 [[package]]
 name = "async-trait"
 version = "0.1.68"
@@ -171,28 +148,6 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
 
-[[package]]
-name = "bitmask-enum"
-version = "2.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afb15541e888071f64592c0b4364fdff21b7cb0a247f984296699351963a8721"
-dependencies = [
- "quote",
- "syn 2.0.58",
-]
-
-[[package]]
-name = "bitvec"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
-dependencies = [
- "funty",
- "radium",
- "tap",
- "wyz",
-]
-
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -202,69 +157,12 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "borsh"
-version = "1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed"
-dependencies = [
- "borsh-derive",
- "cfg_aliases",
-]
-
-[[package]]
-name = "borsh-derive"
-version = "1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b"
-dependencies = [
- "once_cell",
- "proc-macro-crate",
- "proc-macro2",
- "quote",
- "syn 2.0.58",
- "syn_derive",
-]
-
 [[package]]
 name = "bumpalo"
 version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
 
-[[package]]
-name = "byte-unit"
-version = "5.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ac19bdf0b2665407c39d82dbc937e951e7e2001609f0fb32edd0af45a2d63e"
-dependencies = [
- "rust_decimal",
- "serde",
- "utf8-width",
-]
-
-[[package]]
-name = "bytecheck"
-version = "0.6.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627"
-dependencies = [
- "bytecheck_derive",
- "ptr_meta",
- "simdutf8",
-]
-
-[[package]]
-name = "bytecheck_derive"
-version = "0.6.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -289,25 +187,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "cfg_aliases"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
-
-[[package]]
-name = "cgroups-rs"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db7c2f5545da4c12c5701455d9471da5f07db52e49b9cccb4f5512226dd0836"
-dependencies = [
- "libc",
- "log",
- "nix 0.25.1",
- "regex",
- "thiserror",
-]
-
 [[package]]
 name = "chrono"
 version = "0.4.31"
@@ -369,12 +248,6 @@ dependencies = [
  "cc",
 ]
 
-[[package]]
-name = "common-path"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101"
-
 [[package]]
 name = "containerd-client"
 version = "0.4.0"
@@ -489,37 +362,14 @@ dependencies = [
  "typenum",
 ]
 
-[[package]]
-name = "darling"
-version = "0.14.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
-dependencies = [
- "darling_core 0.14.4",
- "darling_macro 0.14.4",
-]
-
 [[package]]
 name = "darling"
 version = "0.20.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1"
 dependencies = [
- "darling_core 0.20.9",
- "darling_macro 0.20.9",
-]
-
-[[package]]
-name = "darling_core"
-version = "0.14.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
-dependencies = [
- "fnv",
- "ident_case",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
+ "darling_core",
+ "darling_macro",
 ]
 
 [[package]]
@@ -536,24 +386,13 @@ dependencies = [
  "syn 2.0.58",
 ]
 
-[[package]]
-name = "darling_macro"
-version = "0.14.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
-dependencies = [
- "darling_core 0.14.4",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "darling_macro"
 version = "0.20.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178"
 dependencies = [
- "darling_core 0.20.9",
+ "darling_core",
  "quote",
  "syn 2.0.58",
 ]
@@ -584,7 +423,7 @@ version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
 dependencies = [
- "darling 0.20.9",
+ "darling",
  "proc-macro2",
  "quote",
  "syn 2.0.58",
@@ -687,17 +526,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "fail"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c"
-dependencies = [
- "log",
- "once_cell",
- "rand",
-]
-
 [[package]]
 name = "fastrand"
 version = "1.9.0"
@@ -783,12 +611,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "funty"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
-
 [[package]]
 name = "futures-channel"
 version = "0.3.28"
@@ -927,12 +749,6 @@ version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
 
-[[package]]
-name = "glob"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
-
 [[package]]
 name = "h2"
 version = "0.3.16"
@@ -957,9 +773,6 @@ name = "hashbrown"
 version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
-dependencies = [
- "ahash",
-]
 
 [[package]]
 name = "hashbrown"
@@ -1003,12 +816,6 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
 
-[[package]]
-name = "hex"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
-
 [[package]]
 name = "hmac"
 version = "0.12.1"
@@ -1136,7 +943,7 @@ dependencies = [
  "iana-time-zone-haiku",
  "js-sys",
  "wasm-bindgen",
- "windows-core 0.50.0",
+ "windows-core",
 ]
 
 [[package]]
@@ -1282,58 +1089,6 @@ dependencies = [
  "tonic-build 0.8.4",
 ]
 
-[[package]]
-name = "kata-sys-util"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "byteorder",
- "cgroups-rs",
- "chrono",
- "common-path",
- "fail",
- "hex",
- "kata-types",
- "lazy_static",
- "libc",
- "nix 0.24.3",
- "oci-spec",
- "once_cell",
- "rand",
- "runtime-spec",
- "safe-path",
- "serde",
- "serde_json",
- "slog",
- "slog-scope",
- "subprocess",
- "thiserror",
-]
-
-[[package]]
-name = "kata-types"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "base64 0.13.1",
- "bitmask-enum",
- "byte-unit",
- "glob",
- "lazy_static",
- "num_cpus",
- "oci-spec",
- "regex",
- "safe-path",
- "serde",
- "serde-enum-str",
- "serde_json",
- "slog",
- "slog-scope",
- "sysinfo",
- "thiserror",
- "toml",
-]
-
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -1404,15 +1159,6 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
-[[package]]
-name = "memoffset"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "memoffset"
 version = "0.7.1"
@@ -1472,30 +1218,6 @@ dependencies = [
  "tempfile",
 ]
 
-[[package]]
-name = "nix"
-version = "0.24.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069"
-dependencies = [
- "bitflags 1.3.2",
- "cfg-if",
- "libc",
- "memoffset 0.6.5",
-]
-
-[[package]]
-name = "nix"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
-dependencies = [
- "autocfg",
- "bitflags 1.3.2",
- "cfg-if",
- "libc",
-]
-
 [[package]]
 name = "nix"
 version = "0.26.4"
@@ -1505,19 +1227,10 @@ dependencies = [
  "bitflags 1.3.2",
  "cfg-if",
  "libc",
- "memoffset 0.7.1",
+ "memoffset",
  "pin-utils",
 ]
 
-[[package]]
-name = "ntapi"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "num"
 version = "0.4.3"
@@ -1809,15 +1522,6 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "proc-macro-crate"
-version = "3.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284"
-dependencies = [
- "toml_edit",
-]
-
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -2026,7 +1730,6 @@ dependencies = [
 name = "protocols"
 version = "0.1.0"
 dependencies = [
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.3.0",
  "serde",
@@ -2035,26 +1738,6 @@ dependencies = [
  "ttrpc-codegen",
 ]
 
-[[package]]
-name = "ptr_meta"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"
-dependencies = [
- "ptr_meta_derive",
-]
-
-[[package]]
-name = "ptr_meta_derive"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "quote"
 version = "1.0.36"
@@ -2064,12 +1747,6 @@ dependencies = [
  "proc-macro2",
 ]
 
-[[package]]
-name = "radium"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
-
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -2100,26 +1777,6 @@ dependencies = [
  "getrandom",
 ]
 
-[[package]]
-name = "rayon"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
-dependencies = [
- "either",
- "rayon-core",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
-dependencies = [
- "crossbeam-deque",
- "crossbeam-utils",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -2184,15 +1841,6 @@ dependencies = [
  "serde_json",
 ]
 
-[[package]]
-name = "rend"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c"
-dependencies = [
- "bytecheck",
-]
-
 [[package]]
 name = "reqwest"
 version = "0.11.14"
@@ -2232,60 +1880,6 @@ dependencies = [
  "winreg",
 ]
 
-[[package]]
-name = "rkyv"
-version = "0.7.42"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58"
-dependencies = [
- "bitvec",
- "bytecheck",
- "hashbrown 0.12.3",
- "ptr_meta",
- "rend",
- "rkyv_derive",
- "seahash",
- "tinyvec",
- "uuid",
-]
-
-[[package]]
-name = "rkyv_derive"
-version = "0.7.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "runtime-spec"
-version = "0.1.0"
-dependencies = [
- "libc",
- "serde",
- "serde_derive",
- "serde_json",
-]
-
-[[package]]
-name = "rust_decimal"
-version = "1.35.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a"
-dependencies = [
- "arrayvec",
- "borsh",
- "bytes",
- "num-traits",
- "rand",
- "rkyv",
- "serde",
- "serde_json",
-]
-
 [[package]]
 name = "rustc-demangle"
 version = "0.1.23"
@@ -2345,13 +1939,6 @@ version = "1.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
 
-[[package]]
-name = "safe-path"
-version = "0.1.0"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "schannel"
 version = "0.1.21"
@@ -2381,12 +1968,6 @@ dependencies = [
  "syn 2.0.58",
 ]
 
-[[package]]
-name = "seahash"
-version = "4.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
-
 [[package]]
 name = "security-framework"
 version = "2.8.2"
@@ -2419,36 +2000,6 @@ dependencies = [
  "serde_derive",
 ]
 
-[[package]]
-name = "serde-attributes"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6eb8ec7724e4e524b2492b510e66957fe1a2c76c26a6975ec80823f2439da685"
-dependencies = [
- "darling_core 0.14.4",
- "serde-rename-rule",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "serde-enum-str"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26416dc95fcd46b0e4b12a3758043a229a6914050aaec2e8191949753ed4e9aa"
-dependencies = [
- "darling 0.14.4",
- "proc-macro2",
- "quote",
- "serde-attributes",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "serde-rename-rule"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "794e44574226fc701e3be5c651feb7939038fc67fb73f6f4dd5c4ba90fd3be70"
-
 [[package]]
 name = "serde-transcode"
 version = "1.1.1"
@@ -2524,12 +2075,6 @@ dependencies = [
  "digest",
 ]
 
-[[package]]
-name = "simdutf8"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
-
 [[package]]
 name = "slab"
 version = "0.4.8"
@@ -2539,23 +2084,6 @@ dependencies = [
  "autocfg",
 ]
 
-[[package]]
-name = "slog"
-version = "2.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06"
-
-[[package]]
-name = "slog-scope"
-version = "4.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786"
-dependencies = [
- "arc-swap",
- "lazy_static",
- "slog",
-]
-
 [[package]]
 name = "socket2"
 version = "0.4.9"
@@ -2607,16 +2135,6 @@ dependencies = [
  "syn 2.0.58",
 ]
 
-[[package]]
-name = "subprocess"
-version = "0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086"
-dependencies = [
- "libc",
- "winapi",
-]
-
 [[package]]
 name = "subtle"
 version = "2.4.1"
@@ -2645,45 +2163,12 @@ dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "syn_derive"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b"
-dependencies = [
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn 2.0.58",
-]
-
 [[package]]
 name = "sync_wrapper"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
-[[package]]
-name = "sysinfo"
-version = "0.30.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3"
-dependencies = [
- "cfg-if",
- "core-foundation-sys",
- "libc",
- "ntapi",
- "once_cell",
- "rayon",
- "windows",
-]
-
-[[package]]
-name = "tap"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
-
 [[package]]
 name = "tar"
 version = "0.4.41"
@@ -2843,32 +2328,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "toml"
-version = "0.5.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "toml_datetime"
-version = "0.6.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"
-
-[[package]]
-name = "toml_edit"
-version = "0.21.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
-dependencies = [
- "indexmap 2.2.3",
- "toml_datetime",
- "winnow",
-]
-
 [[package]]
 name = "tonic"
 version = "0.9.2"
@@ -3004,7 +2463,7 @@ dependencies = [
  "crossbeam",
  "libc",
  "log",
- "nix 0.26.4",
+ "nix",
  "protobuf 3.3.0",
  "protobuf-codegen 3.3.0",
  "thiserror",
@@ -3091,18 +2550,6 @@ dependencies = [
  "percent-encoding",
 ]
 
-[[package]]
-name = "utf8-width"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
-
-[[package]]
-name = "uuid"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314"
-
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
@@ -3263,16 +2710,6 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
-[[package]]
-name = "windows"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
-dependencies = [
- "windows-core 0.52.0",
- "windows-targets 0.52.6",
-]
-
 [[package]]
 name = "windows-core"
 version = "0.50.0"
@@ -3282,15 +2719,6 @@ dependencies = [
  "windows-targets 0.48.0",
 ]
 
-[[package]]
-name = "windows-core"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
-dependencies = [
- "windows-targets 0.52.6",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.42.0"
@@ -3511,15 +2939,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
-[[package]]
-name = "winnow"
-version = "0.5.40"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "winreg"
 version = "0.10.1"
@@ -3529,15 +2948,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "wyz"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed"
-dependencies = [
- "tap",
-]
-
 [[package]]
 name = "xattr"
 version = "1.3.1"

src/tools/genpolicy/rules.rego

@@ -51,7 +51,7 @@ default WriteStreamRequest := false
 # them and inspect OPA logs for the root cause of a failure.
 default AllowRequestsFailingPolicy := false
 
-CreateContainerRequest {
+CreateContainerRequest:= {"ops": ops, "allowed": true} {
     # Check if the input request should be rejected even before checking the
     # policy_data.containers information.
     allow_create_container_input
@@ -60,6 +60,14 @@ CreateContainerRequest {
     i_storages := input.storages
     i_devices := input.devices
 
+    # array of possible state operations
+    ops_builder := []
+
+    # check sandbox name
+    sandbox_name = i_oci.Annotations["io.kubernetes.cri.sandbox-name"]
+    add_sandbox_name_to_state := state_allows("sandbox_name", sandbox_name)
+    ops := concat_op_if_not_null(ops_builder, add_sandbox_name_to_state)
+
     # Check if any element from the policy_data.containers array allows the input request.
     some p_container in policy_data.containers
     print("======== CreateContainerRequest: trying next policy container")
@@ -121,6 +129,47 @@ allow_create_container_input {
     print("allow_create_container_input: true")
 }
 
+# value hasn't been seen before, save it to state
+state_allows(key, value) = action {
+  state := get_state()
+  not state[key]
+  print("state_allows: saving to state key =", key, "value =", value)
+  path := get_state_path(key) 
+  action := {
+    "op": "add",
+    "path": path, 
+    "value": value,
+  }
+}
+
+# value matches what's in state, allow it
+state_allows(key, value) = action {
+  state := get_state()
+  value == state[key]
+  print("state_allows: found key =", key, "value =", value, " in state")
+  action := null
+}
+
+# helper functions to interact with the state
+get_state() = state {
+  state := data["pstate"]
+}
+
+get_state_path(key) = path {
+    path := concat("/", ["", key]) # prepend "/" to key
+}
+
+# Helper functions to conditionally concatenate if op is not null
+concat_op_if_not_null(ops, op) = result {
+    op == null
+    result := ops
+}
+
+concat_op_if_not_null(ops, op) = result {
+    op != null
+    result := array.concat(ops, [op])
+}
+
 # Reject unexpected annotations.
 allow_anno(p_oci, i_oci) {
     print("allow_anno 1: start")

src/tools/kata-ctl/Cargo.lock

@@ -27,7 +27,7 @@ dependencies = [
  "log",
  "logging",
  "nix 0.24.3",
- "oci",
+ "oci-spec",
  "protobuf 3.3.0",
  "protocols",
  "serde",
@@ -329,19 +329,6 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
-[[package]]
-name = "cgroups-rs"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db7c2f5545da4c12c5701455d9471da5f07db52e49b9cccb4f5512226dd0836"
-dependencies = [
- "libc",
- "log",
- "nix 0.25.1",
- "regex",
- "thiserror",
-]
-
 [[package]]
 name = "chrono"
 version = "0.4.31"
@@ -376,7 +363,7 @@ dependencies = [
  "anstream",
  "anstyle",
  "clap_lex",
- "strsim",
+ "strsim 0.10.0",
 ]
 
 [[package]]
@@ -523,12 +510,12 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.20.3"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
+checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
 dependencies = [
- "darling_core 0.20.3",
- "darling_macro 0.20.3",
+ "darling_core 0.20.10",
+ "darling_macro 0.20.10",
 ]
 
 [[package]]
@@ -546,15 +533,15 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.20.3"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621"
+checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
- "strsim",
+ "strsim 0.11.1",
  "syn 2.0.39",
 ]
 
@@ -571,11 +558,11 @@ dependencies = [
 
 [[package]]
 name = "darling_macro"
-version = "0.20.3"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
+checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
- "darling_core 0.20.3",
+ "darling_core 0.20.10",
  "quote",
  "syn 2.0.39",
 ]
@@ -601,6 +588,37 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "derive_builder"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
+dependencies = [
+ "derive_builder_macro",
+]
+
+[[package]]
+name = "derive_builder_core"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
+dependencies = [
+ "darling 0.20.10",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.39",
+]
+
+[[package]]
+name = "derive_builder_macro"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
+dependencies = [
+ "derive_builder_core",
+ "syn 2.0.39",
+]
+
 [[package]]
 name = "dirs-next"
 version = "2.0.0"
@@ -832,6 +850,18 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "getset"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f636605b743120a8d32ed92fc27b6cde1a769f8f936c065151eb66f88ded513c"
+dependencies = [
+ "proc-macro-error2",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.39",
+]
+
 [[package]]
 name = "gimli"
 version = "0.28.1"
@@ -893,6 +923,12 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
 
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.19"
@@ -1167,8 +1203,8 @@ dependencies = [
  "shim-interface",
  "slog",
  "slog-scope",
- "strum",
- "strum_macros",
+ "strum 0.24.1",
+ "strum_macros 0.24.3",
  "sys-info",
  "tempfile",
  "test-utils",
@@ -1186,7 +1222,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -1195,9 +1230,10 @@ dependencies = [
  "lazy_static",
  "libc",
  "nix 0.24.3",
- "oci",
+ "oci-spec",
  "once_cell",
  "rand",
+ "runtime-spec",
  "safe-path",
  "serde",
  "serde_json",
@@ -1218,7 +1254,7 @@ dependencies = [
  "glob",
  "lazy_static",
  "num_cpus",
- "oci",
+ "oci-spec",
  "regex",
  "safe-path",
  "serde",
@@ -1478,20 +1514,27 @@ dependencies = [
 ]
 
 [[package]]
-name = "oci"
-version = "0.1.0"
+name = "oci-spec"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f5a3fe998d50101ae009351fec56d88a69f4ed182e11000e711068c2f5abf72"
 dependencies = [
- "libc",
+ "derive_builder",
+ "getset",
+ "once_cell",
+ "regex",
  "serde",
- "serde_derive",
  "serde_json",
+ "strum 0.26.3",
+ "strum_macros 0.26.4",
+ "thiserror",
 ]
 
 [[package]]
 name = "once_cell"
-version = "1.18.0"
+version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
 [[package]]
 name = "openssl"
@@ -1694,10 +1737,32 @@ dependencies = [
 ]
 
 [[package]]
-name = "proc-macro2"
-version = "1.0.70"
+name = "proc-macro-error-attr2"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b"
+checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+]
+
+[[package]]
+name = "proc-macro-error2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
+dependencies = [
+ "proc-macro-error-attr2",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.39",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a"
 dependencies = [
  "unicode-ident",
 ]
@@ -1879,8 +1944,10 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "oci",
+ "oci-spec",
  "protobuf 3.3.0",
+ "serde",
+ "serde_json",
  "ttrpc 0.8.1",
  "ttrpc-codegen",
 ]
@@ -1917,9 +1984,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.33"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -2011,9 +2078,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.10.2"
+version = "1.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
+checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -2023,9 +2090,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.3"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
+checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -2145,6 +2212,16 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "runtime-spec"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "serde",
+ "serde_derive",
+ "serde_json",
+]
+
 [[package]]
 name = "rust_decimal"
 version = "1.35.0"
@@ -2399,7 +2476,7 @@ version = "2.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f"
 dependencies = [
- "darling 0.20.3",
+ "darling 0.20.10",
  "proc-macro2",
  "quote",
  "syn 2.0.39",
@@ -2569,12 +2646,24 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
 [[package]]
 name = "strum"
 version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
 
+[[package]]
+name = "strum"
+version = "0.26.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
+
 [[package]]
 name = "strum_macros"
 version = "0.24.3"
@@ -2588,6 +2677,19 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "strum_macros"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 2.0.39",
+]
+
 [[package]]
 name = "subprocess"
 version = "0.2.9"

src/tools/runk/Cargo.lock

@@ -1391,7 +1391,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -2068,9 +2067,10 @@ dependencies = [
 name = "protocols"
 version = "0.1.0"
 dependencies = [
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.5.0",
+ "serde",
+ "serde_json",
  "ttrpc",
  "ttrpc-codegen",
 ]

tests/functional/kata-agent-apis/api-tests/test_create_container.bats

@@ -0,0 +1,43 @@
+#!/usr/bin/env bats
+
+# Copyright (c) 2024 Microsoft Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load "${BATS_TEST_DIRNAME}/../../../common.bash"
+load "${BATS_TEST_DIRNAME}/../setup_common.sh"
+
+setup_file() {
+    info "setup"
+}
+
+@test "Test CreateContainer API: Create a container" {
+    info "Create a container"
+    sandbox_id=$RANDOM
+    container_id="test_container_${RANDOM}"
+    
+    local cmds=()
+    cmds+="-c 'CreateSandbox json://{\"sandbox_id\": \"$sandbox_id\"}'"
+    run_agent_ctl "${cmds[@]}"
+
+    local image="ghcr.io/linuxcontainers/alpine:latest"
+    local cmds=()
+    cmds+="-c 'CreateContainer json://{\"image\": \"$image\", \"id\": \"$container_id\"}'"
+    run_agent_ctl "${cmds[@]}"
+    info "Container created successfully."
+
+    local cmds=()
+    cmds+="-c 'StartContainer json://{\"container_id\": \"$container_id\"}'"
+    run_agent_ctl "${cmds[@]}"
+    info "Container process started"
+
+    local cmds=()
+    cmds+="-c 'RemoveContainer json://{\"container_id\": \"$container_id\"}'"
+    run_agent_ctl "${cmds[@]}"
+    info "Container removed."
+}
+
+teardown_file() {
+    info "teardown"
+    sudo rm -r /run/kata-containers/ || echo "Failed to clean /run/kata-containers"
+}

tests/functional/kata-agent-apis/setup_common.sh

@@ -79,7 +79,7 @@ run_agent_ctl()
 
 	[ -n "$cmds" ] || die "need commands for agent control tool"
 
-	local redirect="&>\"${ctl_log_file}\""
+	local redirect=">> ${ctl_log_file} 2>&1"
 
 	local server_address="--server-address ${local_agent_server_addr}"
 

tests/gha-adjust-to-use-prebuilt-components.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo_root_dir="$(cd "${this_script_dir}/../" && pwd)"
+
+base_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build"
+build_dir="${base_dir}/build"
+
+function main() {
+    artifacts_dir="${1:-}"
+    asset="${2:-}"
+
+    if [ -z "${artifacts_dir}" ]; then
+        echo "The artefacts directory must be passed as the first argument to this script."
+        exit 1
+    fi
+
+    if [ -z "${asset}" ]; then
+        echo "The asset must be passed as the second argument to this script."
+        exit 1
+    fi
+
+    mv ${artifacts_dir} ${build_dir}
+    sed -i "s/\(^${asset}-tarball:\).*/\1/g" ${base_dir}/Makefile
+}
+
+main "$@"

tests/gha-run-k8s-common.sh

@@ -101,14 +101,11 @@ function create_cluster() {
 }
 
 function install_bats() {
-	# Installing bats from the lunar repo.
-	# This installs newer version of the bats which supports setup_file and teardown_file functions.
-	# These functions are helpful when adding new tests that require one time setup.
-
+	# Installing bats from the noble repo.
 	sudo apt install -y software-properties-common
-	sudo add-apt-repository 'deb http://archive.ubuntu.com/ubuntu/ lunar universe'
+	sudo add-apt-repository 'deb http://archive.ubuntu.com/ubuntu/ noble universe'
 	sudo apt install -y bats
-	sudo add-apt-repository --remove 'deb http://archive.ubuntu.com/ubuntu/ lunar universe'
+	sudo add-apt-repository --remove 'deb http://archive.ubuntu.com/ubuntu/ noble universe'
 }
 
 function install_kubectl() {

tests/integration/kubernetes/confidential_kbs.sh

@@ -18,6 +18,7 @@ export PATH="${PATH}:/opt/kata/bin"
 
 KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
 ITA_KEY="${ITA_KEY:-}"
+HTTPS_PROXY="${HTTPS_PROXY:-}"
 # Where the trustee (includes kbs) sources will be cloned
 readonly COCO_TRUSTEE_DIR="/tmp/trustee"
 # Where the kbs sources will be cloned
@@ -85,13 +86,12 @@ kbs_set_resource_base64() {
 	fi
 
 	file=$(mktemp -t kbs-resource-XXXXX)
-	trap "rm -f $file" EXIT
-
 	echo "$data" | base64 -d > "$file"
 
 	kbs_set_resource_from_file "$repository" "$type" "$tag" "$file" || \
 		rc=$?
 
+	rm -f "$file"
 	return $rc
 }
 
@@ -117,12 +117,12 @@ kbs_set_resource() {
 	fi
 
 	file=$(mktemp -t kbs-resource-XXXXX)
-	trap "rm -f $file" EXIT
 	echo "$data" > "$file"
 
 	kbs_set_resource_from_file "$repository" "$type" "$tag" "$file" || \
 		rc=$?
 
+	rm -f "$file"
 	return $rc
 }
 
@@ -323,6 +323,20 @@ function kbs_k8s_deploy() {
 			# ITA/ITTS specific configuration
 			sed -i -e "s/tBfd5kKX2x9ahbodKV1.../${ITA_KEY}/g" kbs-config.toml
 		popd
+		
+		if [ -n "${HTTPS_PROXY}" ]; then
+			# Ideally this should be something kustomizable on trustee side.
+			#
+			# However, for now let's take the bullet and do it here, and revert this as
+			# soon as https://github.com/confidential-containers/trustee/issues/567 is
+			# solved.
+			pushd "${COCO_KBS_DIR}/config/kubernetes/base/"
+				ensure_yq
+				
+				yq e ".spec.template.spec.containers[0].env += [{\"name\": \"https_proxy\", \"value\": \"$HTTPS_PROXY\"}]" -i deployment.yaml
+			popd
+		fi
+
 		export DEPLOYMENT_DIR=ita
 	fi
 

tests/integration/kubernetes/filter_out_per_arch/aarch64.yaml

@@ -23,5 +23,4 @@ kubernetes:
   - k8s-limit-range
   - k8s-number-cpus
   - k8s-oom
-  - k8s-qos-pods
-  - k8s-pod-manifest-v1.bats
+  - k8s-qos-pods

tests/integration/kubernetes/filter_out_per_arch/ppc64le.yaml

@@ -11,5 +11,4 @@ kubernetes:
   - k8s-inotify
   - k8s-limit-range
   - k8s-number-cpus
-  - k8s-oom
-  - k8s-pod-manifest-v1.bats
+  - k8s-oom

tests/integration/kubernetes/filter_out_per_arch/s390x.yaml

@@ -6,5 +6,4 @@
 kubernetes:
   - k8s-caps
   - k8s-inotify
-  - k8s-sandbox-vcpus-allocation # see https://github.com/kata-containers/kata-containers/issues/9093
-  - k8s-pod-manifest-v1.bats
+  - k8s-sandbox-vcpus-allocation # see https://github.com/kata-containers/kata-containers/issues/9093

tests/integration/kubernetes/gha-run.sh

@@ -26,6 +26,7 @@ DOCKER_TAG=${DOCKER_TAG:-kata-containers-latest}
 KATA_DEPLOY_WAIT_TIMEOUT=${KATA_DEPLOY_WAIT_TIMEOUT:-600}
 SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT=${SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT:-8m}
 KATA_HYPERVISOR=${KATA_HYPERVISOR:-qemu}
+CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-containerd}
 KBS=${KBS:-false}
 KBS_INGRESS=${KBS_INGRESS:-}
 KUBERNETES="${KUBERNETES:-}"
@@ -80,21 +81,37 @@ EOF
 			containerd_config_file="/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl"
 			sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml "${containerd_config_file}"
 			;;
+		kubeadm)
+			containerd_config_file="/etc/containerd/config.toml"
+			;;
 		*) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;;
 	esac
 
 	# We're not using this with baremetal machines, so we're fine on cutting
 	# corners here and just append this to the configuration file.
-	cat<<EOF | sudo tee -a "${containerd_config_file}"
+	# Check if the "devmapper" plugin section exists in the file
+	if grep -q 'plugins."io.containerd.snapshotter.v1.devmapper"' "${containerd_config_file}"; then
+	    echo "devmapper section found. Updating pool_name and base_image_size..."
+	    sudo sed -i '/\[plugins."io.containerd.snapshotter.v1.devmapper"\]/,/\[plugins\./ {
+	        s/pool_name = ".*"/pool_name = "contd-thin-pool"/
+	        s/base_image_size = ".*"/base_image_size = "4096MB"/
+	    }' "${containerd_config_file}"
+	else
+	    echo "devmapper section not found. Appending to the config file..."
+		cat<<EOF | sudo tee -a "${containerd_config_file}"
 [plugins."io.containerd.snapshotter.v1.devmapper"]
   pool_name = "contd-thin-pool"
   base_image_size = "4096MB"
 EOF
+	fi
 
 	case "${KUBERNETES}" in
 		k3s)
 			sudo sed -i -e 's/snapshotter = "overlayfs"/snapshotter = "devmapper"/g' "${containerd_config_file}"
 			sudo systemctl restart k3s ;;
+		kubeadm)
+			sudo sed -i -e 's/snapshotter = "overlayfs"/snapshotter = "devmapper"/g' "${containerd_config_file}"
+			sudo systemctl restart containerd ;;
 		*) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;;
 	esac
 

tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats

@@ -16,7 +16,7 @@ setup() {
 
     [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one"
 
-    setup_common
+    setup_common || die "setup_common failed"
     AUTHENTICATED_IMAGE="${AUTHENTICATED_IMAGE:-quay.io/kata-containers/confidential-containers-auth:test}"
     AUTHENTICATED_IMAGE_USER=${AUTHENTICATED_IMAGE_USER:-}
     AUTHENTICATED_IMAGE_PASSWORD=${AUTHENTICATED_IMAGE_PASSWORD:-}

tests/integration/kubernetes/k8s-guest-pull-image-encrypted.bats

@@ -20,7 +20,7 @@ setup() {
 
     [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one"
 
-    setup_common
+    setup_common || die "setup_common failed"
     ENCRYPTED_IMAGE="${ENCRYPTED_IMAGE:-ghcr.io/confidential-containers/test-container:multi-arch-encrypted}"
     DECRYPTION_KEY="${DECRYPTION_KEY:-HUlOu8NWz8si11OZUzUJMnjiq/iZyHBJZMSD3BaqgMc=}"
     DECRYPTION_KEY_ID="${DECRYPTION_KEY_ID:-ssh-demo}"

tests/integration/kubernetes/k8s-guest-pull-image-signature.bats

@@ -22,7 +22,7 @@ setup() {
         tag_suffix="-$(uname -m)"
     fi
 
-    setup_common
+    setup_common || die "setup_common failed"
     UNSIGNED_UNPROTECTED_REGISTRY_IMAGE="quay.io/prometheus/busybox:latest"
     UNSIGNED_PROTECTED_REGISTRY_IMAGE="ghcr.io/confidential-containers/test-container-image-rs:unsigned${tag_suffix}"
     COSIGN_SIGNED_PROTECTED_REGISTRY_IMAGE="ghcr.io/confidential-containers/test-container-image-rs:cosign-signed${tag_suffix}"

tests/integration/kubernetes/k8s-guest-pull-image.bats

@@ -15,7 +15,7 @@ setup() {
 
     [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one"
 
-    setup_common
+    setup_common || die "setup_common failed"
     get_pod_config_dir
     unencrypted_image="quay.io/prometheus/busybox:latest"
     image_pulled_time_less_than_default_time="ghcr.io/confidential-containers/test-container:rust-1.79.0" # unpacked size: 1.41GB
@@ -171,10 +171,10 @@ setup() {
     echo "Pod $pod_config file:"
     cat $pod_config
 
-    # The pod should be failed because the default timeout of CreateContainerRequest is 60s
+    # The pod should be failed because the image is too large to be pulled in the timeout
     assert_pod_fail "$pod_config"
-    assert_logs_contain "$node" kata "$node_start_time" \
-		'CreateContainerRequest timed out'
+    assert_logs_contain "$node" kata "$node_start_time" 'createContainer failed'
+    assert_logs_contain "$node" kata "$node_start_time" 'timeout'
 }
 
 @test "Test we can pull a large image inside the guest with large createcontainer timeout" {

tests/integration/kubernetes/k8s-liveness-probes.bats

@@ -14,7 +14,7 @@ setup() {
 	agnhost_name="${container_images_agnhost_name}"
 	agnhost_version="${container_images_agnhost_version}"
 
-	setup_common
+	setup_common || die "setup_common failed"
 	get_pod_config_dir
 }
 

tests/integration/kubernetes/k8s-measured-rootfs.bats

@@ -5,31 +5,33 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+load "${BATS_TEST_DIRNAME}/../../common.bash"
 load "${BATS_TEST_DIRNAME}/lib.sh"
 load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 check_and_skip() {
-	# Currently the kernel-confidential, isn't built withh measured rootfs support, so this test
-	# should be skipped until it is
-	# See https://github.com/kata-containers/kata-containers/issues/9612,
-	# https://github.com/kata-containers/kata-containers/issues/7235
-	# and https://github.com/kata-containers/kata-containers/issues/7415
-	skip "measured rootfs tests not implemented for hypervisor: $KATA_HYPERVISOR"
+	case "${KATA_HYPERVISOR}" in
+		qemu-tdx|qemu-coco-dev)
+			return
+			;;
+		*)
+			skip "measured rootfs tests not implemented for hypervisor: $KATA_HYPERVISOR"
+			;;
+	esac
 }
 
 setup() {
 	check_and_skip
-	setup_common
+	setup_common || die "setup_common failed"
 }
 
 @test "Test cannnot launch pod with measured boot enabled and incorrect hash" {
 	pod_config="$(new_pod_config nginx "kata-${KATA_HYPERVISOR}")"
 
-	incorrect_hash="5180b1568c2ba972e4e06ee0a55976acae8329f2a5d1d2004395635e1ec4a76e"
+	incorrect_hash="1111111111111111111111111111111111111111111111111111111111111111"
 
-	# Despite the kernel being built with support, it is not currently enabled
-	# on configuration.toml. To avoid editing that file on the worker node,
-	# here it will be enabled via pod annotations.
+	# To avoid editing that file on the worker node, here it will be
+	# enabled via pod annotations.
 	set_metadata_annotation "$pod_config" \
 		"io.katacontainers.config.hypervisor.kernel_params" \
 		"rootfs_verity.scheme=dm-verity rootfs_verity.hash=$incorrect_hash"
@@ -45,10 +47,9 @@ setup() {
 	echo "Pod $pod_config file:"
 	cat $pod_config
 
-	assert_pod_fail "$pod_config"
+	kubectl apply -f $pod_config
 
-	assert_logs_contain "$node" kata "$node_start_time" \
-		'verity: .* metadata block .* is corrupted'
+	waitForProcess "60" "3" "exec_host $node journalctl -t kata | grep \"verity: .* metadata block .* is corrupted\""
 }
 
 teardown() {

tests/integration/kubernetes/k8s-nginx-connectivity.bats

@@ -9,6 +9,8 @@ load "${BATS_TEST_DIRNAME}/../../common.bash"
 load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 setup() {
+	[ "${CONTAINER_RUNTIME}" == "crio" ] && skip "test not working see: https://github.com/kata-containers/kata-containers/issues/10414"
+
 	nginx_version="${docker_images_nginx_version}"
 	nginx_image="nginx:$nginx_version"
 	busybox_image="quay.io/prometheus/busybox:latest"
@@ -42,6 +44,8 @@ setup() {
 }
 
 teardown() {
+	[ "${CONTAINER_RUNTIME}" == "crio" ] && skip "test not working see: https://github.com/kata-containers/kata-containers/issues/10414"
+
 	# Debugging information
 	kubectl describe "pod/$busybox_pod"
 	kubectl get "pod/$busybox_pod" -o yaml

tests/integration/kubernetes/k8s-pod-manifest-v1.bats

@@ -1,31 +0,0 @@
-#!/usr/bin/env bats
-#
-# Copyright (c) 2024 Microsoft.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-load "${BATS_TEST_DIRNAME}/../../common.bash"
-load "${BATS_TEST_DIRNAME}/tests_common.sh"
-
-setup() {
-	get_pod_config_dir
-	pod_name="nginxhttps"
-	pod_yaml="${pod_config_dir}/pod-manifest-v1.yaml"
-	auto_generate_policy "${pod_config_dir}" "${pod_yaml}"
-}
-
-@test "Deploy manifest v1 pod" {
-
-	kubectl create -f "${pod_yaml}"
-
-	# Wait for pod to start
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
-}
-
-teardown() {
-	# Debugging information
-	kubectl describe "pod/$pod_name"
-
-	kubectl delete pod "$pod_name"
-}