Merge pull request #10620 from kata-containers/topic/fix-remove-artifact-ordering

workflows: Remove potential timing issues with artifacts

.github/actionlint.yaml

@@ -21,4 +21,5 @@ self-hosted-runner:
     - sev-snp
     - s390x
     - s390x-large
-    - tdx
+    - tdx-no-attestation
+    - tdx-attestation

.github/workflows/actionlint.yaml

@@ -0,0 +1,33 @@
+name: Lint GHA workflows
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths:
+      - '.github/workflows/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-actionlint:
+    env:
+      GH_TOKEN: ${{ github.token }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install actionlint gh extension
+        run: gh extension install https://github.com/cschleiden/gh-actionlint
+
+      - name: Run actionlint
+        run:  gh actionlint

.github/workflows/add-issues-to-project.yaml

@@ -33,7 +33,7 @@ jobs:
         run: |
           # Clone into a temporary directory to avoid overwriting
           # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
+          pushd "$(mktemp -d)" &>/dev/null
           git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
           sudo install hub-util.sh /usr/local/bin
           popd &>/dev/null

.github/workflows/add-pr-sizing-label.yaml

@@ -36,7 +36,7 @@ jobs:
         run: |
           # Clone into a temporary directory to avoid overwriting
           # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
+          pushd "$(mktemp -d)" &>/dev/null
           git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
           sudo install pr-add-size-label.sh /usr/local/bin
           popd &>/dev/null

.github/workflows/basic-ci-amd64.yaml

@@ -138,6 +138,8 @@ jobs:
         run: bash tests/integration/nydus/gha-run.sh run
 
   run-runk:
+    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
+    if: false
     runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: lts
@@ -260,6 +262,8 @@ jobs:
         vmm:
           - clh
           - qemu
+          - dragonball
+          - cloud-hypervisor
     runs-on: ubuntu-22.04
     env:
       KATA_HYPERVISOR: ${{ matrix.vmm }}

.github/workflows/build-checks.yaml

@@ -19,7 +19,6 @@ jobs:
           - runtime-rs
           - agent-ctl
           - kata-ctl
-          - runk
           - trace-forwarder
           - genpolicy
         command:
@@ -40,22 +39,18 @@ jobs:
             component-path: src/tools/agent-ctl
           - component: kata-ctl
             component-path: src/tools/kata-ctl
-          - component: runk
-            component-path: src/tools/runk
           - component: trace-forwarder
             component-path: src/tools/trace-forwarder
           - install-libseccomp: no
           - component: agent
             install-libseccomp: yes
-          - component: runk
-            install-libseccomp: yes
           - component: genpolicy
             component-path: src/tools/genpolicy
     steps:
       - name: Adjust a permission for repo
         run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE $HOME
-          sudo rm -rf $GITHUB_WORKSPACE/* && echo "GITHUB_WORKSPACE removed" || { sleep 10 && sudo rm -rf $GITHUB_WORKSPACE/*; }
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE" "$HOME"
+          sudo rm -rf "$GITHUB_WORKSPACE"/* || { sleep 10 && sudo rm -rf "$GITHUB_WORKSPACE"/*; }
           sudo rm -f /tmp/kata_hybrid*  # Sometime we got leftover from test_setup_hvsock_failed()
 
       - name: Checkout the code
@@ -72,12 +67,12 @@ jobs:
         if: ${{ matrix.component == 'runtime' }}
         run: |
           ./tests/install_go.sh -f -p
-          echo "/usr/local/go/bin" >> $GITHUB_PATH
+          echo "/usr/local/go/bin" >> "$GITHUB_PATH"
       - name: Install rust
         if: ${{ matrix.component != 'runtime' }}
         run: |
           ./tests/install_rust.sh
-          echo "${HOME}/.cargo/bin" >> $GITHUB_PATH
+          echo "${HOME}/.cargo/bin" >> "$GITHUB_PATH"
       - name: Install musl-tools
         if: ${{ matrix.component != 'runtime' }}
         run: sudo apt-get -y install musl-tools
@@ -91,19 +86,19 @@ jobs:
           gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
           ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
           echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-          echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-          echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
+          echo "LIBSECCOMP_LINK_TYPE=static" >> "$GITHUB_ENV"
+          echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> "$GITHUB_ENV"
       - name: Install protobuf-compiler
-        if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'runk' || matrix.component == 'genpolicy') }}
+        if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'genpolicy' || matrix.component == 'agent-ctl') }}
         run: sudo apt-get -y install protobuf-compiler
       - name: Install clang
-        if: ${{ matrix.command == 'make check' && matrix.component == 'agent' }}
+        if: ${{ matrix.command == 'make check' && (matrix.component == 'agent' || matrix.component == 'agent-ctl') }}
         run: sudo apt-get -y install clang
       - name: Setup XDG_RUNTIME_DIR for the `runtime` tests
         if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }}
         run: |
-          XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700))
-          echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> $GITHUB_ENV
+          XDG_RUNTIME_DIR=$(mktemp -d "/tmp/kata-tests-$USER.XXX" | tee >(xargs chmod 0700))
+          echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> "$GITHUB_ENV"
       - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
         run: |
           cd ${{ matrix.component-path }}

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -24,6 +24,11 @@ on:
 jobs:
   build-asset:
     runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
     strategy:
       matrix:
         asset:
@@ -32,6 +37,7 @@ jobs:
           - cloud-hypervisor
           - cloud-hypervisor-glibc
           - coco-guest-components
+          - csi-kata-directvolume
           - firecracker
           - genpolicy
           - kata-ctl
@@ -48,13 +54,6 @@ jobs:
           - qemu
           - qemu-snp-experimental
           - stratovirt
-          - rootfs-image
-          - rootfs-image-confidential
-          - rootfs-initrd
-          - rootfs-initrd-confidential
-          - rootfs-initrd-mariner
-          - runk
-          - shim-v2
           - trace-forwarder
           - virtiofsd
         stage:
@@ -62,6 +61,8 @@ jobs:
         exclude:
           - asset: cloud-hypervisor-glibc
             stage: release
+    env:
+      PERFORM_ATTESTATION: ${{ matrix.asset == 'agent' && inputs.push-to-registry == 'yes' && 'yes' || 'no' }}
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -83,11 +84,104 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Build ${{ matrix.asset }}
+        id: build
         run: |
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: Parse OCI image name and digest
+        id: parse-oci-segments
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        run: |
+          oci_image="$(<"build/${{ matrix.asset }}-oci-image")"
+          echo "oci-name=${oci_image%@*}" >> "$GITHUB_OUTPUT"
+          echo "oci-digest=${oci_image#*@}" >> "$GITHUB_OUTPUT"
+
+      - uses: oras-project/setup-oras@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          version: "1.2.0"
+
+      # for pushing attestations to the registry
+      - uses: docker/login-action@v3
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/attest-build-provenance@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          subject-name: ${{ steps.parse-oci-segments.outputs.oci-name }}
+          subject-digest: ${{ steps.parse-oci-segments.outputs.oci-digest }}
+          push-to-registry: true
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-rootfs:
+    runs-on: ubuntu-22.04
+    needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-image-confidential
+          - rootfs-image-mariner
+          - rootfs-initrd
+          - rootfs-initrd-confidential
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -99,7 +193,6 @@ jobs:
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ matrix.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -107,9 +200,82 @@ jobs:
           retention-days: 15
           if-no-files-found: error
 
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - agent
+          - coco-guest-components
+          - pause-image
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: ubuntu-22.04
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-amd64-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
   create-kata-tarball:
     runs-on: ubuntu-22.04
-    needs: build-asset
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -35,9 +35,6 @@ jobs:
           - nydus
           - qemu
           - stratovirt
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
           - virtiofsd
     steps:
       - name: Login to Kata Containers quay.io
@@ -64,7 +61,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -76,7 +73,6 @@ jobs:
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -84,13 +80,142 @@ jobs:
           retention-days: 15
           if-no-files-found: error
 
-  create-kata-tarball:
+  build-asset-rootfs:
     runs-on: arm64-builder
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-initrd
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - agent
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: arm64-builder
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-arm64-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: arm64-builder
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - name: Adjust a permission for repo
         run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
 
       - uses: actions/checkout@v4
         with:

.github/workflows/build-kata-static-tarball-ppc64le.yaml

@@ -30,16 +30,14 @@ jobs:
           - agent
           - kernel
           - qemu
-          - rootfs-initrd
-          - shim-v2
           - virtiofsd
         stage:
           - ${{ inputs.stage }}
     steps:
       - name: Prepare the self-hosted runner
         run: |
-            ${HOME}/scripts/prepare_runner.sh
-            sudo rm -rf $GITHUB_WORKSPACE/*
+            "${HOME}/scripts/prepare_runner.sh"
+            sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -65,7 +63,7 @@ jobs:
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -77,7 +75,6 @@ jobs:
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || matrix.asset != 'agent' }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -85,13 +82,153 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
-  create-kata-tarball:
+  build-asset-rootfs:
     runs-on: ppc64le
     needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-initrd
+        stage:
+          - ${{ inputs.stage }}
+    steps:
+      - name: Prepare the self-hosted runner
+        run: |
+            "${HOME}/scripts/prepare_runner.sh"
+            sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - agent
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-ppc64le-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: ppc64le
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts]
+    steps:
+      - name: Prepare the self-hosted runner
+        run: |
+            "${HOME}/scripts/prepare_runner.sh"
+            sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-ppc64le-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ppc64le
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
     steps:
       - name: Adjust a permission for repo
         run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
 
       - uses: actions/checkout@v4
         with:

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -24,6 +24,11 @@ on:
 jobs:
   build-asset:
     runs-on: s390x
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
     strategy:
       matrix:
         asset:
@@ -33,12 +38,9 @@ jobs:
           - kernel-confidential
           - pause-image
           - qemu
-          - rootfs-image
-          - rootfs-image-confidential
-          - rootfs-initrd
-          - rootfs-initrd-confidential
-          - shim-v2
           - virtiofsd
+    env:
+      PERFORM_ATTESTATION: ${{ matrix.asset == 'agent' && inputs.push-to-registry == 'yes' && 'yes' || 'no' }}
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -60,11 +62,98 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Build ${{ matrix.asset }}
+        id: build
         run: |
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
-          mkdir -p kata-build && cp "${build_dir}"/kata-static-${KATA_ASSET}*.tar.* kata-build/.
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: Parse OCI image name and digest
+        id: parse-oci-segments
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        run: |
+          oci_image="$(<"build/${{ matrix.asset }}-oci-image")"
+          echo "oci-name=${oci_image%@*}" >> "$GITHUB_OUTPUT"
+          echo "oci-digest=${oci_image#*@}" >> "$GITHUB_OUTPUT"
+
+      # for pushing attestations to the registry
+      - uses: docker/login-action@v3
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/attest-build-provenance@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          subject-name: ${{ steps.parse-oci-segments.outputs.oci-name }}
+          subject-digest: ${{ steps.parse-oci-segments.outputs.oci-digest }}
+          push-to-registry: true
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-rootfs:
+    runs-on: s390x
+    needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-image-confidential
+          - rootfs-initrd
+          - rootfs-initrd-confidential
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
         env:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
@@ -76,7 +165,6 @@ jobs:
           RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
 
       - name: store-artifact ${{ matrix.asset }}
-        if: ${{ inputs.stage != 'release' || (matrix.asset != 'agent' && matrix.asset != 'coco-guest-components' && matrix.asset != 'pause-image') }}
         uses: actions/upload-artifact@v4
         with:
           name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
@@ -86,7 +174,7 @@ jobs:
 
   build-asset-boot-image-se:
     runs-on: s390x
-    needs: build-asset
+    needs: [build-asset, build-asset-rootfs]
     steps:
       - uses: actions/checkout@v4
 
@@ -112,15 +200,11 @@ jobs:
 
       - name: Build boot-image-se
         run: |
-          base_dir=tools/packaging/kata-deploy/local-build/
-          cp -r kata-artifacts ${base_dir}/build
-          # Skip building dependant artifacts of boot-image-se-tarball
-          # because we already have them from the previous build
-          sed -i 's/\(^boot-image-se-tarball:\).*/\1/g' ${base_dir}/Makefile
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "boot-image-se"
           make boot-image-se-tarball
           build_dir=$(readlink -f build)
           sudo cp -r "${build_dir}" "kata-build"
-          sudo chown -R $(id -u):$(id -g) "kata-build"
+          sudo chown -R "$(id -u)":"$(id -g)" "kata-build"
         env:
           HKD_PATH: "host-key-document"
 
@@ -132,9 +216,86 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: [build-asset-rootfs, build-asset-boot-image-se]
+    strategy:
+      matrix:
+        asset:
+          - agent
+          - coco-guest-components
+          - pause-image
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-s390x-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: s390x
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-s390x-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
   create-kata-tarball:
     runs-on: s390x
-    needs: [build-asset, build-asset-boot-image-se]
+    needs:
+      - build-asset
+      - build-asset-rootfs
+      - build-asset-boot-image-se
+      - build-asset-shim-v2
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/cargo-deny-runner.yaml

@@ -24,7 +24,7 @@ jobs:
         run: bash cargo-deny-generator.sh
         working-directory: ./.github/cargo-deny-composite-action/
         env:
-          GOPATH: ${{ runner.workspace }}/kata-containers
+          GOPATH: ${{ github.workspace }}/kata-containers
       - name: Run Action
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         uses: ./.github/cargo-deny-composite-action

.github/workflows/ci-devel.yaml

@@ -0,0 +1,13 @@
+name: Kata Containers CI (manually triggered)
+on:
+  workflow_dispatch:
+
+jobs:
+  kata-containers-ci-on-push:
+    uses: ./.github/workflows/ci.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      pr-number: "dev"
+      tag: ${{ github.sha }}-dev
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit

.github/workflows/ci-nightly-s390x.yaml

@@ -16,6 +16,6 @@ jobs:
     - name: Fetch a test result for {{ matrix.test_title }}
       run: |
         file_name="${TEST_TITLE}-$(date +%Y-%m-%d).log"
-        /home/${USER}/script/handle_test_log.sh download $file_name
+        "/home/${USER}/script/handle_test_log.sh" download "$file_name"
       env:
         TEST_TITLE: ${{ matrix.test_title }}

.github/workflows/ci-nightly.yaml

@@ -2,7 +2,6 @@ name: Kata Containers Nightly CI
 on:
   schedule:
     - cron: '0 0 * * *'
-  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/ci-on-push.yaml

@@ -19,12 +19,21 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  kata-containers-ci-on-push:
+  skipper:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
+    uses: ./.github/workflows/gatekeeper-skipper.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+
+  kata-containers-ci-on-push:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_build != 'yes' }}
     uses: ./.github/workflows/ci.yaml
     with:
       commit-hash: ${{ github.event.pull_request.head.sha }}
       pr-number: ${{ github.event.pull_request.number }}
       tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
       target-branch: ${{ github.event.pull_request.base.ref }}
+      skip-test: ${{ needs.skipper.outputs.skip_test }}
     secrets: inherit

.github/workflows/ci-weekly.yaml

@@ -37,7 +37,7 @@ jobs:
     secrets: inherit
 
   build-and-publish-tee-confidential-unencrypted-image:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -83,4 +83,5 @@ jobs:
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
+      tarball-suffix: -${{ inputs.tag }}
     secrets: inherit

.github/workflows/ci.yaml

@@ -15,6 +15,10 @@ on:
         required: false
         type: string
         default: ""
+      skip-test:
+        required: false
+        type: string
+        default: no
 
 jobs:
   build-kata-static-tarball-amd64:
@@ -131,7 +135,58 @@ jobs:
           platforms: linux/amd64, linux/s390x
           file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
 
+  publish-csi-driver-amd64:
+    needs: build-kata-static-tarball-amd64
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64-${{ inputs.tag }}
+          path: kata-artifacts
+
+      - name: Install tools
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
+      - name: Copy binary into Docker context
+        run: |
+          # Copy to the location where the Dockerfile expects the binary.
+          mkdir -p src/tools/csi-kata-directvolume/bin/
+          cp /opt/kata/bin/csi-kata-directvolume src/tools/csi-kata-directvolume/bin/directvolplugin
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Kata Containers ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker build and push
+        uses: docker/build-push-action@v5
+        with:
+          tags: ghcr.io/kata-containers/csi-kata-directvolume:${{ inputs.pr-number }}
+          push: true
+          context: src/tools/csi-kata-directvolume/
+          platforms: linux/amd64
+          file: src/tools/csi-kata-directvolume/Dockerfile
+
   run-kata-monitor-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-amd64
     uses: ./.github/workflows/run-kata-monitor-tests.yaml
     with:
@@ -140,6 +195,7 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
 
   run-k8s-tests-on-aks:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
     with:
@@ -153,6 +209,7 @@ jobs:
     secrets: inherit
 
   run-k8s-tests-on-amd64:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-k8s-tests-on-amd64.yaml
     with:
@@ -165,9 +222,14 @@ jobs:
     secrets: inherit
 
   run-kata-coco-tests:
-    needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs:
+     - publish-kata-deploy-payload-amd64
+     - build-and-publish-tee-confidential-unencrypted-image
+     - publish-csi-driver-amd64
     uses: ./.github/workflows/run-kata-coco-tests.yaml
     with:
+      tarball-suffix: -${{ inputs.tag }}
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
@@ -177,6 +239,7 @@ jobs:
     secrets: inherit
 
   run-k8s-tests-on-zvsi:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: [publish-kata-deploy-payload-s390x, build-and-publish-tee-confidential-unencrypted-image]
     uses: ./.github/workflows/run-k8s-tests-on-zvsi.yaml
     with:
@@ -189,6 +252,7 @@ jobs:
     secrets: inherit
 
   run-k8s-tests-on-ppc64le:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: publish-kata-deploy-payload-ppc64le
     uses: ./.github/workflows/run-k8s-tests-on-ppc64le.yaml
     with:
@@ -200,6 +264,7 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
 
   run-metrics-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-amd64
     uses: ./.github/workflows/run-metrics.yaml
     with:
@@ -208,6 +273,7 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
 
   run-basic-amd64-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-amd64
     uses: ./.github/workflows/basic-ci-amd64.yaml
     with:
@@ -216,6 +282,7 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
 
   run-cri-containerd-tests-s390x:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-s390x
     uses: ./.github/workflows/run-cri-containerd-tests-s390x.yaml
     with:
@@ -224,6 +291,7 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
 
   run-cri-containerd-tests-ppc64le:
+    if: ${{ inputs.skip-test != 'yes' }}
     needs: build-kata-static-tarball-ppc64le
     uses: ./.github/workflows/run-cri-containerd-tests-ppc64le.yaml
     with:

.github/workflows/darwin-tests.yaml

@@ -16,7 +16,7 @@ jobs:
     runs-on: macos-latest
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.22.2
     - name: Checkout code

.github/workflows/docs-url-alive-check.yaml

@@ -12,15 +12,15 @@ jobs:
       target_branch: ${{ github.base_ref }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.22.2
       env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
+        GOPATH: ${{ github.workspace }}/kata-containers
     - name: Set env
       run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+        echo "GOPATH=${{ github.workspace }}" >> "$GITHUB_ENV"
+        echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Checkout code
       uses: actions/checkout@v4
       with:
@@ -29,4 +29,4 @@ jobs:
     # docs url alive check
     - name: Docs URL Alive Check
       run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make docs-url-alive-check
+        cd "${GOPATH}/src/github.com/${{ github.repository }}" && make docs-url-alive-check

.github/workflows/gatekeeper-skipper.yaml

@@ -0,0 +1,52 @@
+name: Skipper
+
+# This workflow sets various "skip_*" output values that can be used to
+# determine what workflows/jobs are expected to be executed. Sample usage:
+#
+#   skipper:
+#     uses: ./.github/workflows/gatekeeper-skipper.yaml
+#     with:
+#       commit-hash: ${{ github.event.pull_request.head.sha }}
+#       target-branch: ${{ github.event.pull_request.base.ref }}
+#
+#   your-workflow:
+#     needs: skipper
+#     if: ${{ needs.skipper.outputs.skip_build != 'yes' }}
+
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+    outputs:
+      skip_build:
+        value: ${{ jobs.skipper.outputs.skip_build }}
+      skip_test:
+        value: ${{ jobs.skipper.outputs.skip_test }}
+      skip_static:
+        value: ${{ jobs.skipper.outputs.skip_static }}
+
+
+jobs:
+  skipper:
+    runs-on: ubuntu-22.04
+    outputs:
+      skip_build: ${{ steps.skipper.outputs.skip_build }}
+      skip_test: ${{ steps.skipper.outputs.skip_test }}
+      skip_static: ${{ steps.skipper.outputs.skip_static }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - id: skipper
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+        run: |
+          python3 tools/testing/gatekeeper/skips.py | tee -a "$GITHUB_OUTPUT"
+        shell: /usr/bin/bash -x {0}

.github/workflows/gatekeeper.yaml

@@ -0,0 +1,44 @@
+name: Gatekeeper
+
+# Gatekeeper uses the "skips.py" to determine which job names/regexps are
+# required for given PR and waits for them to either complete or fail
+# reporting the status.
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  gatekeeper:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - id: gatekeeper
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMIT_HASH: ${{ github.event.pull_request.head.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          #!/usr/bin/env bash -x
+          mapfile -t lines < <(python3 tools/testing/gatekeeper/skips.py -t)
+          export REQUIRED_JOBS="${lines[0]}"
+          export REQUIRED_REGEXPS="${lines[1]}"
+          export REQUIRED_LABELS="${lines[2]}"
+          echo "REQUIRED_JOBS: $REQUIRED_JOBS"
+          echo "REQUIRED_REGEXPS: $REQUIRED_REGEXPS"
+          echo "REQUIRED_LABELS: $REQUIRED_LABELS"
+          python3 tools/testing/gatekeeper/jobs.py
+          exit $?
+        shell: /usr/bin/bash -x {0}

.github/workflows/kata-runtime-classes-sync.yaml

@@ -20,9 +20,9 @@ jobs:
       run: |
         pushd tools/packaging/kata-deploy/runtimeclasses/
         echo "::group::Combine runtime classes"
-        for runtimeClass in `find . -type f \( -name "*.yaml" -and -not -name "kata-runtimeClasses.yaml" \) | sort`; do
+        for runtimeClass in $(find . -type f \( -name "*.yaml" -and -not -name "kata-runtimeClasses.yaml" \) | sort); do
             echo "Adding ${runtimeClass} to the resultingRuntimeClasses.yaml"
-            cat ${runtimeClass} >> resultingRuntimeClasses.yaml;
+            cat "${runtimeClass}" >> resultingRuntimeClasses.yaml;
         done
         echo "::endgroup::"
         echo "::group::Displaying the content of resultingRuntimeClasses.yaml"

.github/workflows/move-issues-to-in-progress.yaml

@@ -31,7 +31,7 @@ jobs:
         run: |
           # Clone into a temporary directory to avoid overwriting
           # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
+          pushd "$(mktemp -d)" &>/dev/null
           git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
           sudo install hub-util.sh /usr/local/bin
           popd &>/dev/null
@@ -72,9 +72,9 @@ jobs:
           project_type="org"
           project_column="In progress"
 
-          for issue_url in $(echo "$linked_issue_urls")
+          for issue_url in $linked_issue_urls
           do
-            issue=$(echo "$issue_url"| awk -F\/ '{print $NF}' || true)
+            issue=$(echo "$issue_url"| awk -F/ '{print $NF}' || true)
 
             [ -z "$issue" ] && {
               echo "::error::Cannot determine issue number from $issue_url for PR $pr"

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -62,5 +62,5 @@ jobs:
         id: build-and-push-kata-payload
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
+          "$(pwd)"/kata-static.tar.xz \
           ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -28,7 +28,7 @@ jobs:
     steps:
       - name: Adjust a permission for repo
         run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
 
       - uses: actions/checkout@v4
         with:
@@ -66,6 +66,5 @@ jobs:
         id: build-and-push-kata-payload
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
+          "$(pwd)"/kata-static.tar.xz \
           ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}
-

.github/workflows/publish-kata-deploy-payload-ppc64le.yaml

@@ -28,12 +28,12 @@ jobs:
     steps:
       - name: Prepare the self-hosted runner
         run: |
-          ${HOME}/scripts/prepare_runner.sh
-          sudo rm -rf $GITHUB_WORKSPACE/*
+          "${HOME}/scripts/prepare_runner.sh"
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - name: Adjust a permission for repo
         run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
 
       - uses: actions/checkout@v4
         with:
@@ -71,5 +71,5 @@ jobs:
         id: build-and-push-kata-payload
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
+          "$(pwd)"/kata-static.tar.xz \
           ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -62,5 +62,5 @@ jobs:
         id: build-and-push-kata-payload
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
+          "$(pwd)"/kata-static.tar.xz \
           ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/release-amd64.yaml

@@ -42,18 +42,18 @@ jobs:
         run: |
           # We need to do such trick here as the format of the $GITHUB_REF
           # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
           if [ "${tag}" = "main" ]; then
               tag=$(./tools/packaging/release/release.sh release-version)
-              tags=(${tag} "latest")
+              tags=("${tag}" "latest")
           else
-              tags=(${tag})
+              tags=("${tag}")
           fi
-          for tag in ${tags[@]}; do
+          for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
           done

.github/workflows/release-arm64.yaml

@@ -42,18 +42,18 @@ jobs:
         run: |
           # We need to do such trick here as the format of the $GITHUB_REF
           # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
           if [ "${tag}" = "main" ]; then
               tag=$(./tools/packaging/release/release.sh release-version)
-              tags=(${tag} "latest")
+              tags=("${tag}" "latest")
           else
-              tags=(${tag})
+              tags=("${tag}")
           fi
-          for tag in ${tags[@]}; do
+          for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
           done

.github/workflows/release-ppc64le.yaml

@@ -20,8 +20,8 @@ jobs:
     steps:
       - name: Prepare the self-hosted runner
         run: |
-          bash ${HOME}/scripts/prepare_runner.sh
-          sudo rm -rf $GITHUB_WORKSPACE/*
+          bash "${HOME}/scripts/prepare_runner.sh"
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v3
@@ -47,18 +47,18 @@ jobs:
         run: |
           # We need to do such trick here as the format of the $GITHUB_REF
           # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
           if [ "${tag}" = "main" ]; then
               tag=$(./tools/packaging/release/release.sh release-version)
-              tags=(${tag} "latest")
+              tags=("${tag}" "latest")
           else
-              tags=(${tag})
+              tags=("${tag}")
           fi
-          for tag in ${tags[@]}; do
+          for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
           done

.github/workflows/release-s390x.yaml

@@ -42,18 +42,18 @@ jobs:
         run: |
           # We need to do such trick here as the format of the $GITHUB_REF
           # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
           if [ "${tag}" = "main" ]; then
               tag=$(./tools/packaging/release/release.sh release-version)
-              tags=(${tag} "latest")
+              tags=("${tag}" "latest")
           else
-              tags=(${tag})
+              tags=("${tag}")
           fi
-          for tag in ${tags[@]}; do
+          for tag in "${tags[@]}"; do
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
               ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
                   "${tag}-${{ inputs.target-arch }}"
           done

.github/workflows/release.yaml

@@ -175,6 +175,23 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
 
+  upload-helm-chart-tarball:
+    needs: release
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install helm
+        uses: azure/setup-helm@v4.2.0
+        id: install
+
+      - name: Generate and upload helm chart tarball
+        run: |
+          ./tools/packaging/release/release.sh upload-helm-chart-tarball
+        env:
+          GH_TOKEN: ${{ github.token }}
+
   publish-release:
     needs: [ build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x, build-and-push-assets-ppc64le, publish-multi-arch-images, upload-multi-arch-static-tarball, upload-versions-yaml, upload-cargo-vendored-tarball, upload-libseccomp-tarball ]
     runs-on: ubuntu-22.04

.github/workflows/run-cri-containerd-tests-ppc64le.yaml

@@ -30,12 +30,12 @@ jobs:
       KATA_HYPERVISOR: ${{ matrix.vmm }}
     steps:
       - name: Adjust a permission for repo
-        run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-  
+        run: sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
+
       - name: Prepare the self-hosted runner
         run: |
-          bash ${HOME}/scripts/prepare_runner.sh cri-containerd
-          sudo rm -rf $GITHUB_WORKSPACE/*
+          bash "${HOME}/scripts/prepare_runner.sh" cri-containerd
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/checkout@v4
         with:
@@ -62,6 +62,6 @@ jobs:
 
       - name: Run cri-containerd tests
         run: bash tests/integration/cri-containerd/gha-run.sh run
-      
+
       - name: Cleanup actions for the self hosted runner
-        run: ${HOME}/scripts/cleanup_runner.sh
+        run: bash "${HOME}/scripts/cleanup_runner.sh"

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -47,13 +47,16 @@ jobs:
             vmm: clh
             instance-type: small
             genpolicy-pull-method: oci-distribution
+            auto-generate-policy: yes
           - host_os: cbl-mariner
             vmm: clh
             instance-type: small
             genpolicy-pull-method: containerd
+            auto-generate-policy: yes
           - host_os: cbl-mariner
             vmm: clh
             instance-type: normal
+            auto-generate-policy: yes
     runs-on: ubuntu-22.04
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
@@ -66,6 +69,7 @@ jobs:
       USING_NFD: "false"
       K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
       GENPOLICY_PULL_METHOD: ${{ matrix.genpolicy-pull-method }}
+      AUTO_GENERATE_POLICY: ${{ matrix.auto-generate-policy }}
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/run-k8s-tests-on-amd64.yaml

@@ -56,6 +56,7 @@ jobs:
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: "false"
       K8S_TEST_HOST_TYPE: all
+      CONTAINER_RUNTIME: ${{ matrix.container_runtime }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -85,11 +86,11 @@ jobs:
 
       - name: Install `bats`
         run: bash tests/integration/kubernetes/gha-run.sh install-bats
-  
+
       - name: Run tests
         timeout-minutes: 30
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
-  
+
       - name: Collect artifacts ${{ matrix.vmm }}
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh collect-artifacts
@@ -98,7 +99,7 @@ jobs:
       - name: Archive artifacts ${{ matrix.vmm }}
         uses: actions/upload-artifact@v4
         with:
-          name: k8s-tests-${{ matrix.vmm }}-${{ matrix.snapshotter }}-${{ matrix.k8s }}-${{ matrix.instance }}-${{ inputs.tag }}
+          name: k8s-tests-${{ matrix.vmm }}-${{ matrix.snapshotter }}-${{ matrix.k8s }}-${{ inputs.tag }}
           path: /tmp/artifacts
           retention-days: 1
 

.github/workflows/run-k8s-tests-on-ppc64le.yaml

@@ -44,9 +44,9 @@ jobs:
       TARGET_ARCH: "ppc64le"
     steps:
       - name: Prepare the self-hosted runner
-        run: | 
-          bash ${HOME}/scripts/prepare_runner.sh kubernetes
-          sudo rm -rf $GITHUB_WORKSPACE/*
+        run: |
+          bash "${HOME}/scripts/prepare_runner.sh" kubernetes
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/checkout@v4
         with:
@@ -62,13 +62,13 @@ jobs:
       - name: Install golang
         run: |
           ./tests/install_go.sh -f -p
-          echo "/usr/local/go/bin" >> $GITHUB_PATH
+          echo "/usr/local/go/bin" >> "$GITHUB_PATH"
 
       - name: Prepare the runner for k8s cluster creation
-        run: bash ${HOME}/scripts/k8s_cluster_cleanup.sh
+        run: bash "${HOME}/scripts/k8s_cluster_cleanup.sh"
 
       - name: Create k8s cluster using kubeadm
-        run: bash ${HOME}/scripts/k8s_cluster_create.sh
+        run: bash "${HOME}/scripts/k8s_cluster_create.sh"
 
       - name: Deploy Kata
         timeout-minutes: 10
@@ -79,4 +79,4 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete cluster and post cleanup actions
-        run: bash ${HOME}/scripts/k8s_cluster_cleanup.sh
+        run: bash "${HOME}/scripts/k8s_cluster_cleanup.sh"

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -36,7 +36,7 @@ jobs:
           - qemu-runtime-rs
           - qemu-coco-dev
         k8s:
-          - k3s
+          - kubeadm
         include:
           - snapshotter: devmapper
             pull-type: default
@@ -88,18 +88,15 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Set SNAPSHOTTER to empty if overlayfs
-        run: echo "SNAPSHOTTER=" >> $GITHUB_ENV
+        run: echo "SNAPSHOTTER=" >> "$GITHUB_ENV"
         if: ${{ matrix.snapshotter == 'overlayfs' }}
 
       - name: Set KBS and KBS_INGRESS if qemu-coco-dev
         run: |
-          echo "KBS=true" >> $GITHUB_ENV
-          echo "KBS_INGRESS=nodeport" >> $GITHUB_ENV
+          echo "KBS=true" >> "$GITHUB_ENV"
+          echo "KBS_INGRESS=nodeport" >> "$GITHUB_ENV"
         if: ${{ matrix.vmm == 'qemu-coco-dev' }}
 
-      - name: Deploy ${{ matrix.k8s }}
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
-
       # qemu-runtime-rs only works with overlayfs
       # See: https://github.com/kata-containers/kata-containers/issues/10066
       - name: Configure the ${{ matrix.snapshotter }} snapshotter

.github/workflows/run-kata-coco-stability-tests.yaml

@@ -21,6 +21,9 @@ on:
         required: false
         type: string
         default: ""
+      tarball-suffix:
+        required: false
+        type: string
 
 jobs:
   # Generate jobs for testing CoCo on non-TEE environments
@@ -34,13 +37,12 @@ jobs:
           - nydus
         pull-type:
           - guest-pull
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
       DOCKER_TAG: ${{ inputs.tag }}
       GH_PR_NUMBER: ${{ inputs.pr-number }}
-      KATA_HOST_OS: ${{ matrix.host_os }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
       # Some tests rely on that variable to run (or not)
       KBS: "true"
@@ -64,6 +66,15 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
 

.github/workflows/run-kata-coco-tests.yaml

@@ -2,6 +2,9 @@ name: CI | Run kata coco tests
 on:
   workflow_call:
     inputs:
+      tarball-suffix:
+        required: false
+        type: string
       registry:
         required: true
         type: string
@@ -33,7 +36,15 @@ jobs:
           - nydus
         pull-type:
           - guest-pull
-    runs-on: tdx
+        k8s-test-host-type:
+          - baremetal-attestation
+          - baremetal-no-attestation
+        include:
+          - k8s-test-host-type: baremetal-attestation
+            machine: tdx-attestation
+          - k8s-test-host-type: baremetal-no-attestation
+            machine: tdx-no-attestation
+    runs-on: ${{ matrix.machine }}
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
       DOCKER_REPO: ${{ inputs.repo }}
@@ -43,12 +54,14 @@ jobs:
       KUBERNETES: "vanilla"
       USING_NFD: "true"
       KBS: "true"
-      K8S_TEST_HOST_TYPE: "baremetal"
+      K8S_TEST_HOST_TYPE: ${{ matrix.k8s-test-host-type }}
       KBS_INGRESS: "nodeport"
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       PULL_TYPE: ${{ matrix.pull-type }}
       AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      ITA_KEY: ${{ secrets.ITA_KEY }}
+      AUTO_GENERATE_POLICY: "yes"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -70,19 +83,26 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
 
       - name: Uninstall previous `kbs-client`
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
 
       - name: Deploy CoCo KBS
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
 
       - name: Install `kbs-client`
+        if: ${{ matrix.machine != 'tdx-no-attestation' }}
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
       - name: Run tests
-        timeout-minutes: 50
+        timeout-minutes: 100
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete kata-deploy
@@ -94,9 +114,13 @@ jobs:
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
 
       - name: Delete CoCo KBS
-        if: always()
+        if: ${{ always() && matrix.machine != 'tdx-no-attestation' }}
         run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
 
+      - name: Delete CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
   run-k8s-tests-on-sev:
     strategy:
       fail-fast: false
@@ -122,6 +146,7 @@ jobs:
       PULL_TYPE: ${{ matrix.pull-type }}
       AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      AUTO_GENERATE_POLICY: "yes"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -142,10 +167,18 @@ jobs:
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
 
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
       - name: Run tests
         timeout-minutes: 50
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
+      - name: Delete CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
       - name: Delete kata-deploy
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-sev
@@ -181,6 +214,7 @@ jobs:
       PULL_TYPE: ${{ matrix.pull-type }}
       AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      AUTO_GENERATE_POLICY: "yes"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -213,6 +247,10 @@ jobs:
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
       - name: Run tests
         timeout-minutes: 50
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
@@ -229,6 +267,10 @@ jobs:
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
 
+      - name: Delete CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
   # Generate jobs for testing CoCo on non-TEE environments
   run-k8s-tests-coco-nontee:
     strategy:
@@ -246,7 +288,6 @@ jobs:
       DOCKER_REPO: ${{ inputs.repo }}
       DOCKER_TAG: ${{ inputs.tag }}
       GH_PR_NUMBER: ${{ inputs.pr-number }}
-      KATA_HOST_OS: ${{ matrix.host_os }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
       # Some tests rely on that variable to run (or not)
       KBS: "true"
@@ -258,6 +299,7 @@ jobs:
       AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
       SNAPSHOTTER: ${{ matrix.snapshotter }}
       USING_NFD: "false"
+      AUTO_GENERATE_POLICY: "yes"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -270,6 +312,15 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
 
@@ -310,8 +361,12 @@ jobs:
         timeout-minutes: 10
         run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
 
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
       - name: Run tests
-        timeout-minutes: 60
+        timeout-minutes: 80
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete AKS cluster

.github/workflows/run-metrics.yaml

@@ -48,7 +48,7 @@ jobs:
       # all the tests due to a single flaky instance.
       fail-fast: false
       matrix:
-        vmm: ['clh', 'qemu', 'stratovirt']
+        vmm: ['clh', 'qemu']
       max-parallel: 1
     runs-on: metrics
     env:

.github/workflows/run-runk-tests.yaml

@@ -15,6 +15,8 @@ on:
 
 jobs:
   run-runk:
+    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
+    if: false
     runs-on: ubuntu-22.04
     env:
       CONTAINERD_VERSION: lts

.github/workflows/static-checks-self-hosted.yaml

@@ -12,8 +12,16 @@ concurrency:
 
 name: Static checks self-hosted
 jobs:
-  build-checks:
+  skipper:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
+    uses: ./.github/workflows/gatekeeper-skipper.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+
+  build-checks:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     strategy:
       fail-fast: false
       matrix:

.github/workflows/static-checks.yaml

@@ -12,7 +12,15 @@ concurrency:
 
 name: Static checks
 jobs:
+  skipper:
+    uses: ./.github/workflows/gatekeeper-skipper.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+
   check-kernel-config-version:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout the code
@@ -23,8 +31,8 @@ jobs:
         run: |
           kernel_dir="tools/packaging/kernel/"
           kernel_version_file="${kernel_dir}kata_config_version"
-          modified_files=$(git diff --name-only origin/$GITHUB_BASE_REF..HEAD)
-          if git diff --name-only origin/$GITHUB_BASE_REF..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+          modified_files=$(git diff --name-only origin/"$GITHUB_BASE_REF"..HEAD)
+          if git diff --name-only origin/"$GITHUB_BASE_REF"..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
             echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
             if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
               echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
@@ -35,12 +43,16 @@ jobs:
           fi
 
   build-checks:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     uses: ./.github/workflows/build-checks.yaml
     with:
       instance: ubuntu-22.04
 
   build-checks-depending-on-kvm:
     runs-on: ubuntu-22.04
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     strategy:
       fail-fast: false
       matrix:
@@ -78,6 +90,8 @@ jobs:
 
   static-checks:
     runs-on: ubuntu-22.04
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
     strategy:
       fail-fast: false
       matrix:
@@ -93,19 +107,19 @@ jobs:
           path: ./src/github.com/${{ github.repository }}
       - name: Install yq
         run: |
-          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          cd "${GOPATH}/src/github.com/${{ github.repository }}"
           ./ci/install_yq.sh
         env:
           INSTALL_IN_GOPATH: false
       - name: Install golang
         run: |
-          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          cd "${GOPATH}/src/github.com/${{ github.repository }}"
           ./tests/install_go.sh -f -p
-          echo "/usr/local/go/bin" >> $GITHUB_PATH
+          echo "/usr/local/go/bin" >> "$GITHUB_PATH"
       - name: Install system dependencies
         run: |
           sudo apt-get -y install moreutils hunspell hunspell-en-gb hunspell-en-us pandoc
       - name: Run check
         run: |
-          export PATH=${PATH}:${GOPATH}/bin
-          cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}
+          export PATH="${PATH}:${GOPATH}/bin"
+          cd "${GOPATH}/src/github.com/${{ github.repository }}" && ${{ matrix.cmd }}

VERSION

@@ -1 +1 @@
-3.9.0
+3.12.0

ci/README.md

@@ -55,14 +55,14 @@ of a PR review), the following tests will be executed:
 - Run the following tests:
   - Tests depending on the generated tarball
     - Metrics (runs on bare-metal)
-    - `docker` (runs on Azure small instances)
-    - `nerdctl` (runs on Azure small instances)
-    - `kata-monitor` (runs on Azure small instances)
-    - `cri-containerd` (runs on Azure small instances)
-    - `nydus` (runs on Azure small instances)
-    - `vfio` (runs on Azure normal instances)
+    - `docker` (runs on cost free runners)
+    - `nerdctl` (runs on cost free runners)
+    - `kata-monitor` (runs on cost free runners)
+    - `cri-containerd` (runs on cost free runners)
+    - `nydus` (runs on cost free runners)
+    - `vfio` (runs on cost free runners)
   - Tests depending on the generated kata-deploy payload
-    - kata-deploy (runs on Azure small instances)
+    - kata-deploy (runs on cost free runners)
       - Tests are performed using different "Kubernetes flavors", such as k0s, k3s, rke2, and Azure Kubernetes Service (AKS).
     - Kubernetes (runs in Azure small and medium instances depending on what's required by each test, and on TEE bare-metal machines)
       - Tests are performed with different runtime engines, such as CRI-O and containerd.
@@ -77,8 +77,8 @@ them to merely debug issues.
 
 In the previous section we've mentioned using different runners, now in this section we'll go through each type of runner used.
 
-- Cost free runners:  Those are the runners provided by GIthub itself, and
-  those are fairly small machines with no virtualization capabilities enabled - 
+- Cost free runners:  Those are the runners provided by Github itself, and
+  those are fairly small machines with virtualization capabilities enabled. 
 - Azure small instances: Those are runners which have virtualization
   capabilities enabled, 2 CPUs, and 8GB of RAM.  These runners have a "-smaller"
   suffix to their name. 

ci/install_yq.sh

@@ -14,20 +14,38 @@ die() {
 	exit 1
 }
 
+function verify_yq_exists() {
+	local yq_path=$1
+	local yq_version=$2
+	local expected="yq (https://github.com/mikefarah/yq/) version $yq_version"
+	if [ -x  "${yq_path}" ] && [ "$($yq_path --version)"X == "$expected"X ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 # Install the yq yaml query package from the mikefarah github repo
 # Install via binary download, as we may not have golang installed at this point
 function install_yq() {
 	local yq_pkg="github.com/mikefarah/yq"
 	local yq_version=v4.40.7
 	local precmd=""
+	local yq_path=""
 	INSTALL_IN_GOPATH=${INSTALL_IN_GOPATH:-true}
 
-	if [ "${INSTALL_IN_GOPATH}"  == "true" ];then
+	if [ "${INSTALL_IN_GOPATH}" == "true" ]; then
 		GOPATH=${GOPATH:-${HOME}/go}
 		mkdir -p "${GOPATH}/bin"
-		local yq_path="${GOPATH}/bin/yq"
+		yq_path="${GOPATH}/bin/yq"
 	else
 		yq_path="/usr/local/bin/yq"
+	fi
+	if verify_yq_exists "$yq_path" "$yq_version"; then
+		echo "yq is already installed in correct version"
+		return
+	fi
+	if [ "${yq_path}" == "/usr/local/bin/yq" ]; then
 		# Check if we need sudo to install yq
 		if [ ! -w "/usr/local/bin" ]; then
 			# Check if we have sudo privileges
@@ -38,7 +56,6 @@ function install_yq() {
 			fi
 		fi
 	fi
-	[ -x  "${yq_path}" ] && [ "`${yq_path} --version`"X == "yq (https://github.com/mikefarah/yq/) version ${yq_version}"X ] && return
 
 	read -r -a sysInfo <<< "$(uname -sm)"
 

ci/openshift-ci/bisect-range.sh

@@ -16,9 +16,12 @@ REPO="quay.io/kata-containers/kata-deploy-ci"
 TAGS=$(skopeo list-tags "docker://$REPO")
 # Only amd64
 TAGS=$(echo "$TAGS" | jq '.Tags' | jq "map(select(endswith(\"$ARCH\")))" | jq -r '.[]')
-# Tags since $GOOD
-TAGS=$(echo "$TAGS" | sed -n -e "/$GOOD/,$$p")
-# Tags up to $BAD
-[ -n "$BAD" ] && TAGS=$(echo "$TAGS" | sed "/$BAD/q")
+# Sort by git
+SORTED=""
+[ -n "$BAD" ] && LOG_ARGS="$GOOD~1..$BAD" || LOG_ARGS="$GOOD~1.."
+for TAG in $(git log --merges --pretty=format:%H --reverse $LOG_ARGS); do
+	[[ "$TAGS" =~ "$TAG" ]] && SORTED+="
+kata-containers-$TAG-$ARCH"
+done
 # Comma separated tags with repo
-echo "$TAGS" | sed -e "s@^@$REPO:@" | paste -s -d, -
+echo "$SORTED" | tail -n +2 | sed -e "s@^@$REPO:@" | paste -s -d, -

ci/openshift-ci/cluster/deploy_webhook.sh

@@ -13,16 +13,11 @@ set -e
 set -o nounset
 set -o pipefail
 
-script_dir="$(dirname $0)"
+script_dir="$(realpath $(dirname $0))"
 webhook_dir="${script_dir}/../../../tools/testing/kata-webhook"
 source "${script_dir}/../lib.sh"
 KATA_RUNTIME=${KATA_RUNTIME:-kata-ci}
 
-info "Creates the kata-webhook ConfigMap"
-RUNTIME_CLASS="${KATA_RUNTIME}" \
-	envsubst < "${script_dir}/deployments/configmap_kata-webhook.yaml.in" \
-	| oc apply -f -
-
 pushd "${webhook_dir}" >/dev/null
 # Build and deploy the webhook
 #
@@ -30,6 +25,12 @@ info "Builds the kata-webhook"
 ./create-certs.sh
 info "Deploys the kata-webhook"
 oc apply -f deploy/
+
+info "Override our KATA_RUNTIME ConfigMap"
+RUNTIME_CLASS="${KATA_RUNTIME}" \
+	envsubst < "${script_dir}/deployments/configmap_kata-webhook.yaml.in" \
+	| oc apply -f -
+
 # Check the webhook was deployed and is working.
 RUNTIME_CLASS="${KATA_RUNTIME}" ./webhook-check.sh
 popd >/dev/null

ci/openshift-ci/smoke/http-server.yaml.in

@@ -13,7 +13,7 @@ metadata:
 spec:
   containers:
     - name: http-server
-      image: registry.fedoraproject.org/fedora
+      image: docker.io/library/python:3
       ports:
         - containerPort: 8080
       command: ["python3"]

docs/Developer-Guide.md

@@ -499,19 +499,6 @@ If you do not want to install the respective QEMU version, the configuration fil
 
 See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata.
 
-### Build a custom QEMU for aarch64/arm64 - REQUIRED
-> **Note:**
->
-> - You should only do this step if you are on aarch64/arm64.
-> - You should include [Eric Auger's latest PCDIMM/NVDIMM patches](https://patchwork.kernel.org/cover/10647305/) which are
->   under upstream review for supporting NVDIMM on aarch64.
->
-You could build the custom `qemu-system-aarch64` as required with the following command:
-```bash
-$ git clone https://github.com/kata-containers/tests.git
-$ script -fec 'sudo -E tests/.ci/install_qemu.sh'
-```
-
 ## Build `virtiofsd`
 
 When using the file system type virtio-fs (default), `virtiofsd` is required
@@ -640,7 +627,7 @@ the following steps (using rootfs or initrd image).
 >
 > Look for `INIT_PROCESS=systemd` in the `config.sh` osbuilder rootfs config file
 > to verify an osbuilder distro supports systemd for the distro you want to build rootfs for.
-> For an example, see the [Clear Linux config.sh file](../tools/osbuilder/rootfs-builder/clearlinux/config.sh).
+> For an example, see the [Ubuntu config.sh file](../tools/osbuilder/rootfs-builder/ubuntu/config.sh).
 >
 > For a non-systemd-based distro, create an equivalent system
 > service using that distro’s init system syntax. Alternatively, you can build a distro

docs/Release-Process.md

@@ -28,10 +28,22 @@ Bug fixes are released as part of `MINOR` or `MAJOR` releases only. `PATCH` is a
 
 ## Release Process
 
-### Bump the `VERSION` file
+### Bump the `VERSION` and `Chart.yaml` file
 
 When the `kata-containers/kata-containers` repository is ready for a new release,
-first create a PR to set the release in the `VERSION` file and have it merged.
+first create a PR to set the release in the [`VERSION`](./../VERSION) file and update the
+`version` and `appVersion` in the
+[`Chart.yaml`](./../tools/packaging/kata-deploy/helm-chart/kata-deploy/Chart.yaml) file and
+have it merged.
+
+### Lock the `main` branch
+
+In order to prevent any PRs getting merged during the release process, and slowing the release
+process down, by impacting the payload caches, we have recently trailed setting the `main`
+branch to read only whilst the release action runs.
+
+> [!NOTE]
+> Admin permission is needed to complete this task.
 
 ### Check GitHub Actions
 
@@ -40,6 +52,9 @@ We make use of [GitHub actions](https://github.com/features/actions) in the
 file from the `kata-containers/kata-containers` repository to build and upload
 release artifacts.
 
+> [!NOTE]
+> Write permissions to trigger the action.
+
 The action is manually triggered and is responsible for generating a new
 release (including a new tag), pushing those to the
 `kata-containers/kata-containers` repository. The new release is initially
@@ -59,6 +74,11 @@ If for some reason you need to cancel the workflow or re-run it entirely, go fir
 to the [Release page](https://github.com/kata-containers/kata-containers/releases) and
 delete the draft release from the previous run.
 
+### Unlock the `main` branch
+
+After the release process has concluded, either unlock the `main` branch, or ask
+an admin to do it.
+
 ### Improve the release notes
 
 Release notes are auto-generated by the GitHub CLI tool used as part of our

docs/design/architecture/guest-assets.md

@@ -135,7 +135,7 @@ See also the [process overview](README.md#process-overview).
 
 | Image type | Default distro | Init daemon | Reason | Notes |
 |-|-|-|-|-|
-| [image](background.md#root-filesystem-image) | [Clear Linux](https://clearlinux.org) (for x86_64 systems)| systemd | Minimal and highly optimized | systemd offers flexibility |
+| [image](background.md#root-filesystem-image) | [Ubuntu](https://ubuntu.com) (for x86_64 systems) | systemd | Fully tested in our CI |  systemd offers flexibility |
 | [initrd](#initrd-image) | [Alpine Linux](https://alpinelinux.org) | Kata [agent](README.md#agent) (as no systemd support) | Security hardened and tiny C library |
 
 See also:

docs/design/architecture_3.0/README.md

@@ -50,7 +50,7 @@ We provide `Dragonball` Sandbox to enable built-in VMM by integrating VMM's func
 #### How To Support Async
 The kata-runtime is controlled by TOKIO_RUNTIME_WORKER_THREADS to run the OS thread, which is 2 threads by default. For TTRPC and container-related threads run in the `tokio` thread in a unified manner, and related dependencies need to be switched to Async, such as Timer, File, Netlink, etc. With the help of Async, we can easily support no-block I/O and timer. Currently, we only utilize Async for kata-runtime. The built-in VMM keeps the OS thread because it can ensure that the threads are controllable.
 
-**For N tokio worker threads and M containers**
+**For N `tokio` worker threads and M containers**
 
 - Sync runtime(both OS thread and `tokio` task are OS thread but without `tokio` worker thread)  OS thread number:  4 + 12*M
 - Async runtime(only OS thread is OS thread) OS thread number: 2 + N
@@ -103,7 +103,6 @@ In our case, there will be a variety of resources, and every resource has severa
 | `Cgroup V2`                |                     | Stage 2               |  🚧        |
 | Hypervisor                 | `Dragonball`        | Stage 1               |  🚧        |
 |                            | QEMU                | Stage 2               |  🚫        |
-|                            | ACRN                | Stage 3               |  🚫        |
 |                            | Cloud Hypervisor    | Stage 3               |  🚫        |
 |                            | Firecracker         | Stage 3               |  🚫        |
 
@@ -166,4 +165,4 @@ In our case, there will be a variety of resources, and every resource has severa
 
 - What is the security boundary for the monolithic / "Built-in VMM" case?
   
-  It has the security boundary of virtualization. More details will be provided in next stage.
+  It has the security boundary of virtualization. More details will be provided in next stage.

docs/design/virtualization.md

@@ -98,8 +98,7 @@ of Kata Containers, the Cloud Hypervisor configuration supports both CPU
 and memory resize, device hotplug (disk and VFIO), file-system sharing through virtio-fs,
 block-based volumes, booting from VM images backed by pmem device, and
 fine-grained seccomp filters for each VMM threads (e.g. all virtio
-device worker threads). Please check [this GitHub Project](https://github.com/orgs/kata-containers/projects/21)
-for details of ongoing integration efforts.
+device worker threads).
 
 Devices and features used:
 - virtio VSOCK or virtio serial

docs/how-to/README.md

@@ -20,12 +20,6 @@
    for the VM rootfs. Refer to the following guide for additional configuration
    steps:
    - [Setup Kata containers with `firecracker`](how-to-use-kata-containers-with-firecracker.md)
-- `ACRN`
-
-  While `qemu` , `cloud-hypervisor` and `firecracker` work out of the box with installation of Kata,
-  some additional configuration is needed in case of `ACRN`.
-  Refer to the following guides for additional configuration steps:
-- [Kata Containers with ACRN Hypervisor](how-to-use-kata-containers-with-acrn.md)
 
 ## Confidential Containers Policy
 
@@ -52,4 +46,4 @@
 - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
 - [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)
 - [How to use the Kata Agent Policy](how-to-use-the-kata-agent-policy.md)
-- [How to pull images in the guest](how-to-pull-images-in-guest-with-kata.md)
+- [How to pull images in the guest](how-to-pull-images-in-guest-with-kata.md)

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -29,16 +29,16 @@ __SNP-specific steps:__
 - Build the SNP-specific kernel as shown below (see this [guide](../../tools/packaging/kernel/README.md#build-kata-containers-kernel) for more information)
 ```bash
 $ pushd kata-containers/tools/packaging/
-$ ./kernel/build-kernel.sh -a x86_64 -x snp setup
-$ ./kernel/build-kernel.sh -a x86_64 -x snp build
-$ sudo -E PATH="${PATH}" ./kernel/build-kernel.sh -x snp install
+$ ./kernel/build-kernel.sh -a x86_64 -x setup
+$ ./kernel/build-kernel.sh -a x86_64 -x build
+$ sudo -E PATH="${PATH}" ./kernel/build-kernel.sh -x install
 $ popd
 ```
 - Build a current OVMF capable of SEV-SNP:
 ```bash
 $ pushd kata-containers/tools/packaging/static-build/ovmf
-$ ./build.sh
-$ tar -xvf edk2-x86_64.tar.gz
+$ ovmf_build=sev ./build.sh
+$ tar -xvf edk2-sev.tar.gz
 $ popd
 ```
 - Build a custom QEMU
@@ -106,7 +106,7 @@ sev_snp_guest = true
 ```
   - Configure an OVMF (add path)
 ```toml
-firmware = "/path/to/kata-containers/tools/packaging/static-build/ovmf/opt/kata/share/ovmf/OVMF.fd"
+firmware = "/path/to/kata-containers/tools/packaging/static-build/ovmf/opt/kata/share/ovmf/AMDSEV.fd"
 ```
   - SNP attestation (add cert-chain to default path or add the path with cert-chain)
 ```toml

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -46,7 +46,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.block_device_cache_set` | `boolean` | cache-related options will be set to block devices or not |
 | `io.katacontainers.config.hypervisor.block_device_driver` | string | the driver to be used for block device, valid values are `virtio-blk`, `virtio-scsi`, `nvdimm`|
 | `io.katacontainers.config.hypervisor.cpu_features` | `string` | Comma-separated list of CPU features to pass to the CPU (QEMU) |
-| `io.katacontainers.config.hypervisor.ctlpath` (R) | `string` | Path to the `acrnctl` binary for the ACRN hypervisor |
 | `io.katacontainers.config.hypervisor.default_max_vcpus` | uint32| the maximum number of vCPUs allocated for the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.default_memory` | uint32| the memory assigned for a VM by the hypervisor in `MiB` |
 | `io.katacontainers.config.hypervisor.default_vcpus` | float32| the default vCPUs assigned for a VM by the hypervisor |
@@ -95,6 +94,8 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
 | `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
+| `io.katacontainers.config.hypervisor.default_gpus` | uint32 | the minimum number of GPUs required for the VM. Only used by remote hypervisor to help with instance selection |
+| `io.katacontainers.config.hypervisor.default_gpu_model` | string | the GPU model required for the VM. Only used by remote hypervisor to help with instance selection |
 
 ## Container Options
 | Key | Value Type | Comments |
@@ -209,7 +210,6 @@ the configuration entry:
 
 | Key | Config file entry | Comments |
 |-------| ----- | ----- |
-| `ctlpath`  | `valid_ctlpaths` | Valid paths for `acrnctl` binary |
 | `entropy_source` | `valid_entropy_sources` | Valid entropy sources, e.g. `/dev/random` |
 | `file_mem_backend`  | `valid_file_mem_backends` | Valid locations for the file-based memory backend root directory |
 | `jailer_path`  | `valid_jailer_paths`| Valid paths for the jailer constraining the container VM (Firecracker) |

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -1,125 +0,0 @@
-# Kata Containers with ACRN
-
-This document provides an overview on how to run Kata containers with ACRN hypervisor and device model.
-
-## Introduction
-
-ACRN is a flexible, lightweight Type-1 reference hypervisor built with real-time and safety-criticality in mind. ACRN uses an open source platform making it optimized to streamline embedded development.
-
-Some of the key features being:
-
-- Small footprint - Approx. 25K lines of code (LOC).
-- Real Time - Low latency, faster boot time, improves overall responsiveness with hardware.
-- Adaptability - Multi-OS support for guest operating systems like Linux, Android, RTOSes.
-- Rich I/O mediators - Allows sharing of various I/O devices across VMs.
-- Optimized for a variety of IoT (Internet of Things) and embedded device solutions.
-
-Please refer to ACRN [documentation](https://projectacrn.github.io/latest/index.html) for more details on ACRN hypervisor and device model.
-
-## Pre-requisites
-
-This document requires the presence of the ACRN hypervisor and Kata Containers on your system. Install using the instructions available through the following links:
-
-- ACRN supported [Hardware](https://projectacrn.github.io/latest/hardware.html#supported-hardware).
-  > **Note:** Please make sure to have a minimum of 4 logical processors (HT) or cores.
-- ACRN [software](https://projectacrn.github.io/latest/tutorials/run_kata_containers.html) setup.
-- For networking, ACRN supports either MACVTAP or TAP. If MACVTAP is not enabled in the Service OS, please follow the below steps to update the kernel:
-
-  ```sh
-   $ git clone https://github.com/projectacrn/acrn-kernel.git
-   $ cd acrn-kernel
-   $ cp kernel_config_sos .config
-   $ sed -i "s/# CONFIG_MACVLAN is not set/CONFIG_MACVLAN=y/" .config
-   $ sed -i '$ i CONFIG_MACVTAP=y' .config
-   $ make clean && make olddefconfig && make && sudo make modules_install INSTALL_MOD_PATH=out/
-  ```
-  Login into Service OS and update the kernel with MACVTAP support:
-
-  ```sh
-  $ sudo mount /dev/sda1 /mnt
-  $ sudo scp -r <user name>@<host address>:<your workspace>/acrn-kernel/arch/x86/boot/bzImage /mnt/EFI/org.clearlinux/
-  $ sudo scp -r <user name>@<host address>:<your workspace>/acrn-kernel/out/lib/modules/* /lib/modules/
-  $ conf_file=$(sed -n '$ s/default //p' /mnt/loader/loader.conf).conf
-  $ kernel_img=$(sed -n 2p /mnt/loader/entries/$conf_file | cut -d'/' -f4)
-  $ sudo sed -i "s/$kernel_img/bzImage/g" /mnt/loader/entries/$conf_file
-  $ sync && sudo umount /mnt && sudo reboot
-  ```
-- Kata Containers installation: Automated installation does not seem to be supported for Clear Linux, so please use [manual installation](../Developer-Guide.md) steps.
-
-> **Note:** Create rootfs image and not initrd image.
-
-In order to run Kata with ACRN, your container stack must provide block-based storage, such as device-mapper.
-
-> **Note:** Currently, by design you can only launch one VM from Kata Containers using ACRN hypervisor (SDC scenario). Based on feedback from community we can increase number of VMs.
-
-## Configure Docker
-
-To configure Docker for device-mapper and Kata,
-
-1. Stop Docker daemon if it is already running.
-
-```bash
-$ sudo systemctl stop docker
-```
-
-2. Set `/etc/docker/daemon.json` with the following contents.
-
-```
-{
-  "storage-driver": "devicemapper"
-}
-```
-
-3. Restart docker.
-
-```bash
-$ sudo systemctl daemon-reload
-$ sudo systemctl restart docker
-```
-
-4. Configure [Docker](../Developer-Guide.md#update-the-docker-systemd-unit-file) to use `kata-runtime`.
-
-## Configure Kata Containers with ACRN
-
-To configure Kata Containers with ACRN, copy the generated `configuration-acrn.toml` file when building the `kata-runtime` to either `/etc/kata-containers/configuration.toml` or `/usr/share/defaults/kata-containers/configuration.toml`.
-
-The following command shows full paths to the `configuration.toml` files that the runtime loads. It will use the first path that exists. (Please make sure the kernel and image paths are set correctly in the `configuration.toml` file)
-
-```bash
-$ sudo kata-runtime --show-default-config-paths
-```
-
->**Warning:** Please offline CPUs using [this](offline_cpu.sh) script, else VM launches will fail.
-
-```bash
-$ sudo ./offline_cpu.sh
-```
-
-Start an ACRN based Kata Container,
-
-```bash
-$ sudo docker run -ti --runtime=kata-runtime busybox sh
-```
-
-You will see ACRN(`acrn-dm`) is now running on your system, as well as a `kata-shim`. You should obtain an interactive shell prompt. Verify that all the Kata processes terminate once you exit the container.
-
-```bash
-$ ps -ef | grep -E "kata|acrn"
-```
-
-Validate ACRN hypervisor by using `kata-runtime kata-env`,
-
-```sh
-$ kata-runtime kata-env | awk -v RS= '/\[Hypervisor\]/'
-[Hypervisor]
-  MachineType = ""
-  Version = "DM version is: 1.2-unstable-254577a6-dirty (daily tag:acrn-2019w27.4-140000p)
-  Path = "/usr/bin/acrn-dm"
-  BlockDeviceDriver = "virtio-blk"
-  EntropySource = "/dev/urandom"
-  Msize9p = 0
-  MemorySlots = 10
-  Debug = false
-  UseVSock = false
-  SharedFS = ""
-```

docs/how-to/offline_cpu.sh

@@ -18,7 +18,6 @@ for i in $(ls -d /sys/devices/system/cpu/cpu[1-9]*); do
                         echo 0 > $i/online
                         online=`cat $i/online`
                 done
-                echo $idx > /sys/class/vhm/acrn_vhm/offline_cpu
         fi
 done
 

docs/hypervisors.md

@@ -18,7 +18,6 @@ which hypervisors you may wish to investigate further.
 
 | Hypervisor | Written in | Architectures | Type |
 |-|-|-|-|
-|[ACRN] | C | `x86_64` | Type 1 (bare metal) |
 |[Cloud Hypervisor] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) |
 |[Firecracker] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) |
 |[QEMU] | C | all | Type 2 ([KVM]) | `configuration-qemu.toml` |
@@ -38,7 +37,6 @@ the hypervisors:
 
 | Hypervisor | Summary | Features | Limitations | Container Creation speed | Memory density | Use cases | Comment |
 |-|-|-|-|-|-|-|-|
-|[ACRN] | Safety critical and real-time workloads | | | excellent | excellent | Embedded and IOT systems | For advanced users |
 |[Cloud Hypervisor] | Low latency, small memory footprint, small attack surface | Minimal | | excellent | excellent | High performance modern cloud workloads | |
 |[Firecracker] | Very slimline | Extremely minimal | Doesn't support all device types | excellent | excellent | Serverless / FaaS | |
 |[QEMU] | Lots of features | Lots | | good | good | Good option for most users | |
@@ -57,7 +55,6 @@ are available, their default values and how each setting can be used.
 
 | Hypervisor | Golang runtime config file | golang runtime short name | golang runtime default | rust runtime config file | rust runtime short name | rust runtime default |
 |-|-|-|-|-|-|-|
-| [ACRN] | [`configuration-acrn.toml`](../src/runtime/config/configuration-acrn.toml.in) | `acrn` | | | | |
 | [Cloud Hypervisor] | [`configuration-clh.toml`](../src/runtime/config/configuration-clh.toml.in) | `clh` | | [`configuration-cloud-hypervisor.toml`](../src/runtime-rs/config/configuration-cloud-hypervisor.toml.in) | `cloud-hypervisor` | |
 | [Firecracker] | [`configuration-fc.toml`](../src/runtime/config/configuration-fc.toml.in) | `fc` | | | | |
 | [QEMU] | [`configuration-qemu.toml`](../src/runtime/config/configuration-qemu.toml.in) | `qemu` | yes | [`configuration-qemu.toml`](../src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in) | `qemu` | |
@@ -93,10 +90,9 @@ are available, their default values and how each setting can be used.
 To switch the configured hypervisor, you only need to run a single command.
 See [the `kata-manager` documentation](../utils/README.md#choose-a-hypervisor) for further details.
 
-[ACRN]: https://projectacrn.org
 [Cloud Hypervisor]: https://github.com/cloud-hypervisor/cloud-hypervisor
 [Firecracker]: https://github.com/firecracker-microvm/firecracker
 [KVM]: https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine
-[QEMU]: http://www.qemu-project.org
+[QEMU]: http://www.qemu.org
 [`Dragonball`]: https://github.com/kata-containers/kata-containers/blob/main/src/dragonball
 [StratoVirt]: https://gitee.com/openeuler/stratovirt

docs/install/kata-containers-3.0-rust-runtime-installation-guide.md

@@ -83,6 +83,23 @@ $ make && sudo make install
 ```
 After running the command above, the default config file `configuration.toml` will be installed under `/usr/share/defaults/kata-containers/`,  the binary file `containerd-shim-kata-v2` will be installed under `/usr/local/bin/` .
 
+### Install Shim Without Builtin Dragonball VMM
+
+By default, runtime-rs includes the `Dragonball` VMM. To build without the built-in `Dragonball` hypervisor, use `make USE_BUILDIN_DB=false`:
+```bash
+$ cd kata-containers/src/runtime-rs
+$ make USE_BUILDIN_DB=false
+```
+After building, specify the desired hypervisor during installation using `HYPERVISOR`. For example, to use `qemu` or `cloud-hypervisor`:
+
+```
+sudo make install HYPERVISOR=qemu
+```
+or
+```
+sudo make install HYPERVISOR=cloud-hypervisor
+```
+
 ### Build Kata Containers Kernel
 Follow the [Kernel installation guide](/tools/packaging/kernel/README.md).
 

src/agent/Cargo.lock

@@ -64,6 +64,20 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "ahash"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
+dependencies = [
+ "cfg-if 1.0.0",
+ "getrandom",
+ "once_cell",
+ "serde",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -97,6 +111,55 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "anstream"
+version = "0.6.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.86"
@@ -712,6 +775,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "bytecount"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -814,6 +883,30 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cdi"
+version = "0.1.0"
+source = "git+https://github.com/cncf-tags/container-device-interface-rs?rev=fba5677a8e7cc962fc6e495fcec98d7d765e332a#fba5677a8e7cc962fc6e495fcec98d7d765e332a"
+dependencies = [
+ "anyhow",
+ "clap 4.5.13",
+ "const_format",
+ "jsonschema",
+ "lazy_static",
+ "libc",
+ "nix 0.24.3",
+ "notify",
+ "oci-spec",
+ "once_cell",
+ "path-clean",
+ "regex",
+ "semver",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "serde_yaml",
+]
+
 [[package]]
 name = "cesu8"
 version = "1.1.0"
@@ -914,8 +1007,8 @@ checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
 dependencies = [
  "atty",
  "bitflags 1.3.2",
- "clap_derive",
- "clap_lex",
+ "clap_derive 3.2.25",
+ "clap_lex 0.2.4",
  "indexmap 1.9.3",
  "once_cell",
  "strsim 0.10.0",
@@ -923,6 +1016,28 @@ dependencies = [
  "textwrap",
 ]
 
+[[package]]
+name = "clap"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
+dependencies = [
+ "clap_builder",
+ "clap_derive 4.5.13",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex 0.7.2",
+ "strsim 0.11.1",
+]
+
 [[package]]
 name = "clap_derive"
 version = "3.2.25"
@@ -936,6 +1051,18 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "clap_derive"
+version = "4.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.71",
+]
+
 [[package]]
 name = "clap_lex"
 version = "0.2.4"
@@ -945,6 +1072,12 @@ dependencies = [
  "os_str_bytes",
 ]
 
+[[package]]
+name = "clap_lex"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
+
 [[package]]
 name = "cmac"
 version = "0.7.2"
@@ -967,6 +1100,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "colorchoice"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -1741,6 +1880,17 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "fancy-regex"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
+dependencies = [
+ "bit-set",
+ "regex-automata 0.4.7",
+ "regex-syntax 0.8.4",
+]
+
 [[package]]
 name = "fastrand"
 version = "1.9.0"
@@ -1812,6 +1962,15 @@ dependencies = [
  "miniz_oxide",
 ]
 
+[[package]]
+name = "fluent-uri"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17c704e9dbe1ddd863da1e6ff3567795087b1eb201ce80d8fa81162e1516500d"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1827,6 +1986,25 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fraction"
+version = "0.15.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7"
+dependencies = [
+ "lazy_static",
+ "num",
+]
+
+[[package]]
+name = "fsevent-sys"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "funty"
 version = "2.0.0"
@@ -2052,7 +2230,7 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 dependencies = [
- "ahash",
+ "ahash 0.7.8",
 ]
 
 [[package]]
@@ -2605,6 +2783,21 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
+[[package]]
+name = "iso8601"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -2623,15 +2816,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -2690,6 +2874,18 @@ dependencies = [
  "smallvec",
 ]
 
+[[package]]
+name = "json-patch"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b1fb8864823fad91877e6caea0baca82e49e8db50f8e5c9f9a453e27d3330fc"
+dependencies = [
+ "jsonptr",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
 [[package]]
 name = "json-syntax"
 version = "0.12.5"
@@ -2709,6 +2905,47 @@ dependencies = [
  "utf8-decode",
 ]
 
+[[package]]
+name = "jsonptr"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627"
+dependencies = [
+ "fluent-uri",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "jsonschema"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0afd06142c9bcb03f4a8787c77897a87b6be9c4918f1946c33caa714c27578"
+dependencies = [
+ "ahash 0.8.11",
+ "anyhow",
+ "base64 0.22.1",
+ "bytecount",
+ "clap 4.5.13",
+ "fancy-regex",
+ "fraction",
+ "getrandom",
+ "iso8601",
+ "itoa",
+ "memchr",
+ "num-cmp",
+ "once_cell",
+ "parking_lot 0.12.3",
+ "percent-encoding",
+ "regex",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "time",
+ "url",
+ "uuid",
+]
+
 [[package]]
 name = "jwt"
 version = "0.16.0"
@@ -2773,14 +3010,16 @@ dependencies = [
  "async-std",
  "async-trait",
  "capctl",
+ "cdi",
  "cfg-if 1.0.0",
  "cgroups-rs",
- "clap",
+ "clap 3.2.25",
  "const_format",
  "derivative",
  "futures",
  "image-rs",
  "ipnetwork",
+ "json-patch",
  "kata-sys-util",
  "kata-types",
  "lazy_static",
@@ -2835,7 +3074,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -2934,6 +3172,26 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "kqueue"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c"
+dependencies = [
+ "kqueue-sys",
+ "libc",
+]
+
+[[package]]
+name = "kqueue-sys"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+]
+
 [[package]]
 name = "krata-tokio-tar"
 version = "0.4.2"
@@ -3285,6 +3543,18 @@ dependencies = [
  "adler",
 ]
 
+[[package]]
+name = "mio"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+dependencies = [
+ "libc",
+ "log",
+ "wasi",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "mio"
 version = "1.0.2"
@@ -3465,6 +3735,25 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "notify"
+version = "6.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
+dependencies = [
+ "bitflags 2.6.0",
+ "crossbeam-channel",
+ "filetime",
+ "fsevent-sys",
+ "inotify",
+ "kqueue",
+ "libc",
+ "log",
+ "mio 0.8.11",
+ "walkdir",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "ntapi"
 version = "0.4.1"
@@ -3515,6 +3804,12 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "num-cmp"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa"
+
 [[package]]
 name = "num-complex"
 version = "0.4.6"
@@ -3894,6 +4189,12 @@ dependencies = [
  "slash-formatter",
 ]
 
+[[package]]
+name = "path-clean"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
+
 [[package]]
 name = "path-dedot"
 version = "1.2.4"
@@ -4376,7 +4677,6 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.5.1",
  "serde",
@@ -4626,14 +4926,12 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
 
 [[package]]
 name = "regorus"
-version = "0.1.5"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77dd872918e5c172bd42ac49716f89a15e35be513bba3d902e355a531529a87f"
+checksum = "843c3d97f07e3b5ac0955d53ad0af4c91fe4a4f8525843ece5bf014f27829b73"
 dependencies = [
  "anyhow",
- "itertools 0.12.1",
  "lazy_static",
- "num",
  "rand",
  "regex",
  "scientific",
@@ -4666,6 +4964,7 @@ dependencies = [
  "bytes 1.6.1",
  "cookie",
  "cookie_store",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "http",
@@ -5919,7 +6218,7 @@ dependencies = [
  "backtrace",
  "bytes 1.6.1",
  "libc",
- "mio",
+ "mio 1.0.2",
  "parking_lot 0.12.3",
  "pin-project-lite",
  "signal-hook-registry",
@@ -6346,6 +6645,12 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
 [[package]]
 name = "uuid"
 version = "1.10.0"
@@ -7021,6 +7326,26 @@ dependencies = [
  "zvariant",
 ]
 
+[[package]]
+name = "zerocopy"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.71",
+]
+
 [[package]]
 name = "zerofrom"
 version = "0.1.4"

src/agent/Cargo.toml

@@ -80,10 +80,13 @@ strum_macros = "0.26.2"
 image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "v0.10.0", default-features = false, optional = true }
 
 # Agent Policy
-regorus = { version = "0.1.4", default-features = false, features = [
+regorus = { version = "0.2.6", default-features = false, features = [
     "arc",
     "regex",
+    "std",
 ], optional = true }
+cdi = { git = "https://github.com/cncf-tags/container-device-interface-rs", rev = "fba5677a8e7cc962fc6e495fcec98d7d765e332a" }
+json-patch = "2.0.0"
 
 [dev-dependencies]
 tempfile = "3.1.0"

src/agent/rustjail/src/container.rs

@@ -158,7 +158,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(1)
                 .minor(3)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()
@@ -168,7 +168,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(1)
                 .minor(5)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()
@@ -178,7 +178,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(1)
                 .minor(7)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()
@@ -188,7 +188,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(5)
                 .minor(0)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()
@@ -198,7 +198,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(1)
                 .minor(9)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()
@@ -208,7 +208,7 @@ lazy_static! {
                 .typ(oci::LinuxDeviceType::C)
                 .major(1)
                 .minor(8)
-                .file_mode(0o066_u32)
+                .file_mode(0o666_u32)
                 .uid(0xffffffff_u32)
                 .gid(0xffffffff_u32)
                 .build()

src/agent/rustjail/src/mount.rs

@@ -233,7 +233,7 @@ pub fn init_rootfs(
         // bind may be only specified in the oci spec options -> flags update r#type
         let m = &{
             let mut mbind = m.clone();
-            if mbind.typ().is_none() && flags & MsFlags::MS_BIND == MsFlags::MS_BIND {
+            if is_none_mount_type(mbind.typ()) && flags & MsFlags::MS_BIND == MsFlags::MS_BIND {
                 mbind.set_typ(Some("bind".to_string()));
             }
             mbind
@@ -397,6 +397,13 @@ fn mount_cgroups_v2(cfd_log: RawFd, m: &Mount, rootfs: &str, flags: MsFlags) ->
     Ok(())
 }
 
+fn is_none_mount_type(typ: &Option<String>) -> bool {
+    match typ {
+        Some(t) => t == "none",
+        None => true,
+    }
+}
+
 fn mount_cgroups(
     cfd_log: RawFd,
     m: &Mount,

src/agent/src/cdh.rs

@@ -8,13 +8,15 @@
 // https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub
 
 use crate::AGENT_CONFIG;
-use crate::CDH_SOCKET_URI;
-use anyhow::{Context, Result};
+use anyhow::{bail, Context, Result};
 use derivative::Derivative;
 use protocols::{
     confidential_data_hub, confidential_data_hub_ttrpc_async,
     confidential_data_hub_ttrpc_async::{SealedSecretServiceClient, SecureMountServiceClient},
 };
+use std::fs;
+use std::os::unix::fs::symlink;
+use std::path::Path;
 use tokio::sync::OnceCell;
 
 // Nanoseconds
@@ -22,8 +24,14 @@ lazy_static! {
     static ref CDH_API_TIMEOUT: i64 = AGENT_CONFIG.cdh_api_timeout.as_nanos() as i64;
     pub static ref CDH_CLIENT: OnceCell<CDHClient> = OnceCell::new();
 }
+
 const SEALED_SECRET_PREFIX: &str = "sealed.";
 
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "cdh"))
+}
+
 #[derive(Derivative)]
 #[derivative(Clone, Debug)]
 pub struct CDHClient {
@@ -34,8 +42,8 @@ pub struct CDHClient {
 }
 
 impl CDHClient {
-    pub fn new() -> Result<Self> {
-        let client = ttrpc::asynchronous::Client::connect(CDH_SOCKET_URI)?;
+    pub fn new(cdh_socket_uri: &str) -> Result<Self> {
+        let client = ttrpc::asynchronous::Client::connect(cdh_socket_uri)?;
         let sealed_secret_client =
             confidential_data_hub_ttrpc_async::SealedSecretServiceClient::new(client.clone());
         let secure_mount_client =
@@ -78,9 +86,11 @@ impl CDHClient {
     }
 }
 
-pub async fn init_cdh_client() -> Result<()> {
+pub async fn init_cdh_client(cdh_socket_uri: &str) -> Result<()> {
     CDH_CLIENT
-        .get_or_try_init(|| async { CDHClient::new().context("Failed to create CDH Client") })
+        .get_or_try_init(|| async {
+            CDHClient::new(cdh_socket_uri).context("Failed to create CDH Client")
+        })
         .await?;
     Ok(())
 }
@@ -106,6 +116,74 @@ pub async fn unseal_env(env: &str) -> Result<String> {
     Ok((*env.to_owned()).to_string())
 }
 
+pub async fn unseal_file(path: &str) -> Result<()> {
+    let cdh_client = CDH_CLIENT
+        .get()
+        .expect("Confidential Data Hub not initialized");
+
+    if !Path::new(path).exists() {
+        bail!("sealed secret file {:?} does not exist", path);
+    }
+
+    // Iterate over all entries to handle the sealed secret file.
+    // For example, the directory is as follows:
+    // The secret directory in the guest: /run/kata-containers/shared/containers/21bbf0d932b70263d65d7052ecfd72ee46de03f766650cb378e93852ddb30a54-5063be11b6800f96-sealed-secret-target/:
+    // - ..2024_09_30_02_55_58.2237819815
+    // - ..data -> ..2024_09_30_02_55_58.2237819815
+    // - secret -> ..2024_09_30_02_55_58.2237819815/secret
+    //
+    // The directory "..2024_09_30_02_55_58.2237819815":
+    // - secret
+    for entry in fs::read_dir(path)? {
+        let entry = entry?;
+        let entry_type = entry.file_type()?;
+        if !entry_type.is_symlink() && !entry_type.is_file() {
+            debug!(
+                sl(),
+                "skipping sealed source entry {:?} because its file type is {:?}",
+                entry,
+                entry_type
+            );
+            continue;
+        }
+
+        let target_path = fs::canonicalize(&entry.path())?;
+        info!(sl(), "sealed source entry target path: {:?}", target_path);
+
+        // Skip if the target path is not a file (e.g., it's a symlink pointing to the secret file).
+        if !target_path.is_file() {
+            debug!(sl(), "sealed source is not a file: {:?}", target_path);
+            continue;
+        }
+
+        let secret_name = entry.file_name();
+        let contents = fs::read_to_string(&target_path)?;
+        if contents.starts_with(SEALED_SECRET_PREFIX) {
+            // Get the directory name of the sealed secret file
+            let dir_name = target_path
+                .parent()
+                .and_then(|p| p.file_name())
+                .map(|name| name.to_string_lossy().to_string())
+                .unwrap_or_default();
+
+            // Create the unsealed file name in the same directory, which will be written the unsealed data.
+            let unsealed_filename = format!("{}.unsealed", target_path.to_string_lossy());
+            // Create the unsealed file symlink, which is used for reading the unsealed data in the container.
+            let unsealed_filename_symlink =
+                format!("{}/{}.unsealed", dir_name, secret_name.to_string_lossy());
+
+            // Unseal the secret and write it to the unsealed file
+            let unsealed_value = cdh_client.unseal_secret_async(&contents).await?;
+            fs::write(&unsealed_filename, unsealed_value)?;
+
+            // Remove the original sealed symlink and create a symlink to the unsealed file
+            fs::remove_file(&entry.path())?;
+            symlink(unsealed_filename_symlink, &entry.path())?;
+        }
+    }
+    Ok(())
+}
+
 pub async fn secure_mount(
     volume_type: &str,
     options: &std::collections::HashMap<String, String>,
@@ -123,17 +201,15 @@ pub async fn secure_mount(
 }
 
 #[cfg(test)]
-#[cfg(feature = "sealed-secret")]
 mod tests {
-    use crate::cdh::CDHClient;
-    use crate::cdh::CDH_ADDR;
-    use anyhow::anyhow;
+    use super::*;
     use async_trait::async_trait;
-    use protocols::{confidential_data_hub, confidential_data_hub_ttrpc_async};
+    use std::fs::File;
+    use std::io::{Read, Write};
     use std::sync::Arc;
+    use tempfile::tempdir;
     use test_utils::skip_if_not_root;
     use tokio::signal::unix::{signal, SignalKind};
-
     struct TestService;
 
     #[async_trait]
@@ -161,17 +237,17 @@ mod tests {
         Ok(())
     }
 
-    fn start_ttrpc_server() {
+    fn start_ttrpc_server(cdh_socket_uri: String) {
         tokio::spawn(async move {
             let ss = Box::new(TestService {})
                 as Box<dyn confidential_data_hub_ttrpc_async::SealedSecretService + Send + Sync>;
             let ss = Arc::new(ss);
             let ss_service = confidential_data_hub_ttrpc_async::create_sealed_secret_service(ss);
 
-            remove_if_sock_exist(CDH_ADDR).unwrap();
+            remove_if_sock_exist(&cdh_socket_uri).unwrap();
 
             let mut server = ttrpc::asynchronous::Server::new()
-                .bind(CDH_ADDR)
+                .bind(&cdh_socket_uri)
                 .unwrap()
                 .register_service(ss_service);
 
@@ -187,23 +263,58 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_unseal_env() {
+    async fn test_sealed_secret() {
         skip_if_not_root!();
+        let test_dir = tempdir().expect("failed to create tmpdir");
+        let test_dir_path = test_dir.path();
+        let cdh_sock_uri = &format!(
+            "unix://{}",
+            test_dir_path.join("cdh.sock").to_str().unwrap()
+        );
 
         let rt = tokio::runtime::Runtime::new().unwrap();
         let _guard = rt.enter();
-        start_ttrpc_server();
+        start_ttrpc_server(cdh_sock_uri.to_string());
         std::thread::sleep(std::time::Duration::from_secs(2));
+        init_cdh_client(cdh_sock_uri).await.unwrap();
 
-        let cc = Some(CDHClient::new().unwrap());
-        let cdh_client = cc.as_ref().ok_or(anyhow!("get cdh_client failed")).unwrap();
+        // Test sealed secret as env vars
         let sealed_env = String::from("key=sealed.testdata");
-        let unsealed_env = cdh_client.unseal_env(&sealed_env).await.unwrap();
+        let unsealed_env = unseal_env(&sealed_env).await.unwrap();
         assert_eq!(unsealed_env, String::from("key=unsealed"));
         let normal_env = String::from("key=testdata");
-        let unchanged_env = cdh_client.unseal_env(&normal_env).await.unwrap();
+        let unchanged_env = unseal_env(&normal_env).await.unwrap();
         assert_eq!(unchanged_env, String::from("key=testdata"));
 
+        // Test sealed secret as files
+        let sealed_dir = test_dir_path.join("..test");
+        fs::create_dir(&sealed_dir).unwrap();
+        let sealed_filename = sealed_dir.join("secret");
+        let mut sealed_file = File::create(sealed_filename.clone()).unwrap();
+        sealed_file.write_all(b"sealed.testdata").unwrap();
+        let secret_symlink = test_dir_path.join("secret");
+        symlink(&sealed_filename, &secret_symlink).unwrap();
+
+        unseal_file(test_dir_path.to_str().unwrap()).await.unwrap();
+
+        let unsealed_filename = test_dir_path.join("secret");
+        let mut unsealed_file = File::open(unsealed_filename.clone()).unwrap();
+        let mut contents = String::new();
+        unsealed_file.read_to_string(&mut contents).unwrap();
+        assert_eq!(contents, String::from("unsealed"));
+        fs::remove_file(sealed_filename).unwrap();
+        fs::remove_file(unsealed_filename).unwrap();
+
+        let normal_filename = test_dir_path.join("secret");
+        let mut normal_file = File::create(normal_filename.clone()).unwrap();
+        normal_file.write_all(b"testdata").unwrap();
+        unseal_file(test_dir_path.to_str().unwrap()).await.unwrap();
+        let mut contents = String::new();
+        let mut normal_file = File::open(normal_filename.clone()).unwrap();
+        normal_file.read_to_string(&mut contents).unwrap();
+        assert_eq!(contents, String::from("testdata"));
+        fs::remove_file(normal_filename).unwrap();
+
         rt.shutdown_background();
         std::thread::sleep(std::time::Duration::from_secs(2));
     }

src/agent/src/config.rs

@@ -89,7 +89,7 @@ pub enum GuestComponentsFeatures {
     Resource,
 }
 
-#[derive(Clone, Copy, Debug, Default, Display, Deserialize, EnumString, PartialEq)]
+#[derive(Clone, Copy, Debug, Default, Display, Deserialize, EnumString, PartialEq, Eq)]
 /// Attestation-related processes that we want to spawn as children of the agent
 #[strum(serialize_all = "kebab-case")]
 #[serde(rename_all = "kebab-case")]
@@ -630,6 +630,7 @@ mod tests {
 
     use super::*;
     use anyhow::anyhow;
+    use rstest::*;
     use serial_test::serial;
     use std::fs::File;
     use std::io::Write;
@@ -1327,562 +1328,193 @@ mod tests {
         assert_eq!(expected.tracing, config.tracing);
     }
 
-    #[test]
-    fn test_logrus_to_slog_level() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            logrus_level: &'a str,
-            result: Result<slog::Level>,
-        }
-
-        let tests = &[
-            TestData {
-                logrus_level: "",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL)),
-            },
-            TestData {
-                logrus_level: "foo",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL)),
-            },
-            TestData {
-                logrus_level: "debugging",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL)),
-            },
-            TestData {
-                logrus_level: "xdebug",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL)),
-            },
-            TestData {
-                logrus_level: "trace",
-                result: Ok(slog::Level::Trace),
-            },
-            TestData {
-                logrus_level: "debug",
-                result: Ok(slog::Level::Debug),
-            },
-            TestData {
-                logrus_level: "info",
-                result: Ok(slog::Level::Info),
-            },
-            TestData {
-                logrus_level: "warn",
-                result: Ok(slog::Level::Warning),
-            },
-            TestData {
-                logrus_level: "warning",
-                result: Ok(slog::Level::Warning),
-            },
-            TestData {
-                logrus_level: "error",
-                result: Ok(slog::Level::Error),
-            },
-            TestData {
-                logrus_level: "critical",
-                result: Ok(slog::Level::Critical),
-            },
-            TestData {
-                logrus_level: "fatal",
-                result: Ok(slog::Level::Critical),
-            },
-            TestData {
-                logrus_level: "panic",
-                result: Ok(slog::Level::Critical),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = logrus_to_slog_level(d.logrus_level);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_LOG_LEVEL)))]
+    #[case("foo", Err(anyhow!(ERR_INVALID_LOG_LEVEL)))]
+    #[case("debugging", Err(anyhow!(ERR_INVALID_LOG_LEVEL)))]
+    #[case("xdebug", Err(anyhow!(ERR_INVALID_LOG_LEVEL)))]
+    #[case("trace", Ok(slog::Level::Trace))]
+    #[case("debug", Ok(slog::Level::Debug))]
+    #[case("info", Ok(slog::Level::Info))]
+    #[case("warn", Ok(slog::Level::Warning))]
+    #[case("warning", Ok(slog::Level::Warning))]
+    #[case("error", Ok(slog::Level::Error))]
+    #[case("critical", Ok(slog::Level::Critical))]
+    #[case("fatal", Ok(slog::Level::Critical))]
+    #[case("panic", Ok(slog::Level::Critical))]
+    fn test_logrus_to_slog_level(#[case] input: &str, #[case] expected: Result<slog::Level>) {
+        let result = logrus_to_slog_level(input);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_log_level() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<slog::Level>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)),
-            },
-            TestData {
-                param: "=",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "x=",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "=y",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "==",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)),
-            },
-            TestData {
-                param: "= =",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)),
-            },
-            TestData {
-                param: "x=y",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "agent=debug",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "agent.logg=debug",
-                result: Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)),
-            },
-            TestData {
-                param: "agent.log=trace",
-                result: Ok(slog::Level::Trace),
-            },
-            TestData {
-                param: "agent.log=debug",
-                result: Ok(slog::Level::Debug),
-            },
-            TestData {
-                param: "agent.log=info",
-                result: Ok(slog::Level::Info),
-            },
-            TestData {
-                param: "agent.log=warn",
-                result: Ok(slog::Level::Warning),
-            },
-            TestData {
-                param: "agent.log=warning",
-                result: Ok(slog::Level::Warning),
-            },
-            TestData {
-                param: "agent.log=error",
-                result: Ok(slog::Level::Error),
-            },
-            TestData {
-                param: "agent.log=critical",
-                result: Ok(slog::Level::Critical),
-            },
-            TestData {
-                param: "agent.log=fatal",
-                result: Ok(slog::Level::Critical),
-            },
-            TestData {
-                param: "agent.log=panic",
-                result: Ok(slog::Level::Critical),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_log_level(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    #[rstest]
+    #[case("",Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)))]
+    #[case("=",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("x=",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("=y",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("==",Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)))]
+    #[case("= =",Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM)))]
+    #[case("x=y",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("agent=debug",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("agent.logg=debug",Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY)))]
+    #[case("agent.log=trace", Ok(slog::Level::Trace))]
+    #[case("agent.log=debug", Ok(slog::Level::Debug))]
+    #[case("agent.log=info", Ok(slog::Level::Info))]
+    #[case("agent.log=warn", Ok(slog::Level::Warning))]
+    #[case("agent.log=warning", Ok(slog::Level::Warning))]
+    #[case("agent.log=error", Ok(slog::Level::Error))]
+    #[case("agent.log=critical", Ok(slog::Level::Critical))]
+    #[case("agent.log=fatal", Ok(slog::Level::Critical))]
+    #[case("agent.log=panic", Ok(slog::Level::Critical))]
+    fn test_get_log_level(#[case] input: &str, #[case] expected: Result<slog::Level>) {
+        let result = get_log_level(input);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_timeout() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<time::Duration>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_TIMEOUT)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout",
-                result: Err(anyhow!(ERR_INVALID_TIMEOUT)),
-            },
-            TestData {
-                param: "foo=bar",
-                result: Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)),
-            },
-            TestData {
-                param: "agent.hotplug_timeot=1",
-                result: Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)),
-            },
-            TestData {
-                param: "agent.chd_api_timeout=1",
-                result: Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=1",
-                result: Ok(time::Duration::from_secs(1)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=3",
-                result: Ok(time::Duration::from_secs(3)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=3600",
-                result: Ok(time::Duration::from_secs(3600)),
-            },
-            TestData {
-                param: "agent.cdh_api_timeout=600",
-                result: Ok(time::Duration::from_secs(600)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=0",
-                result: Ok(time::Duration::from_secs(0)),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=-1",
-                result: Err(anyhow!(
-                    "unable to parse timeout
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_TIMEOUT)))]
+    #[case("agent.hotplug_timeout", Err(anyhow!(ERR_INVALID_TIMEOUT)))]
+    #[case("foo=bar", Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)))]
+    #[case("agent.hotplug_timeot=1", Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)))]
+    #[case("agent.hotplug_timeout=1", Ok(time::Duration::from_secs(1)))]
+    #[case("agent.hotplug_timeout=3", Ok(time::Duration::from_secs(3)))]
+    #[case("agent.hotplug_timeout=3600", Ok(time::Duration::from_secs(3600)))]
+    #[case("agent.hotplug_timeout=0", Ok(time::Duration::from_secs(0)))]
+    #[case("agent.hotplug_timeout=-1", Err(anyhow!(
+        "unable to parse timeout
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=4jbsdja",
-                result: Err(anyhow!(
-                    "unable to parse timeout
+    )))]
+    #[case("agent.hotplug_timeout=4jbsdja", Err(anyhow!(
+        "unable to parse timeout
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=foo",
-                result: Err(anyhow!(
-                    "unable to parse timeout
+    )))]
+    #[case("agent.hotplug_timeout=foo", Err(anyhow!(
+        "unable to parse timeout
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-            TestData {
-                param: "agent.hotplug_timeout=j",
-                result: Err(anyhow!(
-                    "unable to parse timeout
+    )))]
+    #[case("agent.hotplug_timeout=j", Err(anyhow!(
+        "unable to parse timeout
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_timeout(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    )))]
+    #[case("agent.chd_api_timeout=1", Err(anyhow!(ERR_INVALID_TIMEOUT_KEY)))]
+    #[case("agent.cdh_api_timeout=600", Ok(time::Duration::from_secs(600)))]
+    fn test_timeout(#[case] param: &str, #[case] expected: Result<time::Duration>) {
+        let result = get_timeout(param);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_container_pipe_size() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<i32>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE)),
-            },
-            TestData {
-                param: "agent.container_pipe_size",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE)),
-            },
-            TestData {
-                param: "foo=bar",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE_KEY)),
-            },
-            TestData {
-                param: "agent.container_pip_siz=1",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE_KEY)),
-            },
-            TestData {
-                param: "agent.container_pipe_size=1",
-                result: Ok(1),
-            },
-            TestData {
-                param: "agent.container_pipe_size=3",
-                result: Ok(3),
-            },
-            TestData {
-                param: "agent.container_pipe_size=2097152",
-                result: Ok(2097152),
-            },
-            TestData {
-                param: "agent.container_pipe_size=0",
-                result: Ok(0),
-            },
-            TestData {
-                param: "agent.container_pipe_size=-1",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_NEGATIVE)),
-            },
-            TestData {
-                param: "agent.container_pipe_size=foobar",
-                result: Err(anyhow!(
-                    "unable to parse container pipe size
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE)))]
+    #[case("agent.container_pipe_size", Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE)))]
+    #[case("foo=bar", Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE_KEY)))]
+    #[case("agent.container_pip_siz=1", Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE_KEY)))]
+    #[case("agent.container_pipe_size=1", Ok(1))]
+    #[case("agent.container_pipe_size=3", Ok(3))]
+    #[case("agent.container_pipe_size=2097152", Ok(2097152))]
+    #[case("agent.container_pipe_size=0", Ok(0))]
+    #[case("agent.container_pipe_size=-1", Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_NEGATIVE)))]
+    #[case("agent.container_pipe_size=foobar", Err(anyhow!(
+        "unable to parse container pipe size
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-            TestData {
-                param: "agent.container_pipe_size=j",
-                result: Err(anyhow!(
-                    "unable to parse container pipe size
+    )))]
+    #[case("agent.container_pipe_size=j", Err(anyhow!(
+        "unable to parse container pipe size
 
 Caused by:
     invalid digit found in string",
-                )),
-            },
-            TestData {
-                param: "agent.container_pipe_size=4jbsdja",
-                result: Err(anyhow!(
-                    "unable to parse container pipe size
+    )))]
+    #[case("agent.container_pipe_size=4jbsdja", Err(anyhow!(
+        "unable to parse container pipe size
 
 Caused by:
     invalid digit found in string"
-                )),
-            },
-            TestData {
-                param: "agent.container_pipe_size=4294967296",
-                result: Err(anyhow!(
-                    "unable to parse container pipe size
+    )))]
+    #[case("agent.container_pipe_size=4294967296", Err(anyhow!(
+        "unable to parse container pipe size
 
 Caused by:
     number too large to fit in target type"
-                )),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_container_pipe_size(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    )))]
+    fn test_get_container_pipe_size(#[case] param: &str, #[case] expected: Result<i32>) {
+        let result = get_container_pipe_size(param);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_string_value() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<String>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)),
-            },
-            TestData {
-                param: "=",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "==",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "x=",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_VALUE)),
-            },
-            TestData {
-                param: "x==",
-                result: Ok("=".into()),
-            },
-            TestData {
-                param: "x===",
-                result: Ok("==".into()),
-            },
-            TestData {
-                param: "x==x",
-                result: Ok("=x".into()),
-            },
-            TestData {
-                param: "x=x",
-                result: Ok("x".into()),
-            },
-            TestData {
-                param: "x=x=",
-                result: Ok("x=".into()),
-            },
-            TestData {
-                param: "x=x=x",
-                result: Ok("x=x".into()),
-            },
-            TestData {
-                param: "foo=bar",
-                result: Ok("bar".into()),
-            },
-            TestData {
-                param: "x= =",
-                result: Ok(" =".into()),
-            },
-            TestData {
-                param: "x= =",
-                result: Ok(" =".into()),
-            },
-            TestData {
-                param: "x= = ",
-                result: Ok(" = ".into()),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_string_value(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)))]
+    #[case("=", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("==", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("x=", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_VALUE)))]
+    #[case("x==", Ok("=".into()))]
+    #[case("x===", Ok("==".into()))]
+    #[case("x==x", Ok("=x".into()))]
+    #[case("x=x", Ok("x".into()))]
+    #[case("x=x=", Ok("x=".into()))]
+    #[case("x=x=x", Ok("x=x".into()))]
+    #[case("foo=bar", Ok("bar".into()))]
+    #[case("x= =", Ok(" =".into()))]
+    #[case("x= =", Ok(" =".into()))]
+    #[case("x= = ", Ok(" = ".into()))]
+    fn test_get_string_value(#[case] param: &str, #[case] expected: Result<String>) {
+        let result = get_string_value(param);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_guest_components_features_value() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<GuestComponentsFeatures>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)),
-            },
-            TestData {
-                param: "=",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "==",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "x=all",
-                result: Ok(GuestComponentsFeatures::All),
-            },
-            TestData {
-                param: "x=attestation",
-                result: Ok(GuestComponentsFeatures::Attestation),
-            },
-            TestData {
-                param: "x=resource",
-                result: Ok(GuestComponentsFeatures::Resource),
-            },
-            TestData {
-                param: "x===",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)),
-            },
-            TestData {
-                param: "x==x",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)),
-            },
-            TestData {
-                param: "x=x",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_guest_components_features_value(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)))]
+    #[case("=", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("==", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("x=all", Ok(GuestComponentsFeatures::All))]
+    #[case("x=attestation", Ok(GuestComponentsFeatures::Attestation))]
+    #[case("x=resource", Ok(GuestComponentsFeatures::Resource))]
+    #[case("x===", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)))]
+    #[case("x==x", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)))]
+    #[case("x=x", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)))]
+    fn test_get_guest_components_features_value(
+        #[case] input: &str,
+        #[case] expected: Result<GuestComponentsFeatures>,
+    ) {
+        let result = get_guest_components_features_value(input);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
-    #[test]
-    fn test_get_guest_components_procs_value() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<GuestComponentsProcs>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)),
-            },
-            TestData {
-                param: "=",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "==",
-                result: Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "x=attestation-agent",
-                result: Ok(GuestComponentsProcs::AttestationAgent),
-            },
-            TestData {
-                param: "x=confidential-data-hub",
-                result: Ok(GuestComponentsProcs::ConfidentialDataHub),
-            },
-            TestData {
-                param: "x=none",
-                result: Ok(GuestComponentsProcs::None),
-            },
-            TestData {
-                param: "x=api-server-rest",
-                result: Ok(GuestComponentsProcs::ApiServerRest),
-            },
-            TestData {
-                param: "x===",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
-            },
-            TestData {
-                param: "x==x",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
-            },
-            TestData {
-                param: "x=x",
-                result: Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_guest_components_procs_value(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+    #[rstest]
+    #[case("", Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM)))]
+    #[case("=", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("==", Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME)))]
+    #[case("x=attestation-agent", Ok(GuestComponentsProcs::AttestationAgent))]
+    #[case(
+        "x=confidential-data-hub",
+        Ok(GuestComponentsProcs::ConfidentialDataHub)
+    )]
+    #[case("x=none", Ok(GuestComponentsProcs::None))]
+    #[case("x=api-server-rest", Ok(GuestComponentsProcs::ApiServerRest))]
+    #[case("x===", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)))]
+    #[case("x==x", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)))]
+    #[case("x=x", Err(anyhow!(ERR_INVALID_GUEST_COMPONENTS_PROCS_VALUE)))]
+    fn test_get_guest_components_procs_value(
+        #[case] param: &str,
+        #[case] expected: Result<GuestComponentsProcs>,
+    ) {
+        let result = get_guest_components_procs_value(param);
+        let msg = format!("expected: {:?}, result: {:?}", expected, result);
+        assert_result!(expected, result, msg);
     }
 
     #[test]

src/agent/src/device/mod.rs

@@ -11,6 +11,9 @@ use self::vfio_device_handler::{VfioApDeviceHandler, VfioPciDeviceHandler};
 use crate::pci;
 use crate::sandbox::Sandbox;
 use anyhow::{anyhow, Context, Result};
+use cdi::annotations::parse_annotations;
+use cdi::cache::{new_cache, with_auto_refresh, CdiOption};
+use cdi::spec_dirs::with_spec_dirs;
 use kata_types::device::DeviceHandlerManager;
 use nix::sys::stat;
 use oci::{LinuxDeviceCgroup, Spec};
@@ -25,6 +28,8 @@ use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
+use tokio::time;
+use tokio::time::Duration;
 use tracing::instrument;
 
 pub mod block_device_handler;
@@ -238,6 +243,69 @@ pub async fn add_devices(
     update_spec_devices(logger, spec, dev_updates)
 }
 
+#[instrument]
+pub async fn handle_cdi_devices(
+    logger: &Logger,
+    spec: &mut Spec,
+    spec_dir: &str,
+    cdi_timeout: u64,
+) -> Result<()> {
+    if let Some(container_type) = spec
+        .annotations()
+        .as_ref()
+        .and_then(|a| a.get("io.katacontainers.pkg.oci.container_type"))
+    {
+        if container_type == "pod_sandbox" {
+            return Ok(());
+        }
+    }
+
+    let (_, devices) = parse_annotations(spec.annotations().as_ref().unwrap())?;
+
+    if devices.is_empty() {
+        info!(logger, "no CDI annotations, no devices to inject");
+        return Ok(());
+    }
+    // Explicitly set the cache options to disable auto-refresh and
+    // to use the single spec dir "/var/run/cdi" for tests it can be overridden
+    let options: Vec<CdiOption> = vec![with_auto_refresh(false), with_spec_dirs(&[spec_dir])];
+    let cache: Arc<std::sync::Mutex<cdi::cache::Cache>> = new_cache(options);
+
+    for _ in 0..=cdi_timeout {
+        let inject_result = {
+            // Lock cache within this scope, std::sync::Mutex has no Send
+            // and await will not work with time::sleep
+            let mut cache = cache.lock().unwrap();
+            match cache.refresh() {
+                Ok(_) => {}
+                Err(e) => {
+                    return Err(anyhow!("error refreshing cache: {:?}", e));
+                }
+            }
+            cache.inject_devices(Some(spec), devices.clone())
+        };
+
+        match inject_result {
+            Ok(_) => {
+                info!(
+                    logger,
+                    "all devices injected successfully, modified CDI container spec: {:?}", &spec
+                );
+                return Ok(());
+            }
+            Err(e) => {
+                info!(logger, "error injecting devices: {:?}", e);
+                println!("error injecting devices: {:?}", e);
+            }
+        }
+        time::sleep(Duration::from_millis(1000)).await;
+    }
+    Err(anyhow!(
+        "failed to inject devices after CDI timeout of {} seconds",
+        cdi_timeout
+    ))
+}
+
 #[instrument]
 async fn validate_device(
     logger: &Logger,
@@ -1110,4 +1178,94 @@ mod tests {
         assert!(name.is_ok(), "{}", name.unwrap_err());
         assert_eq!(name.unwrap(), devname);
     }
+
+    #[tokio::test]
+    async fn test_handle_cdi_devices() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let mut spec = Spec::default();
+
+        let mut annotations = HashMap::new();
+        // cdi.k8s.io/vendor1_devices: vendor1.com/device=foo
+        annotations.insert(
+            "cdi.k8s.io/vfio17".to_string(),
+            "kata.com/gpu=0".to_string(),
+        );
+        spec.set_annotations(Some(annotations));
+
+        let temp_dir = tempdir().expect("Failed to create temporary directory");
+        let cdi_file = temp_dir.path().join("kata.json");
+
+        let cdi_version = "0.6.0";
+        let kind = "kata.com/gpu";
+        let device_name = "0";
+        let annotation_whatever = "false";
+        let annotation_whenever = "true";
+        let inner_env = "TEST_INNER_ENV=TEST_INNER_ENV_VALUE";
+        let outer_env = "TEST_OUTER_ENV=TEST_OUTER_ENV_VALUE";
+        let inner_device = "/dev/zero";
+        let outer_device = "/dev/null";
+
+        let cdi_content = format!(
+            r#"{{
+            "cdiVersion": "{cdi_version}",
+            "kind": "{kind}",
+            "devices": [
+                {{
+                    "name": "{device_name}",
+                    "annotations": {{
+                        "whatever": "{annotation_whatever}",
+                        "whenever": "{annotation_whenever}"
+                    }},
+                    "containerEdits": {{
+                        "env": [
+                            "{inner_env}"
+                        ],
+                        "deviceNodes": [
+                            {{
+                                "path": "{inner_device}"
+                            }}
+                        ]
+                    }}
+                }}
+            ],
+            "containerEdits": {{
+                "env": [
+                    "{outer_env}"
+                ],
+                "deviceNodes": [
+                    {{
+                        "path": "{outer_device}"
+                    }}
+                ]
+            }}
+        }}"#
+        );
+
+        fs::write(&cdi_file, cdi_content).expect("Failed to write CDI file");
+
+        let res =
+            handle_cdi_devices(&logger, &mut spec, temp_dir.path().to_str().unwrap(), 0).await;
+        println!("modfied spec {:?}", spec);
+        assert!(res.is_ok(), "{}", res.err().unwrap());
+
+        let linux = spec.linux().as_ref().unwrap();
+        let devices = linux
+            .resources()
+            .as_ref()
+            .unwrap()
+            .devices()
+            .as_ref()
+            .unwrap();
+        assert_eq!(devices.len(), 2);
+
+        let env = spec.process().as_ref().unwrap().env().as_ref().unwrap();
+
+        // find string TEST_OUTER_ENV in env
+        let outer_env = env.iter().find(|e| e.starts_with("TEST_OUTER_ENV"));
+        assert!(outer_env.is_some(), "TEST_OUTER_ENV not found in env");
+
+        // find TEST_INNER_ENV in env
+        let inner_env = env.iter().find(|e| e.starts_with("TEST_INNER_ENV"));
+        assert!(inner_env.is_some(), "TEST_INNER_ENV not found in env");
+    }
 }

src/agent/src/image.rs

@@ -21,6 +21,9 @@ use tokio::sync::Mutex;
 use crate::rpc::CONTAINER_BASE;
 use crate::AGENT_CONFIG;
 
+use kata_types::mount::KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL;
+use protocols::agent::Storage;
+
 pub const KATA_IMAGE_WORK_DIR: &str = "/run/kata-containers/image/";
 const CONFIG_JSON: &str = "config.json";
 const KATA_PAUSE_BUNDLE: &str = "/pause_bundle";
@@ -81,6 +84,28 @@ impl ImageService {
         Self { image_client }
     }
 
+    /// get guest pause image process specification
+    fn get_pause_image_process() -> Result<oci::Process> {
+        let guest_pause_bundle = Path::new(KATA_PAUSE_BUNDLE);
+        if !guest_pause_bundle.exists() {
+            bail!("Pause image not present in rootfs");
+        }
+        let guest_pause_config = scoped_join(guest_pause_bundle, CONFIG_JSON)?;
+
+        let image_oci = oci::Spec::load(guest_pause_config.to_str().ok_or_else(|| {
+            anyhow!(
+                "Failed to load the guest pause image config from {:?}",
+                guest_pause_config
+            )
+        })?)
+        .context("load image config file")?;
+
+        let image_oci_process = image_oci.process().as_ref().ok_or_else(|| {
+            anyhow!("The guest pause image config does not contain a process specification. Please check the pause image.")
+        })?;
+        Ok(image_oci_process.clone())
+    }
+
     /// pause image is packaged in rootfs
     fn unpack_pause_image(cid: &str) -> Result<String> {
         verify_id(cid).context("The guest pause image cid contains invalid characters.")?;
@@ -132,6 +157,20 @@ impl ImageService {
         Ok(pause_rootfs.display().to_string())
     }
 
+    /// check whether the image is for sandbox or for container.
+    fn is_sandbox(image_metadata: &HashMap<String, String>) -> bool {
+        let mut is_sandbox = false;
+        for key in K8S_CONTAINER_TYPE_KEYS.iter() {
+            if let Some(value) = image_metadata.get(key as &str) {
+                if value == "sandbox" {
+                    is_sandbox = true;
+                    break;
+                }
+            }
+        }
+        is_sandbox
+    }
+
     /// pull_image is used for call image-rs to pull image in the guest.
     /// # Parameters
     /// - `image`: Image name (exp: quay.io/prometheus/busybox:latest)
@@ -147,18 +186,7 @@ impl ImageService {
     ) -> Result<String> {
         info!(sl(), "image metadata: {image_metadata:?}");
 
-        //Check whether the image is for sandbox or for container.
-        let mut is_sandbox = false;
-        for key in K8S_CONTAINER_TYPE_KEYS.iter() {
-            if let Some(value) = image_metadata.get(key as &str) {
-                if value == "sandbox" {
-                    is_sandbox = true;
-                    break;
-                }
-            }
-        }
-
-        if is_sandbox {
+        if Self::is_sandbox(image_metadata) {
             let mount_path = Self::unpack_pause_image(cid)?;
             return Ok(mount_path);
         }
@@ -194,6 +222,32 @@ impl ImageService {
     }
 }
 
+/// get_process overrides the OCI process spec with pause image process spec if needed
+pub fn get_process(
+    ocip: &oci::Process,
+    oci: &oci::Spec,
+    storages: Vec<Storage>,
+) -> Result<oci::Process> {
+    let mut guest_pull = false;
+    for storage in storages {
+        if storage.driver == KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL {
+            guest_pull = true;
+            break;
+        }
+    }
+    if guest_pull {
+        match oci.annotations() {
+            Some(a) => {
+                if ImageService::is_sandbox(a) {
+                    return ImageService::get_pause_image_process();
+                }
+            }
+            None => {}
+        }
+    }
+    Ok(ocip.clone())
+}
+
 /// Set proxy environment from AGENT_CONFIG
 pub async fn set_proxy_env_vars() {
     if env::var("HTTPS_PROXY").is_err() {

src/agent/src/main.rs

@@ -21,7 +21,7 @@ extern crate slog;
 use anyhow::{anyhow, Context, Result};
 use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
-use const_format::concatcp;
+use const_format::{concatcp, formatcp};
 use nix::fcntl::OFlag;
 use nix::sys::reboot::{reboot, RebootMode};
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
@@ -29,7 +29,7 @@ use nix::unistd::{self, dup, sync, Pid};
 use std::env;
 use std::ffi::OsStr;
 use std::fs::{self, File};
-use std::os::unix::fs as unixfs;
+use std::os::unix::fs::{self as unixfs, FileTypeExt};
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
 use std::process::exit;
@@ -109,7 +109,18 @@ const CDH_SOCKET_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, CDH_SOCKET);
 const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest";
 
 /// Path of ocicrypt config file. This is used by image-rs when decrypting image.
-const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+const OCICRYPT_CONFIG_PATH: &str = "/run/confidential-containers/ocicrypt_config.json";
+
+const OCICRYPT_CONFIG: &str = formatcp!(
+    r#"{{
+    "key-providers": {{
+        "attestation-agent": {{
+            "ttrpc": "{}"
+        }}
+    }}
+}}"#,
+    CDH_SOCKET_URI
+);
 
 const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6;
 
@@ -408,15 +419,13 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     let gc_procs = config.guest_components_procs;
-    if gc_procs != GuestComponentsProcs::None {
-        if !attestation_binaries_available(logger, &gc_procs) {
-            warn!(
-                logger,
-                "attestation binaries requested for launch not available"
-            );
-        } else {
-            init_attestation_components(logger, config).await?;
-        }
+    if !attestation_binaries_available(logger, &gc_procs) {
+        warn!(
+            logger,
+            "attestation binaries requested for launch not available"
+        );
+    } else {
+        init_attestation_components(logger, config).await?;
     }
 
     // vsock:///dev/vsock, port
@@ -447,12 +456,7 @@ fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs)
     true
 }
 
-// Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
-// and the corresponding procs are enabled in the agent configuration. the process will be
-// launched in the background and the function will return immediately.
-// If the CDH is started, a CDH client will be instantiated and returned.
-async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> {
-    // skip launch of any guest-component
+async fn launch_guest_component_procs(logger: &Logger, config: &AgentConfig) -> Result<()> {
     if config.guest_components_procs == GuestComponentsProcs::None {
         return Ok(());
     }
@@ -472,17 +476,6 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
         return Ok(());
     }
 
-    let ocicrypt_config = serde_json::json!({
-        "key-providers": {
-            "attestation-agent":{
-                "ttrpc":CDH_SOCKET_URI
-            }
-        }
-    });
-
-    fs::write(OCICRYPT_CONFIG_PATH, ocicrypt_config.to_string().as_bytes())?;
-    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
-
     debug!(
         logger,
         "spawning confidential-data-hub process {}", CDH_PATH
@@ -497,9 +490,6 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
     )
     .map_err(|e| anyhow!("launch_process {} failed: {:?}", CDH_PATH, e))?;
 
-    // initialize cdh client
-    cdh::init_cdh_client().await?;
-
     // skip launch of api-server-rest
     if config.guest_components_procs == GuestComponentsProcs::ConfidentialDataHub {
         return Ok(());
@@ -522,6 +512,33 @@ async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> R
     Ok(())
 }
 
+// Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
+// and the corresponding procs are enabled in the agent configuration. the process will be
+// launched in the background and the function will return immediately.
+// If the CDH is started, a CDH client will be instantiated and returned.
+async fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> {
+    launch_guest_component_procs(logger, config).await?;
+
+    // If a CDH socket exists, initialize the CDH client and enable ocicrypt
+    match tokio::fs::metadata(CDH_SOCKET).await {
+        Ok(md) => {
+            if md.file_type().is_socket() {
+                cdh::init_cdh_client(CDH_SOCKET_URI).await?;
+                fs::write(OCICRYPT_CONFIG_PATH, OCICRYPT_CONFIG.as_bytes())?;
+                env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
+            } else {
+                debug!(logger, "File {} is not a socket", CDH_SOCKET);
+            }
+        }
+        Err(err) => warn!(
+            logger,
+            "Failed to probe CDH socket file {}: {:?}", CDH_SOCKET, err
+        ),
+    }
+
+    Ok(())
+}
+
 fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> {
     let p = Path::new(path);
     let mut attempts = 0;

src/agent/src/policy.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::Result;
+use anyhow::{bail, Result};
 use protobuf::MessageDyn;
 use tokio::io::AsyncWriteExt;
 
@@ -68,6 +68,12 @@ pub struct AgentPolicy {
     engine: regorus::Engine,
 }
 
+#[derive(serde::Deserialize, Debug)]
+struct MetadataResponse {
+    allowed: bool,
+    ops: Option<json_patch::Patch>,
+}
+
 impl AgentPolicy {
     /// Create AgentPolicy object.
     pub fn new() -> Self {
@@ -82,6 +88,17 @@ impl AgentPolicy {
         let mut engine = regorus::Engine::new();
         engine.set_strict_builtin_errors(false);
         engine.set_gather_prints(true);
+        // assign a slice of the engine data "pstate" to be used as policy state
+        engine
+            .add_data(
+                regorus::Value::from_json_str(
+                    r#"{
+                        "pstate": {}
+                    }"#,
+                )
+                .unwrap(),
+            )
+            .unwrap();
         engine
     }
 
@@ -112,6 +129,23 @@ impl AgentPolicy {
         Ok(())
     }
 
+    async fn apply_patch_to_state(&mut self, patch: json_patch::Patch) -> Result<()> {
+        // Convert the current engine data to a JSON value
+        let mut state = serde_json::to_value(self.engine.get_data())?;
+
+        // Apply the patch to the state
+        json_patch::patch(&mut state, &patch)?;
+
+        // Clear the existing data in the engine
+        self.engine.clear_data();
+
+        // Add the patched state back to the engine
+        self.engine
+            .add_data(regorus::Value::from_json_str(&state.to_string())?)?;
+
+        Ok(())
+    }
+
     /// Ask regorus if an API call should be allowed or not.
     async fn allow_request(&mut self, ep: &str, ep_input: &str) -> Result<(bool, String)> {
         debug!(sl!(), "policy check: {ep}");
@@ -120,13 +154,56 @@ impl AgentPolicy {
         let query = format!("data.agent_policy.{ep}");
         self.engine.set_input_json(ep_input)?;
 
-        let mut allow = match self.engine.eval_bool_query(query, false) {
-            Ok(a) => a,
-            Err(e) => {
-                if !self.allow_failures {
-                    return Err(e);
+        let results = self.engine.eval_query(query, false)?;
+
+        let prints = match self.engine.take_prints() {
+            Ok(p) => p.join(" "),
+            Err(e) => format!("Failed to get policy log: {e}"),
+        };
+
+        if results.result.len() != 1 {
+            // Results are empty when AllowRequestsFailingPolicy is used to allow a Request that hasn't been defined in the policy
+            if self.allow_failures {
+                return Ok((true, prints));
+            }
+            bail!(
+                "policy check: unexpected eval_query result len {:?}",
+                results
+            );
+        }
+
+        if results.result[0].expressions.len() != 1 {
+            bail!(
+                "policy check: unexpected eval_query result expressions {:?}",
+                results
+            );
+        }
+
+        let mut allow = match &results.result[0].expressions[0].value {
+            regorus::Value::Bool(b) => *b,
+
+            // Match against a specific variant that could be interpreted as MetadataResponse
+            regorus::Value::Object(obj) => {
+                let json_str = serde_json::to_string(obj)?;
+
+                self.log_eval_input(ep, &json_str).await;
+
+                let metadata_response: MetadataResponse = serde_json::from_str(&json_str)?;
+
+                if metadata_response.allowed {
+                    if let Some(ops) = metadata_response.ops {
+                        self.apply_patch_to_state(ops).await?;
+                    }
                 }
-                false
+                metadata_response.allowed
+            }
+
+            _ => {
+                error!(sl!(), "allow_request: unexpected eval_query result type");
+                bail!(
+                    "policy check: unexpected eval_query result type {:?}",
+                    results
+                );
             }
         };
 
@@ -135,11 +212,6 @@ impl AgentPolicy {
             allow = true;
         }
 
-        let prints = match self.engine.take_prints() {
-            Ok(p) => p.join(" "),
-            Err(e) => format!("Failed to get policy log: {e}"),
-        };
-
         Ok((allow, prints))
     }
 

src/agent/src/rpc.rs

@@ -58,7 +58,7 @@ use rustjail::process::ProcessOperations;
 use crate::cdh;
 use crate::device::block_device_handler::get_virtio_blk_pci_device_name;
 use crate::device::network_device_handler::wait_for_net_interface;
-use crate::device::{add_devices, update_env_pci};
+use crate::device::{add_devices, handle_cdi_devices, update_env_pci};
 use crate::features::get_build_features;
 use crate::image::KATA_IMAGE_WORK_DIR;
 use crate::linux_abi::*;
@@ -130,6 +130,8 @@ const ERR_NO_SANDBOX_PIDNS: &str = "Sandbox does not have sandbox_pidns";
 // not available.
 const IPTABLES_RESTORE_WAIT_SEC: u64 = 5;
 
+const CDI_TIMEOUT_LIMIT: u64 = 100;
+
 // Convenience function to obtain the scope logger.
 fn sl() -> slog::Logger {
     slog_scope::logger()
@@ -224,54 +226,16 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&sl(), &req.devices, &mut oci, &self.sandbox).await?;
 
-        let process = oci
-            .process_mut()
-            .as_mut()
-            .ok_or_else(|| anyhow!("Spec didn't contain process field"))?;
-        if cdh::is_cdh_client_initialized().await {
-            if let Some(envs) = process.env_mut().as_mut() {
-                for env in envs.iter_mut() {
-                    match cdh::unseal_env(env).await {
-                        Ok(unsealed_env) => *env = unsealed_env.to_string(),
-                        Err(e) => {
-                            warn!(sl(), "Failed to unseal secret: {}", e)
-                        }
-                    }
-                }
-            }
-        }
+        // In guest-kernel mode some devices need extra handling. Taking the
+        // GPU as an example the shim will inject CDI annotations that will
+        // be used by the kata-agent to do containerEdits according to the
+        // CDI spec coming from a registry that is created on the fly by UDEV
+        // or other entities for a specifc device.
+        // In Kata we only consider the directory "/var/run/cdi", "/etc" may be
+        // readonly
+        handle_cdi_devices(&sl(), &mut oci, "/var/run/cdi", CDI_TIMEOUT_LIMIT).await?;
 
-        let linux = oci
-            .linux()
-            .as_ref()
-            .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?;
-
-        if cdh::is_cdh_client_initialized().await {
-            if let Some(devices) = linux.devices() {
-                for specdev in devices.iter() {
-                    if specdev.path().as_path().to_str() == Some(TRUSTED_IMAGE_STORAGE_DEVICE) {
-                        let dev_major_minor = format!("{}:{}", specdev.major(), specdev.minor());
-                        let secure_storage_integrity =
-                            AGENT_CONFIG.secure_storage_integrity.to_string();
-                        info!(
-                            sl(),
-                            "trusted_store device major:min {}, enable data integrity {}",
-                            dev_major_minor,
-                            secure_storage_integrity
-                        );
-
-                        let options = std::collections::HashMap::from([
-                            ("deviceId".to_string(), dev_major_minor),
-                            ("encryptType".to_string(), "LUKS".to_string()),
-                            ("dataIntegrity".to_string(), secure_storage_integrity),
-                        ]);
-                        cdh::secure_mount("BlockDevice", &options, vec![], KATA_IMAGE_WORK_DIR)
-                            .await?;
-                        break;
-                    }
-                }
-            }
-        }
+        cdh_handler(&mut oci).await?;
 
         // Both rootfs and volumes (invoked with --volume for instance) will
         // be processed the same way. The idea is to always mount any provided
@@ -280,7 +244,13 @@ impl AgentService {
         // After all those storages have been processed, no matter the order
         // here, the agent will rely on rustjail (using the oci.Mounts
         // list) to bind mount all of them inside the container.
-        let m = add_storages(sl(), req.storages, &self.sandbox, Some(req.container_id)).await?;
+        let m = add_storages(
+            sl(),
+            req.storages.clone(),
+            &self.sandbox,
+            Some(req.container_id),
+        )
+        .await?;
 
         let mut s = self.sandbox.lock().await;
         s.container_mounts.insert(cid.clone(), m);
@@ -335,6 +305,13 @@ impl AgentService {
         let pipe_size = AGENT_CONFIG.container_pipe_size;
 
         let p = if let Some(p) = oci.process() {
+            #[cfg(feature = "guest-pull")]
+            {
+                let new_p = image::get_process(p, &oci, req.storages.clone())?;
+                Process::new(&sl(), &new_p, cid.as_str(), true, pipe_size, proc_io)?
+            }
+
+            #[cfg(not(feature = "guest-pull"))]
             Process::new(&sl(), p, cid.as_str(), true, pipe_size, proc_io)?
         } else {
             info!(sl(), "no process configurations!");
@@ -1726,13 +1703,19 @@ fn update_container_namespaces(
     if let Some(namespaces) = linux.namespaces_mut() {
         for namespace in namespaces.iter_mut() {
             if namespace.typ().to_string() == NSTYPEIPC {
-                namespace.set_path(Some(PathBuf::from(&sandbox.shared_ipcns.path.clone())));
-                namespace.set_path(None);
+                namespace.set_path(if !sandbox.shared_ipcns.path.is_empty() {
+                    Some(PathBuf::from(&sandbox.shared_ipcns.path))
+                } else {
+                    None
+                });
                 continue;
             }
             if namespace.typ().to_string() == NSTYPEUTS {
-                namespace.set_path(Some(PathBuf::from(&sandbox.shared_utsns.path.clone())));
-                namespace.set_path(None);
+                namespace.set_path(if !sandbox.shared_utsns.path.is_empty() {
+                    Some(PathBuf::from(&sandbox.shared_utsns.path))
+                } else {
+                    None
+                });
                 continue;
             }
         }
@@ -1750,7 +1733,7 @@ fn update_container_namespaces(
                 if !pidns.path.is_empty() {
                     pid_ns.set_path(Some(PathBuf::from(&pidns.path)));
                 }
-            } else {
+            } else if !sandbox.containers.is_empty() {
                 return Err(anyhow!(ERR_NO_SANDBOX_PIDNS));
             }
         }
@@ -2093,6 +2076,76 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> {
     }
 }
 
+async fn cdh_handler(oci: &mut Spec) -> Result<()> {
+    if !cdh::is_cdh_client_initialized().await {
+        return Ok(());
+    }
+    let process = oci
+        .process_mut()
+        .as_mut()
+        .ok_or_else(|| anyhow!("Spec didn't contain process field"))?;
+    if let Some(envs) = process.env_mut().as_mut() {
+        for env in envs.iter_mut() {
+            match cdh::unseal_env(env).await {
+                Ok(unsealed_env) => *env = unsealed_env.to_string(),
+                Err(e) => {
+                    warn!(sl(), "Failed to unseal secret: {}", e)
+                }
+            }
+        }
+    }
+
+    let mounts = oci
+        .mounts_mut()
+        .as_mut()
+        .ok_or_else(|| anyhow!("Spec didn't contain mounts field"))?;
+
+    for m in mounts.iter_mut() {
+        if m.destination().starts_with("/sealed") {
+            info!(
+                sl(),
+                "sealed mount destination: {:?} source: {:?}",
+                m.destination(),
+                m.source()
+            );
+            if let Some(source_str) = m.source().as_ref().and_then(|p| p.to_str()) {
+                cdh::unseal_file(source_str).await?;
+            } else {
+                warn!(sl(), "Failed to unseal: Mount source is None or invalid");
+            }
+        }
+    }
+
+    let linux = oci
+        .linux()
+        .as_ref()
+        .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?;
+
+    if let Some(devices) = linux.devices() {
+        for specdev in devices.iter() {
+            if specdev.path().as_path().to_str() == Some(TRUSTED_IMAGE_STORAGE_DEVICE) {
+                let dev_major_minor = format!("{}:{}", specdev.major(), specdev.minor());
+                let secure_storage_integrity = AGENT_CONFIG.secure_storage_integrity.to_string();
+                info!(
+                    sl(),
+                    "trusted_store device major:min {}, enable data integrity {}",
+                    dev_major_minor,
+                    secure_storage_integrity
+                );
+
+                let options = std::collections::HashMap::from([
+                    ("deviceId".to_string(), dev_major_minor),
+                    ("encryptType".to_string(), "LUKS".to_string()),
+                    ("dataIntegrity".to_string(), secure_storage_integrity),
+                ]);
+                cdh::secure_mount("BlockDevice", &options, vec![], KATA_IMAGE_WORK_DIR).await?;
+                break;
+            }
+        }
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 #[allow(dead_code)]
 mod tests {
@@ -2527,14 +2580,6 @@ mod tests {
                     .unwrap()],
                 ..Default::default()
             },
-            TestData {
-                namespaces: vec![],
-                sandbox_pidns_path: None,
-                use_sandbox_pidns: true,
-                result: Err(anyhow!(ERR_NO_SANDBOX_PIDNS)),
-                expected_namespaces: vec![],
-                ..Default::default()
-            },
             TestData {
                 has_linux_in_spec: false,
                 result: Err(anyhow!(ERR_NO_LINUX_FIELD)),

src/agent/src/storage/ephemeral_handler.rs

@@ -170,7 +170,7 @@ impl EphemeralHandler {
         let size = size_str
             .unwrap()
             .parse::<u64>()
-            .context(format!("parse size: {:?}", &pagesize_str))?;
+            .context(format!("parse size: {:?}", &size_str))?;
 
         Ok((pagesize, size))
     }

src/libs/Cargo.lock

@@ -240,19 +240,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "cgroups-rs"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
-dependencies = [
- "libc",
- "log",
- "nix 0.25.1",
- "regex",
- "thiserror",
-]
-
 [[package]]
 name = "chrono"
 version = "0.4.20"
@@ -814,7 +801,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -975,18 +961,6 @@ dependencies = [
  "memoffset 0.6.5",
 ]
 
-[[package]]
-name = "nix"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
-dependencies = [
- "autocfg",
- "bitflags",
- "cfg-if",
- "libc",
-]
-
 [[package]]
 name = "nix"
 version = "0.26.4"
@@ -1316,7 +1290,6 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.2.0",
  "serde",

src/libs/kata-sys-util/Cargo.toml

@@ -13,7 +13,6 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0.31"
 byteorder = "1.4.3"
-cgroups = { package = "cgroups-rs", version = "0.3.2" }
 chrono = "0.4.0"
 common-path = "=1.0.0"
 fail = "0.5.0"

src/libs/kata-sys-util/README.md

@@ -1,10 +1,9 @@
-# kata-sys-util
+# `kata-sys-util`
 
 This crate is a collection of utilities and helpers for 
 [Kata Containers](https://github.com/kata-containers/kata-containers/) components to access system services.
 
 It provides safe wrappers over system services, such as:
-- cgroups
 - file systems
 - mount
 - NUMA

src/libs/kata-sys-util/src/mount.rs

@@ -53,6 +53,7 @@ use std::time::Instant;
 use lazy_static::lazy_static;
 use nix::mount::{mount, MntFlags, MsFlags};
 use nix::{unistd, NixPath};
+use oci_spec::runtime as oci;
 
 use crate::fs::is_symlink;
 use crate::sl;
@@ -799,8 +800,20 @@ pub fn get_mount_options(options: &Option<Vec<String>>) -> Vec<String> {
     }
 }
 
-pub fn get_mount_type(typ: &Option<String>) -> String {
-    typ.clone().unwrap_or("bind".to_string())
+pub fn get_mount_type(m: &oci::Mount) -> String {
+    m.typ()
+        .clone()
+        .map(|typ| {
+            if typ.as_str() == "none" {
+                if let Some(opts) = m.options() {
+                    if opts.iter().any(|opt| opt == "bind" || opt == "rbind") {
+                        return "bind".to_string();
+                    }
+                }
+            }
+            typ
+        })
+        .unwrap_or("bind".to_string())
 }
 
 #[cfg(test)]

src/libs/kata-sys-util/src/spec.rs

@@ -97,11 +97,3 @@ pub fn load_oci_spec() -> Result<oci::Spec, OciSpecError> {
 
     oci::Spec::load(spec_file.to_str().unwrap_or_default())
 }
-
-/// handle string parsing for input possibly be JSON string.
-pub fn parse_json_string(input: &str) -> &str {
-    let json_str: &str = serde_json::from_str(input).unwrap_or(input);
-    let stripped_str = json_str.strip_prefix("CAP_").unwrap_or(json_str);
-
-    stripped_str
-}

src/libs/kata-types/src/annotations/cri_containerd.rs

@@ -11,7 +11,12 @@ pub const CONTAINER_NAME_LABEL_KEY: &str = "io.kubernetes.cri.container-name";
 pub const SANDBOX: &str = "sandbox";
 pub const CONTAINER: &str = "container";
 
+// SandboxID is the sandbox ID annotation
 pub const SANDBOX_ID_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-id";
+// SandboxName is the name of the sandbox (pod)
+pub const SANDBOX_NAME_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-name";
+// SandboxNamespace is the name of the namespace of the sandbox (pod)
+pub const SANDBOX_NAMESPACE_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-namespace";
 
 // Ref: https://pkg.go.dev/github.com/containerd/containerd@v1.6.7/pkg/cri/annotations
 // SandboxCPU annotations are based on the initial CPU configuration for the sandbox. This is calculated as the

src/libs/kata-types/src/annotations/mod.rs

@@ -88,12 +88,6 @@ pub const KATA_ANNO_CFG_HYPERVISOR_PREFIX: &str = "io.katacontainers.config.hype
 pub const KATA_ANNO_CFG_HYPERVISOR_PATH: &str = "io.katacontainers.config.hypervisor.path";
 /// A sandbox annotation for passing a container hypervisor binary SHA-512 hash value.
 pub const KATA_ANNO_CFG_HYPERVISOR_HASH: &str = "io.katacontainers.config.hypervisor.path_hash";
-/// A sandbox annotation for passing a per container path pointing at the hypervisor control binary
-/// that will run the container VM.
-pub const KATA_ANNO_CFG_HYPERVISOR_CTLPATH: &str = "io.katacontainers.config.hypervisor.ctlpath";
-/// A sandbox annotation for passing a container hypervisor control binary SHA-512 hash value.
-pub const KATA_ANNO_CFG_HYPERVISOR_CTLHASH: &str =
-    "io.katacontainers.config.hypervisor.hypervisorctl_hash";
 /// A sandbox annotation for passing a per container path pointing at the jailer that will constrain
 /// the container VM.
 pub const KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH: &str =
@@ -506,10 +500,6 @@ impl Annotation {
                         hv.validate_hypervisor_path(value)?;
                         hv.path = value.to_string();
                     }
-                    KATA_ANNO_CFG_HYPERVISOR_CTLPATH => {
-                        hv.validate_hypervisor_ctlpath(value)?;
-                        hv.ctlpath = value.to_string();
-                    }
 
                     KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH => {
                         hv.validate_jailer_path(value)?;

src/libs/kata-types/src/config/default.rs

@@ -98,3 +98,11 @@ pub const DEFAULT_FIRECRACKER_GUEST_KERNEL_IMAGE: &str = "vmlinux";
 pub const DEFAULT_FIRECRACKER_GUEST_KERNEL_PARAMS: &str = "";
 pub const MAX_FIRECRACKER_VCPUS: u32 = 32;
 pub const MIN_FIRECRACKER_MEMORY_SIZE_MB: u32 = 128;
+
+// Default configuration for remote
+pub const DEFAULT_REMOTE_HYPERVISOR_SOCKET: &str = "/run/peerpod/hypervisor.sock";
+pub const DEFAULT_REMOTE_HYPERVISOR_TIMEOUT: i32 = 600; // 600 Seconds
+pub const MAX_REMOTE_VCPUS: u32 = 32;
+pub const MIN_REMOTE_MEMORY_SIZE_MB: u32 = 64;
+pub const DEFAULT_REMOTE_MEMORY_SIZE_MB: u32 = 128;
+pub const DEFAULT_REMOTE_MEMORY_SLOTS: u32 = 128;

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -44,6 +44,9 @@ pub use self::qemu::{QemuConfig, HYPERVISOR_NAME_QEMU};
 mod ch;
 pub use self::ch::{CloudHypervisorConfig, HYPERVISOR_NAME_CH};
 
+mod remote;
+pub use self::remote::{RemoteConfig, HYPERVISOR_NAME_REMOTE};
+
 /// Virtual PCI block device driver.
 pub const VIRTIO_BLK_PCI: &str = "virtio-blk-pci";
 
@@ -540,6 +543,7 @@ impl TopologyConfigInfo {
             HYPERVISOR_NAME_CH,
             HYPERVISOR_NAME_DRAGONBALL,
             HYPERVISOR_NAME_FIRECRACKER,
+            HYPERVISOR_NAME_REMOTE,
         ];
         let hypervisor_name = toml_config.runtime.hypervisor_name.as_str();
         if !hypervisor_names.contains(&hypervisor_name) {
@@ -1040,6 +1044,18 @@ impl SharedFsInfo {
     }
 }
 
+/// Configuration information for remote hypervisor type.
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub struct RemoteInfo {
+    /// Remote hypervisor socket path
+    #[serde(default)]
+    pub hypervisor_socket: String,
+
+    /// Remote hyperisor timeout of creating (in seconds)
+    #[serde(default)]
+    pub hypervisor_timeout: i32,
+}
+
 /// Common configuration information for hypervisors.
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct Hypervisor {
@@ -1123,6 +1139,10 @@ pub struct Hypervisor {
     #[serde(default, flatten)]
     pub shared_fs: SharedFsInfo,
 
+    /// Remote hypervisor configuration information.
+    #[serde(default, flatten)]
+    pub remote_info: RemoteInfo,
+
     /// A sandbox annotation used to specify prefetch_files.list host path container image
     /// being used, and runtime will pass it to Hypervisor to  search for corresponding
     /// prefetch list file:
@@ -1164,6 +1184,10 @@ impl ConfigOps for Hypervisor {
     fn adjust_config(conf: &mut TomlConfig) -> Result<()> {
         HypervisorVendor::adjust_config(conf)?;
         let hypervisors: Vec<String> = conf.hypervisor.keys().cloned().collect();
+        info!(
+            sl!(),
+            "Adjusting hypervisor configuration {:?}", hypervisors
+        );
         for hypervisor in hypervisors.iter() {
             if let Some(plugin) = get_hypervisor_plugin(hypervisor) {
                 plugin.adjust_config(conf)?;

src/libs/kata-types/src/config/hypervisor/remote.rs

@@ -0,0 +1,116 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use byte_unit::{Byte, Unit};
+use std::io::Result;
+use std::path::Path;
+use std::sync::Arc;
+use sysinfo::System;
+
+use crate::{
+    config::{
+        default::{self, MAX_REMOTE_VCPUS, MIN_REMOTE_MEMORY_SIZE_MB},
+        ConfigPlugin,
+    }, device::DRIVER_NVDIMM_TYPE, eother, resolve_path
+};
+
+use super::register_hypervisor_plugin;
+
+/// Hypervisor name for remote, used to index `TomlConfig::hypervisor`.
+pub const HYPERVISOR_NAME_REMOTE: &str = "remote";
+
+/// Configuration information for remote.
+#[derive(Default, Debug)]
+pub struct RemoteConfig {}
+
+impl RemoteConfig {
+    /// Create a new instance of `RemoteConfig`
+    pub fn new() -> Self {
+        RemoteConfig {}
+    }
+
+    /// Register the remote plugin.
+    pub fn register(self) {
+        let plugin = Arc::new(self);
+        register_hypervisor_plugin(HYPERVISOR_NAME_REMOTE, plugin);
+    }
+}
+
+impl ConfigPlugin for RemoteConfig {
+    fn name(&self) -> &str {
+        HYPERVISOR_NAME_REMOTE
+    }
+
+    /// Adjust the configuration information after loading from configuration file.
+    fn adjust_config(&self, conf: &mut crate::config::TomlConfig) -> Result<()> {
+        if let Some(remote) = conf.hypervisor.get_mut(HYPERVISOR_NAME_REMOTE) {
+            if remote.remote_info.hypervisor_socket.is_empty() {
+                remote.remote_info.hypervisor_socket =
+                    default::DEFAULT_REMOTE_HYPERVISOR_SOCKET.to_string();
+            }
+            resolve_path!(
+                remote.remote_info.hypervisor_socket,
+                "Remote hypervisor socket `{}` is invalid: {}"
+            )?;
+            if remote.remote_info.hypervisor_timeout == 0 {
+                remote.remote_info.hypervisor_timeout = default::DEFAULT_REMOTE_HYPERVISOR_TIMEOUT;
+            }
+            if remote.memory_info.default_memory == 0 {
+                remote.memory_info.default_memory = default::MIN_REMOTE_MEMORY_SIZE_MB;
+            }
+            if remote.memory_info.memory_slots == 0 {
+                remote.memory_info.memory_slots = default::DEFAULT_REMOTE_MEMORY_SLOTS
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Validate the configuration information.
+    fn validate(&self, conf: &crate::config::TomlConfig) -> Result<()> {
+        if let Some(remote) = conf.hypervisor.get(HYPERVISOR_NAME_REMOTE) {
+            let s = System::new_all();
+            let total_memory = Byte::from_u64(s.total_memory())
+                .get_adjusted_unit(Unit::MiB)
+                .get_value() as u32;
+            if remote.memory_info.default_maxmemory != total_memory {
+                return Err(eother!(
+                    "Remote hypervisor does not support memory hotplug, default_maxmemory must be equal to the total system memory",
+                ));
+            }
+            let cpus = num_cpus::get() as u32;
+            if remote.cpu_info.default_maxvcpus != cpus {
+                return Err(eother!(
+                    "Remote hypervisor does not support CPU hotplug, default_maxvcpus must be equal to the total system CPUs",
+                ));
+            }
+            if !remote.boot_info.initrd.is_empty() {
+                return Err(eother!("Remote hypervisor does not support initrd"));
+            }
+            if !remote.boot_info.rootfs_type.is_empty() {
+                return Err(eother!("Remote hypervisor does not support rootfs_type"));
+            }
+            if remote.blockdev_info.block_device_driver.as_str() == DRIVER_NVDIMM_TYPE {
+                return Err(eother!("Remote hypervisor does not support nvdimm"));
+            }
+            if remote.memory_info.default_memory < MIN_REMOTE_MEMORY_SIZE_MB {
+                return Err(eother!(
+                    "Remote hypervisor has minimal memory limitation {}",
+                    MIN_REMOTE_MEMORY_SIZE_MB
+                ));
+            }
+        }
+
+        Ok(())
+    }
+
+    fn get_min_memory(&self) -> u32 {
+        MIN_REMOTE_MEMORY_SIZE_MB
+    }
+
+    fn get_max_cpus(&self) -> u32 {
+        MAX_REMOTE_VCPUS
+    }
+}

src/libs/kata-types/src/config/mod.rs

@@ -26,7 +26,7 @@ pub use self::agent::Agent;
 use self::default::DEFAULT_AGENT_DBG_CONSOLE_PORT;
 pub use self::hypervisor::{
     BootInfo, CloudHypervisorConfig, DragonballConfig, FirecrackerConfig, Hypervisor, QemuConfig,
-    HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_FIRECRACKER, HYPERVISOR_NAME_QEMU,
+    RemoteConfig, HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_FIRECRACKER, HYPERVISOR_NAME_QEMU,
 };
 
 mod runtime;

src/libs/protocols/Cargo.toml

@@ -7,19 +7,17 @@ license = "Apache-2.0"
 
 [features]
 default = []
-with-serde = [ "serde", "serde_json" ]
+with-serde = []
 async = ["ttrpc/async", "async-trait"]
 
 [dependencies]
 ttrpc = "0.8"
 async-trait = { version = "0.1.42", optional = true }
 protobuf = { version = "3.2.0" }
-serde = { version = "1.0.130", features = ["derive"], optional = true }
-serde_json = { version = "1.0.68", optional = true }
+serde = { version = "1.0.130", features = ["derive"] }
+serde_json = "1.0.68"
 oci-spec = { version = "0.6.8", features = ["runtime"] }
 
-kata-sys-util = { path = "../kata-sys-util" }
-
 [build-dependencies]
 ttrpc-codegen = "0.4.2"
 protobuf = { version = "3.2.0" }

src/libs/protocols/build.rs

@@ -204,6 +204,7 @@ fn real_main() -> Result<(), std::io::Error> {
                 "protos/agent.proto",
                 "protos/health.proto",
                 "protos/confidential_data_hub.proto",
+                "protos/remote.proto",
             ],
             true,
         )?;
@@ -214,6 +215,7 @@ fn real_main() -> Result<(), std::io::Error> {
             "src/confidential_data_hub_ttrpc.rs",
             "src/confidential_data_hub_ttrpc_async.rs",
         )?;
+        fs::rename("src/remote_ttrpc.rs", "src/remote_ttrpc_async.rs")?;
     }
 
     codegen(
@@ -222,6 +224,7 @@ fn real_main() -> Result<(), std::io::Error> {
             "protos/agent.proto",
             "protos/health.proto",
             "protos/confidential_data_hub.proto",
+            "protos/remote.proto",
         ],
         false,
     )?;

src/libs/protocols/protos/remote.proto

@@ -0,0 +1,47 @@
+// Copyright 2024 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+syntax = "proto3";
+
+package remote;
+
+service Hypervisor {
+	rpc CreateVM(CreateVMRequest) returns (CreateVMResponse) {}
+	rpc StartVM(StartVMRequest) returns (StartVMResponse) {}
+	rpc StopVM(StopVMRequest) returns (StopVMResponse) {}
+	rpc Version(VersionRequest) returns (VersionResponse) {}
+}
+
+message VersionRequest {
+	string version = 1;
+}
+
+message VersionResponse {
+	string version = 1;
+}
+
+message CreateVMRequest {
+	string id = 1;
+	map<string, string> annotations = 2;
+	string networkNamespacePath = 3;
+}
+
+message CreateVMResponse {
+	string agentSocketPath = 1;
+}
+
+message StartVMRequest {
+	string id = 1;
+}
+
+message StartVMResponse {
+}
+
+message StopVMRequest {
+	string id = 1;
+}
+
+message StopVMResponse {
+}

src/libs/protocols/src/lib.rs

@@ -21,6 +21,10 @@ pub mod oci;
 mod serde_config;
 pub mod trans;
 pub mod types;
+pub mod remote;
+pub mod remote_ttrpc;
+#[cfg(feature = "async")]
+pub mod remote_ttrpc_async;
 
 #[cfg(feature = "with-serde")]
 pub use serde_config::{

src/libs/protocols/src/trans.rs

@@ -10,7 +10,6 @@ use std::convert::TryFrom;
 use std::path::PathBuf;
 
 use crate::oci as grpc;
-use kata_sys_util::spec::parse_json_string;
 use oci_spec::runtime as oci;
 
 // translate from interface to ttprc tools
@@ -41,8 +40,9 @@ fn cap_hashset2vec(hash_set: &Option<HashSet<oci::Capability>>) -> Vec<String> {
 fn cap_vec2hashset(caps: Vec<String>) -> HashSet<oci::Capability> {
     caps.iter()
         .map(|cap: &String| {
-            let cap_str = parse_json_string(cap);
-            cap_str
+            // cap might be JSON-encoded
+            let decoded: &str = serde_json::from_str(cap).unwrap_or(cap);
+            decoded.strip_prefix("CAP_").unwrap_or(decoded)
                 .parse::<oci::Capability>()
                 .unwrap_or_else(|_| panic!("Failed to parse {:?} to Enum Capability", cap))
         })
@@ -97,6 +97,8 @@ impl From<oci::LinuxCapabilities> for grpc::LinuxCapabilities {
     }
 }
 
+// TODO(burgerdev): remove condition here and below after upgrading to oci_spec > 0.7.
+#[cfg(target_os = "linux")]
 impl From<oci::PosixRlimit> for grpc::POSIXRlimit {
     fn from(from: oci::PosixRlimit) -> Self {
         grpc::POSIXRlimit {
@@ -118,6 +120,7 @@ impl From<oci::Process> for grpc::Process {
             Env: option_vec_to_vec(from.env()),
             Cwd: from.cwd().display().to_string(),
             Capabilities: from_option(from.capabilities().clone()),
+            #[cfg(target_os = "linux")]
             Rlimits: from_option_vec(from.rlimits().clone()),
             NoNewPrivileges: from.no_new_privileges().unwrap_or_default(),
             ApparmorProfile: from
@@ -993,6 +996,7 @@ impl From<grpc::Linux> for oci::Linux {
     }
 }
 
+#[cfg(target_os = "linux")]
 impl From<grpc::POSIXRlimit> for oci::PosixRlimit {
     fn from(proto: grpc::POSIXRlimit) -> Self {
         oci::PosixRlimitBuilder::default()
@@ -1078,6 +1082,8 @@ impl From<grpc::Process> for oci::Process {
         } else {
             process.set_capabilities(None);
         }
+
+        #[cfg(target_os = "linux")]
         if !from.Rlimits().is_empty() {
             process.set_rlimits(Some(
                 from.Rlimits().iter().cloned().map(|r| r.into()).collect(),
@@ -1238,6 +1244,11 @@ impl From<grpc::LinuxIntelRdt> for oci::LinuxIntelRdt {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashSet;
+
+    use super::cap_vec2hashset;
+    use super::oci;
+
     fn from_vec<F: Sized, T: From<F>>(from: Vec<F>) -> Vec<T> {
         let mut to: Vec<T> = vec![];
         for data in from {
@@ -1289,4 +1300,26 @@ mod tests {
         assert_eq!(from.len(), to.len());
         assert_eq!(from[0].from, to[0].to);
     }
+
+    #[test]
+    fn test_cap_vec2hashset_good() {
+        let expected: HashSet<oci::Capability> =
+            vec![oci::Capability::NetAdmin, oci::Capability::Mknod]
+                .into_iter()
+                .collect();
+        let actual = cap_vec2hashset(vec![
+            "CAP_NET_ADMIN".to_string(),
+            "\"CAP_MKNOD\"".to_string(),
+        ]);
+
+        assert_eq!(expected, actual);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_cap_vec2hashset_bad() {
+        cap_vec2hashset(vec![
+            "CAP_DOES_NOT_EXIST".to_string(),
+        ]);
+    }
 }

src/runtime-rs/Cargo.lock

@@ -1685,8 +1685,11 @@ dependencies = [
  "libc",
  "logging",
  "nix 0.24.3",
+ "oci-spec",
  "path-clean",
  "persist",
+ "protobuf 3.2.0",
+ "protocols",
  "qapi",
  "qapi-qmp",
  "qapi-spec",
@@ -1706,6 +1709,8 @@ dependencies = [
  "thiserror",
  "tokio",
  "tracing",
+ "ttrpc",
+ "ttrpc-codegen",
  "vmm-sys-util 0.11.1",
 ]
 
@@ -1839,7 +1844,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "byteorder",
- "cgroups-rs",
  "chrono",
  "common-path",
  "fail",
@@ -2994,9 +2998,10 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "kata-sys-util",
  "oci-spec",
  "protobuf 3.2.0",
+ "serde",
+ "serde_json",
  "ttrpc",
  "ttrpc-codegen",
 ]

src/runtime-rs/Makefile

@@ -83,16 +83,18 @@ LOCALSTATEDIR := /var
 CONFIG_FILE = configuration.toml
 RUNTIMENAME := virt_container
 HYPERVISOR_DB = dragonball
-HYPERVISOR_ACRN = acrn
 HYPERVISOR_FC = firecracker
 HYPERVISOR_QEMU = qemu
 HYPERVISOR_CLH = cloud-hypervisor
+HYPERVISOR_REMOTE = remote
 
+# When set to true, builds the built-in Dragonball hypervisor
+USE_BUILDIN_DB := true
 
-DEFAULT_HYPERVISOR ?= $(HYPERVISOR_DB)
+HYPERVISOR ?= $(HYPERVISOR_DB)
 
 ##VAR HYPERVISOR=<hypervisor_name> List of hypervisors this build system can generate configuration for.
-HYPERVISORS := $(HYPERVISOR_DB) $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
+HYPERVISORS := $(HYPERVISOR_DB) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_REMOTE)
 
 CLHPATH := $(CLHBINDIR)/$(CLHCMD)
 CLHVALIDHYPERVISORPATHS := [\"$(CLHPATH)\"]
@@ -187,8 +189,6 @@ CONFIG_PATHS =
 SYSCONFIG_PATHS =
 # List of hypervisors known for the current architecture
 KNOWN_HYPERVISORS =
-# List of hypervisors known for the current architecture
-KNOWN_HYPERVISORS =
 
 CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR)/runtime-rs
 SYSCONFDIR := $(SYSCONFDIR)/$(PROJECT_DIR)
@@ -318,16 +318,33 @@ ifneq (,$(FCCMD))
     DEFSTATICRESOURCEMGMT_FC := true
 endif
 
-ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_DB))
+ifneq (,$(REMOTE))
+    KNOWN_HYPERVISORS += $(HYPERVISOR_REMOTE)
+    CONFIG_FILE_REMOTE = configuration-remote.toml
+    CONFIG_REMOTE = config/$(CONFIG_FILE_REMOTE)
+    CONFIG_REMOTE_IN = $(CONFIG_REMOTE).in
+    CONFIG_PATH_REMOTE = $(abspath $(CONFDIR)/$(CONFIG_FILE_REMOTE))
+    CONFIG_PATHS += $(CONFIG_PATH_REMOTE)
+    SYSCONFDIR_REMOTE = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_REMOTE))
+    SYSCONFIG_PATHS += $(SYSCONFDIR_REMOTE)
+    CONFIGS += $(CONFIG_REMOTE)
+    # remote-specific options (all should be suffixed by "_REMOTE")
+    DEFSANDBOXCGROUPONLY_REMOTE := false
+endif
+
+ifeq ($(HYPERVISOR),$(HYPERVISOR_DB))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_DB)
 endif
 
-ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_QEMU))
+ifeq ($(HYPERVISOR),$(HYPERVISOR_QEMU))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_QEMU)
 endif
-ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_FC))
+ifeq ($(HYPERVISOR),$(HYPERVISOR_FC))
     DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_FC)
 endif
+ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_REMOTE))
+    DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_REMOTE)
+endif
 # list of variables the user may wish to override
 USER_VARS += ARCH
 USER_VARS += BINDIR
@@ -335,8 +352,10 @@ USER_VARS += CONFIG_DB_IN
 USER_VARS += CONFIG_FC_IN
 USER_VARS += CONFIG_PATH
 USER_VARS += CONFIG_QEMU_IN
+USER_VARS += CONFIG_REMOTE_IN
 USER_VARS += DESTDIR
-USER_VARS += DEFAULT_HYPERVISOR
+USER_VARS += HYPERVISOR
+USER_VARS += USE_BUILDIN_DB
 USER_VARS += DBCMD
 USER_VARS += DBCTLCMD
 USER_VARS += FCCTLCMD
@@ -399,7 +418,6 @@ USER_VARS += SYSCONFDIR
 USER_VARS += DEFVCPUS
 USER_VARS += DEFVCPUS_QEMU
 USER_VARS += DEFMAXVCPUS
-USER_VARS += DEFMAXVCPUS_ACRN
 USER_VARS += DEFMAXVCPUS_DB
 USER_VARS += DEFMAXVCPUS_QEMU
 USER_VARS += DEFMEMSZ
@@ -444,6 +462,7 @@ USER_VARS += DEFSANDBOXCGROUPONLY_QEMU
 USER_VARS += DEFSANDBOXCGROUPONLY_DB
 USER_VARS += DEFSANDBOXCGROUPONLY_FC
 USER_VARS += DEFSANDBOXCGROUPONLY_CLH
+USER_VARS += DEFSANDBOXCGROUPONLY_REMOTE
 USER_VARS += DEFSTATICRESOURCEMGMT_DB
 USER_VARS += DEFSTATICRESOURCEMGMT_FC
 USER_VARS += DEFSTATICRESOURCEMGMT_CLH
@@ -475,6 +494,11 @@ COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown)
 
 EXTRA_RUSTFEATURES :=
 
+# if use dragonball hypervisor, add the feature to build dragonball in runtime
+ifeq ($(USE_BUILDIN_DB),true)
+    EXTRA_RUSTFEATURES +=  dragonball
+endif
+
 ifneq ($(EXTRA_RUSTFEATURES),)
     override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES)
 endif
@@ -556,7 +580,7 @@ static-checks-build: $(GENERATED_FILES)
 $(TARGET): $(GENERATED_FILES) $(TARGET_PATH)
 
 $(TARGET_PATH): $(SOURCES) | show-summary
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed \
@@ -614,7 +638,7 @@ show-summary: show-header
 	@printf "  %s\n" "$(call get_toolchain_version)"
 	@printf "\n"
 	@printf "• Hypervisors:\n"
-	@printf "\tDefault: $(DEFAULT_HYPERVISOR)\n"
+	@printf "\tDefault: $(HYPERVISOR)\n"
 	@printf "\tKnown: $(sort $(HYPERVISORS))\n"
 	@printf "\tAvailable for this architecture: $(sort $(KNOWN_HYPERVISORS))\n"
 	@printf "\n"
@@ -634,7 +658,7 @@ show-summary: show-header
 	@printf "\talternate config paths (SYSCONFIG_PATHS) : %s\n"
 	@printf \
 	  "$(foreach c,$(sort $(SYSCONFIG_PATHS)),$(shell printf "\\t - $(c)\\\n"))"
-	@printf "\tdefault install path for $(DEFAULT_HYPERVISOR) (CONFIG_PATH) : %s\n" $(abspath $(CONFIG_PATH))
+	@printf "\tdefault install path for $(HYPERVISOR) (CONFIG_PATH) : %s\n" $(abspath $(CONFIG_PATH))
 	@printf "\tdefault alternate config path (SYSCONFIG) : %s\n" $(abspath $(SYSCONFIG))
 ifneq (,$(findstring $(HYPERVISOR_QEMU),$(KNOWN_HYPERVISORS)))
 	@printf "\t$(HYPERVISOR_QEMU) hypervisor path (QEMUPATH) : %s\n" $(abspath $(QEMUPATH))
@@ -647,9 +671,6 @@ ifneq (,$(findstring $(HYPERVISOR_CLH),$(KNOWN_HYPERVISORS)))
 endif
 ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS)))
 	@printf "\t$(HYPERVISOR_FC) hypervisor path (FCPATH) : %s\n" $(abspath $(FCPATH))
-endif
-ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS)))
-	@printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH))
 endif
 	@printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR))
 	@printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR))

src/runtime-rs/arch/x86_64-options.mk

@@ -20,3 +20,5 @@ CLHCMD := cloud-hypervisor
 # firecracker binary (vmm and jailer)
 FCCMD := firecracker
 FCJAILERCMD := jailer
+
+REMOTE := remote

src/runtime-rs/config/configuration-remote.toml.in

@@ -1,43 +1,47 @@
-# Copyright (c) 2017-2019 Intel Corporation
-# Copyright (c) 2021 Adobe Inc.
+# Copyright 2024 Kata Contributors
 #
 # SPDX-License-Identifier: Apache-2.0
 #
 
+
 # XXX: WARNING: this file is auto-generated.
 # XXX:
-# XXX: Source file: "@CONFIG_ACRN_IN@"
+# XXX: Source file: "@CONFIG_REMOTE_IN@"
 # XXX: Project:
 # XXX:   Name: @PROJECT_NAME@
 # XXX:   Type: @PROJECT_TYPE@
 
-[hypervisor.acrn]
-path = "@ACRNPATH@"
-ctlpath = "@ACRNCTLPATH@"
-kernel = "@KERNELPATH_ACRN@"
-image = "@IMAGEPATH@"
 
-# rootfs filesystem type:
-#   - ext4 (default)
-#   - xfs
-#   - erofs
-rootfs_type=@DEFROOTFSTYPE@
+[hypervisor.remote]
+# Default VM information query service unix domain socket, created by cloud-api-adaptor
+# Ref: https://github.com/confidential-containers/cloud-api-adaptor/blob/main/src/cloud-api-adaptor/docs/vminfo.md
+remote_hypervisor_socket = "/run/peerpod/hypervisor.sock"
+# Timeout in seconds for creating a remote hypervisor, 600s(10min) by default
+remote_hypervisor_timeout = 600
+
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+# confidential_guest = true
+
 
 # List of valid annotation names for the hypervisor
 # Each member of the list is a regular expression, which is the base name
 # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
-enable_annotations = @DEFENABLEANNOTATIONS@
-
-# List of valid annotations values for the hypervisor
-# Each member of the list is a path pattern as described by glob(3).
-# The default if not set is empty (all annotations rejected.)
-# Your distribution recommends: @ACRNVALIDHYPERVISORPATHS@
-valid_hypervisor_paths = @ACRNVALIDHYPERVISORPATHS@
-
-# List of valid annotations values for ctlpath
-# The default if not set is empty (all annotations rejected.)
-# Your distribution recommends: @ACRNVALIDCTLPATHS@
-valid_ctlpaths = @ACRNVALIDCTLPATHS@
+# Note: Remote hypervisor is only handling the following annotations
+enable_annotations = ["machine_type", "default_memory", "default_vcpus"]
 
 # Optional space-separated list of options to pass to the guest kernel.
 # For example, use `kernel_params = "vsyscall=emulate"` if you are having
@@ -49,12 +53,20 @@ valid_ctlpaths = @ACRNVALIDCTLPATHS@
 # may stop the virtual machine from booting.
 # To see the list of default parameters, enable hypervisor debug, create a
 # container and look for 'default-kernel-parameters' log entries.
-kernel_params = "@KERNELPARAMS@"
+# NOTE: kernel_params are not currently passed over in remote hypervisor
+# kernel_params = ""
 
 # Path to the firmware.
-# If you want that acrn uses the default firmware leave this option empty
+# If you want that qemu uses the default firmware leave this option empty
 firmware = "@FIRMWAREPATH@"
 
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+# default_vcpus = 1
+
 # Default maximum number of vCPUs per SB/VM:
 # unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
 #                                     of vCPUs supported by KVM if that number is exceeded
@@ -69,14 +81,15 @@ firmware = "@FIRMWAREPATH@"
 # `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
 # vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
 # unless you know what are you doing.
-default_maxvcpus = @DEFMAXVCPUS_ACRN@
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+# default_maxvcpus = @DEFMAXVCPUS@
 
 # Bridges can be used to hot plug devices.
 # Limitations:
 # * Currently only pci bridges are supported
 # * Until 30 devices per bridge can be hot plugged.
 # * Until 5 PCI bridges can be cold plugged per VM.
-#   This limitation could be a bug in the kernel
+#   This limitation could be a bug in qemu or in the kernel
 # Default number of bridges per SB/VM:
 # unspecified or 0   --> will be set to @DEFBRIDGES@
 # > 1 <= 5           --> will be set to the specified number
@@ -85,27 +98,20 @@ default_bridges = @DEFBRIDGES@
 
 # Default memory size in MiB for SB/VM.
 # If unspecified then it will be set @DEFMEMSZ@ MiB.
-default_memory = @DEFMEMSZ@
-
-# Block storage driver to be used for the hypervisor in case the container
-# rootfs is backed by a block device. ACRN only supports virtio-blk.
-block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@"
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+# default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+#memory_slots = @DEFMEMSLOTS@
 
 # This option changes the default hypervisor and kernel parameters
-# to enable debug output where available.
+# to enable debug output where available. And Debug also enable the hmp socket.
 #
 # Default false
-#enable_debug = true
-
-# Disable the customizations done in the runtime when it detects
-# that it is running on top a VMM. This will result in the runtime
-# behaving as it would when running on bare metal.
-#
-#disable_nesting_checks = true
-
-# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics.
-# Default false
-#disable_vhost_net = true
+# enable_debug = true
 
 # Path to OCI hook binaries in the *guest rootfs*.
 # This does not affect host-side hooks which must instead be added to
@@ -127,10 +133,18 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@"
 # disable applying SELinux on the VMM process (default false)
 disable_selinux=@DEFDISABLESELINUX@
 
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+# Note: The remote hypervisor has a different guest, so currently requires this to be disabled
+disable_guest_selinux = true
+
 [agent.@PROJECT_TYPE@]
 # If enabled, make the agent display debug-level messages.
 # (default: disabled)
-#enable_debug = true
+# enable_debug = true
 
 # Enable agent tracing.
 #
@@ -144,7 +158,7 @@ disable_selinux=@DEFDISABLESELINUX@
 #   increasing the container shutdown time slightly.
 #
 # (default: disabled)
-#enable_tracing = true
+# enable_tracing = true
 
 # Enable debug console.
 
@@ -154,31 +168,20 @@ disable_selinux=@DEFDISABLESELINUX@
 #debug_console_enabled = true
 
 # Agent connection dialing timeout value in seconds
-# (default: 45)
-dial_timeout = 45
-
-# Confidential Data Hub API timeout value in seconds
-# (default: 50)
-#cdh_api_timeout = 50
+# (default: 30)
+#dial_timeout = 30
 
 [runtime]
 # If enabled, the runtime will log additional debug messages to the
 # system log
 # (default: disabled)
-#enable_debug = true
+# enable_debug = true
 #
 # Internetworking model
 # Determines how the VM should be connected to the
 # the container network interface
 # Options:
 #
-#   - bridged (Deprecated)
-#     Uses a linux bridge to interconnect the container interface to
-#     the VM. Works for most cases except macvlan and ipvlan.
-#     ***NOTE: This feature has been deprecated with plans to remove this
-#     feature in the future. Please use other network models listed below.
-#
-#
 #   - macvtap
 #     Used when the Container network interface can be bridged using
 #     macvtap.
@@ -190,14 +193,29 @@ dial_timeout = 45
 #     Uses tc filter rules to redirect traffic from the network interface
 #     provided by plugin to a tap interface connected to the VM.
 #
-internetworking_model="@DEFNETWORKMODEL_ACRN@"
+# Note: The remote hypervisor, uses it's own network, so "none" is required
+internetworking_model="none"
+
+name="virt_container"
+hypervisor_name="remote"
+agent_name="kata"
 
 # disable guest seccomp
 # Determines whether container seccomp profiles are passed to the virtual
 # machine and applied by the kata agent. If set to true, seccomp is not applied
 # within the guest
 # (default: true)
-disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+# Note: The remote hypervisor has a different guest, so currently requires this to be set to true
+disable_guest_seccomp=true
+
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
 
 # If enabled, the runtime will create opentracing.io traces and spans.
 # (See https://www.jaegertracing.io/docs/getting-started).
@@ -216,11 +234,12 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 
 # If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
 # This option may have some potential impacts to your host. It should only be used when you know what you're doing.
-# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
 # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
 # (like OVS) directly.
 # (default: false)
-#disable_new_netns = true
+# Note: The remote hypervisor has a different networking model, which requires true 
+disable_new_netns = false
 
 # if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
 # The container cgroups in the host are not created, just one single cgroup per sandbox.
@@ -228,11 +247,43 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 # The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
 # The sandbox cgroup is constrained if there is no container type annotation.
 # See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
-sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY_REMOTE@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+# Note: the remote hypervisor uses the peer pod config to determine the sandbox size, so requires this to be set to true
+static_sandbox_resource_mgmt=true
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
 
 # If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
 # be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
-disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+# Note: remote hypervisor has no sharing of emptydir mounts from host to guest
+disable_guest_empty_dir=false
 
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
@@ -244,20 +295,3 @@ experimental=@DEFAULTEXPFEATURES@
 # If enabled, user can run pprof tools with shim v2 process through kata-monitor.
 # (default: false)
 # enable_pprof = true
-
-# Indicates the CreateContainer request timeout needed for the workload(s)
-# It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to @DEFCREATECONTAINERTIMEOUT@ second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
-# In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
-create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
-
-# Base directory of directly attachable network config.
-# Network devices for VM-based containers are allowed to be placed in the
-# host netns to eliminate as many hops as possible, which is what we
-# called a "Directly Attachable Network". The config, set by special CNI
-# plugins, is used to tell the Kata containers what devices are attached
-# to the hypervisor.
-# (default: /run/kata-containers/dans)
-dan_conf = "@DEFDANCONF@"

src/runtime-rs/crates/agent/src/sock/mod.rs

@@ -8,6 +8,8 @@ mod hybrid_vsock;
 pub use hybrid_vsock::HybridVsock;
 mod vsock;
 pub use vsock::Vsock;
+mod remote;
+pub use remote::Remote;
 
 use std::{
     pin::Pin,
@@ -28,6 +30,7 @@ use url::Url;
 
 const VSOCK_SCHEME: &str = "vsock";
 const HYBRID_VSOCK_SCHEME: &str = "hvsock";
+const REMOTE_SCHEME: &str = "remote";
 
 /// Socket stream
 pub enum Stream {
@@ -98,6 +101,7 @@ impl ConnectConfig {
 enum SockType {
     Vsock(Vsock),
     HybridVsock(HybridVsock),
+    Remote(Remote),
 }
 
 #[async_trait]
@@ -114,6 +118,7 @@ pub fn new(address: &str, port: u32) -> Result<Arc<dyn Sock>> {
     match parse(address, port).context("parse url")? {
         SockType::Vsock(sock) => Ok(Arc::new(sock)),
         SockType::HybridVsock(sock) => Ok(Arc::new(sock)),
+        SockType::Remote(sock) => Ok(Arc::new(sock)),
     }
 }
 
@@ -136,6 +141,13 @@ fn parse(address: &str, port: u32) -> Result<SockType> {
             let uds = path[0];
             Ok(SockType::HybridVsock(HybridVsock::new(uds, port)))
         }
+        REMOTE_SCHEME => {
+            let path: Vec<&str> = url.path().split(':').collect();
+            if path.len() != 1 {
+                return Err(anyhow!("invalid path {:?}", path));
+            }
+            Ok(SockType::Remote(Remote::new(path[0].to_string())))
+        }
         _ => Err(anyhow!("Unsupported scheme")),
     }
 }

src/runtime-rs/crates/agent/src/sock/remote.rs

@@ -0,0 +1,61 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::{os::unix::prelude::AsRawFd, path::Path};
+
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use tokio::{io::Interest, net::UnixStream};
+
+use super::{ConnectConfig, Sock, Stream};
+
+#[derive(Debug, PartialEq)]
+pub struct Remote {
+    path: String,
+}
+
+impl Remote {
+    pub fn new(path: String) -> Self {
+        Self { path }
+    }
+}
+
+#[async_trait]
+impl Sock for Remote {
+    async fn connect(&self, config: &ConnectConfig) -> Result<Stream> {
+        let retry_times = config.reconnect_timeout_ms / config.dial_timeout_ms;
+        for i in 0..retry_times {
+            match connect_helper(&self.path).await {
+                Ok(stream) => {
+                    info!(
+                        sl!(),
+                        "remote connect success on {} current client fd {}",
+                        i,
+                        stream.as_raw_fd()
+                    );
+                    return Ok(Stream::Unix(stream));
+                }
+                Err(err) => {
+                    debug!(sl!(), "remote connect on {} err : {:?}", i, err);
+                    tokio::time::sleep(std::time::Duration::from_millis(config.dial_timeout_ms))
+                        .await;
+                    continue;
+                }
+            }
+        }
+        Err(anyhow!("cannot connect to agent ttrpc server {:?}", config))
+    }
+}
+
+async fn connect_helper(address: &str) -> Result<UnixStream> {
+    let stream = UnixStream::connect(Path::new(&address))
+        .await
+        .context("failed to create UnixAddr")?;
+    stream
+        .ready(Interest::READABLE | Interest::WRITABLE)
+        .await?;
+    Ok(stream)
+}

src/runtime-rs/crates/agent/src/types.rs

@@ -42,7 +42,7 @@ pub struct StringUser {
     pub additional_gids: Vec<String>,
 }
 
-#[derive(PartialEq, Clone, Default)]
+#[derive(PartialEq, Clone, Debug, Default)]
 pub struct Device {
     pub id: String,
     pub field_type: String,

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -28,12 +28,15 @@ rand = "0.8.4"
 path-clean = "1.0.1"
 lazy_static = "1.4"
 tracing = "0.1.36"
+ttrpc = {version = "0.8.1", features = ["async"] }
+protobuf = "3.1.0"
 
-dbs-utils = { path = "../../../dragonball/src/dbs_utils" }
 kata-sys-util = { path = "../../../libs/kata-sys-util" }
 kata-types = { path = "../../../libs/kata-types" }
 logging = { path = "../../../libs/logging" }
+protocols = { path = "../../../libs/protocols", features = ["async"] }
 shim-interface = { path = "../../../libs/shim-interface" }
+oci-spec = { version = "0.6.8", features = ["runtime"] }
 
 ch-config = { path = "ch-config", optional = true }
 tests_utils = { path = "../../tests/utils" }
@@ -48,7 +51,7 @@ qapi-spec = "0.3.1"
 qapi-qmp = "0.14.0"
 
 [target.'cfg(not(target_arch = "s390x"))'.dependencies]
-dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall", "virtio-mem", "virtio-balloon", "vhost-user-net", "host-device"] }
+dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall", "virtio-mem", "virtio-balloon", "vhost-user-net", "host-device"], optional = true }
 dbs-utils = { path = "../../../dragonball/src/dbs_utils" }
 hyperlocal = "0.8.0"
 hyper = {version = "0.14.18", features = ["client"]}
@@ -56,6 +59,7 @@ hyper = {version = "0.14.18", features = ["client"]}
 [features]
 default = []
 
+dragonball = ["dep:dragonball"]
 # Feature is not yet complete, so not enabled by default.
 # See https://github.com/kata-containers/kata-containers/issues/6264.
 cloud-hypervisor = ["ch-config"]
@@ -68,3 +72,7 @@ hypervisor = { path = ".", features = ["cloud-hypervisor"] }
 test-utils = { path = "../../../libs/test-utils" }
 
 serial_test = "2.0.0"
+
+
+[build-dependencies]
+ttrpc-codegen = "0.4.2"

src/runtime-rs/crates/hypervisor/src/ch/inner.rs

@@ -16,7 +16,6 @@ use persist::sandbox_persist::Persist;
 use std::collections::HashMap;
 use std::os::unix::net::UnixStream;
 use tokio::sync::watch::{channel, Receiver, Sender};
-use tokio::sync::Mutex;
 use tokio::task::JoinHandle;
 use tokio::{process::Child, sync::mpsc};
 
@@ -79,13 +78,12 @@ pub struct CloudHypervisorInner {
     pub(crate) _guest_memory_block_size_mb: u32,
 
     pub(crate) exit_notify: Option<mpsc::Sender<i32>>,
-    pub(crate) exit_waiter: Mutex<(mpsc::Receiver<i32>, i32)>,
 }
 
 const CH_DEFAULT_TIMEOUT_SECS: u32 = 10;
 
 impl CloudHypervisorInner {
-    pub fn new() -> Self {
+    pub fn new(exit_notify: Option<mpsc::Sender<i32>>) -> Self {
         let mut capabilities = Capabilities::new();
         capabilities.set(
             CapabilityBits::BlockDeviceSupport
@@ -95,7 +93,6 @@ impl CloudHypervisorInner {
         );
 
         let (tx, rx) = channel(true);
-        let (exit_notify, exit_waiter) = mpsc::channel(1);
 
         Self {
             api_socket: None,
@@ -122,8 +119,7 @@ impl CloudHypervisorInner {
             ch_features: None,
             _guest_memory_block_size_mb: 0,
 
-            exit_notify: Some(exit_notify),
-            exit_waiter: Mutex::new((exit_waiter, 0)),
+            exit_notify,
         }
     }
 
@@ -138,14 +134,14 @@ impl CloudHypervisorInner {
 
 impl Default for CloudHypervisorInner {
     fn default() -> Self {
-        Self::new()
+        Self::new(None)
     }
 }
 
 #[async_trait]
 impl Persist for CloudHypervisorInner {
     type State = HypervisorState;
-    type ConstructorArgs = ();
+    type ConstructorArgs = mpsc::Sender<i32>;
 
     // Return a state object that will be saved by the caller.
     async fn save(&self) -> Result<Self::State> {
@@ -166,11 +162,10 @@ impl Persist for CloudHypervisorInner {
 
     // Set the hypervisor state to the specified state
     async fn restore(
-        _hypervisor_args: Self::ConstructorArgs,
+        exit_notify: mpsc::Sender<i32>,
         hypervisor_state: Self::State,
     ) -> Result<Self> {
         let (tx, rx) = channel(true);
-        let (exit_notify, exit_waiter) = mpsc::channel(1);
 
         let mut ch = Self {
             config: Some(hypervisor_state.config),
@@ -190,7 +185,6 @@ impl Persist for CloudHypervisorInner {
             jailer_root: String::default(),
             ch_features: None,
             exit_notify: Some(exit_notify),
-            exit_waiter: Mutex::new((exit_waiter, 0)),
 
             ..Default::default()
         };
@@ -207,7 +201,9 @@ mod tests {
 
     #[actix_rt::test]
     async fn test_save_clh() {
-        let mut clh = CloudHypervisorInner::new();
+        let (exit_notify, _exit_waiter) = mpsc::channel(1);
+
+        let mut clh = CloudHypervisorInner::new(Some(exit_notify.clone()));
         clh.id = String::from("123456");
         clh.netns = Some(String::from("/var/run/netns/testnet"));
         clh.vm_path = String::from("/opt/kata/bin/cloud-hypervisor");
@@ -229,7 +225,7 @@ mod tests {
         assert!(!state.jailed);
         assert_eq!(state.hypervisor_type, HYPERVISOR_NAME_CH.to_string());
 
-        let clh = CloudHypervisorInner::restore((), state.clone())
+        let clh = CloudHypervisorInner::restore(exit_notify, state.clone())
             .await
             .unwrap();
         assert_eq!(clh.id, state.id);

src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs

@@ -19,7 +19,6 @@ use ch_config::ch_api::{
 };
 use ch_config::{guest_protection_is_tdx, NamedHypervisorConfig, VmConfig};
 use core::future::poll_fn;
-use futures::executor::block_on;
 use futures::future::join_all;
 use kata_sys_util::protection::{available_guest_protection, GuestProtection};
 use kata_types::capabilities::{Capabilities, CapabilityBits};
@@ -640,7 +639,7 @@ impl CloudHypervisorInner {
         Ok(())
     }
 
-    pub(crate) fn stop_vm(&mut self) -> Result<()> {
+    pub(crate) async fn stop_vm(&mut self) -> Result<()> {
         // If the container workload exits, this method gets called. However,
         // the container manager always makes a ShutdownContainer request,
         // which results in this method being called potentially a second
@@ -652,19 +651,14 @@ impl CloudHypervisorInner {
 
         self.state = VmmState::NotReady;
 
-        block_on(self.cloud_hypervisor_shutdown()).map_err(|e| anyhow!(e))?;
+        self.cloud_hypervisor_shutdown().await?;
 
         Ok(())
     }
 
+    #[allow(dead_code)]
     pub(crate) async fn wait_vm(&self) -> Result<i32> {
-        debug!(sl!(), "Waiting CH vmm");
-        let mut waiter = self.exit_waiter.lock().await;
-        if let Some(exitcode) = waiter.0.recv().await {
-            waiter.1 = exitcode;
-        }
-
-        Ok(waiter.1)
+        Ok(0)
     }
 
     pub(crate) fn pause_vm(&self) -> Result<()> {

src/runtime-rs/crates/hypervisor/src/ch/mod.rs

@@ -11,8 +11,9 @@ use async_trait::async_trait;
 use kata_types::capabilities::{Capabilities, CapabilityBits};
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 use persist::sandbox_persist::Persist;
+use std::collections::HashMap;
 use std::sync::Arc;
-use tokio::sync::RwLock;
+use tokio::sync::{mpsc, Mutex, RwLock};
 
 // Convenience macro to obtain the scope logger
 #[macro_export]
@@ -29,15 +30,19 @@ mod utils;
 
 use inner::CloudHypervisorInner;
 
-#[derive(Debug, Default, Clone)]
+#[derive(Debug)]
 pub struct CloudHypervisor {
     inner: Arc<RwLock<CloudHypervisorInner>>,
+    exit_waiter: Mutex<(mpsc::Receiver<i32>, i32)>,
 }
 
 impl CloudHypervisor {
     pub fn new() -> Self {
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
         Self {
-            inner: Arc::new(RwLock::new(CloudHypervisorInner::new())),
+            inner: Arc::new(RwLock::new(CloudHypervisorInner::new(Some(exit_notify)))),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
         }
     }
 
@@ -47,9 +52,20 @@ impl CloudHypervisor {
     }
 }
 
+impl Default for CloudHypervisor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 #[async_trait]
 impl Hypervisor for CloudHypervisor {
-    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+    async fn prepare_vm(
+        &self,
+        id: &str,
+        netns: Option<String>,
+        _annotations: &HashMap<String, String>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_vm(id, netns).await
     }
@@ -61,12 +77,17 @@ impl Hypervisor for CloudHypervisor {
 
     async fn stop_vm(&self) -> Result<()> {
         let mut inner = self.inner.write().await;
-        inner.stop_vm()
+        inner.stop_vm().await
     }
 
     async fn wait_vm(&self) -> Result<i32> {
-        let inner = self.inner.read().await;
-        inner.wait_vm().await
+        debug!(sl!(), "Waiting CH vmm");
+        let mut waiter = self.exit_waiter.lock().await;
+        if let Some(exitcode) = waiter.0.recv().await {
+            waiter.1 = exitcode;
+        }
+
+        Ok(waiter.1)
     }
 
     async fn pause_vm(&self) -> Result<()> {
@@ -204,12 +225,15 @@ impl Persist for CloudHypervisor {
     }
 
     async fn restore(
-        hypervisor_args: Self::ConstructorArgs,
+        _hypervisor_args: Self::ConstructorArgs,
         hypervisor_state: Self::State,
     ) -> Result<Self> {
-        let inner = CloudHypervisorInner::restore(hypervisor_args, hypervisor_state).await?;
+        let (exit_notify, exit_waiter) = mpsc::channel(1);
+
+        let inner = CloudHypervisorInner::restore(exit_notify, hypervisor_state).await?;
         Ok(Self {
             inner: Arc::new(RwLock::new(inner)),
+            exit_waiter: Mutex::new((exit_waiter, 0)),
         })
     }
 }