Merge pull request #6661 from fidencio/topic/update-deps-for-v0.5.0-release

Update deps for the v0.5.0 release of Confidential Containers

.github/workflows/add-backport-label.yaml

@@ -62,15 +62,15 @@ jobs:
           has_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'needs-backport') }}
           has_no_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'no-backport-needed') }}
 
-          echo "::set-output name=add_backport_label::false"
+          echo "add_backport_label=false" >> $GITHUB_OUTPUT
           if [ $has_backport_needed_label  = true ] || [ $has_bug  = true ]; then
             if [[ $has_no_backport_needed_label = false ]]; then
-              echo "::set-output name=add_backport_label::true"
+              echo "add_backport_label=true" >> $GITHUB_OUTPUT
             fi
           fi
 
           # Do not spam comment, only if auto-backport label is going to be newly added.
-          echo "::set-output name=auto_backport_added::$CONTAINS_AUTO_BACKPORT"
+          echo "auto_backport_added=$CONTAINS_AUTO_BACKPORT" >> $GITHUB_OUTPUT
 
       - name: Add comment
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'true' && steps.add_label.outputs.auto_backport_added == 'false' }}
@@ -97,4 +97,4 @@ jobs:
         uses: andymckay/labeler@e6c4322d0397f3240f0e7e30a33b5c5df2d39e90
         with:
           add-labels: "auto-backport"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/cc-payload-after-push-amd64.yaml

@@ -0,0 +1,161 @@
+name: CI | Publish CC runtime payload for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - cc-cloud-hypervisor
+          - cc-kernel
+          - cc-qemu
+          - cc-rootfs-image
+          - cc-virtiofsd
+          - cc-sev-kernel
+          - cc-sev-ovmf
+          - cc-x86_64-ovmf
+          - cc-snp-qemu
+          - cc-sev-rootfs-initrd
+          - cc-tdx-kernel
+          - cc-tdx-rootfs-image
+          - cc-tdx-qemu
+          - cc-tdx-td-shim
+          - cc-tdx-tdvf
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_tdx.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/root_hash_tdx.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_tdx.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push-s390x.yaml

@@ -0,0 +1,164 @@
+name: CI | Publish CC runtime payload for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        asset:
+          - cc-kernel
+          - cc-qemu
+          - cc-rootfs-image
+          - cc-rootfs-initrd
+          - cc-se-image
+          - cc-virtiofsd
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Place a host key document
+        run: |
+          mkdir -p "host-key-document"
+          cp "${CI_HKD_PATH}" "host-key-document"
+        env:
+          CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+          sudo chown -R $(id -u):$(id -g) "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+          HKD_PATH: "host-key-document"
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push.yaml

@@ -0,0 +1,39 @@
+name: CI | Publish Kata Containers payload for Confidential Containers
+on:
+  push:
+    branches:
+      - CCv0
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/cc-payload-after-push-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/cc-payload-after-push-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-assets-amd64, build-assets-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest

.github/workflows/cc-payload-amd64.yaml

@@ -0,0 +1,144 @@
+name: Publish Kata Containers payload for Confidential Containers (amd64)
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - cc-cloud-hypervisor
+          - cc-kernel
+          - cc-qemu
+          - cc-rootfs-image
+          - cc-virtiofsd
+          - cc-sev-kernel
+          - cc-sev-ovmf
+          - cc-x86_64-ovmf
+          - cc-snp-qemu
+          - cc-sev-rootfs-initrd
+          - cc-tdx-kernel
+          - cc-tdx-rootfs-image
+          - cc-tdx-qemu
+          - cc-tdx-td-shim
+          - cc-tdx-tdvf
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_tdx.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/root_hash_tdx.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_tdx.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          "quay.io/confidential-containers/runtime-payload" \
+          "kata-containers-${{ inputs.target-arch }}"
+

.github/workflows/cc-payload-s390x.yaml

@@ -0,0 +1,134 @@
+name: Publish Kata Containers payload for Confidential Containers (s390x)
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        asset:
+          - cc-kernel
+          - cc-qemu
+          - cc-rootfs-image
+          - cc-virtiofsd
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          "quay.io/confidential-containers/runtime-payload" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload.yaml

@@ -0,0 +1,39 @@
+name: Publish Kata Containers payload for Confidential Containers
+on:
+  push:
+    tags:
+      - 'CC\-[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/cc-payload-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/cc-payload-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-assets-amd64, build-assets-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-latest \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-latest

.github/workflows/commit-message-check.yaml

@@ -47,7 +47,7 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^.{0,75}(\n.*)*$'
+        pattern: '^.{0,75}(\n.*)*$|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
         error: 'Subject too long (max 75)'
         post_error: ${{ env.error_msg }}
 
@@ -95,6 +95,6 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:'
+        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
         error: 'Failed to find subsystem in subject'
         post_error: ${{ env.error_msg }}

.github/workflows/deploy-ccv0-demo.yaml

@@ -0,0 +1,124 @@
+on:
+  issue_comment:
+    types: [created, edited]
+
+name: deploy-ccv0-demo
+
+jobs:
+  check-comment-and-membership:
+    runs-on: ubuntu-latest
+    if: |
+      github.event.issue.pull_request
+      && github.event_name == 'issue_comment'
+      && github.event.action == 'created'
+      && startsWith(github.event.comment.body, '/deploy-ccv0-demo')
+    steps:
+      - name: Check membership
+        uses: kata-containers/is-organization-member@1.0.1
+        id: is_organization_member
+        with:
+          organization: kata-containers
+          username: ${{ github.event.comment.user.login }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Fail if not member
+        run: |
+          result=${{ steps.is_organization_member.outputs.result }}
+          if [ $result == false ]; then
+              user=${{ github.event.comment.user.login }}
+              echo Either ${user} is not part of the kata-containers organization
+              echo or ${user} has its Organization Visibility set to Private at
+              echo https://github.com/orgs/kata-containers/people?query=${user}
+              echo 
+              echo Ensure you change your Organization Visibility to Public and
+              echo trigger the test again.
+              exit 1
+          fi
+
+  build-asset:
+    runs-on: ubuntu-latest
+    needs: check-comment-and-membership
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
+    steps:
+      - uses: actions/checkout@v2
+      - name: Prepare confidential container rootfs
+        if: ${{ matrix.asset == 'rootfs-initrd' }}
+        run: |
+          pushd include_rootfs/etc
+          curl -LO https://raw.githubusercontent.com/confidential-containers/documentation/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json
+          mkdir kata-containers
+          envsubst < docs/how-to/data/confidential-agent-config.toml.in > kata-containers/agent.toml
+          popd
+        env:
+          AA_KBC_PARAMS: offline_fs_kbc::null
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          AA_KBC: offline_fs_kbc
+          INCLUDE_ROOTFS: include_rootfs
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+
+  kata-deploy:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-static-tarball
+      - name: build-and-push-kata-deploy-ci
+        id: build-and-push-kata-deploy-ci
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/confidential-containers/runtime-payload:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/confidential-containers/runtime-payload:$pkg_sha
+          mkdir -p packaging/kata-deploy
+          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
+          echo "::set-output name=PKG_SHA::${pkg_sha}"

.github/workflows/kata-deploy-push.yaml

@@ -72,6 +72,10 @@ jobs:
   make-kata-tarball:
     runs-on: ubuntu-latest
     steps:
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - uses: actions/checkout@v2
       - name: make kata-tarball
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

.github/workflows/kata-deploy-test.yaml

@@ -67,7 +67,7 @@ jobs:
                 ref="refs/pull/${{ github.event.inputs.pr }}/merge"
             fi
             echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
-            echo "##[set-output name=pr-ref;]${ref}"
+            echo "pr-ref=${ref}" >> $GITHUB_OUTPUT
       - uses: actions/checkout@v2
         with:
           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
@@ -102,7 +102,7 @@ jobs:
                 ref="refs/pull/${{ github.event.inputs.pr }}/merge"
             fi
             echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
-            echo "##[set-output name=pr-ref;]${ref}"
+            echo "pr-ref=${ref}" >> $GITHUB_OUTPUT
       - uses: actions/checkout@v2
         with:
           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
@@ -133,7 +133,7 @@ jobs:
                 ref="refs/pull/${{ github.event.inputs.pr }}/merge"
             fi
             echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
-            echo "##[set-output name=pr-ref;]${ref}"
+            echo "pr-ref=${ref}" >> $GITHUB_OUTPUT
       - uses: actions/checkout@v2
         with:
           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
@@ -151,7 +151,7 @@ jobs:
           docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${PR_SHA}"
+          echo "PKG_SHA=${PR_SHA}" >> $GITHUB_OUTPUT
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:

.github/workflows/payload-after-push-amd64.yaml

@@ -0,0 +1,96 @@
+name: CI | Publish kata-deploy payload for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - nydus
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - virtiofsd
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-amd64
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-amd64
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-amd64
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/payload-after-push-arm64.yaml

@@ -0,0 +1,108 @@
+name: CI | Publish kata-deploy payload for arm64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: arm64
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - nydus
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - virtiofsd
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-arm64
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: arm64
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-arm64
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-arm64
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: arm64
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/payload-after-push-s390x.yaml

@@ -0,0 +1,107 @@
+name: CI | Publish kata-deploy payload for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        asset:
+          - kernel
+          - shim-v2
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - virtiofsd
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+          sudo chown -R $(id -u):$(id -g) "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/payload-after-push.yaml

@@ -0,0 +1,47 @@
+name: CI | Publish Kata Containers payload
+on:
+  push:
+    branches:
+      - main
+      - stable-*
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/payload-after-push-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-assets-arm64:
+    uses: ./.github/workflows/payload-after-push-arm64.yaml
+    with:
+      target-arch: arm64
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/payload-after-push-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-assets-amd64, build-assets-arm64, build-assets-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
+        run: |
+          docker manifest create quay.io/kata-containers/kata-deploy-ci:kata-containers-latest \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-amd64 \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-arm64 \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-s390x
+          docker manifest push quay.io/kata-containers/kata-deploy-ci:kata-containers-latest

.github/workflows/release.yaml

@@ -84,7 +84,7 @@ jobs:
           docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
+          echo "PKG_SHA=${pkg_sha}" >> $GITHUB_OUTPUT
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:

.github/workflows/static-checks.yaml

@@ -19,13 +19,14 @@ jobs:
           - "make test"
           - "sudo -E PATH=\"$PATH\" make test"
     env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
       RUST_BACKTRACE: "1"
       target_branch: ${{ github.base_ref }}
+      GOPATH: ${{ github.workspace }}
     steps:
+    - name: Free disk space
+      run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
     - name: Checkout code
       uses: actions/checkout@v3
       with:
@@ -35,16 +36,13 @@ jobs:
       uses: actions/setup-go@v3
       with:
         go-version: 1.19.3
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Check kernel config version
       run: |
         cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
         kernel_dir="tools/packaging/kernel/"
         kernel_version_file="${kernel_dir}kata_config_version"
-        modified_files=$(git diff --name-only origin/main..HEAD)
-        result=$(git whatchanged origin/main..HEAD "${kernel_dir}" >>"/dev/null")
-        if git whatchanged origin/main..HEAD "${kernel_dir}" >>"/dev/null"; then
+        modified_files=$(git diff --name-only origin/CCv0..HEAD)
+        if git diff --name-only origin/CCv0..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
           echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
           if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
             echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
@@ -53,29 +51,14 @@ jobs:
           fi
           echo "Check passed"
         fi
-    - name: Setup GOPATH
+    - name: Set PATH
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
     - name: Setup
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Installing rust
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |

VERSION

@@ -1 +1 @@
-3.1.2
+3.1.0-rc0

ci/install_libseccomp.sh

@@ -72,7 +72,8 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    # Unset $CC for configure, we will always use native for gperf
+    # gperf is a build time dependency of libseccomp and not to be used in the target.
+    # Unset $CC since that might point to a cross compiler.
     CC= ./configure --prefix="${gperf_install_dir}"
     make
     make install

ci/lib.sh

@@ -64,3 +64,86 @@ run_get_pr_changed_file_details()
 	source "$tests_repo_dir/.ci/lib.sh"
 	get_pr_changed_file_details
 }
+
+# Check if the 1st argument version is greater than and equal to 2nd one
+# Version format: [0-9]+ separated by period (e.g. 2.4.6, 1.11.3 and etc.)
+#
+# Parameters:
+#	$1	- a version to be tested
+#	$2	- a target version
+#
+# Return:
+# 	0 if $1 is greater than and equal to $2
+#	1 otherwise
+version_greater_than_equal() {
+	local current_version=$1
+	local target_version=$2
+	smaller_version=$(echo -e "$current_version\n$target_version" | sort -V | head -1)
+	if [ "${smaller_version}" = "${target_version}" ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
+# Build a IBM zSystem secure execution (SE) image
+#
+# Parameters:
+#	$1	- kernel_parameters
+#	$2	- a source directory where kernel and initrd are located
+#	$3	- a destination directory where a SE image is built
+#
+# Return:
+# 	0 if the image is successfully built
+#	1 otherwise
+build_secure_image() {
+	kernel_params="${1:-}"
+	install_src_dir="${2:-}"
+	install_dest_dir="${3:-}"
+
+	if [ ! -f "${install_src_dir}/vmlinuz.container" ] ||
+		[ ! -f "${install_src_dir}/kata-containers-initrd.img" ]; then
+		cat << EOF >&2
+Either kernel or initrd does not exist or is mistakenly named
+A file name for kernel must be vmlinuz.container (raw binary)
+A file name for initrd must be kata-containers-initrd.img
+EOF
+		return 1
+	fi
+
+	cmdline="${kernel_params} panic=1 scsi_mod.scan=none swiotlb=262144"
+	parmfile="$(mktemp --suffix=-cmdline)"
+	echo "${cmdline}" > "${parmfile}"
+	chmod 600 "${parmfile}"
+
+	[ -n "${HKD_PATH:-}" ] || (echo >&2 "No host key document specified." && return 1)
+	cert_list=($(ls -1 $HKD_PATH))
+	declare hkd_options
+	eval "for cert in ${cert_list[*]}; do
+		hkd_options+=\"--host-key-document=\\\"\$HKD_PATH/\$cert\\\" \"
+	done"
+
+	command -v genprotimg > /dev/null 2>&1 || { apt update; apt install -y s390-tools; }
+	extra_arguments=""
+	genprotimg_version=$(genprotimg --version | grep -Po '(?<=version )[^-]+')
+	if ! version_greater_than_equal "${genprotimg_version}" "2.17.0"; then
+		extra_arguments="--x-pcf '0xe0'"
+	fi
+
+	eval genprotimg \
+		"${extra_arguments}" \
+		"${hkd_options}" \
+		--output="${install_dest_dir}/kata-containers-secure.img" \
+		--image="${install_src_dir}/vmlinuz.container" \
+		--ramdisk="${install_src_dir}/kata-containers-initrd.img" \
+		--parmfile="${parmfile}" \
+		--no-verify # no verification for CI testing purposes
+
+	build_result=$?
+	rm -f "${parmfile}"
+	if [ $build_result -eq 0 ]; then
+		return 0
+	else
+		return 1
+	fi
+}

docs/Developer-Guide.md

@@ -2,6 +2,8 @@
 
 This document is written **specifically for developers**: it is not intended for end users.
 
+If you want to contribute changes that you have made, please read the [community guidelines](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md) for information about our processes.
+
 # Assumptions
 
 - You are working on a non-critical test or development system.
@@ -654,7 +656,7 @@ section when using rootfs, or when using initrd, complete the steps in the [Buil
 
 Install the image:
 
->**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img` 
+>**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img`
 >with the initrd image name `kata-containers-initrd.img`.
 
 ```bash
@@ -688,25 +690,25 @@ $ sudo crictl run -r kata container.yaml pod.yaml
 
 The steps required to enable debug console for QEMU slightly differ with
 those for firecracker / cloud-hypervisor.
- 
+
 ##### Enabling debug console for QEMU
 
-Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console. 
+Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console.
 
 ```bash
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console"/g' "${kata_configuration_file}"
 ```
 
-Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml` 
+Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml`
 or `/usr/share/defaults/kata-containers/configuration.toml`
 or `/opt/kata/share/defaults/kata-containers/configuration-{hypervisor}.toml`, if
 you installed Kata Containers using `kata-deploy`.
 
 ##### Enabling debug console for cloud-hypervisor / firecracker
 
-Slightly different configuration is required in case of firecracker and cloud hypervisor. 
-Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`. 
-Hence, the kernel command line option `agent.debug_console` will not work for them. 
+Slightly different configuration is required in case of firecracker and cloud hypervisor.
+Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`.
+Hence, the kernel command line option `agent.debug_console` will not work for them.
 These hypervisors support `hybrid vsocks`,  which can be used for communication
 between the host and the guest. The kernel command line option `agent.debug_console_vport`
  was added to allow developers specify on which `vsock` port the debugging console should be connected.
@@ -719,7 +721,7 @@ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_cons
 ```
 
 > **Note** Ports 1024 and 1025 are reserved for communication with the agent
-> and gathering of agent logs respectively. 
+> and gathering of agent logs respectively.
 
 ##### Connecting to the debug console
 

docs/Release-Process.md

@@ -28,23 +28,6 @@
   $ ./update-repository-version.sh -p "$NEW_VERSION" "$BRANCH"
   ```
 
-### Point tests repository to stable branch
-
-  If you create a new stable branch, i.e. if your release changes a major or minor version number (not a patch release), then
-  you should modify the `tests` repository to point to that newly created stable branch and not the `main` branch.
-  The objective is that changes in the CI on the main branch will not impact the stable branch.
-
-  In the test directory, change references the main branch in:
-  * `README.md`
-  * `versions.yaml`
-  * `cmd/github-labels/labels.yaml.in`
-  * `cmd/pmemctl/pmemctl.sh`
-  * `.ci/lib.sh`
-  * `.ci/static-checks.sh`
-
-  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
-
-
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
@@ -63,6 +46,24 @@
   $ ./tag_repos.sh -p -b "$BRANCH" tag
   ```
 
+### Point tests repository to stable branch
+
+  If your release changes a major or minor version number(not a patch release), then the above 
+  `./tag_repos.sh` script will create a new stable branch in all the repositories in addition to tagging them.
+  This happens when you are making the first `rc` release for a new major or minor version in Kata.
+  In this case, you should modify the `tests` repository to point to the newly created stable branch and not the `main` branch.
+  The objective is that changes in the CI on the main branch will not impact the stable branch.
+
+  In the test directory, change references of the `main` branch to the new stable branch in:
+  * `README.md`
+  * `versions.yaml`
+  * `cmd/github-labels/labels.yaml.in`
+  * `cmd/pmemctl/pmemctl.sh`
+  * `.ci/lib.sh`
+  * `.ci/static-checks.sh`
+
+  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
+
 ### Check Git-hub Actions
 
   We make use of [GitHub actions](https://github.com/features/actions) in this [file](../.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.

docs/design/README.md

@@ -11,6 +11,7 @@ Kata Containers design documents:
 - [Host cgroups](host-cgroups.md)
 - [Agent systemd cgroup](agent-systemd-cgroup.md)
 - [`Inotify` support](inotify.md)
+- [`Hooks` support](hooks-handling.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 - [Design for direct-assigned volume](direct-blk-device-assignment.md)

docs/design/hooks-handling.md

@@ -0,0 +1,63 @@
+# Kata Containers support for `Hooks`
+
+## Introduction
+
+During container's lifecycle, different Hooks can be executed to do custom actions. In Kata Containers, we support two types of Hooks, `OCI Hooks` and `Kata Hooks`.
+
+### OCI Hooks
+
+The OCI Spec stipulates six hooks that can be executed at different time points and namespaces, including `Prestart Hooks`, `CreateRuntime Hooks`, `CreateContainer Hooks`, `StartContainer Hooks`, `Poststart Hooks` and `Poststop Hooks`. We support these types of Hooks as compatible as possible in Kata Containers.
+
+The path and arguments of these hooks will be passed to Kata for execution via `bundle/config.json`. For example:
+```
+...
+"hooks": {
+  "prestart": [
+    {
+      "path": "/usr/bin/prestart-hook",
+      "args": ["prestart-hook", "arg1", "arg2"],
+      "env":  [ "key1=value1"]
+    }
+  ],
+  "createRuntime": [
+    {
+      "path": "/usr/bin/createRuntime-hook",
+      "args": ["createRuntime-hook", "arg1", "arg2"],
+      "env":  [ "key1=value1"]
+    }
+  ]
+}
+...
+```
+
+### Kata Hooks
+
+In Kata, we support another three kinds of hooks executed in guest VM, including `Guest Prestart Hook`, `Guest Poststart Hook`, `Guest Poststop Hook`.
+
+The executable files for Kata Hooks must be packaged in the *guest rootfs*. The file path to those guest hooks should be specified in the configuration file, and guest hooks must be stored in a subdirectory of `guest_hook_path` according to their hook type. For example:
+
++ In configuration file:
+```
+guest_hook_path="/usr/share/hooks"
+```
++ In guest rootfs, prestart-hook is stored in `/usr/share/hooks/prestart/prestart-hook`.
+
+## Execution
+The table below summarized when and where those different hooks will be executed in Kata Containers:
+
+| Hook Name | Hook Type | Hook Path | Exec Place | Exec Time |
+|---|---|---|---|---|
+| `Prestart(deprecated)` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created. |
+| `CreateRuntime` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created, after `Prestart` hooks. |
+| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace* | After VM is started, before container is created, after `CreateRuntime` hooks. |
+| `StartContainer` | OCI hook | guest container namespace | guest container namespace | After container is created, before container is started. |
+| `Poststart` | OCI hook | host runtime namespace | host runtime namespace | After container is started, before start operation returns. |
+| `Poststop` | OCI hook | host runtime namespace | host runtime namespace | After container is deleted, before delete operation returns. |
+| `Guest Prestart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, before container command is executed. |
+| `Guest Poststart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, after container command is executed, before start operation returns. |
+| `Guest Poststop` | Kata hook | guest agent namespace | guest agent namespace | During delete operation, after container is deleted, before delete operation returns. |
+
++ `Hook Path` specifies where hook's path be resolved.
++ `Exec Place` specifies in which namespace those hooks can be executed.
+  + For `CreateContainer` Hooks, OCI requires to run them inside the container namespace while the hook executable path is in the host runtime, which is a non-starter for VM-based containers. So we design to keep them running in the *host vmm namespace.* 
++ `Exec Time` specifies at which time point those hooks can be executed.

docs/how-to/README.md

@@ -45,3 +45,6 @@
 - [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
 - [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
 - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
+## Confidential Containers
+- [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
+- [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md)

docs/how-to/ccv0.sh

@@ -0,0 +1,640 @@
+#!/bin/bash -e
+#
+# Copyright (c) 2021, 2022 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# Disclaimer: This script is work in progress for supporting the CCv0 prototype
+# It shouldn't be considered supported by the Kata Containers community, or anyone else
+
+# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md,
+# but with elements of the tests/.ci scripts used
+
+readonly script_name="$(basename "${BASH_SOURCE[0]}")"
+
+# By default in Golang >= 1.16 GO111MODULE is set to "on", but not all modules support it, so overwrite to "auto"
+export GO111MODULE="auto"
+
+# Setup kata containers environments if not set - we default to use containerd
+export CRI_CONTAINERD=${CRI_CONTAINERD:-"yes"}
+export CRI_RUNTIME=${CRI_RUNTIME:-"containerd"}
+export CRIO=${CRIO:-"no"}
+export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
+export KUBERNETES=${KUBERNETES:-"no"}
+export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}"
+export AA_KBC="${AA_KBC:-offline_fs_kbc}"
+
+# Allow the user to overwrite the default repo and branch names if they want to build from a fork
+export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}"
+export katacontainers_branch="${katacontainers_branch:-CCv0}" 
+export kata_default_branch=${katacontainers_branch}
+export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
+export tests_branch="${tests_branch:-CCv0}"
+export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
+
+# if .bash_profile exists then use it, otherwise fall back to .profile
+export PROFILE="${HOME}/.profile"
+if [ -r "${HOME}/.bash_profile" ]; then
+    export PROFILE="${HOME}/.bash_profile"
+fi
+# Stop PS1: unbound variable error happening
+export PS1=${PS1:-}
+
+# Create a bunch of common, derived values up front so we don't need to create them in all the different functions
+. ${PROFILE}
+if [ -z ${GOPATH} ]; then
+    export GOPATH=${HOME}/go
+fi
+export tests_repo_dir="${GOPATH}/src/${tests_repo}"
+export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
+export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
+export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication
+export CONTAINER_ID="${CONTAINER_ID:-0123456789}"
+source /etc/os-release || source /usr/lib/os-release
+grep -Eq "\<fedora\>" /etc/os-release 2> /dev/null && export USE_PODMAN=true
+
+
+# If we've already checked out the test repo then source the confidential scripts
+if [ "${KUBERNETES}" == "yes" ]; then
+    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/kubernetes/confidential"
+    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
+else
+    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/containerd/confidential"
+    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
+fi
+
+[ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+
+export RUNTIME_CONFIG_PATH=/etc/kata-containers/configuration.toml
+
+usage() {
+    exit_code="$1"
+    cat <<EOF
+Overview:
+    Build and test kata containers from source
+    Optionally set kata-containers and tests repo and branch as exported variables before running
+    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all
+Usage:
+    ${script_name} [options] <command>
+Commands:
+- agent_create_container:           Run CreateContainer command against the agent with agent-ctl
+- agent_pull_image:                 Run PullImage command against the agent with agent-ctl
+- all:                              Build and install everything, test kata with containerd and capture the logs
+- build_and_add_agent_to_rootfs:    Builds the kata-agent and adds it to the rootfs
+- build_and_install_all:            Build and install everything
+- build_and_install_rootfs:         Builds and installs the rootfs image
+- build_kata_runtime:               Build and install the kata runtime
+- build_cloud_hypervisor            Checkout, patch, build and install Cloud Hypervisor
+- build_qemu:                       Checkout, patch, build and install QEMU
+- configure:                        Configure Kata to use rootfs and enable debug
+- connect_to_ssh_demo_pod:          Ssh into the ssh demo pod, showing that the decryption succeeded
+- copy_signature_files_to_guest     Copies signature verification files to guest
+- create_rootfs:                    Create a local rootfs
+- crictl_create_cc_container        Use crictl to create a new busybox container in the kata cc pod
+- crictl_create_cc_pod              Use crictl to create a new kata cc pod
+- crictl_delete_cc                  Use crictl to delete the kata cc pod sandbox and container in it
+- help:                             Display this help
+- init_kubernetes:                  initialize a Kubernetes cluster on this system
+- initialize:                       Install dependencies and check out kata-containers source
+- install_guest_kernel:             Setup, build and install the guest kernel
+- kubernetes_create_cc_pod:         Create a Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_create_ssh_demo_pod:   Create a Kata CC runtime pod based on the ssh demo
+- kubernetes_delete_cc_pod:         Delete the Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_delete_ssh_demo_pod:   Delete the Kata CC runtime pod based on the ssh demo
+- open_kata_shell:                  Open a shell into the kata runtime
+- rebuild_and_install_kata:         Rebuild the kata runtime and agent and build and install the image
+- shim_pull_image:                  Run PullImage command against the shim with ctr
+- test_capture_logs:                Test using kata with containerd and capture the logs in the user's home directory
+- test:                             Test using kata with containerd
+
+Options:
+    -d: Enable debug
+    -h: Display this help
+EOF
+    # if script sourced don't exit as this will exit the main shell, just return instead
+    [[ $_ != $0 ]] && return "$exit_code" || exit "$exit_code"
+}
+
+build_and_install_all() {
+    initialize
+    build_and_install_kata_runtime
+    configure
+    create_a_local_rootfs
+    build_and_install_rootfs
+    install_guest_kernel_image
+    case "$KATA_HYPERVISOR" in
+        "qemu") 
+            build_qemu
+            ;;
+        "cloud-hypervisor") 
+            build_cloud_hypervisor
+            ;;
+        *)
+            echo "Invalid option: $KATA_HYPERVISOR is not supported." >&2
+            ;;
+    esac
+
+    check_kata_runtime
+    if [ "${KUBERNETES}" == "yes" ]; then
+        init_kubernetes
+    fi
+}
+
+rebuild_and_install_kata() {
+    checkout_tests_repo
+    checkout_kata_containers_repo
+    build_and_install_kata_runtime
+    build_and_add_agent_to_rootfs
+    build_and_install_rootfs
+    check_kata_runtime
+}
+
+# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
+initialize() {
+    # We need git to checkout and bootstrap the ci scripts and some other packages used in testing
+    sudo apt-get update && sudo apt-get install -y curl git qemu-utils
+    
+    grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
+    grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
+    grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}"
+    
+    # Load the new go and PATH parameters from the profile
+    . ${PROFILE}
+    mkdir -p "${GOPATH}"
+
+    checkout_tests_repo
+
+    pushd "${tests_repo_dir}"
+    local ci_dir_name=".ci"
+    sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f
+    sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh"
+    # Need to change ownership of rustup so later process can create temp files there
+    sudo chown -R ${USER}:${USER} "${HOME}/.rustup"
+
+    checkout_kata_containers_repo
+
+    # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
+    export INSTALL_KATA="no"
+    sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh
+    # Reload the profile to pick up installed dependencies
+    . ${PROFILE}
+    popd
+}
+
+checkout_tests_repo() {
+    echo "Creating repo: ${tests_repo} and branch ${tests_branch} into ${tests_repo_dir}..."
+    # Due to git https://github.blog/2022-04-12-git-security-vulnerability-announced/ the tests repo needs
+    # to be owned by root as it is re-checked out in rootfs.sh
+    mkdir -p $(dirname "${tests_repo_dir}")
+    [ -d "${tests_repo_dir}" ] || sudo -E git clone "https://${tests_repo}.git" "${tests_repo_dir}"
+    sudo -E chown -R root:root "${tests_repo_dir}"
+    pushd "${tests_repo_dir}"
+    sudo -E git fetch
+    if [ -n "${tests_branch}" ]; then
+        sudo -E git checkout ${tests_branch}
+    fi
+    sudo -E git reset --hard origin/${tests_branch}
+    popd
+
+    source "${BATS_TEST_DIRNAME}/lib.sh"
+    source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+}
+
+# Note: clone_katacontainers_repo using go, so that needs to be installed first
+checkout_kata_containers_repo() {
+    source "${tests_repo_dir}/.ci/lib.sh"
+    echo "Creating repo: ${katacontainers_repo} and branch ${kata_default_branch} into ${katacontainers_repo_dir}..."
+    clone_katacontainers_repo
+    sudo -E chown -R ${USER}:${USER} "${katacontainers_repo_dir}"
+}
+
+build_and_install_kata_runtime() {
+    pushd ${katacontainers_repo_dir}/src/runtime
+    make clean && make DEFAULT_HYPERVISOR=${KATA_HYPERVISOR} && sudo -E PATH=$PATH make DEFAULT_HYPERVISOR=${KATA_HYPERVISOR} install
+    popd
+}
+
+configure() {
+    configure_kata_to_use_rootfs
+    enable_full_debug
+    enable_agent_console
+
+    # Switch image offload to true in kata config
+    switch_image_service_offload "on"
+
+    configure_cc_containerd
+    # From crictl v1.24.1 the default timoout leads to the pod creation failing, so update it
+    sudo crictl config --set timeout=10
+}
+
+configure_kata_to_use_rootfs() {
+    sudo mkdir -p /etc/kata-containers/
+    sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+    sudo sed -i 's/^\(initrd =.*\)/# \1/g' ${RUNTIME_CONFIG_PATH}
+}
+
+build_and_add_agent_to_rootfs() {
+    build_a_custom_kata_agent
+    add_custom_agent_to_rootfs
+}
+
+build_a_custom_kata_agent() {
+    # Install libseccomp for static linking
+    sudo -E PATH=$PATH GOPATH=$GOPATH ${katacontainers_repo_dir}/ci/install_libseccomp.sh /tmp/kata-libseccomp /tmp/kata-gperf
+    export LIBSECCOMP_LINK_TYPE=static
+    export LIBSECCOMP_LIB_PATH=/tmp/kata-libseccomp/lib
+
+    . "$HOME/.cargo/env"
+    pushd ${katacontainers_repo_dir}/src/agent
+    sudo -E PATH=$PATH make
+
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+
+    # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
+    sudo -E PATH=$PATH make install DESTDIR="${ROOTFS_DIR}"
+    popd
+}
+
+create_a_local_rootfs() {
+    sudo rm -rf "${ROOTFS_DIR}"
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
+    export distro="ubuntu"
+    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
+    sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG:-}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} AA_KBC=${AA_KBC:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
+
+     # Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now
+    pushd "${tests_repo_dir}"
+    sudo -E git checkout ${tests_branch}
+    popd
+    # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
+    sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
+
+    popd
+}
+
+add_custom_agent_to_rootfs() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
+
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+
+    sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent
+    sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
+    sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
+    popd
+}
+
+build_and_install_rootfs() {
+    build_rootfs_image
+    install_rootfs_image
+}
+
+build_rootfs_image() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
+    # Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker
+    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
+    sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR}
+    popd
+}
+
+install_rootfs_image() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
+    local commit=$(git log --format=%h -1 HEAD)
+    local date=$(date +%Y-%m-%d-%T.%N%z)
+    local image="kata-containers-${date}-${commit}"
+    sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
+    (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+    echo "Built Rootfs from ${ROOTFS_DIR} to /usr/share/kata-containers/${image}"
+    ls -al /usr/share/kata-containers/
+    popd
+}
+
+install_guest_kernel_image() {
+    pushd ${katacontainers_repo_dir}/tools/packaging/kernel
+    sudo -E PATH=$PATH ./build-kernel.sh setup
+    sudo -E PATH=$PATH ./build-kernel.sh build
+    sudo chmod u+wrx /usr/share/kata-containers/ # Give user permission to install kernel
+    sudo -E PATH=$PATH ./build-kernel.sh install
+    popd
+}
+
+build_qemu() {
+    ${tests_repo_dir}/.ci/install_virtiofsd.sh
+    ${tests_repo_dir}/.ci/install_qemu.sh
+}
+
+build_cloud_hypervisor() {
+    ${tests_repo_dir}/.ci/install_virtiofsd.sh
+    ${tests_repo_dir}/.ci/install_cloud_hypervisor.sh
+}
+
+check_kata_runtime() {
+    sudo kata-runtime check
+}
+
+k8s_pod_file="${HOME}/busybox-cc.yaml"
+init_kubernetes() {
+    # Check that kubeadm was installed and install it otherwise
+    if ! [ -x "$(command -v kubeadm)" ]; then
+        pushd "${tests_repo_dir}/.ci"
+        sudo -E PATH=$PATH -s install_kubernetes.sh
+        if [ "${CRI_CONTAINERD}" == "yes" ]; then
+            sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh"
+        fi
+        popd
+    fi
+
+    # If kubernetes init has previously run we need to clean it by removing the image and resetting k8s
+    local cid=$(sudo docker ps -a -q -f name=^/kata-registry$)
+    if [ -n "${cid}" ]; then
+        sudo docker stop ${cid} && sudo docker rm ${cid}
+    fi
+    local k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true)
+    if [ -n "${k8s_nodes}" ]; then
+        sudo kubeadm reset -f
+    fi
+
+    export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh
+    sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config"
+    cat << EOF > ${k8s_pod_file}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: busybox-cc
+spec:
+  runtimeClassName: kata
+  containers:
+  - name: nginx
+    image: quay.io/kata-containers/confidential-containers:signed
+    imagePullPolicy: Always  
+EOF
+}
+
+call_kubernetes_create_cc_pod() {
+    kubernetes_create_cc_pod ${k8s_pod_file}
+}
+
+call_kubernetes_delete_cc_pod() {
+    pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
+    kubernetes_delete_cc_pod $pod_name
+}
+
+call_kubernetes_create_ssh_demo_pod() {
+    setup_decryption_files_in_guest
+    kubernetes_create_ssh_demo_pod
+}
+
+call_connect_to_ssh_demo_pod() {
+    connect_to_ssh_demo_pod
+}
+
+call_kubernetes_delete_ssh_demo_pod() {
+    pod=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
+    kubernetes_delete_ssh_demo_pod $pod
+}
+
+crictl_sandbox_name=kata-cc-busybox-sandbox
+call_crictl_create_cc_pod() {
+    # Update iptables to allow forwarding to the cni0 bridge avoiding issues caused by the docker0 bridge
+    sudo iptables -P FORWARD ACCEPT
+    
+    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
+    get_pod_config
+
+    crictl_delete_cc_pod_if_exists "${crictl_sandbox_name}"
+    crictl_create_cc_pod "${pod_config}"
+    sudo crictl pods
+}
+
+call_crictl_create_cc_container() {
+    # Create container configuration yaml based on our test copy of busybox
+    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
+    get_pod_config
+
+    local container_config="${FIXTURES_DIR}/${CONTAINER_CONFIG_FILE:-container-config.yaml}"
+    local pod_name=${crictl_sandbox_name}
+    crictl_create_cc_container ${pod_name} ${pod_config} ${container_config}
+    sudo crictl ps -a
+}
+
+crictl_delete_cc() {
+    crictl_delete_cc_pod ${crictl_sandbox_name}
+}
+
+test_kata_runtime() {
+    echo "Running ctr with the kata runtime..."
+    local test_image="quay.io/kata-containers/confidential-containers:signed"
+    if [ -z $(ctr images ls -q name=="${test_image}") ]; then
+        sudo ctr image pull "${test_image}"
+    fi
+    sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
+}
+
+run_kata_and_capture_logs() {
+    echo "Clearing systemd journal..."
+    sudo systemctl stop systemd-journald
+    sudo rm -f /var/log/journal/*/* /run/log/journal/*/*
+    sudo systemctl start systemd-journald
+    test_kata_runtime
+    echo "Collecting logs..."
+    sudo journalctl -q -o cat -a -t kata-runtime > ${HOME}/kata-runtime.log
+    sudo journalctl -q -o cat -a -t kata > ${HOME}/shimv2.log
+    echo "Logs output to ${HOME}/kata-runtime.log and ${HOME}/shimv2.log"
+}
+
+get_ids() {
+    guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
+    sandbox_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
+}
+
+open_kata_shell() {
+    get_ids
+    sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id}
+}
+
+build_bundle_dir_if_necessary() {
+    bundle_dir="/tmp/bundle"
+    if [ ! -d "${bundle_dir}" ]; then
+        rootfs_dir="$bundle_dir/rootfs"
+        image="quay.io/kata-containers/confidential-containers:signed"
+        mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec)
+        sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf -
+    fi
+    # There were errors in create container agent-ctl command due to /bin/ seemingly not being on the path, so hardcode it
+    sudo sed -i -e 's%^\(\t*\)"sh"$%\1"/bin/sh"%g' "${bundle_dir}/config.json"
+}
+
+build_agent_ctl() {
+    cd ${GOPATH}/src/${katacontainers_repo}/src/tools/agent-ctl/
+    if [ -e "${HOME}/.cargo/registry" ]; then
+        sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
+    fi
+    sudo -E PATH=$PATH -s  make
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+    cd "./target/${ARCH}-unknown-linux-${LIBC}/release/"
+}
+
+run_agent_ctl_command() {
+    get_ids
+    build_bundle_dir_if_necessary
+    command=$1
+    # If kata-agent-ctl pre-built in this directory, use it directly, otherwise build it first and switch to release
+    if [ ! -x kata-agent-ctl ]; then
+        build_agent_ctl
+    fi 
+     ./kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c "${command}"
+}
+
+agent_pull_image() {
+    run_agent_ctl_command "PullImage image=${PULL_IMAGE} cid=${CONTAINER_ID} source_creds=${SOURCE_CREDS}"
+}
+
+agent_create_container() {
+    run_agent_ctl_command "CreateContainer cid=${CONTAINER_ID}"
+}
+
+shim_pull_image() {
+    get_ids
+    local ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
+    echo "Issuing command '${ctr_shim_command}'"
+    ${ctr_shim_command}
+}
+
+call_copy_signature_files_to_guest() {
+    # TODO #5173 - remove this once the kernel_params aren't ignored by the agent config
+    export DEBUG_CONSOLE="true"
+    
+    if [ "${SKOPEO:-}" = "yes" ]; then
+        add_kernel_params "agent.container_policy_file=/etc/containers/quay_verification/quay_policy.json"
+        setup_skopeo_signature_files_in_guest
+    else
+        # TODO #4888 - set config to specifically enable signature verification to be on in ImageClient
+        setup_offline_fs_kbc_signature_files_in_guest
+    fi
+}
+
+main() {
+    while getopts "dh" opt; do
+        case "$opt" in
+            d) 
+                export DEBUG="-d"
+                set -x
+                ;;
+            h) 
+                usage 0
+                ;;
+            \?)
+                echo "Invalid option: -$OPTARG" >&2
+                usage 1
+                ;;
+        esac
+    done
+
+    shift $((OPTIND - 1))
+
+    subcmd="${1:-}"
+
+    [ -z "${subcmd}" ] && usage 1
+
+    case "${subcmd}" in
+        all)
+            build_and_install_all
+            run_kata_and_capture_logs
+            ;;
+        build_and_install_all)
+            build_and_install_all
+            ;;
+        rebuild_and_install_kata)
+            rebuild_and_install_kata
+            ;;
+        initialize)
+            initialize
+            ;;
+        build_kata_runtime)
+            build_and_install_kata_runtime
+            ;;
+        configure)
+            configure
+            ;;
+        create_rootfs)
+            create_a_local_rootfs
+            ;;
+        build_and_add_agent_to_rootfs)
+            build_and_add_agent_to_rootfs
+            ;;
+        build_and_install_rootfs)
+            build_and_install_rootfs
+            ;;
+        install_guest_kernel)
+            install_guest_kernel_image
+            ;;
+        build_cloud_hypervisor)
+            build_cloud_hypervisor
+            ;;
+        build_qemu)
+            build_qemu
+            ;;
+        init_kubernetes)
+            init_kubernetes
+            ;;
+        crictl_create_cc_pod)
+            call_crictl_create_cc_pod
+            ;;
+        crictl_create_cc_container)
+            call_crictl_create_cc_container
+            ;;
+        crictl_delete_cc)
+            crictl_delete_cc
+            ;;
+        kubernetes_create_cc_pod)
+            call_kubernetes_create_cc_pod
+            ;;
+        kubernetes_delete_cc_pod)
+            call_kubernetes_delete_cc_pod
+            ;;
+        kubernetes_create_ssh_demo_pod)
+            call_kubernetes_create_ssh_demo_pod
+            ;;
+        connect_to_ssh_demo_pod)
+            call_connect_to_ssh_demo_pod
+            ;;
+        kubernetes_delete_ssh_demo_pod)
+            call_kubernetes_delete_ssh_demo_pod
+            ;;
+        test)
+            test_kata_runtime
+            ;;
+        test_capture_logs)
+            run_kata_and_capture_logs
+            ;;
+        open_kata_console)
+            open_kata_console
+            ;;
+        open_kata_shell)
+            open_kata_shell
+            ;;
+        agent_pull_image)
+            agent_pull_image
+            ;;
+        shim_pull_image)
+            shim_pull_image
+            ;;
+        agent_create_container)
+            agent_create_container
+            ;;
+        copy_signature_files_to_guest)
+            call_copy_signature_files_to_guest
+            ;;
+        *)
+            usage 1
+            ;;
+    esac
+}
+
+main $@

docs/how-to/data/confidential-agent-config.toml.in

@@ -0,0 +1,45 @@
+# Copyright (c) 2021 IBM Corp.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+aa_kbc_params = "$AA_KBC_PARAMS"
+https_proxy = "$HTTPS_PROXY"
+[endpoints]
+allowed = [
+"AddARPNeighborsRequest",
+"AddSwapRequest",
+"CloseStdinRequest",
+"CopyFileRequest",
+"CreateContainerRequest",
+"CreateSandboxRequest",
+"DestroySandboxRequest",
+#"ExecProcessRequest",
+"GetMetricsRequest",
+"GetOOMEventRequest",
+"GuestDetailsRequest",
+"ListInterfacesRequest",
+"ListRoutesRequest",
+"MemHotplugByProbeRequest",
+"OnlineCPUMemRequest",
+"PauseContainerRequest",
+"PullImageRequest",
+"ReadStreamRequest",
+"RemoveContainerRequest",
+#"ReseedRandomDevRequest",
+"ResizeVolumeRequest",
+"ResumeContainerRequest",
+"SetGuestDateTimeRequest",
+"SignalProcessRequest",
+"StartContainerRequest",
+"StartTracingRequest",
+"StatsContainerRequest",
+"StopTracingRequest",
+"TtyWinResizeRequest",
+"UpdateContainerRequest",
+"UpdateInterfaceRequest",
+"UpdateRoutesRequest",
+"VolumeStatsRequest",
+"WaitProcessRequest",
+"WriteStreamRequest"
+]

docs/how-to/how-to-build-and-test-ccv0.md

@@ -0,0 +1,475 @@
+# How to build, run and test Kata CCv0
+
+## Introduction and Background
+
+In order to try and make building (locally) and demoing the Kata Containers `CCv0` code base as simple as possible I've
+shared a script [`ccv0.sh`](./ccv0.sh). This script was originally my attempt to automate the steps of the 
+[Developer Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md) so that I could do
+different sections of them repeatedly and reliably as I was playing around with make changes to different parts of the 
+Kata code base. I then tried to weave in some of the [`tests/.ci`](https://github.com/kata-containers/tests/tree/main/.ci) 
+scripts in order to have less duplicated code.
+As we're progress on the confidential containers journey I hope to add more features to demonstrate the functionality
+we have working.
+
+*Disclaimer: This script has mostly just been used and tested by me ([@stevenhorsman](https://github.com/stevenhorsman)),*
+*so there might be issues with it. I'm happy to try and help solve these if possible, but this shouldn't be considered a*
+*fully supported process by the Kata Containers community.*
+
+### Basic script set-up and optional environment variables
+
+In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take:
+- Provision a new VM
+    - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed*
+    *packages, so these will need re-working to be compatible with other platforms.*
+- Copy the script over to your VM *(I put it in the home directory)* and ensure it has execute permission by running 
+```bash
+$ chmod u+x ccv0.sh
+```
+- Optionally set up some environment variables
+    - By default the script checks out the `CCv0` branches of the `kata-containers/kata-containers` and 
+      `kata-containers/tests` repositories, but it is designed to be used to test of personal forks and branches as well. 
+      If you want to build and run these you can export the `katacontainers_repo`, `katacontainers_branch`, `tests_repo`
+      and `tests_branch` variables e.g.
+      ```bash
+      $ export katacontainers_repo=github.com/stevenhorsman/kata-containers
+      $ export katacontainers_branch=stevenh/agent-pull-image-endpoint
+      $ export tests_repo=github.com/stevenhorsman/tests
+      $ export tests_branch=stevenh/add-ccv0-changes-to-build
+      ```
+      before running the script.
+    - By default the build and configuration are using `QEMU` as the hypervisor. In order to use `Cloud Hypervisor` instead
+      set:
+      ```
+      $ export KATA_HYPERVISOR="cloud-hypervisor"
+      ```
+      before running the build.
+
+- At this point you can provision a Kata confidential containers pod and container with either
+  [`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image),
+  or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+  and then test and use it. 
+
+### Using crictl for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
+
+- Run the full build process with Kubernetes turned off, so its configuration doesn't interfere with `crictl` using:
+  ```bash
+  $ export KUBERNETES="no"
+  $ export KATA_HYPERVISOR="qemu"
+  $ ~/ccv0.sh -d build_and_install_all
+  ```
+    > **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password.
+    - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest*
+      *of the process without having to reload the session.*
+    - The steps that `build_and_install_all` takes is:
+        - Checkout the git repos for the `tests` and `kata-containers` repos as specified by the environment variables
+        (default to `CCv0` branches if they are not supplied)
+        - Use the `tests/.ci` scripts to install the build dependencies
+        - Build and install the Kata runtime
+        - Configure Kata to use containerd and for debug and confidential containers features to be enabled (including
+          enabling console access to the Kata guest shell, which should only be done in development)
+        - Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Ubuntu
+        20.04.
+        - Build the Kata guest kernel
+        - Install the hypervisor (in order to select which hypervisor will be used, the `KATA_HYPERVISOR` environment
+        variable can be used to select between `qemu` or `cloud-hypervisor`)
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull ubuntu
+  >  ```
+  >  then re-run the command.
+    - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already
+      installed and they can be further cut down by not running all the above steps 
+      [see "Additional script usage" below](#additional-script-usage)
+   
+- Create a new Kata sandbox pod using `crictl` with:
+  ```bash
+  $ ~/ccv0.sh crictl_create_cc_pod
+  ```
+    - This creates a pod configuration file, creates the pod from this using 
+    `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod
+- Create a new Kata confidential container with:
+  ```bash
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+    - This creates a container (based on `busybox:1.33.1`) in the Kata cc sandbox and prints a list of containers.
+      This will have been created based on an image pulled in the Kata pod sandbox/guest, not on the host machine.
+
+As this point you should have a `crictl` pod and container that is using the Kata confidential containers runtime.
+You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
+or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
+
+#### Clean up the `crictl` pod sandbox and container
+- When the testing is complete you can delete the container and pod by running:
+  ```bash
+  $ ~/ccv0.sh crictl_delete_cc
+  ```
+### Using Kubernetes for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
+
+- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
+  configured and created using the VM
+  as a single node cluster: 
+  ```bash
+  $ export KUBERNETES="yes"
+  $ ~/ccv0.sh build_and_install_all
+  ```
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull registry:2
+  >  $ sudo docker pull ubuntu:20.04
+  >  ```
+  >  then re-run the command.
+- Check that your Kubernetes cluster has been correctly set-up by running : 
+  ```bash
+  $ kubectl get nodes
+  ```
+  and checking that you see a single node e.g.
+  ```text
+  NAME                             STATUS   ROLES                  AGE   VERSION
+  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
+  ```
+- Create a Kata confidential containers pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_create_cc_pod
+  ```
+- Wait a few seconds for pod to start then check that the pod's status is `Running` with 
+  ```bash
+  $ kubectl get pods
+  ```
+  which should show something like:
+  ```text
+  NAME         READY   STATUS    RESTARTS   AGE
+  busybox-cc   1/1     Running   0          54s
+  ```
+
+- As this point you should have a Kubernetes pod and container running, that is using the Kata 
+confidential containers runtime.
+You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
+or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
+
+#### Clean up the Kubernetes pod sandbox and container
+- When the testing is complete you can delete the container and pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_delete_cc_pod
+  ```
+
+### Validate that the container image was pulled on the guest
+
+There are a couple of ways we can check that the container pull image action was offloaded to the guest, by checking
+the guest's file system for the unpacked bundle and checking the host's directories to ensure it wasn't also pulled
+there.
+- To check the guest's file system:
+    - Open a shell into the Kata guest with:
+      ```bash
+      $ ~/ccv0.sh open_kata_shell
+      ```
+    - List the files in the directory that the container image bundle should have been unpacked to with:
+      ```bash
+      $ ls -ltr /run/kata-containers/confidential-containers_signed/
+      ```
+    - This should give something like
+        ```
+        total 72
+        -rw-r--r--  1 root root  2977 Jan 20 10:03 config.json
+        drwxr-xr-x 12 root root   240 Jan 20 10:03 rootfs
+        ```
+        which shows how the image has been pulled and then unbundled on the guest.
+    - Leave the Kata guest shell by running:
+      ```bash
+      $ exit
+      ```
+- To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we
+  should only see a single bundle for the pause container as the `busybox` based container image should have been
+  pulled on the guest:
+    - Find all the `rootfs` directories under in the pod's shared directory with:
+      ```bash
+      $ pod_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
+      $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs
+      ```
+      which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host
+    - Looking that `rootfs` directory with
+      ```bash
+      $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs)
+      ```
+      shows something similar to
+      ```
+      total 668
+      -rwxr-xr-x 1 root root 682696 Aug 25 13:58 pause
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 proc
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 dev
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 sys
+      drwxr-xr-x 2 root root     25 Jan 20 02:01 etc
+      ```
+      which is clearly the pause container indicating that the `busybox` based container image is not exposed to the host.
+
+### Using a Kata pod sandbox for testing with `agent-ctl` or `ctr shim`
+
+Once you have a kata pod sandbox created as described above, either using 
+[`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image), or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+, you can use this to test specific components of the Kata confidential
+containers architecture. This can be useful for development and debugging to isolate and test features
+that aren't broadly supported end-to-end. Here are some examples:
+
+- In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`).
+This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to
+interact with the shim directly. The command takes the form
+`ctr --namespace k8s.io shim --id <sandbox-id> pull-image <image> <new-container-id>` and can been run directly, or through
+the `ccv0.sh` script to automatically fill in the variables:
+  - Optionally, set up some environment variables to set the image and credentials used:
+    - By default the shim pull image test in `ccv0.sh` will use the `busybox:1.33.1` based test image
+     `quay.io/kata-containers/confidential-containers:signed` which requires no authentication. To use a different
+      image, set the `PULL_IMAGE` environment variable e.g. 
+      ```bash
+      $ export PULL_IMAGE="docker.io/library/busybox:latest"
+      ```
+      Currently the containerd shim pull image
+      code doesn't support using a container registry that requires authentication, so if this is required, see the 
+      below steps to run the pull image command against the agent directly.
+  - Run the pull image agent endpoint with:
+    ```bash
+    $ ~/ccv0.sh shim_pull_image
+    ```
+    which we print the `ctr shim` command for reference
+- Alternatively you can issue the command directly to the `kata-agent` pull image endpoint, which also supports
+  credentials in order to pull from an authenticated registry:
+    - Optionally set up some environment variables to set the image and credentials used:
+        - Set the `PULL_IMAGE` environment variable e.g. `export PULL_IMAGE="docker.io/library/busybox:latest"`
+          if a specific container image is required.
+        - If the container registry for the image requires authentication then this can be set with an environment
+          variable `SOURCE_CREDS`. For example to use Docker Hub (`docker.io`) as an authenticated user first run 
+          `export SOURCE_CREDS="<dockerhub username>:<dockerhub api key>"`
+            > **Note**: the credentials support on the agent request is a tactical solution for the short-term
+              proof of concept to allow more images to be pulled and tested. Once we have support for getting
+              keys into the Kata guest image using the attestation-agent and/or KBS I'd expect container registry
+              credentials to be looked up using that mechanism.
+    - Run the pull image agent endpoint with
+      ```bash
+      $ ~/ccv0.sh agent_pull_image
+      ```
+      and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate
+      that the `PullImage` request was successful e.g.
+      ```
+      Finished release [optimized] target(s) in 0.21s
+      {"msg":"announce","level":"INFO","ts":"2021-09-15T08:40:14.189360410-07:00","subsystem":"rpc","name":"kata-agent-ctl","pid":"830920","version":"0.1.0","source":"kata-agent-ctl","config":"Config { server_address: \"vsock://1970354082:1024\", bundle_dir: \"/tmp/bundle\", timeout_nano: 0, interactive: false, ignore_errors: false }"}
+      {"msg":"client setup complete","level":"INFO","ts":"2021-09-15T08:40:14.193639057-07:00","pid":"830920","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","server-address":"vsock://1970354082:1024"}
+      {"msg":"Run command PullImage (1 of 1)","level":"INFO","ts":"2021-09-15T08:40:14.196643765-07:00","pid":"830920","source":"kata-agent-ctl","subsystem":"rpc","name":"kata-agent-ctl","version":"0.1.0"}
+      {"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""}
+      {"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"}
+      ```
+      > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built
+      which may take a few minutes.
+- To validate that the image pull was successful, you can open a shell into the Kata guest with:
+  ```bash
+  $ ~/ccv0.sh open_kata_shell
+  ```
+- Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory
+  named either `01234556789` (for the container id), or the container image name, e.g.
+  ```bash
+  $ ls -ltr /run/kata-containers/confidential-containers_signed/
+  ```
+  which should show something like
+  ```
+  total 72
+  drwxr-xr-x 10 root root   200 Jan  1  1970 rootfs
+  -rw-r--r--  1 root root  2977 Jan 20 16:45 config.json
+  ```
+- Leave the Kata shell by running:
+  ```bash
+  $ exit
+  ```
+
+## Verifying signed images
+
+For this sample demo, we use local attestation to pass through the required
+configuration to do container image signature verification. Due to this, the ability to verify images is limited
+to a pre-created selection of test images in our test
+repository [`quay.io/kata-containers/confidential-containers`](https://quay.io/repository/kata-containers/confidential-containers?tab=tags).
+For pulling images not in this test repository (called an *unprotected* registry below), we fall back to the behaviour
+of not enforcing signatures. More documentation on how to customise this to match your own containers through local,
+or remote attestation will be available in future.
+
+In our test repository there are three tagged images:
+
+| Test Image | Base Image used | Signature status | GPG key status |
+| --- | --- | --- | --- |
+| `quay.io/kata-containers/confidential-containers:signed` | `busybox:1.33.1` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs |  [public key](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/public.gpg) embedded in kata rootfs |
+| `quay.io/kata-containers/confidential-containers:unsigned` | `busybox:1.33.1` | not signed | not signed |
+| `quay.io/kata-containers/confidential-containers:other_signed` | `nginx:1.21.3` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs | GPG key not kept |
+
+Using a standard unsigned `busybox` image that can be pulled from another, *unprotected*, `quay.io` repository we can
+test a few scenarios.
+
+In this sample, with local attestation, we pass in the the public GPG key and signature files, and the [`offline_fs_kbc`
+configuration](https://github.com/confidential-containers/attestation-agent/blob/main/src/kbc_modules/offline_fs_kbc/README.md)
+into the guest image which specifies that any container image from `quay.io/kata-containers`
+must be signed with the embedded GPG key and the agent configuration needs updating to enable this. 
+With this policy set a few tests of image verification can be done to test different scenarios by attempting
+to create containers from these images using `crictl`:
+
+- If you don't already have the Kata Containers CC code built and configured for `crictl`, then follow the 
+[instructions above](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+up to the `~/ccv0.sh crictl_create_cc_pod` command.
+
+- In order to enable the guest image, you will need to setup the required configuration, policy and signature files
+needed by running 
+`~/ccv0.sh copy_signature_files_to_guest` and then run `~/ccv0.sh crictl_create_cc_pod` which will delete and recreate 
+your pod - adding in the new files.
+ 
+- To test the fallback behaviour works using an unsigned image from an *unprotected* registry we can pull the `busybox`
+image by running:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-unprotected.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This finishes showing the running container e.g.
+  ```text
+  CONTAINER           IMAGE                               CREATED                  STATE               NAME                        ATTEMPT             POD ID
+  98c70fefe997a       quay.io/prometheus/busybox:latest   Less than a second ago   Running             prometheus-busybox-signed   0                   70119e0539238
+  ```
+- To test that an unsigned image from our *protected* test container registry is rejected we can run:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-protected.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This correctly results in an error message from `crictl`:
+  `PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [Match reference failed.]" image="quay.io/kata-containers/confidential-containers:unsigned"`
+- To test that the signed image our *protected* test container registry is accepted we can run:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This finishes by showing a new `kata-cc-busybox-signed` running container e.g.
+  ```text
+  CONTAINER           IMAGE                                                    CREATED                  STATE               NAME                        ATTEMPT             POD ID
+  b4d85c2132ed9       quay.io/kata-containers/confidential-containers:signed   Less than a second ago   Running             kata-cc-busybox-signed      0                   70119e0539238
+  ...
+  ```
+- Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really
+want to protect with the attestation agent in future) fails we can run: 
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_signed-protected-other.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - Again this results in an error message from `crictl`:
+  `"PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [signature verify failed! There is no pubkey can verify the signature!]" image="quay.io/kata-containers/confidential-containers:other_signed"`
+
+### Using Kubernetes to create a Kata confidential containers pod from the encrypted ssh demo sample image
+
+The [ssh-demo](https://github.com/confidential-containers/documentation/tree/main/demos/ssh-demo) explains how to
+demonstrate creating a Kata confidential containers pod from an encrypted image with the runtime created by the
+[confidential-containers operator](https://github.com/confidential-containers/documentation/blob/main/demos/operator-demo).
+To be fully confidential, this should be run on a Trusted Execution Environment, but it can be tested on generic
+hardware as well.
+
+If you wish to build the Kata confidential containers runtime to do this yourself, then you can using the following
+steps:
+
+- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
+  configured and created using the VM as a single node cluster and with `AA_KBC` set to `offline_fs_kbc`. 
+  ```bash
+  $ export KUBERNETES="yes"
+  $ export AA_KBC=offline_fs_kbc
+  $ ~/ccv0.sh build_and_install_all
+  ```
+    - The `AA_KBC=offline_fs_kbc` mode will ensure that, when creating the rootfs of the Kata guest, the 
+      [attestation-agent](https://github.com/confidential-containers/attestation-agent) will be added along with the
+      [sample offline KBC](https://github.com/confidential-containers/documentation/blob/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json)
+      and an agent configuration file
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull registry:2
+  >  $ sudo docker pull ubuntu:20.04
+  >  ```
+  >  then re-run the command.
+- Check that your Kubernetes cluster has been correctly set-up by running : 
+  ```bash
+  $ kubectl get nodes
+  ```
+  and checking that you see a single node e.g.
+  ```text
+  NAME                             STATUS   ROLES                  AGE   VERSION
+  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
+  ```
+- Create a sample Kata confidential containers ssh pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_create_ssh_demo_pod
+  ```
+- As this point you should have a Kubernetes pod running the Kata confidential containers runtime that has pulled
+the [sample image](https://hub.docker.com/r/katadocker/ccv0-ssh) which was encrypted by the key file that we included
+in the rootfs.
+During the pod deployment the image was pulled and then decrypted using the key file, on the Kata guest image, without
+it ever being available to the host.
+
+- To validate that the container is working you, can connect to the image via SSH by running:
+  ```bash
+  $ ~/ccv0.sh connect_to_ssh_demo_pod
+  ```
+  - During this connection the host key fingerprint is shown and should match:
+    `ED25519 key fingerprint is SHA256:wK7uOpqpYQczcgV00fGCh+X97sJL3f6G1Ku4rvlwtR0.`
+  - After you are finished connecting then run:
+    ```bash
+    $ exit
+    ```
+  
+- To delete the sample SSH demo pod run:
+  ```bash
+  $ ~/ccv0.sh kubernetes_delete_ssh_demo_pod
+  ```
+
+## Additional script usage
+
+As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just
+re-build bits of it by running the script with different parameters. For example after the first build you will often
+not need to re-install the dependencies, the hypervisor or the Guest kernel, but just test code changes made to the
+runtime and agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout*
+*from git, so if your changes are only made locally it is better to do the individual steps e.g.* 
+`~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`).
+There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running
+`~/ccv0.sh help`:
+```
+$ ~/ccv0.sh help
+Overview:
+    Build and test kata containers from source
+    Optionally set kata-containers and tests repo and branch as exported variables before running
+    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/ccv0.sh build_and_install_all
+Usage:
+    ccv0.sh [options] <command>
+Commands:
+- help:                         Display this help
+- all:                          Build and install everything, test kata with containerd and capture the logs
+- build_and_install_all:        Build and install everything
+- initialize:                   Install dependencies and check out kata-containers source
+- rebuild_and_install_kata:     Rebuild the kata runtime and agent and build and install the image
+- build_kata_runtime:           Build and install the kata runtime
+- configure:                    Configure Kata to use rootfs and enable debug
+- create_rootfs:                Create a local rootfs
+- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs
+- build_and_install_rootfs:     Builds and installs the rootfs image
+- install_guest_kernel:         Setup, build and install the guest kernel
+- build_cloud_hypervisor        Checkout, patch, build and install Cloud Hypervisor
+- build_qemu:                   Checkout, patch, build and install QEMU
+- init_kubernetes:              initialize a Kubernetes cluster on this system
+- crictl_create_cc_pod          Use crictl to create a new kata cc pod
+- crictl_create_cc_container    Use crictl to create a new busybox container in the kata cc pod
+- crictl_delete_cc              Use crictl to delete the kata cc pod sandbox and container in it
+- kubernetes_create_cc_pod:     Create a Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_delete_cc_pod:     Delete the Kata CC runtime busybox-based pod in Kubernetes
+- open_kata_shell:              Open a shell into the kata runtime
+- agent_pull_image:             Run PullImage command against the agent with agent-ctl
+- shim_pull_image:              Run PullImage command against the shim with ctr
+- agent_create_container:       Run CreateContainer command against the agent with agent-ctl
+- test:                         Test using kata with containerd
+- test_capture_logs:            Test using kata with containerd and capture the logs in the user's home directory
+
+Options:
+    -d: Enable debug
+    -h: Display this help
+```

docs/how-to/how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md

@@ -0,0 +1,44 @@
+# Generating a Kata Containers payload for the Confidential Containers Operator
+
+[Confidential Containers
+Operator](https://github.com/confidential-containers/operator) consumes a Kata
+Containers payload, generated from the `CCv0` branch, and here one can find all
+the necessary info on how to build such a payload.
+
+## Requirements
+
+* `make` installed in the machine
+* Docker installed in the machine
+* `sudo` access to the machine
+
+## Process
+
+* Clone [Kata Containers](https://github.com/kata-containers/kata-containers)
+  ```sh
+  git clone --branch CCv0 https://github.com/kata-containers/kata-containers
+  ```
+  * In case you've already cloned the repo, make sure to switch to the `CCv0` branch
+    ```sh
+    git checkout CCv0
+    ```
+  * Ensure your tree is clean and in sync with upstream `CCv0`
+    ```sh
+    git clean -xfd
+    git reset --hard <upstream>/CCv0
+    ```
+* Make sure you're authenticated to `quay.io`
+    ```sh
+    sudo docker login quay.io
+    ```
+* From the top repo directory, run:
+  ```sh
+  sudo make cc-payload
+  ```
+* Make sure the image was upload to the [Confidential Containers
+  runtime-payload
+registry](https://quay.io/repository/confidential-containers/runtime-payload?tab=tags)
+
+## Notes
+
+Make sure to run it on a machine that's not the one you're hacking on, prepare a
+cup of tea, and get back to it an hour later (at least).

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -94,6 +94,16 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
 | `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
 
+## Confidential Computing Options
+| Key | Value Type | Comments |
+|-------| ----- | ----- |
+| `io.katacontainers.config.pre_attestation.enabled"` | `bool` |
+determines if SEV/-ES attestation is enabled |
+| `io.katacontainers.config.pre_attestation.uri"` | `string` |
+specify the location of the attestation server |
+| `io.katacontainers.config.sev.policy"` | `uint32` |
+specify the SEV guest policy |
+
 ## Container Options
 | Key | Value Type | Comments |
 |-------| ----- | ----- |

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -27,6 +27,8 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
+  uid: $(uuidgen)
+  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -32,6 +32,7 @@ The `nydus-sandbox.yaml` looks like below:
 metadata:
   attempt: 1
   name: nydus-sandbox
+  uid: nydus-uid
   namespace: default
 log_directory: /tmp
 linux:

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -42,6 +42,8 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
+  uid: $(uuidgen)
+  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/install/aws-installation-guide.md

@@ -123,7 +123,7 @@ Refer to [this guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-l
 SSH into the machine
 
 ```bash
-$ ssh -i MyKeyPair.pen ubuntu@${IP}
+$ ssh -i MyKeyPair.pem ubuntu@${IP}
 ```
 
 Go onto the next step.

src/agent/Cargo.toml

@@ -23,6 +23,7 @@ regex = "1.5.6"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
+url = "2.2.2"
 
 # Async helpers
 async-trait = "0.1.42"
@@ -30,7 +31,7 @@ async-recursion = "0.3.2"
 futures = "0.3.17"
 
 # Async runtime
-tokio = { version = "1.28.1", features = ["full"] }
+tokio = { version = "1.21.2", features = ["full"] }
 tokio-vsock = "0.3.1"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
@@ -48,6 +49,7 @@ slog-scope = "4.1.2"
 slog-stdlog = "4.0.0"
 log = "0.4.11"
 
+cfg-if = "1.0.0"
 prometheus = { version = "0.13.0", features = ["process"] }
 procfs = "0.12.0"
 anyhow = "1.0.32"
@@ -65,12 +67,26 @@ serde = { version = "1.0.129", features = ["derive"] }
 toml = "0.5.8"
 clap = { version = "3.0.1", features = ["derive"] }
 
+# "vendored" feature for openssl is required by musl build
+openssl = { version = "0.10.38", features = ["vendored"] }
+
+# Image pull/decrypt
+[target.'cfg(target_arch = "s390x")'.dependencies]
+image-rs = { git = "https://github.com/confidential-containers/image-rs", tag = "v0.5.1", default-features = false, features = ["kata-cc-s390x"] }
+
+[target.'cfg(not(target_arch = "s390x"))'.dependencies]
+image-rs = { git = "https://github.com/confidential-containers/image-rs", tag = "v0.5.1", default-features = false, features = ["kata-cc"] }
+
+[patch.crates-io]
+oci-distribution = { git = "https://github.com/krustlet/oci-distribution.git", rev = "f44124c" }
+
 [dev-dependencies]
 tempfile = "3.1.0"
 test-utils = { path = "../libs/test-utils" }
 which = "4.3.0"
 
 [workspace]
+resolver = "2"
 members = [
     "rustjail",
 ]

src/agent/rustjail/Cargo.toml

@@ -29,7 +29,7 @@ cgroups = { package = "cgroups-rs", version = "0.3.2" }
 rlimit = "0.5.3"
 cfg-if = "0.1.0"
 
-tokio = { version = "1.28.1", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
+tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
 futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -545,8 +545,11 @@ fn linux_device_to_cgroup_device(d: &LinuxDevice) -> Option<DeviceResource> {
 }
 
 fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> Option<DeviceResource> {
-    let dev_type = match DeviceType::from_char(d.r#type.chars().next()) {
-        Some(t) => t,
+    let dev_type = match &d.r#type {
+        Some(t_s) => match DeviceType::from_char(t_s.chars().next()) {
+            Some(t_c) => t_c,
+            None => return None,
+        },
         None => return None,
     };
 
@@ -603,7 +606,7 @@ lazy_static! {
             // all mknod to all char devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -612,7 +615,7 @@ lazy_static! {
             // all mknod to all block devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "b".to_string(),
+                r#type: Some("b".to_string()),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -621,7 +624,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/console
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(5),
                 minor: Some(1),
                 access: "rwm".to_string(),
@@ -630,7 +633,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/pts/<N>
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(136),
                 minor: Some(WILDCARD),
                 access: "rwm".to_string(),
@@ -639,7 +642,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/ptmx
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(5),
                 minor: Some(2),
                 access: "rwm".to_string(),
@@ -648,7 +651,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/net/tun
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(10),
                 minor: Some(200),
                 access: "rwm".to_string(),

src/agent/rustjail/src/cgroups/systemd/dbus_client.rs

@@ -26,7 +26,7 @@ pub trait SystemdInterface {
 
     fn get_version(&self) -> Result<String>;
 
-    fn unit_exists(&self, unit_name: &str) -> Result<bool>;
+    fn unit_exist(&self, unit_name: &str) -> Result<bool>;
 
     fn add_process(&self, pid: i32, unit_name: &str) -> Result<()>;
 }
@@ -36,9 +36,8 @@ pub struct DBusClient {}
 
 impl DBusClient {
     fn build_proxy(&self) -> Result<SystemManager<'static>> {
-        let connection =
-            zbus::blocking::Connection::system().context("Establishing a D-Bus connection")?;
-        let proxy = SystemManager::new(&connection).context("Building a D-Bus proxy manager")?;
+        let connection = zbus::blocking::Connection::system()?;
+        let proxy = SystemManager::new(&connection)?;
         Ok(proxy)
     }
 }
@@ -109,10 +108,8 @@ impl SystemdInterface for DBusClient {
         Ok(systemd_version)
     }
 
-    fn unit_exists(&self, unit_name: &str) -> Result<bool> {
-        let proxy = self
-            .build_proxy()
-            .with_context(|| format!("Checking if systemd unit {} exists", unit_name))?;
+    fn unit_exist(&self, unit_name: &str) -> Result<bool> {
+        let proxy = self.build_proxy()?;
 
         Ok(proxy.get_unit(unit_name).is_ok())
     }

src/agent/rustjail/src/cgroups/systemd/manager.rs

@@ -41,7 +41,7 @@ pub struct Manager {
 impl CgroupManager for Manager {
     fn apply(&self, pid: pid_t) -> Result<()> {
         let unit_name = self.unit_name.as_str();
-        if self.dbus_client.unit_exists(unit_name)? {
+        if self.dbus_client.unit_exist(unit_name).unwrap() {
             self.dbus_client.add_process(pid, self.unit_name.as_str())?;
         } else {
             self.dbus_client.start_unit(

src/agent/rustjail/src/cgroups/systemd/subsystem/cpu.rs

@@ -71,7 +71,7 @@ impl Cpu {
     }
 
     // v2:
-    // cpu.shares <-> CPUWeight
+    // cpu.shares <-> CPUShares
     // cpu.period <-> CPUQuotaPeriodUSec
     // cpu.period & cpu.quota <-> CPUQuotaPerSecUSec
     fn unified_apply(
@@ -80,8 +80,8 @@ impl Cpu {
         systemd_version: &str,
     ) -> Result<()> {
         if let Some(shares) = cpu_resources.shares {
-            let weight = shares_to_weight(shares);
-            properties.push(("CPUWeight", Value::U64(weight)));
+            let unified_shares = get_unified_cpushares(shares);
+            properties.push(("CPUShares", Value::U64(unified_shares)));
         }
 
         if let Some(period) = cpu_resources.period {
@@ -104,7 +104,7 @@ impl Cpu {
 
 // ref: https://github.com/containers/crun/blob/main/crun.1.md#cgroup-v2
 // [2-262144] to [1-10000]
-fn shares_to_weight(shares: u64) -> u64 {
+fn get_unified_cpushares(shares: u64) -> u64 {
     if shares == 0 {
         return 100;
     }

src/agent/rustjail/src/container.rs

@@ -374,13 +374,18 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let buf = read_sync(crfd)?;
     let spec_str = std::str::from_utf8(&buf)?;
     let spec: oci::Spec = serde_json::from_str(spec_str)?;
-
     log_child!(cfd_log, "notify parent to send oci process");
     write_sync(cwfd, SYNC_SUCCESS, "")?;
 
     let buf = read_sync(crfd)?;
     let process_str = std::str::from_utf8(&buf)?;
     let oci_process: oci::Process = serde_json::from_str(process_str)?;
+    log_child!(cfd_log, "notify parent to send oci state");
+    write_sync(cwfd, SYNC_SUCCESS, "")?;
+
+    let buf = read_sync(crfd)?;
+    let state_str = std::str::from_utf8(&buf)?;
+    let mut state: oci::State = serde_json::from_str(state_str)?;
     log_child!(cfd_log, "notify parent to send cgroup manager");
     write_sync(cwfd, SYNC_SUCCESS, "")?;
 
@@ -743,6 +748,19 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::read(fd, buf)?;
     }
 
+    if init {
+        // StartContainer Hooks:
+        // * should be run in container namespace
+        // * should be run after container is created and before container is started (before user-specific command is executed)
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#startcontainer-hooks
+        state.pid = std::process::id() as i32;
+        state.status = oci::ContainerState::Created;
+        if let Some(hooks) = spec.hooks.as_ref() {
+            let mut start_container_states = HookStates::new();
+            start_container_states.execute_hooks(&hooks.start_container, Some(state))?;
+        }
+    }
+
     // With NoNewPrivileges, we should set seccomp as close to
     // do_exec as possible in order to reduce the amount of
     // system calls in the seccomp profiles.
@@ -1323,7 +1341,6 @@ async fn join_namespaces(
     write_async(pipe_w, SYNC_DATA, spec_str.as_str()).await?;
 
     info!(logger, "wait child received oci spec");
-
     read_async(pipe_r).await?;
 
     info!(logger, "send oci process from parent to child");
@@ -1333,6 +1350,13 @@ async fn join_namespaces(
     info!(logger, "wait child received oci process");
     read_async(pipe_r).await?;
 
+    info!(logger, "try to send state from parent to child");
+    let state_str = serde_json::to_string(st)?;
+    write_async(pipe_w, SYNC_DATA, state_str.as_str()).await?;
+
+    info!(logger, "wait child received oci state");
+    read_async(pipe_r).await?;
+
     let cm_str = if use_systemd_cgroup {
         serde_json::to_string(cm.as_any()?.downcast_ref::<SystemdManager>().unwrap())
     } else {

src/agent/rustjail/src/lib.rs

@@ -153,13 +153,17 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
 
 fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks {
     let prestart = hook_grpc_to_oci(h.Prestart.as_ref());
-
+    let create_runtime = hook_grpc_to_oci(h.CreateRuntime.as_ref());
+    let create_container = hook_grpc_to_oci(h.CreateContainer.as_ref());
+    let start_container = hook_grpc_to_oci(h.StartContainer.as_ref());
     let poststart = hook_grpc_to_oci(h.Poststart.as_ref());
-
     let poststop = hook_grpc_to_oci(h.Poststop.as_ref());
 
     oci::Hooks {
         prestart,
+        create_runtime,
+        create_container,
+        start_container,
         poststart,
         poststop,
     }
@@ -237,6 +241,12 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
     let devices = {
         let mut d = Vec::new();
         for dev in res.Devices.iter() {
+            let dev_type = if dev.Type.is_empty() {
+                None
+            } else {
+                Some(dev.Type.clone())
+            };
+
             let major = if dev.Major == -1 {
                 None
             } else {
@@ -250,7 +260,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             };
             d.push(oci::LinuxDeviceCgroup {
                 allow: dev.Allow,
-                r#type: dev.Type.clone(),
+                r#type: dev_type,
                 major,
                 minor,
                 access: dev.Access.clone(),
@@ -831,6 +841,45 @@ mod tests {
                         Timeout: 10,
                         ..Default::default()
                     }])),
+                    CreateRuntime: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createruntimepath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    CreateContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    StartContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("startcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -860,6 +909,24 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
+                    create_runtime: Vec::from([oci::Hook {
+                        path: String::from("createruntimepath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    create_container: Vec::from([oci::Hook {
+                        path: String::from("createcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    start_container: Vec::from([oci::Hook {
+                        path: String::from("startcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
                 },
             },
             TestData {
@@ -892,6 +959,45 @@ mod tests {
                         Timeout: 10,
                         ..Default::default()
                     }])),
+                    CreateRuntime: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createruntimepath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    CreateContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    StartContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("startcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -908,6 +1014,24 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
+                    create_runtime: Vec::from([oci::Hook {
+                        path: String::from("createruntimepath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    create_container: Vec::from([oci::Hook {
+                        path: String::from("createcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    start_container: Vec::from([oci::Hook {
+                        path: String::from("startcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
                 },
             },
         ];

src/agent/src/ap.rs

@@ -0,0 +1,79 @@
+// Copyright (c) IBM Corp. 2023
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+use std::fmt;
+use std::str::FromStr;
+
+use anyhow::{anyhow, Context};
+
+// IBM Adjunct Processor (AP) is used for cryptographic operations
+// by IBM Crypto Express hardware security modules on IBM zSystem & LinuxONE (s390x).
+// In Linux, virtual cryptographic devices are called AP queues.
+// The name of an AP queue respects a format <xx>.<xxxx> in hexadecimal notation [1, p.467]:
+//   - <xx> is an adapter ID
+//   - <xxxx> is an adapter domain ID
+// [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku5dd05.pdf
+
+#[derive(Debug)]
+pub struct Address {
+    pub adapter_id: u8,
+    pub adapter_domain: u16,
+}
+
+impl Address {
+    pub fn new(adapter_id: u8, adapter_domain: u16) -> Address {
+        Address {
+            adapter_id,
+            adapter_domain,
+        }
+    }
+}
+
+impl FromStr for Address {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> anyhow::Result<Self> {
+        let split: Vec<&str> = s.split('.').collect();
+        if split.len() != 2 {
+            return Err(anyhow!(
+                "Wrong AP bus format. It needs to be in the form <xx>.<xxxx> (e.g. 0a.003f), got {:?}",
+                s
+            ));
+        }
+
+        let adapter_id = u8::from_str_radix(split[0], 16).context(format!(
+            "Wrong AP bus format. AP ID needs to be in the form <xx> (e.g. 0a), got {:?}",
+            split[0]
+        ))?;
+        let adapter_domain = u16::from_str_radix(split[1], 16).context(format!(
+            "Wrong AP bus format. AP domain needs to be in the form <xxxx> (e.g. 003f), got {:?}",
+            split[1]
+        ))?;
+
+        Ok(Address::new(adapter_id, adapter_domain))
+    }
+}
+
+impl fmt::Display for Address {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "{:02x}.{:04x}", self.adapter_id, self.adapter_domain)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_from_str() {
+        let device = Address::from_str("a.1").unwrap();
+        assert_eq!(format!("{}", device), "0a.0001");
+
+        assert!(Address::from_str("").is_err());
+        assert!(Address::from_str(".").is_err());
+        assert!(Address::from_str("0.0.0").is_err());
+        assert!(Address::from_str("0g.0000").is_err());
+        assert!(Address::from_str("0a.10000").is_err());
+    }
+}

src/agent/src/config.rs

@@ -11,6 +11,7 @@ use std::fs;
 use std::str::FromStr;
 use std::time;
 use tracing::instrument;
+use url::Url;
 
 use kata_types::config::default::DEFAULT_AGENT_VSOCK_PORT;
 
@@ -25,6 +26,11 @@ const LOG_VPORT_OPTION: &str = "agent.log_vport";
 const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
 const CONFIG_FILE: &str = "agent.config_file";
+const AA_KBC_PARAMS: &str = "agent.aa_kbc_params";
+const HTTPS_PROXY: &str = "agent.https_proxy";
+const NO_PROXY: &str = "agent.no_proxy";
+const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity";
+const ENABLE_SIGNATURE_VERIFICATION: &str = "agent.enable_signature_verification";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
@@ -77,6 +83,12 @@ pub struct AgentConfig {
     pub tracing: bool,
     pub endpoints: AgentEndpoints,
     pub supports_seccomp: bool,
+    pub container_policy_path: String,
+    pub aa_kbc_params: String,
+    pub https_proxy: String,
+    pub no_proxy: String,
+    pub data_integrity: bool,
+    pub enable_signature_verification: bool,
 }
 
 #[derive(Debug, Deserialize)]
@@ -92,6 +104,12 @@ pub struct AgentConfigBuilder {
     pub unified_cgroup_hierarchy: Option<bool>,
     pub tracing: Option<bool>,
     pub endpoints: Option<EndpointsConfig>,
+    pub container_policy_path: Option<String>,
+    pub aa_kbc_params: Option<String>,
+    pub https_proxy: Option<String>,
+    pub no_proxy: Option<String>,
+    pub data_integrity: Option<bool>,
+    pub enable_signature_verification: Option<bool>,
 }
 
 macro_rules! config_override {
@@ -153,6 +171,12 @@ impl Default for AgentConfig {
             tracing: false,
             endpoints: Default::default(),
             supports_seccomp: rpc::have_seccomp(),
+            container_policy_path: String::from(""),
+            aa_kbc_params: String::from(""),
+            https_proxy: String::from(""),
+            no_proxy: String::from(""),
+            data_integrity: false,
+            enable_signature_verification: true,
         }
     }
 }
@@ -181,6 +205,16 @@ impl FromStr for AgentConfig {
         config_override!(agent_config_builder, agent_config, server_addr);
         config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
         config_override!(agent_config_builder, agent_config, tracing);
+        config_override!(agent_config_builder, agent_config, container_policy_path);
+        config_override!(agent_config_builder, agent_config, aa_kbc_params);
+        config_override!(agent_config_builder, agent_config, https_proxy);
+        config_override!(agent_config_builder, agent_config, no_proxy);
+        config_override!(agent_config_builder, agent_config, data_integrity);
+        config_override!(
+            agent_config_builder,
+            agent_config,
+            enable_signature_verification
+        );
 
         // Populate the allowed endpoints hash set, if we got any from the config file.
         if let Some(endpoints) = agent_config_builder.endpoints {
@@ -209,6 +243,10 @@ impl AgentConfig {
         let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
+
+        let mut using_config_file = false;
+        // Check if there is config file before parsing params that might
+        // override values from the config file.
         for param in params.iter() {
             // If we get a configuration file path from the command line, we
             // generate our config from it.
@@ -216,9 +254,13 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                return AgentConfig::from_config_file(&config_file);
+                config = AgentConfig::from_config_file(&config_file)?;
+                using_config_file = true;
+                break;
             }
+        }
 
+        for param in params.iter() {
             // parse cmdline flags
             parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, config.debug_console);
             parse_cmdline_param!(param, DEV_MODE_FLAG, config.dev_mode);
@@ -278,6 +320,23 @@ impl AgentConfig {
                 config.unified_cgroup_hierarchy,
                 get_bool_value
             );
+
+            parse_cmdline_param!(param, AA_KBC_PARAMS, config.aa_kbc_params, get_string_value);
+            parse_cmdline_param!(param, HTTPS_PROXY, config.https_proxy, get_url_value);
+            parse_cmdline_param!(param, NO_PROXY, config.no_proxy, get_string_value);
+            parse_cmdline_param!(
+                param,
+                ENABLE_DATA_INTEGRITY,
+                config.data_integrity,
+                get_bool_value
+            );
+
+            parse_cmdline_param!(
+                param,
+                ENABLE_SIGNATURE_VERIFICATION,
+                config.enable_signature_verification,
+                get_bool_value
+            );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
@@ -297,7 +356,9 @@ impl AgentConfig {
         }
 
         // We did not get a configuration file: allow all endpoints.
-        config.endpoints.all_allowed = true;
+        if !using_config_file {
+            config.endpoints.all_allowed = true;
+        }
 
         Ok(config)
     }
@@ -431,6 +492,12 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
     Ok(value)
 }
 
+#[instrument]
+fn get_url_value(param: &str) -> Result<String> {
+    let value = get_string_value(param)?;
+    Ok(Url::parse(&value)?.to_string())
+}
+
 #[cfg(test)]
 mod tests {
     use test_utils::assert_result;
@@ -449,6 +516,8 @@ mod tests {
         assert!(!config.dev_mode);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
+        assert_eq!(config.container_policy_path, "");
+        assert!(config.enable_signature_verification);
     }
 
     #[test]
@@ -467,6 +536,12 @@ mod tests {
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
             tracing: bool,
+            container_policy_path: &'a str,
+            aa_kbc_params: &'a str,
+            https_proxy: &'a str,
+            no_proxy: &'a str,
+            data_integrity: bool,
+            enable_signature_verification: bool,
         }
 
         impl Default for TestData<'_> {
@@ -482,6 +557,12 @@ mod tests {
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
                     tracing: false,
+                    container_policy_path: "",
+                    aa_kbc_params: "",
+                    https_proxy: "",
+                    no_proxy: "",
+                    data_integrity: false,
+                    enable_signature_verification: true,
                 }
             }
         }
@@ -851,6 +932,81 @@ mod tests {
                 tracing: true,
                 ..Default::default()
             },
+            TestData {
+                contents: "agent.aa_kbc_params=offline_fs_kbc::null",
+                aa_kbc_params: "offline_fs_kbc::null",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.aa_kbc_params=eaa_kbc::127.0.0.1:50000",
+                aa_kbc_params: "eaa_kbc::127.0.0.1:50000",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.https_proxy=http://proxy.url.com:81/",
+                https_proxy: "http://proxy.url.com:81/",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.https_proxy=http://192.168.1.100:81/",
+                https_proxy: "http://192.168.1.100:81/",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.no_proxy=*.internal.url.com",
+                no_proxy: "*.internal.url.com",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.no_proxy=192.168.1.0/24,172.16.0.0/12",
+                no_proxy: "192.168.1.0/24,172.16.0.0/12",
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=true",
+                data_integrity: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=false",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=1",
+                data_integrity: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=0",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=false",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=0",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=1",
+                enable_signature_verification: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=foo",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -898,6 +1054,20 @@ mod tests {
             assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
             assert_eq!(d.server_addr, config.server_addr, "{}", msg);
             assert_eq!(d.tracing, config.tracing, "{}", msg);
+            assert_eq!(
+                d.container_policy_path, config.container_policy_path,
+                "{}",
+                msg
+            );
+            assert_eq!(d.aa_kbc_params, config.aa_kbc_params, "{}", msg);
+            assert_eq!(d.https_proxy, config.https_proxy, "{}", msg);
+            assert_eq!(d.no_proxy, config.no_proxy, "{}", msg);
+            assert_eq!(d.data_integrity, config.data_integrity, "{}", msg);
+            assert_eq!(
+                d.enable_signature_verification, config.enable_signature_verification,
+                "{}",
+                msg
+            );
 
             for v in vars_to_unset {
                 env::remove_var(v);
@@ -1399,4 +1569,50 @@ Caused by:
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
+
+    #[test]
+    fn test_config_from_cmdline_and_config_file() {
+        let dir = tempdir().expect("failed to create tmpdir");
+
+        let agent_config = r#"
+               dev_mode = false
+               server_addr = 'vsock://8:2048'
+
+               [endpoints]
+               allowed = ["CreateContainer", "StartContainer"]
+              "#;
+
+        let config_path = dir.path().join("agent-config.toml");
+        let config_filename = config_path.to_str().expect("failed to get config filename");
+
+        fs::write(config_filename, agent_config).expect("failed to write agen config");
+
+        let cmdline = format!("agent.devmode agent.config_file={}", config_filename);
+
+        let cmdline_path = dir.path().join("cmdline");
+        let cmdline_filename = cmdline_path
+            .to_str()
+            .expect("failed to get cmdline filename");
+
+        fs::write(cmdline_filename, cmdline).expect("failed to write agen config");
+
+        let config = AgentConfig::from_cmdline(cmdline_filename, vec![])
+            .expect("failed to parse command line");
+
+        // Should be overwritten by cmdline
+        assert!(config.dev_mode);
+
+        // Should be from agent config
+        assert_eq!(config.server_addr, "vsock://8:2048");
+
+        // Should be from agent config
+        assert_eq!(
+            config.endpoints.allowed,
+            vec!["CreateContainer".to_string(), "StartContainer".to_string()]
+                .iter()
+                .cloned()
+                .collect()
+        );
+        assert!(!config.endpoints.all_allowed);
+    }
 }

src/agent/src/device.rs

@@ -16,13 +16,12 @@ use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
 
-#[cfg(target_arch = "s390x")]
-use crate::ccw;
 use crate::linux_abi::*;
 use crate::pci;
 use crate::sandbox::Sandbox;
 use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
 use anyhow::{anyhow, Context, Result};
+use cfg_if::cfg_if;
 use oci::{LinuxDeviceCgroup, LinuxResources, Spec};
 use protocols::agent::Device;
 use tracing::instrument;
@@ -46,14 +45,22 @@ pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
 pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
 pub const DRIVER_LOCAL_TYPE: &str = "local";
 pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
-// VFIO device to be bound to a guest kernel driver
-pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
-// VFIO device to be bound to vfio-pci and made available inside the
+// VFIO PCI device to be bound to a guest kernel driver
+pub const DRIVER_VFIO_PCI_GK_TYPE: &str = "vfio-pci-gk";
+// VFIO PCI device to be bound to vfio-pci and made available inside the
 // container as a VFIO device node
-pub const DRIVER_VFIO_TYPE: &str = "vfio";
+pub const DRIVER_VFIO_PCI_TYPE: &str = "vfio-pci";
+pub const DRIVER_VFIO_AP_TYPE: &str = "vfio-ap";
 pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs";
 pub const FS_TYPE_HUGETLB: &str = "hugetlbfs";
 
+cfg_if! {
+    if #[cfg(target_arch = "s390x")] {
+        use crate::ap;
+        use crate::ccw;
+    }
+}
+
 #[instrument]
 pub fn online_device(path: &str) -> Result<()> {
     fs::write(path, "1")?;
@@ -280,7 +287,7 @@ pub async fn get_virtio_blk_ccw_device_name(
     sandbox: &Arc<Mutex<Sandbox>>,
     device: &ccw::Device,
 ) -> Result<String> {
-    let matcher = VirtioBlkCCWMatcher::new(&create_ccw_root_bus_path(), device);
+    let matcher = VirtioBlkCCWMatcher::new(CCW_ROOT_BUS_PATH, device);
     let uev = wait_for_uevent(sandbox, matcher).await?;
     let devname = uev.devname;
     return match Path::new(SYSTEM_DEV_PATH).join(&devname).to_str() {
@@ -401,6 +408,39 @@ async fn get_vfio_device_name(sandbox: &Arc<Mutex<Sandbox>>, grp: IommuGroup) ->
     Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
 }
 
+#[cfg(target_arch = "s390x")]
+#[derive(Debug)]
+struct ApMatcher {
+    syspath: String,
+}
+
+#[cfg(target_arch = "s390x")]
+impl ApMatcher {
+    fn new(address: ap::Address) -> ApMatcher {
+        ApMatcher {
+            syspath: format!(
+                "{}/card{:02x}/{}",
+                AP_ROOT_BUS_PATH, address.adapter_id, address
+            ),
+        }
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+impl UeventMatcher for ApMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.action == "add" && uev.devpath == self.syspath
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+#[instrument]
+async fn wait_for_ap_device(sandbox: &Arc<Mutex<Sandbox>>, address: ap::Address) -> Result<()> {
+    let matcher = ApMatcher::new(address);
+    wait_for_uevent(sandbox, matcher).await?;
+    Ok(())
+}
+
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 #[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
@@ -571,13 +611,15 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
 
     if let Some(resources) = linux.resources.as_mut() {
         for r in &mut resources.devices {
-            if let (Some(host_major), Some(host_minor)) = (r.major, r.minor) {
-                if let Some(update) = res_updates.get(&(r.r#type.as_str(), host_major, host_minor))
+            if let (Some(host_type), Some(host_major), Some(host_minor)) =
+                (r.r#type.as_ref(), r.major, r.minor)
+            {
+                if let Some(update) = res_updates.get(&(host_type.as_str(), host_major, host_minor))
                 {
                     info!(
                         sl!(),
                         "update_spec_devices() updating resource";
-                        "type" => &r.r#type,
+                        "type" => &host_type,
                         "host_major" => host_major,
                         "host_minor" => host_minor,
                         "guest_major" => update.guest_major,
@@ -699,7 +741,7 @@ async fn virtio_nvdimm_device_handler(
     Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into())
 }
 
-fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
+fn split_vfio_pci_option(opt: &str) -> Option<(&str, &str)> {
     let mut tokens = opt.split('=');
     let hostbdf = tokens.next()?;
     let path = tokens.next()?;
@@ -714,14 +756,18 @@ fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
 // Each option should have the form "DDDD:BB:DD.F=<pcipath>"
 //     DDDD:BB:DD.F is the device's PCI address in the host
 //     <pcipath> is a PCI path to the device in the guest (see pci.rs)
-async fn vfio_device_handler(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
-    let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE;
+#[instrument]
+async fn vfio_pci_device_handler(
+    device: &Device,
+    sandbox: &Arc<Mutex<Sandbox>>,
+) -> Result<SpecUpdate> {
+    let vfio_in_guest = device.field_type != DRIVER_VFIO_PCI_GK_TYPE;
     let mut pci_fixups = Vec::<(pci::Address, pci::Address)>::new();
     let mut group = None;
 
     for opt in device.options.iter() {
-        let (host, pcipath) =
-            split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?;
+        let (host, pcipath) = split_vfio_pci_option(opt)
+            .ok_or_else(|| anyhow!("Malformed VFIO PCI option {:?}", opt))?;
         let host =
             pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?;
         let pcipath = pci::Path::from_str(pcipath)?;
@@ -763,6 +809,28 @@ async fn vfio_device_handler(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) ->
     })
 }
 
+// The VFIO AP (Adjunct Processor) device handler takes all the APQNs provided as device options
+// and awaits them. It sets the minimum AP rescan time of 5 seconds and temporarily adds that
+// amount to the hotplug timeout.
+#[cfg(target_arch = "s390x")]
+#[instrument]
+async fn vfio_ap_device_handler(
+    device: &Device,
+    sandbox: &Arc<Mutex<Sandbox>>,
+) -> Result<SpecUpdate> {
+    // Force AP bus rescan
+    fs::write(AP_SCANS_PATH, "1")?;
+    for apqn in device.options.iter() {
+        wait_for_ap_device(sandbox, ap::Address::from_str(apqn)?).await?;
+    }
+    Ok(Default::default())
+}
+
+#[cfg(not(target_arch = "s390x"))]
+async fn vfio_ap_device_handler(_: &Device, _: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
+    Err(anyhow!("AP is only supported on s390x"))
+}
+
 #[instrument]
 pub async fn add_devices(
     devices: &[Device],
@@ -828,7 +896,10 @@ async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<Sp
         DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, sandbox).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, sandbox).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, sandbox).await,
-        DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => vfio_device_handler(device, sandbox).await,
+        DRIVER_VFIO_PCI_GK_TYPE | DRIVER_VFIO_PCI_TYPE => {
+            vfio_pci_device_handler(device, sandbox).await
+        }
+        DRIVER_VFIO_AP_TYPE => vfio_ap_device_handler(device, sandbox).await,
         _ => Err(anyhow!("Unknown device type {}", device.field_type)),
     }
 }
@@ -854,7 +925,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
         allow: false,
         major: Some(major),
         minor: Some(minor),
-        r#type: String::from("b"),
+        r#type: Some(String::from("b")),
         access: String::from("rw"),
     });
 
@@ -1017,13 +1088,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         oci::LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major_a),
                             minor: Some(host_minor_a),
                             ..oci::LinuxDeviceCgroup::default()
                         },
                         oci::LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major_b),
                             minor: Some(host_minor_b),
                             ..oci::LinuxDeviceCgroup::default()
@@ -1116,13 +1187,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
                         },
                         LinuxDeviceCgroup {
-                            r#type: "b".to_string(),
+                            r#type: Some("b".to_string()),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
@@ -1378,7 +1449,7 @@ mod tests {
     #[cfg(target_arch = "s390x")]
     #[tokio::test]
     async fn test_virtio_blk_ccw_matcher() {
-        let root_bus = create_ccw_root_bus_path();
+        let root_bus = CCW_ROOT_BUS_PATH;
         let subsystem = "block";
         let devname = "vda";
         let relpath = "0.0.0002";
@@ -1487,13 +1558,13 @@ mod tests {
     }
 
     #[test]
-    fn test_split_vfio_option() {
+    fn test_split_vfio_pci_option() {
         assert_eq!(
-            split_vfio_option("0000:01:00.0=02/01"),
+            split_vfio_pci_option("0000:01:00.0=02/01"),
             Some(("0000:01:00.0", "02/01"))
         );
-        assert_eq!(split_vfio_option("0000:01:00.0=02/01=rubbish"), None);
-        assert_eq!(split_vfio_option("0000:01:00.0"), None);
+        assert_eq!(split_vfio_pci_option("0000:01:00.0=02/01=rubbish"), None);
+        assert_eq!(split_vfio_pci_option("0000:01:00.0"), None);
     }
 
     #[test]
@@ -1572,4 +1643,35 @@ mod tests {
         // Test dev2
         assert!(pci_iommu_group(&syspci, dev2).is_err());
     }
+
+    #[cfg(target_arch = "s390x")]
+    #[tokio::test]
+    async fn test_vfio_ap_matcher() {
+        let subsystem = "ap";
+        let card = "0a";
+        let relpath = format!("{}.0001", card);
+
+        let mut uev = Uevent::default();
+        uev.action = U_EVENT_ACTION_ADD.to_string();
+        uev.subsystem = subsystem.to_string();
+        uev.devpath = format!("{}/card{}/{}", AP_ROOT_BUS_PATH, card, relpath);
+
+        let ap_address = ap::Address::from_str(&relpath).unwrap();
+        let matcher = ApMatcher::new(ap_address);
+
+        assert!(matcher.is_match(&uev));
+
+        let mut uev_remove = uev.clone();
+        uev_remove.action = U_EVENT_ACTION_REMOVE.to_string();
+        assert!(!matcher.is_match(&uev_remove));
+
+        let mut uev_other_device = uev.clone();
+        uev_other_device.devpath = format!(
+            "{}/card{}/{}",
+            AP_ROOT_BUS_PATH,
+            card,
+            format!("{}.0002", card)
+        );
+        assert!(!matcher.is_match(&uev_other_device));
+    }
 }

src/agent/src/image_rpc.rs

@@ -0,0 +1,352 @@
+// Copyright (c) 2021 Alibaba Cloud
+// Copyright (c) 2021, 2023 IBM Corporation
+// Copyright (c) 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::env;
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+use std::sync::atomic::{AtomicBool, AtomicU16, Ordering};
+use std::sync::Arc;
+
+use anyhow::{anyhow, Result};
+use async_trait::async_trait;
+use protocols::image;
+use tokio::sync::Mutex;
+use ttrpc::{self, error::get_rpc_status as ttrpc_error};
+
+use crate::rpc::{verify_cid, CONTAINER_BASE};
+use crate::sandbox::Sandbox;
+use crate::AGENT_CONFIG;
+
+use image_rs::image::ImageClient;
+use std::io::Write;
+
+const AA_PATH: &str = "/usr/local/bin/attestation-agent";
+
+const AA_KEYPROVIDER_URI: &str =
+    "unix:///run/confidential-containers/attestation-agent/keyprovider.sock";
+const AA_GETRESOURCE_URI: &str =
+    "unix:///run/confidential-containers/attestation-agent/getresource.sock";
+
+const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+// kata rootfs is readonly, use tmpfs before CC storage is implemented.
+const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/";
+const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle";
+const CONFIG_JSON: &str = "config.json";
+
+// Convenience macro to obtain the scope logger
+macro_rules! sl {
+    () => {
+        slog_scope::logger()
+    };
+}
+
+pub struct ImageService {
+    sandbox: Arc<Mutex<Sandbox>>,
+    attestation_agent_started: AtomicBool,
+    image_client: Arc<Mutex<ImageClient>>,
+    container_count: Arc<AtomicU16>,
+}
+
+impl ImageService {
+    pub fn new(sandbox: Arc<Mutex<Sandbox>>) -> Self {
+        env::set_var("CC_IMAGE_WORK_DIR", KATA_CC_IMAGE_WORK_DIR);
+        Self {
+            sandbox,
+            attestation_agent_started: AtomicBool::new(false),
+            image_client: Arc::new(Mutex::new(ImageClient::default())),
+            container_count: Arc::new(AtomicU16::new(0)),
+        }
+    }
+
+    // pause image is packaged in rootfs for CC
+    fn unpack_pause_image(cid: &str) -> Result<()> {
+        let cc_pause_bundle = Path::new(KATA_CC_PAUSE_BUNDLE);
+        if !cc_pause_bundle.exists() {
+            return Err(anyhow!("Pause image not present in rootfs"));
+        }
+
+        info!(sl!(), "use guest pause image cid {:?}", cid);
+        let pause_bundle = Path::new(CONTAINER_BASE).join(cid);
+        let pause_rootfs = pause_bundle.join("rootfs");
+        let pause_config = pause_bundle.join(CONFIG_JSON);
+        let pause_binary = pause_rootfs.join("pause");
+        fs::create_dir_all(&pause_rootfs)?;
+        if !pause_config.exists() {
+            fs::copy(
+                cc_pause_bundle.join(CONFIG_JSON),
+                pause_bundle.join(CONFIG_JSON),
+            )?;
+        }
+        if !pause_binary.exists() {
+            fs::copy(cc_pause_bundle.join("rootfs").join("pause"), pause_binary)?;
+        }
+
+        Ok(())
+    }
+
+    // If we fail to start the AA, ocicrypt won't be able to unwrap keys
+    // and container decryption will fail.
+    fn init_attestation_agent() -> Result<()> {
+        let config_path = OCICRYPT_CONFIG_PATH;
+
+        // The image will need to be encrypted using a keyprovider
+        // that has the same name (at least according to the config).
+        let ocicrypt_config = serde_json::json!({
+            "key-providers": {
+                "attestation-agent":{
+                    "ttrpc":AA_KEYPROVIDER_URI
+                }
+            }
+        });
+
+        let mut config_file = fs::File::create(config_path)?;
+        config_file.write_all(ocicrypt_config.to_string().as_bytes())?;
+
+        // The Attestation Agent will run for the duration of the guest.
+        Command::new(AA_PATH)
+            .arg("--keyprovider_sock")
+            .arg(AA_KEYPROVIDER_URI)
+            .arg("--getresource_sock")
+            .arg(AA_GETRESOURCE_URI)
+            .spawn()?;
+        Ok(())
+    }
+
+    /// Determines the container id (cid) to use for a given request.
+    ///
+    /// If the request specifies a non-empty id, use it; otherwise derive it from the image path.
+    /// In either case, verify that the chosen id is valid.
+    fn cid_from_request(&self, req: &image::PullImageRequest) -> Result<String> {
+        let req_cid = req.get_container_id();
+        let cid = if !req_cid.is_empty() {
+            req_cid.to_string()
+        } else if let Some(last) = req.get_image().rsplit('/').next() {
+            // Support multiple containers with same image
+            let index = self.container_count.fetch_add(1, Ordering::Relaxed);
+
+            // ':' not valid for container id
+            format!("{}_{}", last.replace(':', "_"), index)
+        } else {
+            return Err(anyhow!("Invalid image name. {}", req.get_image()));
+        };
+        verify_cid(&cid)?;
+        Ok(cid)
+    }
+
+    async fn pull_image(&self, req: &image::PullImageRequest) -> Result<String> {
+        env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
+
+        let https_proxy = &AGENT_CONFIG.read().await.https_proxy;
+        if !https_proxy.is_empty() {
+            env::set_var("HTTPS_PROXY", https_proxy);
+        }
+
+        let no_proxy = &AGENT_CONFIG.read().await.no_proxy;
+        if !no_proxy.is_empty() {
+            env::set_var("NO_PROXY", no_proxy);
+        }
+
+        let cid = self.cid_from_request(req)?;
+        let image = req.get_image();
+        if cid.starts_with("pause") {
+            Self::unpack_pause_image(&cid)?;
+
+            let mut sandbox = self.sandbox.lock().await;
+            sandbox.images.insert(String::from(image), cid);
+            return Ok(image.to_owned());
+        }
+
+        let aa_kbc_params = &AGENT_CONFIG.read().await.aa_kbc_params;
+        if !aa_kbc_params.is_empty() {
+            match self.attestation_agent_started.compare_exchange_weak(
+                false,
+                true,
+                Ordering::SeqCst,
+                Ordering::SeqCst,
+            ) {
+                Ok(_) => Self::init_attestation_agent()?,
+                Err(_) => info!(sl!(), "Attestation Agent already running"),
+            }
+        }
+        // If the attestation-agent is being used, then enable the authenticated credentials support
+        info!(
+            sl!(),
+            "image_client.config.auth set to: {}",
+            !aa_kbc_params.is_empty()
+        );
+        self.image_client.lock().await.config.auth = !aa_kbc_params.is_empty();
+
+        // Read enable signature verification from the agent config and set it in the image_client
+        let enable_signature_verification =
+            &AGENT_CONFIG.read().await.enable_signature_verification;
+        info!(
+            sl!(),
+            "enable_signature_verification set to: {}", enable_signature_verification
+        );
+        self.image_client.lock().await.config.security_validate = *enable_signature_verification;
+
+        let source_creds = (!req.get_source_creds().is_empty()).then(|| req.get_source_creds());
+
+        let bundle_path = Path::new(CONTAINER_BASE).join(&cid);
+        fs::create_dir_all(&bundle_path)?;
+
+        let decrypt_config = format!("provider:attestation-agent:{}", aa_kbc_params);
+
+        info!(sl!(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
+        // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles
+        // with rootfs and config.json will store under CONTAINER_BASE/cid.
+        let res = self
+            .image_client
+            .lock()
+            .await
+            .pull_image(image, &bundle_path, &source_creds, &Some(&decrypt_config))
+            .await;
+
+        match res {
+            Ok(image) => {
+                info!(
+                    sl!(),
+                    "pull and unpack image {:?}, cid: {:?}, with image-rs succeed. ", image, cid
+                );
+            }
+            Err(e) => {
+                error!(
+                    sl!(),
+                    "pull and unpack image {:?}, cid: {:?}, with image-rs failed with {:?}. ",
+                    image,
+                    cid,
+                    e.to_string()
+                );
+                return Err(e);
+            }
+        };
+
+        let mut sandbox = self.sandbox.lock().await;
+        sandbox.images.insert(String::from(image), cid);
+        Ok(image.to_owned())
+    }
+}
+
+#[async_trait]
+impl protocols::image_ttrpc_async::Image for ImageService {
+    async fn pull_image(
+        &self,
+        _ctx: &ttrpc::r#async::TtrpcContext,
+        req: image::PullImageRequest,
+    ) -> ttrpc::Result<image::PullImageResponse> {
+        match self.pull_image(&req).await {
+            Ok(r) => {
+                let mut resp = image::PullImageResponse::new();
+                resp.image_ref = r;
+                return Ok(resp);
+            }
+            Err(e) => {
+                return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()));
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::ImageService;
+    use crate::sandbox::Sandbox;
+    use protocols::image;
+    use std::sync::Arc;
+    use tokio::sync::Mutex;
+
+    #[tokio::test]
+    async fn test_cid_from_request() {
+        struct Case {
+            cid: &'static str,
+            image: &'static str,
+            result: Option<&'static str>,
+        }
+
+        let cases = [
+            Case {
+                cid: "",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "..",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "..",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "abc/..",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "abc/",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "../abc",
+                result: Some("abc_4"),
+            },
+            Case {
+                cid: "",
+                image: "../9abc",
+                result: Some("9abc_5"),
+            },
+            Case {
+                cid: "some-string.1_2",
+                image: "",
+                result: Some("some-string.1_2"),
+            },
+            Case {
+                cid: "0some-string.1_2",
+                image: "",
+                result: Some("0some-string.1_2"),
+            },
+            Case {
+                cid: "a:b",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "prefix/a:b",
+                result: Some("a_b_6"),
+            },
+            Case {
+                cid: "",
+                image: "/a/b/c/d:e",
+                result: Some("d_e_7"),
+            },
+        ];
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let s = Sandbox::new(&logger).unwrap();
+        let image_service = ImageService::new(Arc::new(Mutex::new(s)));
+        for case in &cases {
+            let mut req = image::PullImageRequest::new();
+            req.set_image(case.image.to_string());
+            req.set_container_id(case.cid.to_string());
+            let ret = image_service.cid_from_request(&req);
+            match (case.result, ret) {
+                (Some(expected), Ok(actual)) => assert_eq!(expected, actual),
+                (None, Err(_)) => (),
+                (None, Ok(r)) => panic!("Expected an error, got {}", r),
+                (Some(expected), Err(e)) => {
+                    panic!("Expected {} but got an error ({})", expected, e)
+                }
+            }
+        }
+    }
+}

src/agent/src/linux_abi.rs

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use cfg_if::cfg_if;
+
 /// Linux ABI related constants.
 
 #[cfg(target_arch = "aarch64")]
@@ -64,10 +66,14 @@ pub fn create_pci_root_bus_path() -> String {
     ret
 }
 
-#[cfg(target_arch = "s390x")]
-pub fn create_ccw_root_bus_path() -> String {
-    String::from("/devices/css0")
+cfg_if! {
+    if #[cfg(target_arch = "s390x")] {
+        pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0";
+        pub const AP_ROOT_BUS_PATH: &str = "/devices/ap";
+        pub const AP_SCANS_PATH: &str = "/sys/bus/ap/scans";
+    }
 }
+
 // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
 // The Linux kernel's core ACPI subsystem creates struct acpi_device
 // objects for ACPI namespace objects representing devices, power resources

src/agent/src/main.rs

@@ -20,6 +20,7 @@ extern crate scopeguard;
 extern crate slog;
 
 use anyhow::{anyhow, Context, Result};
+use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
 use nix::fcntl::OFlag;
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
@@ -34,8 +35,6 @@ use std::process::exit;
 use std::sync::Arc;
 use tracing::{instrument, span};
 
-#[cfg(target_arch = "s390x")]
-mod ccw;
 mod config;
 mod console;
 mod device;
@@ -71,9 +70,17 @@ use tokio::{
     task::JoinHandle,
 };
 
+mod image_rpc;
 mod rpc;
 mod tracer;
 
+cfg_if! {
+    if #[cfg(target_arch = "s390x")] {
+        mod ap;
+        mod ccw;
+    }
+}
+
 const NAME: &str = "kata-agent";
 
 lazy_static! {
@@ -436,8 +443,9 @@ mod tests {
             let msg = format!("test[{}]: {:?}", i, d);
             let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap();
             defer!({
-                // XXX: Never try to close rfd, because it will be closed by PipeStream in
-                // create_logger_task() and it's not safe to close the same fd twice time.
+                // rfd is closed by the use of PipeStream in the crate_logger_task function,
+                // but we will attempt to close in case of a failure
+                let _ = unistd::close(rfd);
                 unistd::close(wfd).unwrap();
             });
 

src/agent/src/mount.rs

@@ -240,6 +240,70 @@ async fn ephemeral_storage_handler(
     Ok("".to_string())
 }
 
+// update_ephemeral_mounts takes a list of ephemeral mounts and remounts them
+// with mount options passed by the caller
+#[instrument]
+pub async fn update_ephemeral_mounts(
+    logger: Logger,
+    storages: Vec<Storage>,
+    sandbox: Arc<Mutex<Sandbox>>,
+) -> Result<()> {
+    for (_, storage) in storages.iter().enumerate() {
+        let handler_name = storage.driver.clone();
+        let logger = logger.new(o!(
+	    "msg" => "updating tmpfs storage",
+            "subsystem" => "storage",
+            "storage-type" => handler_name.to_owned()));
+
+        match handler_name.as_str() {
+            DRIVER_EPHEMERAL_TYPE => {
+                fs::create_dir_all(Path::new(&storage.mount_point))?;
+
+                if storage.options.is_empty() {
+                    continue;
+                } else {
+                    // assume that fsGid has already been set
+                    let mut opts = Vec::<&str>::new();
+                    for (_, opt) in storage.options.iter().enumerate() {
+                        if opt.starts_with(FS_GID) {
+                            continue;
+                        }
+                        opts.push(opt)
+                    }
+                    let mount_path = Path::new(&storage.mount_point);
+                    let src_path = Path::new(&storage.source);
+
+                    let (flags, options) = parse_mount_flags_and_options(opts);
+
+                    info!(logger, "mounting storage";
+                    "mount-source" => src_path.display(),
+                    "mount-destination" => mount_path.display(),
+                    "mount-fstype"  => storage.fstype.as_str(),
+                    "mount-options" => options.as_str(),
+                    );
+
+                    return baremount(
+                        src_path,
+                        mount_path,
+                        storage.fstype.as_str(),
+                        flags,
+                        options.as_str(),
+                        &logger,
+                    );
+                }
+            }
+            _ => {
+                return Err(anyhow!(
+                    "Unsupported storage type for syncing mounts {}. Only ephemeral storage update is supported",
+                    storage.driver.to_owned()
+                ));
+            }
+        };
+    }
+
+    Ok(())
+}
+
 #[instrument]
 async fn overlayfs_storage_handler(
     logger: &Logger,

src/agent/src/network.rs

@@ -7,6 +7,7 @@ use anyhow::{anyhow, Result};
 use nix::mount::{self, MsFlags};
 use slog::Logger;
 use std::fs;
+use std::path;
 
 const KATA_GUEST_SANDBOX_DNS_FILE: &str = "/run/kata-containers/sandbox/resolv.conf";
 const GUEST_DNS_FILE: &str = "/etc/resolv.conf";
@@ -64,6 +65,12 @@ fn do_setup_guest_dns(logger: Logger, dns_list: Vec<String>, src: &str, dst: &st
         .map(|x| x.trim())
         .collect::<Vec<&str>>()
         .join("\n");
+
+    // make sure the src file's parent path exist.
+    let file_path = path::Path::new(src);
+    if let Some(p) = file_path.parent() {
+        fs::create_dir_all(p)?;
+    }
     fs::write(src, content)?;
 
     // bind mount to /etc/resolv.conf

src/agent/src/rpc.rs

@@ -34,7 +34,10 @@ use protocols::health::{
     HealthCheckResponse, HealthCheckResponse_ServingStatus, VersionCheckResponse,
 };
 use protocols::types::Interface;
-use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc};
+use protocols::{
+    agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc,
+    image_ttrpc_async as image_ttrpc,
+};
 use rustjail::cgroups::notifier;
 use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT};
 use rustjail::process::Process;
@@ -49,9 +52,10 @@ use rustjail::process::ProcessOperations;
 use crate::device::{
     add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci,
 };
+use crate::image_rpc;
 use crate::linux_abi::*;
 use crate::metrics::get_metrics;
-use crate::mount::{add_storages, baremount, STORAGE_HANDLER_LIST};
+use crate::mount::{add_storages, baremount, update_ephemeral_mounts, STORAGE_HANDLER_LIST};
 use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS};
 use crate::network::setup_guest_dns;
 use crate::pci;
@@ -80,8 +84,12 @@ use std::io::{BufRead, BufReader, Write};
 use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
 
-const CONTAINER_BASE: &str = "/run/kata-containers";
+pub const CONTAINER_BASE: &str = "/run/kata-containers";
 const MODPROBE_PATH: &str = "/sbin/modprobe";
+const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name";
+const CONFIG_JSON: &str = "config.json";
+const INIT_TRUSTED_STORAGE: &str = "/usr/bin/kata-init-trusted-storage";
+const TRUSTED_STORAGE_DEVICE: &str = "/dev/trusted_store";
 
 /// the iptables seriers binaries could appear either in /sbin
 /// or /usr/sbin, we need to check both of them
@@ -140,6 +148,41 @@ pub struct AgentService {
     init_mode: bool,
 }
 
+// A container ID must match this regex:
+//
+//     ^[a-zA-Z0-9][a-zA-Z0-9_.-]+$
+//
+pub fn verify_cid(id: &str) -> Result<()> {
+    let mut chars = id.chars();
+
+    let valid = matches!(chars.next(), Some(first) if first.is_alphanumeric()
+                && id.len() > 1
+                && chars.all(|c| c.is_alphanumeric() || ['.', '-', '_'].contains(&c)));
+
+    match valid {
+        true => Ok(()),
+        false => Err(anyhow!("invalid container ID: {:?}", id)),
+    }
+}
+
+// Partially merge an OCI process specification into another one.
+fn merge_oci_process(target: &mut oci::Process, source: &oci::Process) {
+    if target.args.is_empty() && !source.args.is_empty() {
+        target.args.append(&mut source.args.clone());
+    }
+
+    if target.cwd == "/" && source.cwd != "/" {
+        target.cwd = String::from(&source.cwd);
+    }
+
+    for source_env in &source.env {
+        let variable_name: Vec<&str> = source_env.split('=').collect();
+        if !target.env.iter().any(|i| i.contains(variable_name[0])) {
+            target.env.push(source_env.to_string());
+        }
+    }
+}
+
 impl AgentService {
     #[instrument]
     async fn do_create_container(
@@ -170,6 +213,9 @@ impl AgentService {
             "receive createcontainer, storages: {:?}", &req.storages
         );
 
+        // Merge the image bundle OCI spec into the container creation request OCI spec.
+        self.merge_bundle_oci(&mut oci).await?;
+
         // Some devices need some extra processing (the ones invoked with
         // --device for instance), and that's what this call is doing. It
         // updates the devices listed in the OCI spec, so that they actually
@@ -177,6 +223,30 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&req.devices.to_vec(), &mut oci, &self.sandbox).await?;
 
+        let linux = oci
+            .linux
+            .as_mut()
+            .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?;
+
+        for specdev in &mut linux.devices {
+            let dev_major_minor = format!("{}:{}", specdev.major, specdev.minor);
+
+            if specdev.path == TRUSTED_STORAGE_DEVICE {
+                let data_integrity = AGENT_CONFIG.read().await.data_integrity;
+                info!(
+                    sl!(),
+                    "trusted_store device major:min {}, enable data integrity {}",
+                    dev_major_minor,
+                    data_integrity.to_string()
+                );
+
+                Command::new(INIT_TRUSTED_STORAGE)
+                    .args([&dev_major_minor, &data_integrity.to_string()])
+                    .output()
+                    .expect("Failed to initialize confidential storage");
+            }
+        }
+
         // Both rootfs and volumes (invoked with --volume for instance) will
         // be processed the same way. The idea is to always mount any provided
         // storage to the specified MountPoint, so that it will match what's
@@ -638,6 +708,54 @@ impl AgentService {
             }
         }
     }
+
+    // When being passed an image name through a container annotation, merge its
+    // corresponding bundle OCI specification into the passed container creation one.
+    async fn merge_bundle_oci(&self, container_oci: &mut oci::Spec) -> Result<()> {
+        if let Some(image_name) = container_oci
+            .annotations
+            .get(&ANNO_K8S_IMAGE_NAME.to_string())
+        {
+            if let Some(container_id) = self.sandbox.clone().lock().await.images.get(image_name) {
+                let image_oci_config_path = Path::new(CONTAINER_BASE)
+                    .join(container_id)
+                    .join(CONFIG_JSON);
+                debug!(
+                    sl!(),
+                    "Image bundle config path: {:?}", image_oci_config_path
+                );
+
+                let image_oci =
+                    oci::Spec::load(image_oci_config_path.to_str().ok_or_else(|| {
+                        anyhow!(
+                            "Invalid container image OCI config path {:?}",
+                            image_oci_config_path
+                        )
+                    })?)
+                    .context("load image bundle")?;
+
+                if let Some(container_root) = container_oci.root.as_mut() {
+                    if let Some(image_root) = image_oci.root.as_ref() {
+                        let root_path = Path::new(CONTAINER_BASE)
+                            .join(container_id)
+                            .join(image_root.path.clone());
+                        container_root.path =
+                            String::from(root_path.to_str().ok_or_else(|| {
+                                anyhow!("Invalid container image root path {:?}", root_path)
+                            })?);
+                    }
+                }
+
+                if let Some(container_process) = container_oci.process.as_mut() {
+                    if let Some(image_process) = image_oci.process.as_ref() {
+                        merge_oci_process(container_process, image_process);
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
 }
 
 #[async_trait]
@@ -997,6 +1115,23 @@ impl agent_ttrpc::AgentService for AgentService {
         })
     }
 
+    async fn update_ephemeral_mounts(
+        &self,
+        ctx: &TtrpcContext,
+        req: protocols::agent::UpdateEphemeralMountsRequest,
+    ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "update_mounts", req);
+        is_allowed!(req);
+
+        match update_ephemeral_mounts(sl!(), req.storages.to_vec(), self.sandbox.clone()).await {
+            Ok(_) => Ok(Empty::new()),
+            Err(e) => Err(ttrpc_error!(
+                ttrpc::Code::INTERNAL,
+                format!("Failed to update mounts: {:?}", e),
+            )),
+        }
+    }
+
     async fn get_ip_tables(
         &self,
         ctx: &TtrpcContext,
@@ -1687,7 +1822,7 @@ async fn read_stream(reader: Arc<Mutex<ReadHalf<PipeStream>>>, l: usize) -> Resu
 
 pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str, init_mode: bool) -> Result<TtrpcServer> {
     let agent_service = Box::new(AgentService {
-        sandbox: s,
+        sandbox: s.clone(),
         init_mode,
     }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
 
@@ -1696,14 +1831,20 @@ pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str, init_mode: bool) -> R
     let health_service = Box::new(HealthService {}) as Box<dyn health_ttrpc::Health + Send + Sync>;
     let health_worker = Arc::new(health_service);
 
+    let image_service =
+        Box::new(image_rpc::ImageService::new(s)) as Box<dyn image_ttrpc::Image + Send + Sync>;
+
     let aservice = agent_ttrpc::create_agent_service(agent_worker);
 
     let hservice = health_ttrpc::create_health(health_worker);
 
+    let iservice = image_ttrpc::create_image(Arc::new(image_service));
+
     let server = TtrpcServer::new()
         .bind(server_address)?
         .register_service(aservice)
-        .register_service(hservice);
+        .register_service(hservice)
+        .register_service(iservice);
 
     info!(sl!(), "ttRPC server started"; "address" => server_address);
 
@@ -1891,23 +2032,50 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
         ));
     }
 
-    let parent = path.parent();
-
-    let dir = if let Some(parent) = parent {
-        parent.to_path_buf()
-    } else {
-        PathBuf::from("/")
-    };
-
-    fs::create_dir_all(&dir).or_else(|e| {
-        if e.kind() != std::io::ErrorKind::AlreadyExists {
-            return Err(e);
+    if let Some(parent) = path.parent() {
+        if !parent.exists() {
+            let dir = parent.to_path_buf();
+            if let Err(e) = fs::create_dir_all(&dir) {
+                if e.kind() != std::io::ErrorKind::AlreadyExists {
+                    return Err(e.into());
+                }
+            } else {
+                std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(req.dir_mode))?;
+            }
         }
+    }
 
-        Ok(())
-    })?;
+    let sflag = stat::SFlag::from_bits_truncate(req.file_mode);
 
-    std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(req.dir_mode))?;
+    if sflag.contains(stat::SFlag::S_IFDIR) {
+        fs::create_dir(path.clone()).or_else(|e| {
+            if e.kind() != std::io::ErrorKind::AlreadyExists {
+                return Err(e);
+            }
+            Ok(())
+        })?;
+
+        std::fs::set_permissions(path.clone(), std::fs::Permissions::from_mode(req.file_mode))?;
+
+        unistd::chown(
+            &path,
+            Some(Uid::from_raw(req.uid as u32)),
+            Some(Gid::from_raw(req.gid as u32)),
+        )?;
+
+        return Ok(());
+    }
+
+    if sflag.contains(stat::SFlag::S_IFLNK) {
+        let src = PathBuf::from(String::from_utf8(req.data.clone()).unwrap());
+
+        unistd::symlinkat(&src, None, &path)?;
+        let path_str = CString::new(path.to_str().unwrap())?;
+        let ret = unsafe { libc::lchown(path_str.as_ptr(), req.uid as u32, req.gid as u32) };
+        Errno::result(ret).map(drop)?;
+
+        return Ok(());
+    }
 
     let mut tmpfile = path.clone();
     tmpfile.set_extension("tmp");
@@ -1974,18 +2142,26 @@ pub fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
     let spec_root_path = Path::new(&spec_root.path);
 
     let bundle_path = Path::new(CONTAINER_BASE).join(cid);
-    let config_path = bundle_path.join("config.json");
+    let config_path = bundle_path.join(CONFIG_JSON);
     let rootfs_path = bundle_path.join("rootfs");
 
-    fs::create_dir_all(&rootfs_path)?;
-    baremount(
-        spec_root_path,
-        &rootfs_path,
-        "bind",
-        MsFlags::MS_BIND,
-        "",
-        &sl!(),
-    )?;
+    let rootfs_exists = Path::new(&rootfs_path).exists();
+    info!(
+        sl!(),
+        "The rootfs_path is {:?} and exists: {}", rootfs_path, rootfs_exists
+    );
+
+    if !rootfs_exists {
+        fs::create_dir_all(&rootfs_path)?;
+        baremount(
+            spec_root_path,
+            &rootfs_path,
+            "bind",
+            MsFlags::MS_BIND,
+            "",
+            &sl!(),
+        )?;
+    }
 
     let rootfs_path_name = rootfs_path
         .to_str()
@@ -2966,4 +3142,135 @@ COMMIT
             "We should see the resulting rule"
         );
     }
+
+    #[tokio::test]
+    async fn test_merge_cwd() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            container_process_cwd: &'a str,
+            image_process_cwd: &'a str,
+            expected: &'a str,
+        }
+
+        let tests = &[
+            // Image cwd should override blank container cwd
+            // TODO - how can we tell the user didn't specifically set it to `/` vs not setting at all? Is that scenario valid?
+            TestData {
+                container_process_cwd: "/",
+                image_process_cwd: "/imageDir",
+                expected: "/imageDir",
+            },
+            // Container cwd should override image cwd
+            TestData {
+                container_process_cwd: "/containerDir",
+                image_process_cwd: "/imageDir",
+                expected: "/containerDir",
+            },
+            // Container cwd should override blank image cwd
+            TestData {
+                container_process_cwd: "/containerDir",
+                image_process_cwd: "/",
+                expected: "/containerDir",
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mut container_process = oci::Process {
+                cwd: d.container_process_cwd.to_string(),
+                ..Default::default()
+            };
+
+            let image_process = oci::Process {
+                cwd: d.image_process_cwd.to_string(),
+                ..Default::default()
+            };
+
+            merge_oci_process(&mut container_process, &image_process);
+
+            assert_eq!(d.expected, container_process.cwd, "{}", msg);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_merge_env() {
+        #[derive(Debug)]
+        struct TestData {
+            container_process_env: Vec<String>,
+            image_process_env: Vec<String>,
+            expected: Vec<String>,
+        }
+
+        let tests = &[
+            // Test that the pods environment overrides the images
+            TestData {
+                container_process_env: vec!["ISPRODUCTION=true".to_string()],
+                image_process_env: vec!["ISPRODUCTION=false".to_string()],
+                expected: vec!["ISPRODUCTION=true".to_string()],
+            },
+            // Test that multiple environment variables can be overrided
+            TestData {
+                container_process_env: vec![
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=false".to_string(),
+                ],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=false".to_string(),
+                ],
+            },
+            // Test that when none of the variables match do not override them
+            TestData {
+                container_process_env: vec!["ANOTHERENV=TEST".to_string()],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+            },
+            // Test a mix of both overriding and not
+            TestData {
+                container_process_env: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=true".to_string(),
+                ],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mut container_process = oci::Process {
+                env: d.container_process_env.clone(),
+                ..Default::default()
+            };
+
+            let image_process = oci::Process {
+                env: d.image_process_env.clone(),
+                ..Default::default()
+            };
+
+            merge_oci_process(&mut container_process, &image_process);
+
+            assert_eq!(d.expected, container_process.env, "{}", msg);
+        }
+    }
 }

src/agent/src/sandbox.rs

@@ -60,6 +60,7 @@ pub struct Sandbox {
     pub event_tx: Option<Sender<String>>,
     pub bind_watcher: BindWatcher,
     pub pcimap: HashMap<pci::Address, pci::Address>,
+    pub images: HashMap<String, String>,
 }
 
 impl Sandbox {
@@ -93,6 +94,7 @@ impl Sandbox {
             event_tx: Some(tx),
             bind_watcher: BindWatcher::new(),
             pcimap: HashMap::new(),
+            images: HashMap::new(),
         })
     }
 

src/agent/src/signal.rs

@@ -24,7 +24,7 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
     loop {
         // Avoid reaping the undesirable child's signal, e.g., execute_hook's
         // The lock should be released immediately.
-        let _locker = rustjail::container::WAIT_PID_LOCKER.lock().await;
+        let _ = rustjail::container::WAIT_PID_LOCKER.lock().await;
         let result = wait::waitpid(
             Some(Pid::from_raw(-1)),
             Some(WaitPidFlag::WNOHANG | WaitPidFlag::__WALL),

src/agent/vsock-exporter/Cargo.toml

@@ -18,4 +18,4 @@ bincode = "1.3.3"
 byteorder = "1.4.3"
 slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
 async-trait = "0.1.50"
-tokio = "1.28.1"
+tokio = "1.2.0"

src/dragonball/Cargo.toml

@@ -19,11 +19,11 @@ dbs-boot = "0.3.0"
 dbs-device = "0.2.0"
 dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] }
 dbs-legacy-devices = "0.1.0"
-dbs-upcall = { version = "0.1.0", optional = true }
+dbs-upcall = { version = "0.2.0", optional = true }
 dbs-utils = "0.2.0"
-dbs-virtio-devices = { version = "0.1.0", optional = true, features = ["virtio-mmio"] }
-kvm-bindings = "0.5.0"
-kvm-ioctls = "0.11.0"
+dbs-virtio-devices = { version = "0.2.0", optional = true, features = ["virtio-mmio"] }
+kvm-bindings = "0.6.0"
+kvm-ioctls = "0.12.0"
 lazy_static = "1.2"
 libc = "0.2.39"
 linux-loader = "0.6.0"
@@ -37,8 +37,9 @@ slog = "2.5.2"
 slog-scope = "4.4.0"
 thiserror = "1"
 vmm-sys-util = "0.11.0"
-virtio-queue = { version = "0.4.0", optional = true }
+virtio-queue = { version = "0.6.0", optional = true }
 vm-memory = { version = "0.9.0", features = ["backend-mmap"] }
+crossbeam-channel = "0.5.6"
 
 [dev-dependencies]
 slog-term = "2.9.0"
@@ -47,7 +48,7 @@ test-utils = { path = "../libs/test-utils" }
 
 [features]
 acpi = []
-atomic-guest-memory = [ "vm-memory/backend-atomic" ]
+atomic-guest-memory = ["vm-memory/backend-atomic"]
 hotplug = ["virtio-vsock"]
 virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"]
 virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]

src/dragonball/src/api/v1/instance_info.rs

@@ -5,6 +5,15 @@
 
 use serde_derive::{Deserialize, Serialize};
 
+/// This struct represents the strongly typed equivalent of the json body
+/// from confidential container related requests.
+#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub enum ConfidentialVmType {
+    /// Intel Trusted Domain
+    TDX = 2,
+}
+
 /// The microvm state.
 ///
 /// When Dragonball starts, the instance state is Uninitialized. Once start_microvm method is
@@ -50,27 +59,38 @@ pub struct InstanceInfo {
     pub vmm_version: String,
     /// The pid of the current VMM process.
     pub pid: u32,
+    /// The tid of the current VMM master thread.
+    pub master_tid: u32,
     /// The state of async actions.
     pub async_state: AsyncState,
     /// List of tids of vcpu threads (vcpu index, tid)
     pub tids: Vec<(u8, u32)>,
     /// Last instance downtime
     pub last_instance_downtime: u64,
+    /// confidential vm type
+    pub confidential_vm_type: Option<ConfidentialVmType>,
 }
 
 impl InstanceInfo {
-    /// create instance info object with given id, version, and platform type
+    /// create instance info object with given id, version, platform type and confidential vm type.
     pub fn new(id: String, vmm_version: String) -> Self {
         InstanceInfo {
             id,
             state: InstanceState::Uninitialized,
             vmm_version,
             pid: std::process::id(),
+            master_tid: 0,
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
+            confidential_vm_type: None,
         }
     }
+
+    /// return true if VM confidential type is TDX
+    pub fn is_tdx_enabled(&self) -> bool {
+        matches!(self.confidential_vm_type, Some(ConfidentialVmType::TDX))
+    }
 }
 
 impl Default for InstanceInfo {
@@ -80,9 +100,11 @@ impl Default for InstanceInfo {
             state: InstanceState::Uninitialized,
             vmm_version: env!("CARGO_PKG_VERSION").to_string(),
             pid: std::process::id(),
+            master_tid: 0,
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
+            confidential_vm_type: None,
         }
     }
 }

src/dragonball/src/api/v1/mod.rs

@@ -12,7 +12,7 @@ pub use self::boot_source::{BootSourceConfig, BootSourceConfigError, DEFAULT_KER
 
 /// Wrapper over the microVM general information.
 mod instance_info;
-pub use self::instance_info::{InstanceInfo, InstanceState};
+pub use self::instance_info::{ConfidentialVmType, InstanceInfo, InstanceState};
 
 /// Wrapper for configuring the memory and CPU of the microVM.
 mod machine_config;

src/dragonball/src/api/v1/vmm_action.rs

@@ -7,8 +7,8 @@
 // found in the THIRD-PARTY file.
 
 use std::fs::File;
-use std::sync::mpsc::{Receiver, Sender, TryRecvError};
 
+use crossbeam_channel::{Receiver, Sender, TryRecvError};
 use log::{debug, error, info, warn};
 
 use crate::error::{Result, StartMicroVmError, StopMicrovmError};
@@ -676,9 +676,9 @@ fn handle_cpu_topology(
 
 #[cfg(test)]
 mod tests {
-    use std::sync::mpsc::channel;
     use std::sync::{Arc, Mutex};
 
+    use crossbeam_channel::unbounded;
     use dbs_utils::epoll_manager::EpollManager;
     use test_utils::skip_if_not_root;
     use vmm_sys_util::tempfile::TempFile;
@@ -702,8 +702,8 @@ mod tests {
         }
 
         fn check_request(&mut self) {
-            let (to_vmm, from_api) = channel();
-            let (to_api, from_vmm) = channel();
+            let (to_vmm, from_api) = unbounded();
+            let (to_api, from_vmm) = unbounded();
 
             let epoll_mgr = EpollManager::default();
             let vmm = Arc::new(Mutex::new(create_vmm_instance(epoll_mgr.clone())));
@@ -728,8 +728,8 @@ mod tests {
     fn test_vmm_action_receive_unknown() {
         skip_if_not_root!();
 
-        let (_to_vmm, from_api) = channel();
-        let (to_api, _from_vmm) = channel();
+        let (_to_vmm, from_api) = unbounded();
+        let (to_api, _from_vmm) = unbounded();
         let epoll_mgr = EpollManager::default();
         let vmm = Arc::new(Mutex::new(create_vmm_instance(epoll_mgr.clone())));
         let mut vservice = VmmService::new(from_api, to_api);
@@ -742,8 +742,8 @@ mod tests {
     #[should_panic]
     #[test]
     fn test_vmm_action_disconnected() {
-        let (to_vmm, from_api) = channel();
-        let (to_api, _from_vmm) = channel();
+        let (to_vmm, from_api) = unbounded();
+        let (to_api, _from_vmm) = unbounded();
         let epoll_mgr = EpollManager::default();
         let vmm = Arc::new(Mutex::new(create_vmm_instance(epoll_mgr.clone())));
         let mut vservice = VmmService::new(from_api, to_api);

src/dragonball/src/config_manager.rs

@@ -231,7 +231,7 @@ where
                         info.config.check_conflicts(config)?;
                     }
                 }
-                self.info_list[index] = device_info;
+                self.info_list[index].config = config.clone();
                 index
             }
             None => {

src/dragonball/src/device_manager/blk_dev_mgr.rs

@@ -776,3 +776,587 @@ impl Default for BlockDeviceMgr {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use test_utils::skip_if_not_root;
+    use vmm_sys_util::tempfile::TempFile;
+
+    use super::*;
+    use crate::test_utils::tests::create_vm_for_test;
+
+    #[test]
+    fn test_block_device_type() {
+        let dev_type = BlockDeviceType::get_type("spool:/device1");
+        assert_eq!(dev_type, BlockDeviceType::Spool);
+        let dev_type = BlockDeviceType::get_type("/device1");
+        assert_eq!(dev_type, BlockDeviceType::RawBlock);
+    }
+
+    #[test]
+    fn test_create_block_devices_configs() {
+        let mgr = BlockDeviceMgr::default();
+        assert!(!mgr.has_root_block_device());
+        assert!(!mgr.has_part_uuid_root());
+        assert!(!mgr.is_read_only_root());
+        assert_eq!(mgr.get_index_of_drive_id(""), None);
+        assert_eq!(mgr.info_list.len(), 0);
+    }
+
+    #[test]
+    fn test_add_non_root_block_device() {
+        skip_if_not_root!();
+        let dummy_file = TempFile::new().unwrap();
+        let dummy_path = dummy_file.as_path().to_owned();
+        let dummy_id = String::from("1");
+        let dummy_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path.clone(),
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: dummy_id.clone(),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        assert!(BlockDeviceMgr::insert_device(
+            vm.device_manager_mut(),
+            ctx,
+            dummy_block_device.clone(),
+        )
+        .is_ok());
+
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
+        assert!(!vm.device_manager().block_manager.has_root_block_device());
+        assert!(!vm.device_manager().block_manager.has_part_uuid_root());
+        assert!(!vm.device_manager().block_manager.is_read_only_root());
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
+        assert_eq!(
+            vm.device_manager().block_manager.info_list[0]
+                .config
+                .device_type(),
+            BlockDeviceType::RawBlock
+        );
+        assert_eq!(
+            vm.device_manager().block_manager.info_list[0]
+                .config
+                .queue_sizes(),
+            [128u16]
+        );
+
+        let dev_config = vm.device_manager().block_manager.iter().next().unwrap();
+        assert_eq!(dev_config.config, dummy_block_device);
+        assert!(vm
+            .device_manager()
+            .block_manager
+            .get_index_of_drive_path(&dummy_path)
+            .is_some());
+        assert!(vm
+            .device_manager()
+            .block_manager
+            .get_index_of_drive_id(&dummy_id)
+            .is_some());
+    }
+
+    #[test]
+    fn test_update_blk_device_ratelimiters() {
+        skip_if_not_root!();
+        //Init vm for test.
+        let mut vm = create_vm_for_test();
+        let device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            None,
+            false,
+        );
+
+        let dummy_file = TempFile::new().unwrap();
+        let dummy_path = dummy_file.as_path().to_owned();
+
+        let dummy_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: true,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), device_op_ctx, dummy_block_device)
+            .unwrap();
+
+        let cfg = BlockDeviceConfigUpdateInfo {
+            drive_id: String::from("1"),
+            rate_limiter: None,
+        };
+
+        let mut device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            None,
+            false,
+        );
+
+        vm.device_manager_mut()
+            .block_manager
+            .attach_devices(&mut device_op_ctx)
+            .unwrap();
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
+
+        //Patch while the epoll handler is invalid.
+        let expected_error = "could not send patch message to the block epoll handler".to_string();
+
+        assert_eq!(
+            BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg)
+                .unwrap_err()
+                .to_string(),
+            expected_error
+        );
+
+        //Invalid drive id
+        let cfg2 = BlockDeviceConfigUpdateInfo {
+            drive_id: String::from("2"),
+            rate_limiter: None,
+        };
+
+        let expected_error = format!("invalid block device id '{0}'", cfg2.drive_id);
+
+        assert_eq!(
+            BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg2)
+                .unwrap_err()
+                .to_string(),
+            expected_error
+        );
+    }
+
+    #[test]
+    fn test_add_one_root_block_device() {
+        skip_if_not_root!();
+        let dummy_file = TempFile::new().unwrap();
+        let dummy_path = dummy_file.as_path().to_owned();
+        let dummy_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: true,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        assert!(BlockDeviceMgr::insert_device(
+            vm.device_manager_mut(),
+            ctx,
+            dummy_block_device.clone(),
+        )
+        .is_ok());
+
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
+        assert!(vm.device_manager().block_manager.has_root_block);
+        assert!(!vm.device_manager().block_manager.has_part_uuid_root);
+        assert!(vm.device_manager().block_manager.read_only_root);
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
+
+        let dev_config = vm.device_manager().block_manager.iter().next().unwrap();
+        assert_eq!(dev_config.config, dummy_block_device);
+        assert!(vm.device_manager().block_manager.is_read_only_root());
+    }
+
+    #[test]
+    fn test_add_two_root_block_devices_configs() {
+        skip_if_not_root!();
+        let dummy_file_1 = TempFile::new().unwrap();
+        let dummy_path_1 = dummy_file_1.as_path().to_owned();
+        let root_block_device_1 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_1,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_2 = TempFile::new().unwrap();
+        let dummy_path_2 = dummy_file_2.as_path().to_owned();
+        let root_block_device_2 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_2,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("2"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_1).unwrap();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        assert!(
+            BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_2)
+                .is_err()
+        );
+    }
+
+    #[test]
+    // Test BlockDevicesConfigs::add when you first add the root device and then the other devices.
+    fn test_add_root_block_device_first() {
+        skip_if_not_root!();
+        let dummy_file_1 = TempFile::new().unwrap();
+        let dummy_path_1 = dummy_file_1.as_path().to_owned();
+        let root_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_1,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_2 = TempFile::new().unwrap();
+        let dummy_path_2 = dummy_file_2.as_path().to_owned();
+        let dummy_block_device_2 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_2,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("2"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_3 = TempFile::new().unwrap();
+        let dummy_path_3 = dummy_file_3.as_path().to_owned();
+        let dummy_block_device_3 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_3,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("3"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+        vm.device_manager_mut()
+            .block_manager
+            .create(root_block_device.clone())
+            .unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .create(dummy_block_device_2.clone())
+            .unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .create(dummy_block_device_3.clone())
+            .unwrap();
+
+        assert!(vm.device_manager().block_manager.has_root_block_device(),);
+        assert!(!vm.device_manager().block_manager.has_part_uuid_root());
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 3);
+
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap();
+
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2).unwrap();
+
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3).unwrap();
+    }
+
+    #[test]
+    // Test BlockDevicesConfigs::add when you add other devices first and then the root device.
+    fn test_root_block_device_add_last() {
+        skip_if_not_root!();
+        let dummy_file_1 = TempFile::new().unwrap();
+        let dummy_path_1 = dummy_file_1.as_path().to_owned();
+        let root_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_1,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_2 = TempFile::new().unwrap();
+        let dummy_path_2 = dummy_file_2.as_path().to_owned();
+        let dummy_block_device_2 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_2,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("2"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_3 = TempFile::new().unwrap();
+        let dummy_path_3 = dummy_file_3.as_path().to_owned();
+        let dummy_block_device_3 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_3,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("3"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+            .unwrap();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3.clone())
+            .unwrap();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device.clone())
+            .unwrap();
+
+        assert!(vm.device_manager().block_manager.has_root_block_device(),);
+        assert!(!vm.device_manager().block_manager.has_part_uuid_root());
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 3);
+
+        let mut block_dev_iter = vm.device_manager().block_manager.iter();
+        // The root device should be first in the list no matter of the order in
+        // which the devices were added.
+        assert_eq!(
+            block_dev_iter.next().unwrap().config.drive_id,
+            root_block_device.drive_id
+        );
+        assert_eq!(
+            block_dev_iter.next().unwrap().config.drive_id,
+            dummy_block_device_2.drive_id
+        );
+        assert_eq!(
+            block_dev_iter.next().unwrap().config.drive_id,
+            dummy_block_device_3.drive_id
+        );
+    }
+
+    #[test]
+    fn test_block_device_update() {
+        skip_if_not_root!();
+        let dummy_file_1 = TempFile::new().unwrap();
+        let dummy_path_1 = dummy_file_1.as_path().to_owned();
+        let root_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_1.clone(),
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_2 = TempFile::new().unwrap();
+        let dummy_path_2 = dummy_file_2.as_path().to_owned();
+        let mut dummy_block_device_2 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_2.clone(),
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("2"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+
+        // Add 2 block devices.
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+            .unwrap();
+
+        // Get index zero.
+        assert_eq!(
+            vm.device_manager()
+                .block_manager
+                .get_index_of_drive_id(&String::from("1"))
+                .unwrap(),
+            0
+        );
+
+        // Get None.
+        assert!(vm
+            .device_manager()
+            .block_manager
+            .get_index_of_drive_id(&String::from("foo"))
+            .is_none());
+
+        // Test several update cases using dummy_block_device_2.
+        // Validate `dummy_block_device_2` is already in the list
+        assert!(vm
+            .device_manager()
+            .block_manager
+            .get_index_of_drive_id(&dummy_block_device_2.drive_id)
+            .is_some());
+        // Update OK.
+        dummy_block_device_2.is_read_only = true;
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+            .unwrap();
+
+        let index = vm
+            .device_manager()
+            .block_manager
+            .get_index_of_drive_id(&dummy_block_device_2.drive_id)
+            .unwrap();
+        // Validate update was successful.
+        assert!(
+            vm.device_manager().block_manager.info_list[index]
+                .config
+                .is_read_only
+        );
+
+        // Update with invalid path.
+        let dummy_filename_3 = String::from("test_update_3");
+        let dummy_path_3 = PathBuf::from(dummy_filename_3);
+        dummy_block_device_2.path_on_host = dummy_path_3;
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        assert!(BlockDeviceMgr::insert_device(
+            vm.device_manager_mut(),
+            ctx,
+            dummy_block_device_2.clone(),
+        )
+        .is_err());
+
+        // Update with 2 root block devices.
+        dummy_block_device_2.path_on_host = dummy_path_2.clone();
+        dummy_block_device_2.is_root_device = true;
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        assert!(
+            BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2,)
+                .is_err(),
+        );
+
+        // Switch roots and add a PARTUUID for the new one.
+        let root_block_device_old = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_1,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+        let root_block_device_new = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_2,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: Some("0eaa91a0-01".to_string()),
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("2"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_old).unwrap();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_new).unwrap();
+        assert!(vm.device_manager().block_manager.has_part_uuid_root);
+    }
+}

src/dragonball/src/device_manager/mod.rs

@@ -147,17 +147,13 @@ pub type Result<T> = ::std::result::Result<T, DeviceMgrError>;
 /// Type of the dragonball virtio devices.
 #[cfg(feature = "dbs-virtio-devices")]
 pub type DbsVirtioDevice = Box<
-    dyn VirtioDevice<
-        GuestAddressSpaceImpl,
-        virtio_queue::QueueStateSync,
-        vm_memory::GuestRegionMmap,
-    >,
+    dyn VirtioDevice<GuestAddressSpaceImpl, virtio_queue::QueueSync, vm_memory::GuestRegionMmap>,
 >;
 
 /// Type of the dragonball virtio mmio devices.
 #[cfg(feature = "dbs-virtio-devices")]
 pub type DbsMmioV2Device =
-    MmioV2Device<GuestAddressSpaceImpl, virtio_queue::QueueStateSync, vm_memory::GuestRegionMmap>;
+    MmioV2Device<GuestAddressSpaceImpl, virtio_queue::QueueSync, vm_memory::GuestRegionMmap>;
 
 /// Struct to support transactional operations for device management.
 pub struct DeviceManagerTx {

src/dragonball/src/error.rs

@@ -9,6 +9,8 @@
 
 //! Error codes for the virtual machine monitor subsystem.
 
+#[cfg(target_arch = "aarch64")]
+use dbs_arch::pmu::PmuError;
 #[cfg(feature = "dbs-virtio-devices")]
 use dbs_virtio_devices::Error as VirtIoError;
 
@@ -61,6 +63,11 @@ pub enum Error {
     #[error("failed to write MP table to guest memory: {0}")]
     MpTableSetup(#[source] dbs_boot::mptable::Error),
 
+    /// Create pmu device error
+    #[cfg(target_arch = "aarch64")]
+    #[error("Create pmu device error: {0}")]
+    PmuDeviceError(#[source] PmuError),
+
     /// Fail to boot system
     #[error("failed to boot system: {0}")]
     BootSystem(#[source] dbs_boot::Error),
@@ -68,6 +75,10 @@ pub enum Error {
     /// Cannot open the VM file descriptor.
     #[error(transparent)]
     Vm(vm::VmError),
+
+    /// confidential vm type Error
+    #[error("confidential-vm-type can only be used in x86_64 now")]
+    ConfidentialVmType,
 }
 
 /// Errors associated with starting the instance.

src/dragonball/src/kvm_context.rs

@@ -10,7 +10,7 @@ use kvm_bindings::KVM_API_VERSION;
 use kvm_ioctls::{Cap, Kvm, VmFd};
 use std::os::unix::io::{FromRawFd, RawFd};
 
-use crate::error::{Error, Result};
+use crate::error::{Error as VmError, Result};
 
 /// Describes a KVM context that gets attached to the micro VM instance.
 /// It gives access to the functionality of the KVM wrapper as long as every required
@@ -29,11 +29,11 @@ impl KvmContext {
             // Safe because we expect kvm_fd to contain a valid fd number when is_some() == true.
             unsafe { Kvm::from_raw_fd(fd) }
         } else {
-            Kvm::new().map_err(Error::Kvm)?
+            Kvm::new().map_err(VmError::Kvm)?
         };
 
         if kvm.get_api_version() != KVM_API_VERSION as i32 {
-            return Err(Error::KvmApiVersion(kvm.get_api_version()));
+            return Err(VmError::KvmApiVersion(kvm.get_api_version()));
         }
 
         Self::check_cap(&kvm, Cap::Irqchip)?;
@@ -44,7 +44,8 @@ impl KvmContext {
         Self::check_cap(&kvm, Cap::SetTssAddr)?;
 
         #[cfg(target_arch = "x86_64")]
-        let supported_msrs = dbs_arch::msr::supported_guest_msrs(&kvm).map_err(Error::GuestMSRs)?;
+        let supported_msrs =
+            dbs_arch::msr::supported_guest_msrs(&kvm).map_err(VmError::GuestMSRs)?;
         let max_memslots = kvm.get_nr_memslots();
 
         Ok(KvmContext {
@@ -67,7 +68,7 @@ impl KvmContext {
 
     /// Create a virtual machine object.
     pub fn create_vm(&self) -> Result<VmFd> {
-        self.kvm.create_vm().map_err(Error::Kvm)
+        self.kvm.create_vm().map_err(VmError::Kvm)
     }
 
     /// Get the max vcpu count supported by kvm
@@ -75,9 +76,9 @@ impl KvmContext {
         self.kvm.get_max_vcpus()
     }
 
-    fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), Error> {
+    fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), VmError> {
         if !kvm.check_extension(cap) {
-            return Err(Error::KvmCap(cap));
+            return Err(VmError::KvmCap(cap));
         }
         Ok(())
     }
@@ -91,6 +92,18 @@ mod x86_64 {
     use std::collections::HashSet;
 
     impl KvmContext {
+        /// Create a virtual machine object with specific type.
+        /// vm_type: u64
+        ///      0: legacy vm
+        ///      2: tdx vm
+        pub fn create_vm_with_type(&self, vm_type: u64) -> Result<VmFd> {
+            let fd = self
+                .kvm
+                .create_vm_with_type(vm_type)
+                .map_err(VmError::Kvm)?;
+            Ok(fd)
+        }
+
         /// Get information about supported CPUID of x86 processor.
         pub fn supported_cpuid(
             &self,
@@ -110,7 +123,7 @@ mod x86_64 {
         // It's very sensible to manipulate MSRs, so please be careful to change code below.
         fn build_msrs_list(kvm: &Kvm) -> Result<Msrs> {
             let mut mset: HashSet<u32> = HashSet::new();
-            let supported_msr_list = kvm.get_msr_index_list().map_err(super::Error::Kvm)?;
+            let supported_msr_list = kvm.get_msr_index_list().map_err(VmError::Kvm)?;
             for msr in supported_msr_list.as_slice() {
                 mset.insert(*msr);
             }
@@ -203,7 +216,7 @@ mod x86_64 {
                 })
                 .collect();
 
-            Msrs::from_entries(&msrs).map_err(super::Error::Msr)
+            Msrs::from_entries(&msrs).map_err(VmError::Msr)
         }
     }
 }
@@ -257,4 +270,20 @@ mod tests {
 
         let _ = c.create_vm().unwrap();
     }
+
+    #[test]
+    fn test_create_vm_with_type() {
+        let c = KvmContext::new(None).unwrap();
+        #[cfg(not(target_arch = "aarch64"))]
+        let _ = c.create_vm_with_type(0_u64).unwrap();
+        #[cfg(target_arch = "aarch64")]
+        {
+            /// aarch64 is using ipa_size to create vm
+            let mut ipa_size = 0; // Create using default VM type
+            if c.check_extension(kvm_ioctls::Cap::ArmVmIPASize) {
+                ipa_size = c.kvm.get_host_ipa_limit();
+            }
+            let _ = c.create_vm_with_type(ipa_size as u64).unwrap();
+        }
+    }
 }

src/dragonball/src/vcpu/aarch64.rs

@@ -11,7 +11,7 @@ use std::sync::mpsc::{channel, Sender};
 use std::sync::Arc;
 
 use crate::IoManagerCached;
-use dbs_arch::regs;
+use dbs_arch::{regs, VpmuFeatureLevel};
 use dbs_boot::get_fdt_addr;
 use dbs_utils::time::TimestampUs;
 use kvm_ioctls::{VcpuFd, VmFd};
@@ -81,7 +81,7 @@ impl Vcpu {
     /// * `_pgtable_addr` - pgtable address for ap vcpu (not used in aarch64)
     pub fn configure(
         &mut self,
-        _vcpu_config: &VcpuConfig,
+        vcpu_config: &VcpuConfig,
         vm_fd: &VmFd,
         vm_as: &GuestAddressSpaceImpl,
         kernel_load_addr: Option<GuestAddress>,
@@ -99,6 +99,9 @@ impl Vcpu {
         if self.id > 0 {
             kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF;
         }
+        if vcpu_config.vpmu_feature == VpmuFeatureLevel::FullyEnabled {
+            kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PMU_V3;
+        }
 
         self.fd.vcpu_init(&kvi).map_err(VcpuError::VcpuArmInit)?;
 

src/dragonball/src/vcpu/mod.rs

@@ -7,9 +7,7 @@ mod sm;
 mod vcpu_impl;
 mod vcpu_manager;
 
-#[cfg(target_arch = "x86_64")]
-use dbs_arch::cpuid::VpmuFeatureLevel;
-
+use dbs_arch::VpmuFeatureLevel;
 pub use vcpu_manager::{VcpuManager, VcpuManagerError, VcpuResizeInfo};
 
 #[cfg(feature = "hotplug")]
@@ -32,6 +30,6 @@ pub struct VcpuConfig {
     /// if vpmu feature is Disabled, it means vpmu feature is off (by default)
     /// if vpmu feature is LimitedlyEnabled, it means minimal vpmu counters are supported (cycles and instructions)
     /// if vpmu feature is FullyEnabled, it means all vpmu counters are supported
-    #[cfg(target_arch = "x86_64")]
+    /// For aarch64, VpmuFeatureLevel only supports Disabled and FullyEnabled.
     pub vpmu_feature: VpmuFeatureLevel,
 }

src/dragonball/src/vcpu/vcpu_impl.rs

@@ -441,75 +441,77 @@ impl Vcpu {
     /// Returns error or enum specifying whether emulation was handled or interrupted.
     fn run_emulation(&mut self) -> Result<VcpuEmulation> {
         match Vcpu::emulate(&self.fd) {
-            Ok(run) => match run {
-                #[cfg(target_arch = "x86_64")]
-                VcpuExit::IoIn(addr, data) => {
-                    let _ = self.io_mgr.pio_read(addr, data);
-                    METRICS.vcpu.exit_io_in.inc();
-                    Ok(VcpuEmulation::Handled)
-                }
-                #[cfg(target_arch = "x86_64")]
-                VcpuExit::IoOut(addr, data) => {
-                    if !self.check_io_port_info(addr, data)? {
-                        let _ = self.io_mgr.pio_write(addr, data);
+            Ok(run) => {
+                match run {
+                    #[cfg(target_arch = "x86_64")]
+                    VcpuExit::IoIn(addr, data) => {
+                        let _ = self.io_mgr.pio_read(addr, data);
+                        METRICS.vcpu.exit_io_in.inc();
+                        Ok(VcpuEmulation::Handled)
                     }
-                    METRICS.vcpu.exit_io_out.inc();
-                    Ok(VcpuEmulation::Handled)
-                }
-                VcpuExit::MmioRead(addr, data) => {
-                    let _ = self.io_mgr.mmio_read(addr, data);
-                    METRICS.vcpu.exit_mmio_read.inc();
-                    Ok(VcpuEmulation::Handled)
-                }
-                VcpuExit::MmioWrite(addr, data) => {
-                    let _ = self.io_mgr.mmio_write(addr, data);
-                    METRICS.vcpu.exit_mmio_write.inc();
-                    Ok(VcpuEmulation::Handled)
-                }
-                VcpuExit::Hlt => {
-                    info!("Received KVM_EXIT_HLT signal");
-                    Err(VcpuError::VcpuUnhandledKvmExit)
-                }
-                VcpuExit::Shutdown => {
-                    info!("Received KVM_EXIT_SHUTDOWN signal");
-                    Err(VcpuError::VcpuUnhandledKvmExit)
-                }
-                // Documentation specifies that below kvm exits are considered errors.
-                VcpuExit::FailEntry => {
-                    METRICS.vcpu.failures.inc();
-                    error!("Received KVM_EXIT_FAIL_ENTRY signal");
-                    Err(VcpuError::VcpuUnhandledKvmExit)
-                }
-                VcpuExit::InternalError => {
-                    METRICS.vcpu.failures.inc();
-                    error!("Received KVM_EXIT_INTERNAL_ERROR signal");
-                    Err(VcpuError::VcpuUnhandledKvmExit)
-                }
-                VcpuExit::SystemEvent(event_type, event_flags) => match event_type {
-                    KVM_SYSTEM_EVENT_RESET | KVM_SYSTEM_EVENT_SHUTDOWN => {
-                        info!(
-                            "Received KVM_SYSTEM_EVENT: type: {}, event: {}",
-                            event_type, event_flags
-                        );
-                        Ok(VcpuEmulation::Stopped)
+                    #[cfg(target_arch = "x86_64")]
+                    VcpuExit::IoOut(addr, data) => {
+                        if !self.check_io_port_info(addr, data)? {
+                            let _ = self.io_mgr.pio_write(addr, data);
+                        }
+                        METRICS.vcpu.exit_io_out.inc();
+                        Ok(VcpuEmulation::Handled)
                     }
-                    _ => {
-                        METRICS.vcpu.failures.inc();
-                        error!(
-                            "Received KVM_SYSTEM_EVENT signal type: {}, flag: {}",
-                            event_type, event_flags
-                        );
+                    VcpuExit::MmioRead(addr, data) => {
+                        let _ = self.io_mgr.mmio_read(addr, data);
+                        METRICS.vcpu.exit_mmio_read.inc();
+                        Ok(VcpuEmulation::Handled)
+                    }
+                    VcpuExit::MmioWrite(addr, data) => {
+                        let _ = self.io_mgr.mmio_write(addr, data);
+                        METRICS.vcpu.exit_mmio_write.inc();
+                        Ok(VcpuEmulation::Handled)
+                    }
+                    VcpuExit::Hlt => {
+                        info!("Received KVM_EXIT_HLT signal");
+                        Err(VcpuError::VcpuUnhandledKvmExit)
+                    }
+                    VcpuExit::Shutdown => {
+                        info!("Received KVM_EXIT_SHUTDOWN signal");
+                        Err(VcpuError::VcpuUnhandledKvmExit)
+                    }
+                    // Documentation specifies that below kvm exits are considered errors.
+                    VcpuExit::FailEntry(reason, cpu) => {
+                        METRICS.vcpu.failures.inc();
+                        error!("Received KVM_EXIT_FAIL_ENTRY signal, reason {reason}, cpu number {cpu}");
+                        Err(VcpuError::VcpuUnhandledKvmExit)
+                    }
+                    VcpuExit::InternalError => {
+                        METRICS.vcpu.failures.inc();
+                        error!("Received KVM_EXIT_INTERNAL_ERROR signal");
+                        Err(VcpuError::VcpuUnhandledKvmExit)
+                    }
+                    VcpuExit::SystemEvent(event_type, event_flags) => match event_type {
+                        KVM_SYSTEM_EVENT_RESET | KVM_SYSTEM_EVENT_SHUTDOWN => {
+                            info!(
+                                "Received KVM_SYSTEM_EVENT: type: {}, event: {}",
+                                event_type, event_flags
+                            );
+                            Ok(VcpuEmulation::Stopped)
+                        }
+                        _ => {
+                            METRICS.vcpu.failures.inc();
+                            error!(
+                                "Received KVM_SYSTEM_EVENT signal type: {}, flag: {}",
+                                event_type, event_flags
+                            );
+                            Err(VcpuError::VcpuUnhandledKvmExit)
+                        }
+                    },
+                    r => {
+                        METRICS.vcpu.failures.inc();
+                        // TODO: Are we sure we want to finish running a vcpu upon
+                        // receiving a vm exit that is not necessarily an error?
+                        error!("Unexpected exit reason on vcpu run: {:?}", r);
                         Err(VcpuError::VcpuUnhandledKvmExit)
                     }
-                },
-                r => {
-                    METRICS.vcpu.failures.inc();
-                    // TODO: Are we sure we want to finish running a vcpu upon
-                    // receiving a vm exit that is not necessarily an error?
-                    error!("Unexpected exit reason on vcpu run: {:?}", r);
-                    Err(VcpuError::VcpuUnhandledKvmExit)
                 }
-            },
+            }
             // The unwrap on raw_os_error can only fail if we have a logic
             // error in our code in which case it is better to panic.
             Err(ref e) => {
@@ -758,6 +760,11 @@ impl Vcpu {
         // State machine reached its end.
         StateMachine::finish(Self::exited)
     }
+
+    /// Get vcpu file descriptor.
+    pub fn vcpu_fd(&self) -> &VcpuFd {
+        self.fd.as_ref()
+    }
 }
 
 impl Drop for Vcpu {
@@ -786,7 +793,7 @@ pub mod tests {
         MmioWrite,
         Hlt,
         Shutdown,
-        FailEntry,
+        FailEntry(u64, u32),
         InternalError,
         Unknown,
         SystemEvent(u32, u64),
@@ -807,7 +814,9 @@ pub mod tests {
                 EmulationCase::MmioWrite => Ok(VcpuExit::MmioWrite(0, &[])),
                 EmulationCase::Hlt => Ok(VcpuExit::Hlt),
                 EmulationCase::Shutdown => Ok(VcpuExit::Shutdown),
-                EmulationCase::FailEntry => Ok(VcpuExit::FailEntry),
+                EmulationCase::FailEntry(error_type, cpu_num) => {
+                    Ok(VcpuExit::FailEntry(*error_type, *cpu_num))
+                }
                 EmulationCase::InternalError => Ok(VcpuExit::InternalError),
                 EmulationCase::Unknown => Ok(VcpuExit::Unknown),
                 EmulationCase::SystemEvent(event_type, event_flags) => {
@@ -850,6 +859,8 @@ pub mod tests {
 
     #[cfg(target_arch = "aarch64")]
     fn create_vcpu() -> (Vcpu, Receiver<VcpuStateEvent>) {
+        use kvm_ioctls::Kvm;
+        use std::os::fd::AsRawFd;
         // Call for kvm too frequently would cause error in some host kernel.
         std::thread::sleep(std::time::Duration::from_millis(5));
 
@@ -918,7 +929,7 @@ pub mod tests {
         assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit)));
 
         // KVM_EXIT_FAIL_ENTRY signal
-        *(EMULATE_RES.lock().unwrap()) = EmulationCase::FailEntry;
+        *(EMULATE_RES.lock().unwrap()) = EmulationCase::FailEntry(0, 0);
         let res = vcpu.run_emulation();
         assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit)));
 

src/dragonball/src/vcpu/vcpu_manager.rs

@@ -15,6 +15,7 @@ use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender};
 use std::sync::{Arc, Barrier, Mutex, RwLock};
 use std::time::Duration;
 
+use dbs_arch::VpmuFeatureLevel;
 #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
 use dbs_upcall::{DevMgrService, UpcallClient};
 use dbs_utils::epoll_manager::{EpollManager, EventOps, EventSet, Events, MutEventSubscriber};
@@ -281,11 +282,20 @@ impl VcpuManager {
         let supported_cpuid = kvm_context
             .supported_cpuid(kvm_bindings::KVM_MAX_CPUID_ENTRIES)
             .map_err(VcpuManagerError::Kvm)?;
-        #[cfg(target_arch = "x86_64")]
+        #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
         let vpmu_feature_level = match vm_config_info.vpmu_feature {
-            1 => dbs_arch::cpuid::VpmuFeatureLevel::LimitedlyEnabled,
-            2 => dbs_arch::cpuid::VpmuFeatureLevel::FullyEnabled,
-            _ => dbs_arch::cpuid::VpmuFeatureLevel::Disabled,
+            #[cfg(target_arch = "x86_64")]
+            1 => VpmuFeatureLevel::LimitedlyEnabled,
+            #[cfg(target_arch = "aarch64")]
+            1 => {
+                log::warn!(
+                    "Limitedly enabled vpmu feature isn't supported on aarch64 for now.\
+                       This will be supported in the future. The vpmu_feature will be set disabled!"
+                );
+                VpmuFeatureLevel::Disabled
+            }
+            2 => VpmuFeatureLevel::FullyEnabled,
+            _ => VpmuFeatureLevel::Disabled,
         };
 
         let vcpu_manager = Arc::new(Mutex::new(VcpuManager {
@@ -297,7 +307,6 @@ impl VcpuManager {
                 cores_per_die: vm_config_info.cpu_topology.cores_per_die,
                 dies_per_socket: vm_config_info.cpu_topology.dies_per_socket,
                 sockets: vm_config_info.cpu_topology.sockets,
-                #[cfg(target_arch = "x86_64")]
                 vpmu_feature: vpmu_feature_level,
             },
             vcpu_seccomp_filter,
@@ -799,6 +808,11 @@ impl VcpuManager {
         )
         .map_err(VcpuManagerError::Vcpu)
     }
+
+    /// get vpmu_feature config
+    pub fn vpmu_feature(&self) -> VpmuFeatureLevel {
+        self.vcpu_config.vpmu_feature
+    }
 }
 
 #[cfg(feature = "hotplug")]
@@ -887,7 +901,9 @@ mod hotplug {
             cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]);
             let req = DevMgrRequest::AddVcpu(CpuDevRequest {
                 count: cpu_ids.len() as u8,
+                #[cfg(target_arch = "x86_64")]
                 apic_ids: cpu_ids_array,
+                #[cfg(target_arch = "x86_64")]
                 apic_ver: APIC_VERSION,
             });
             self.send_upcall_action(upcall_client, req)?;
@@ -924,7 +940,9 @@ mod hotplug {
             cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]);
             let req = DevMgrRequest::DelVcpu(CpuDevRequest {
                 count: cpu_num_to_be_del as u8,
+                #[cfg(target_arch = "x86_64")]
                 apic_ids: cpu_ids_array,
+                #[cfg(target_arch = "x86_64")]
                 apic_ver: APIC_VERSION,
             });
             self.send_upcall_action(upcall_client, req)?;
@@ -969,7 +987,10 @@ mod hotplug {
                                 vcpu_state_sender
                                     .send(VcpuStateEvent::Hotplug((
                                         result,
+                                        #[cfg(target_arch = "x86_64")]
                                         resp.info.apic_id_index,
+                                        #[cfg(target_arch = "aarch64")]
+                                        resp.info.cpu_id,
                                     )))
                                     .unwrap();
                                 vcpu_state_event.write(1).unwrap();

src/dragonball/src/vm/aarch64.rs

@@ -11,7 +11,8 @@ use std::fmt::Debug;
 use std::ops::Deref;
 
 use dbs_arch::gic::GICDevice;
-use dbs_arch::{DeviceInfoForFDT, DeviceType};
+use dbs_arch::pmu::initialize_pmu;
+use dbs_arch::{DeviceInfoForFDT, DeviceType, VpmuFeatureLevel};
 use dbs_boot::InitrdConfig;
 use dbs_utils::epoll_manager::EpollManager;
 use dbs_utils::time::TimestampUs;
@@ -43,6 +44,7 @@ fn configure_system<T: DeviceInfoForFDT + Clone + Debug, M: GuestMemory>(
     device_info: Option<&HashMap<(DeviceType, String), T>>,
     gic_device: &Box<dyn GICDevice>,
     initrd: &Option<super::InitrdConfig>,
+    vpmu_feature: &VpmuFeatureLevel,
 ) -> super::Result<()> {
     dbs_boot::fdt::create_fdt(
         guest_mem,
@@ -51,6 +53,7 @@ fn configure_system<T: DeviceInfoForFDT + Clone + Debug, M: GuestMemory>(
         device_info,
         gic_device,
         initrd,
+        vpmu_feature,
     )
     .map_err(Error::BootSystem)?;
     Ok(())
@@ -76,6 +79,23 @@ impl Vm {
         Ok(())
     }
 
+    /// Setup pmu devices for guest vm.
+    pub fn setup_pmu_devices(&mut self) -> std::result::Result<(), StartMicroVmError> {
+        let vm = self.vm_fd();
+        let mut vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?;
+        let vpmu_feature = vcpu_manager.vpmu_feature();
+        if vpmu_feature == VpmuFeatureLevel::Disabled {
+            return Ok(());
+        }
+
+        for vcpu in vcpu_manager.vcpus_mut() {
+            initialize_pmu(vm, vcpu.vcpu_fd())
+                .map_err(|e| StartMicroVmError::ConfigureVm(VmError::SetupPmu(e)))?;
+        }
+
+        Ok(())
+    }
+
     /// Initialize the virtual machine instance.
     ///
     /// It initialize the virtual machine instance by:
@@ -113,6 +133,7 @@ impl Vm {
             .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load)
             .map_err(StartMicroVmError::Vcpu)?;
         self.setup_interrupt_controller()?;
+        self.setup_pmu_devices()?;
         self.init_devices(epoll_mgr)?;
 
         Ok(())
@@ -129,6 +150,7 @@ impl Vm {
         initrd: Option<InitrdConfig>,
     ) -> std::result::Result<(), StartMicroVmError> {
         let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?;
+        let vpmu_feature = vcpu_manager.vpmu_feature();
         let vcpu_mpidr = vcpu_manager
             .vcpus()
             .into_iter()
@@ -143,6 +165,7 @@ impl Vm {
             self.device_manager.get_mmio_device_info(),
             self.get_irqchip(),
             &initrd,
+            &vpmu_feature,
         )
         .map_err(StartMicroVmError::ConfigureSystem)
     }

src/dragonball/src/vm/mod.rs

@@ -10,6 +10,8 @@ use std::sync::{Arc, Mutex, RwLock};
 use dbs_address_space::AddressSpace;
 #[cfg(target_arch = "aarch64")]
 use dbs_arch::gic::GICDevice;
+#[cfg(target_arch = "aarch64")]
+use dbs_arch::pmu::PmuError;
 use dbs_boot::InitrdConfig;
 use dbs_utils::epoll_manager::EpollManager;
 use dbs_utils::time::TimestampUs;
@@ -69,6 +71,11 @@ pub enum VmError {
     #[cfg(target_arch = "aarch64")]
     #[error("failed to configure GIC")]
     SetupGIC(GICError),
+
+    /// Cannot setup pmu device
+    #[cfg(target_arch = "aarch64")]
+    #[error("failed to setup pmu device")]
+    SetupPmu(#[source] PmuError),
 }
 
 /// Configuration information for user defined NUMA nodes.
@@ -208,7 +215,26 @@ impl Vm {
         let id = api_shared_info.read().unwrap().id.clone();
         let logger = slog_scope::logger().new(slog::o!("id" => id));
         let kvm = KvmContext::new(kvm_fd)?;
-        let vm_fd = Arc::new(kvm.create_vm()?);
+        let vm_fd = match api_shared_info
+            .as_ref()
+            .read()
+            .unwrap()
+            .confidential_vm_type
+        {
+            None => Arc::new(kvm.create_vm()?),
+            Some(confidential_vm_type) => {
+                #[cfg(not(any(target_arch = "x86_64")))]
+                {
+                    error!(
+                        "confidential-vm-type {} only can be used in x86_64",
+                        confidential_vm_type as u64
+                    );
+                    return Err(Error::ConfidentialVmType);
+                }
+                #[cfg(target_arch = "x86_64")]
+                Arc::new(kvm.create_vm_with_type(confidential_vm_type as u64)?)
+            }
+        };
         let resource_manager = Arc::new(ResourceManager::new(Some(kvm.max_memslots())));
         let device_manager = DeviceManager::new(
             vm_fd.clone(),
@@ -336,6 +362,15 @@ impl Vm {
         instance_state == InstanceState::Running
     }
 
+    /// return true if VM confidential type is TDX
+    pub fn is_tdx_enabled(&self) -> bool {
+        let shared_info = self
+            .shared_info()
+            .read()
+            .expect("failed to get instance state, because shared info is poisoned lock");
+        shared_info.is_tdx_enabled()
+    }
+
     /// Save VM instance exit state
     pub fn vm_exit(&self, exit_code: i32) {
         if let Ok(mut info) = self.shared_info.write() {
@@ -1005,6 +1040,7 @@ pub mod tests {
         assert!(vm.remove_devices().is_ok());
     }
 
+    #[cfg(target_arch = "x86_64")]
     #[test]
     fn test_run_code() {
         skip_if_not_root!();

src/libs/kata-types/src/config/agent.rs

@@ -9,14 +9,16 @@ use crate::config::{ConfigOps, TomlConfig};
 
 pub use vendor::AgentVendor;
 
-use super::default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT};
+use super::default::{
+    DEFAULT_AGENT_DIAL_TIMEOUT_MS, DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT,
+};
 use crate::eother;
 
 /// agent name of Kata agent.
 pub const AGENT_NAME_KATA: &str = "kata";
 
 /// Kata agent configuration information.
-#[derive(Debug, Default, Deserialize, Serialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct Agent {
     /// If enabled, the agent will log additional debug messages to the system log.
     #[serde(default, rename = "enable_debug")]
@@ -81,6 +83,24 @@ pub struct Agent {
     pub container_pipe_size: u32,
 }
 
+impl std::default::Default for Agent {
+    fn default() -> Self {
+        Self {
+            debug: true,
+            enable_tracing: false,
+            debug_console_enabled: false,
+            server_port: DEFAULT_AGENT_VSOCK_PORT,
+            log_port: DEFAULT_AGENT_LOG_PORT,
+            dial_timeout_ms: DEFAULT_AGENT_DIAL_TIMEOUT_MS,
+            reconnect_timeout_ms: 3_000,
+            request_timeout_ms: 30_000,
+            health_check_request_timeout_ms: 90_000,
+            kernel_modules: Default::default(),
+            container_pipe_size: 0,
+        }
+    }
+}
+
 fn default_server_port() -> u32 {
     DEFAULT_AGENT_VSOCK_PORT
 }

src/libs/kata-types/src/config/default.rs

@@ -24,6 +24,7 @@ pub const DEFAULT_AGENT_VSOCK_PORT: u32 = 1024;
 pub const DEFAULT_AGENT_LOG_PORT: u32 = 1025;
 pub const DEFAULT_AGENT_DBG_CONSOLE_PORT: u32 = 1026;
 pub const DEFAULT_AGENT_TYPE_NAME: &str = AGENT_NAME_KATA;
+pub const DEFAULT_AGENT_DIAL_TIMEOUT_MS: u32 = 10;
 
 pub const DEFAULT_RUNTIME_NAME: &str = RUNTIME_NAME_VIRTCONTAINER;
 pub const DEFAULT_HYPERVISOR: &str = HYPERVISOR_NAME_DRAGONBALL;
@@ -42,6 +43,7 @@ pub const MIN_SHARED_9PFS_SIZE_MB: u32 = 4 * 1024;
 pub const MAX_SHARED_9PFS_SIZE_MB: u32 = 8 * 1024 * 1024;
 
 pub const DEFAULT_GUEST_HOOK_PATH: &str = "/opt/kata/hooks";
+pub const DEFAULT_GUEST_DNS_FILE: &str = "/etc/resolv.conf";
 
 pub const DEFAULT_GUEST_VCPUS: u32 = 1;
 
@@ -67,3 +69,18 @@ pub const DEFAULT_QEMU_PCI_BRIDGES: u32 = 2;
 pub const MAX_QEMU_PCI_BRIDGES: u32 = 5;
 pub const MAX_QEMU_VCPUS: u32 = 256;
 pub const MIN_QEMU_MEMORY_SIZE_MB: u32 = 64;
+
+// Default configuration for Cloud Hypervisor (CH)
+pub const DEFAULT_CH_BINARY_PATH: &str = "/usr/bin/cloud-hypervisor";
+pub const DEFAULT_CH_ROOTFS_TYPE: &str = "ext4";
+pub const DEFAULT_CH_CONTROL_PATH: &str = "";
+pub const DEFAULT_CH_ENTROPY_SOURCE: &str = "/dev/urandom";
+pub const DEFAULT_CH_GUEST_KERNEL_IMAGE: &str = "vmlinuz";
+pub const DEFAULT_CH_GUEST_KERNEL_PARAMS: &str = "";
+pub const DEFAULT_CH_FIRMWARE_PATH: &str = "";
+pub const DEFAULT_CH_MEMORY_SIZE_MB: u32 = 128;
+pub const DEFAULT_CH_MEMORY_SLOTS: u32 = 128;
+pub const DEFAULT_CH_PCI_BRIDGES: u32 = 2;
+pub const MAX_CH_PCI_BRIDGES: u32 = 5;
+pub const MAX_CH_VCPUS: u32 = 256;
+pub const MIN_CH_MEMORY_SIZE_MB: u32 = 64;

src/libs/kata-types/src/config/hypervisor/ch.rs

@@ -0,0 +1,146 @@
+// Copyright (c) 2019-2021 Alibaba Cloud
+// Copyright (c) 2022-2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::io::Result;
+use std::path::Path;
+use std::sync::Arc;
+
+use super::{default, register_hypervisor_plugin};
+
+use crate::config::default::MAX_CH_VCPUS;
+use crate::config::default::MIN_CH_MEMORY_SIZE_MB;
+
+use crate::config::hypervisor::VIRTIO_BLK_MMIO;
+use crate::config::{ConfigPlugin, TomlConfig};
+use crate::{eother, resolve_path, validate_path};
+
+/// Hypervisor name for CH, used to index `TomlConfig::hypervisor`.
+pub const HYPERVISOR_NAME_CH: &str = "cloud-hypervisor";
+
+/// Configuration information for CH.
+#[derive(Default, Debug)]
+pub struct CloudHypervisorConfig {}
+
+impl CloudHypervisorConfig {
+    /// Create a new instance of `CloudHypervisorConfig`.
+    pub fn new() -> Self {
+        CloudHypervisorConfig {}
+    }
+
+    /// Register the CH plugin.
+    pub fn register(self) {
+        let plugin = Arc::new(self);
+        register_hypervisor_plugin(HYPERVISOR_NAME_CH, plugin);
+    }
+}
+
+impl ConfigPlugin for CloudHypervisorConfig {
+    fn get_max_cpus(&self) -> u32 {
+        MAX_CH_VCPUS
+    }
+
+    fn get_min_memory(&self) -> u32 {
+        MIN_CH_MEMORY_SIZE_MB
+    }
+
+    fn name(&self) -> &str {
+        HYPERVISOR_NAME_CH
+    }
+
+    /// Adjust the configuration information after loading from configuration file.
+    fn adjust_config(&self, conf: &mut TomlConfig) -> Result<()> {
+        if let Some(ch) = conf.hypervisor.get_mut(HYPERVISOR_NAME_CH) {
+            if ch.path.is_empty() {
+                ch.path = default::DEFAULT_CH_BINARY_PATH.to_string();
+            }
+            resolve_path!(ch.path, "CH binary path `{}` is invalid: {}")?;
+            if ch.ctlpath.is_empty() {
+                ch.ctlpath = default::DEFAULT_CH_CONTROL_PATH.to_string();
+            }
+            resolve_path!(ch.ctlpath, "CH ctlpath `{}` is invalid: {}")?;
+
+            if ch.boot_info.kernel.is_empty() {
+                ch.boot_info.kernel = default::DEFAULT_CH_GUEST_KERNEL_IMAGE.to_string();
+            }
+            if ch.boot_info.kernel_params.is_empty() {
+                ch.boot_info.kernel_params = default::DEFAULT_CH_GUEST_KERNEL_PARAMS.to_string();
+            }
+            if ch.boot_info.firmware.is_empty() {
+                ch.boot_info.firmware = default::DEFAULT_CH_FIRMWARE_PATH.to_string();
+            }
+
+            if ch.device_info.default_bridges == 0 {
+                ch.device_info.default_bridges = default::DEFAULT_CH_PCI_BRIDGES;
+            }
+
+            if ch.machine_info.entropy_source.is_empty() {
+                ch.machine_info.entropy_source = default::DEFAULT_CH_ENTROPY_SOURCE.to_string();
+            }
+
+            if ch.memory_info.default_memory == 0 {
+                ch.memory_info.default_memory = default::DEFAULT_CH_MEMORY_SIZE_MB;
+            }
+            if ch.memory_info.memory_slots == 0 {
+                ch.memory_info.memory_slots = default::DEFAULT_CH_MEMORY_SLOTS;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Validate the configuration information.
+    fn validate(&self, conf: &TomlConfig) -> Result<()> {
+        if let Some(ch) = conf.hypervisor.get(HYPERVISOR_NAME_CH) {
+            validate_path!(ch.path, "CH binary path `{}` is invalid: {}")?;
+            validate_path!(ch.ctlpath, "CH control path `{}` is invalid: {}")?;
+            if !ch.jailer_path.is_empty() {
+                return Err(eother!("Path for CH jailer should be empty"));
+            }
+            if !ch.valid_jailer_paths.is_empty() {
+                return Err(eother!("Valid CH jailer path list should be empty"));
+            }
+
+            if !ch.blockdev_info.disable_block_device_use
+                && ch.blockdev_info.block_device_driver == VIRTIO_BLK_MMIO
+            {
+                return Err(eother!("CH doesn't support virtio-blk-mmio"));
+            }
+
+            if ch.boot_info.kernel.is_empty() {
+                return Err(eother!("Guest kernel image for CH is empty"));
+            }
+            if ch.boot_info.image.is_empty() && ch.boot_info.initrd.is_empty() {
+                return Err(eother!("Both guest boot image and initrd for CH are empty"));
+            }
+
+            if (ch.cpu_info.default_vcpus > 0
+                && ch.cpu_info.default_vcpus as u32 > default::MAX_CH_VCPUS)
+                || ch.cpu_info.default_maxvcpus > default::MAX_CH_VCPUS
+            {
+                return Err(eother!(
+                    "CH hypervisor cannot support {} vCPUs",
+                    ch.cpu_info.default_maxvcpus
+                ));
+            }
+
+            if ch.device_info.default_bridges > default::MAX_CH_PCI_BRIDGES {
+                return Err(eother!(
+                    "CH hypervisor cannot support {} PCI bridges",
+                    ch.device_info.default_bridges
+                ));
+            }
+
+            if ch.memory_info.default_memory < MIN_CH_MEMORY_SIZE_MB {
+                return Err(eother!(
+                    "CH hypervisor has minimal memory limitation {}",
+                    MIN_CH_MEMORY_SIZE_MB
+                ));
+            }
+        }
+
+        Ok(())
+    }
+}

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -40,6 +40,9 @@ pub use self::dragonball::{DragonballConfig, HYPERVISOR_NAME_DRAGONBALL};
 mod qemu;
 pub use self::qemu::{QemuConfig, HYPERVISOR_NAME_QEMU};
 
+mod ch;
+pub use self::ch::{CloudHypervisorConfig, HYPERVISOR_NAME_CH};
+
 const VIRTIO_BLK: &str = "virtio-blk";
 const VIRTIO_BLK_MMIO: &str = "virtio-mmio";
 const VIRTIO_BLK_CCW: &str = "virtio-blk-ccw";

src/libs/kata-types/src/config/mod.rs

@@ -25,8 +25,8 @@ pub mod hypervisor;
 pub use self::agent::Agent;
 use self::default::DEFAULT_AGENT_DBG_CONSOLE_PORT;
 pub use self::hypervisor::{
-    BootInfo, DragonballConfig, Hypervisor, QemuConfig, HYPERVISOR_NAME_DRAGONBALL,
-    HYPERVISOR_NAME_QEMU,
+    BootInfo, CloudHypervisorConfig, DragonballConfig, Hypervisor, QemuConfig,
+    HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_QEMU,
 };
 
 mod runtime;

src/libs/kata-types/src/config/runtime.rs

@@ -103,8 +103,9 @@ pub struct Runtime {
     pub enable_pprof: bool,
 
     /// If enabled, static resource management will calculate the vcpu and memory for the sandbox/container
+    /// And pod configured this will not be able to further update its CPU/Memory resource
     #[serde(default)]
-    pub static_resource_mgmt: bool,
+    pub static_sandbox_resource_mgmt: bool,
 
     /// Determines whether container seccomp profiles are passed to the virtual machine and
     /// applied by the kata agent. If set to true, seccomp is not applied within the guest.

src/libs/oci/src/lib.rs

@@ -193,6 +193,12 @@ pub struct Hooks {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub prestart: Vec<Hook>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub create_runtime: Vec<Hook>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub create_container: Vec<Hook>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub start_container: Vec<Hook>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub poststart: Vec<Hook>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub poststop: Vec<Hook>,
@@ -494,8 +500,8 @@ pub struct LinuxDevice {
 pub struct LinuxDeviceCgroup {
     #[serde(default)]
     pub allow: bool,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
-    pub r#type: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub r#type: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub major: Option<i64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -1401,6 +1407,7 @@ mod tests {
                     env: vec![],
                     timeout: None,
                 }],
+                ..Default::default()
             }),
             annotations: [
                 ("com.example.key1".to_string(), "value1".to_string()),
@@ -1431,21 +1438,21 @@ mod tests {
                     devices: vec![
                         crate::LinuxDeviceCgroup {
                             allow: false,
-                            r#type: "".to_string(),
+                            r#type: None,
                             major: None,
                             minor: None,
                             access: "rwm".to_string(),
                         },
                         crate::LinuxDeviceCgroup {
                             allow: true,
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(10),
                             minor: Some(229),
                             access: "rw".to_string(),
                         },
                         crate::LinuxDeviceCgroup {
                             allow: true,
-                            r#type: "b".to_string(),
+                            r#type: Some("b".to_string()),
                             major: Some(8),
                             minor: Some(0),
                             access: "r".to_string(),

src/libs/protocols/.gitignore

@@ -7,5 +7,8 @@ src/empty.rs
 src/health.rs
 src/health_ttrpc.rs
 src/health_ttrpc_async.rs
+src/image.rs
+src/image_ttrpc.rs
+src/image_ttrpc_async.rs
 src/oci.rs
 src/types.rs

src/libs/protocols/build.rs

@@ -157,13 +157,30 @@ fn real_main() -> Result<(), std::io::Error> {
     // generate async
     #[cfg(feature = "async")]
     {
-        codegen("src", &["protos/agent.proto", "protos/health.proto"], true)?;
+        codegen(
+            "src",
+            &[
+                "protos/agent.proto",
+                "protos/health.proto",
+                "protos/image.proto",
+            ],
+            true,
+        )?;
 
         fs::rename("src/agent_ttrpc.rs", "src/agent_ttrpc_async.rs")?;
         fs::rename("src/health_ttrpc.rs", "src/health_ttrpc_async.rs")?;
+        fs::rename("src/image_ttrpc.rs", "src/image_ttrpc_async.rs")?;
     }
 
-    codegen("src", &["protos/agent.proto", "protos/health.proto"], false)?;
+    codegen(
+        "src",
+        &[
+            "protos/agent.proto",
+            "protos/health.proto",
+            "protos/image.proto",
+        ],
+        false,
+    )?;
 
     // There is a message named 'Box' in oci.proto
     // so there is a struct named 'Box', we should replace Box<Self> to ::std::boxed::Box<Self>

src/libs/protocols/protos/agent.proto

@@ -34,6 +34,7 @@ service AgentService {
 	rpc SignalProcess(SignalProcessRequest) returns (google.protobuf.Empty);
 	rpc WaitProcess(WaitProcessRequest) returns (WaitProcessResponse); // wait & reap like waitpid(2)
 	rpc UpdateContainer(UpdateContainerRequest) returns (google.protobuf.Empty);
+	rpc UpdateEphemeralMounts(UpdateEphemeralMountsRequest) returns (google.protobuf.Empty);
 	rpc StatsContainer(StatsContainerRequest) returns (StatsContainerResponse);
 	rpc PauseContainer(PauseContainerRequest) returns (google.protobuf.Empty);
 	rpc ResumeContainer(ResumeContainerRequest) returns (google.protobuf.Empty);
@@ -316,6 +317,10 @@ message UpdateRoutesRequest {
 	Routes routes = 1;
 }
 
+message UpdateEphemeralMountsRequest {
+	repeated Storage storages = 1;
+}
+
 message ListInterfacesRequest {
 }
 

src/libs/protocols/protos/image.proto

@@ -0,0 +1,31 @@
+//
+// Copyright (c) 2021 Alibaba Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+syntax = "proto3";
+
+option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
+
+package grpc;
+
+// Image defines the public APIs for managing images.
+service Image {
+    // PullImage pulls an image with authentication config.
+    rpc PullImage(PullImageRequest) returns (PullImageResponse) {}
+}
+
+message PullImageRequest {
+    // Image name (e.g. docker.io/library/busybox:latest).
+    string image = 1;
+    // Unique image identifier, used to avoid duplication when unpacking the image layers.
+    string container_id = 2;
+    // Use USERNAME[:PASSWORD] for accessing the registry
+    string source_creds = 3;
+}
+
+message PullImageResponse {
+    // Reference to the image in use. For most runtimes, this should be an
+    // image ID or digest.
+    string image_ref = 1;
+}

src/libs/protocols/protos/oci.proto

@@ -166,6 +166,15 @@ message Hooks {
 
 	// Poststop is a list of hooks to be run after the container process exits.
 	repeated Hook Poststop = 3  [(gogoproto.nullable) = false];
+
+	// Createruntime is a list of hooks to be run during the creation of runtime(sandbox).
+	repeated Hook CreateRuntime = 4  [(gogoproto.nullable) = false];
+
+	// CreateContainer is a list of hooks to be run after VM is started, and before container is created.
+	repeated Hook CreateContainer = 5  [(gogoproto.nullable) = false];
+
+	// StartContainer is a list of hooks to be run after container is created, but before it is started.
+	repeated Hook StartContainer = 6  [(gogoproto.nullable) = false];
 }
 
 message Hook {

src/libs/protocols/src/lib.rs

@@ -15,6 +15,10 @@ pub mod health;
 pub mod health_ttrpc;
 #[cfg(feature = "async")]
 pub mod health_ttrpc_async;
+pub mod image;
+pub mod image_ttrpc;
+#[cfg(feature = "async")]
+pub mod image_ttrpc_async;
 pub mod oci;
 pub mod trans;
 pub mod types;

src/libs/protocols/src/trans.rs

@@ -105,7 +105,7 @@ impl From<oci::LinuxDeviceCgroup> for crate::oci::LinuxDeviceCgroup {
     fn from(from: oci::LinuxDeviceCgroup) -> Self {
         crate::oci::LinuxDeviceCgroup {
             Allow: from.allow,
-            Type: from.r#type,
+            Type: from.r#type.map_or("".to_string(), |t| t as String),
             Major: from.major.map_or(0, |t| t),
             Minor: from.minor.map_or(0, |t| t),
             Access: from.access,
@@ -294,6 +294,9 @@ impl From<oci::Hooks> for crate::oci::Hooks {
     fn from(from: Hooks) -> Self {
         crate::oci::Hooks {
             Prestart: from_vec(from.prestart),
+            CreateRuntime: from_vec(from.create_runtime),
+            CreateContainer: from_vec(from.create_container),
+            StartContainer: from_vec(from.start_container),
             Poststart: from_vec(from.poststart),
             Poststop: from_vec(from.poststop),
             unknown_fields: Default::default(),
@@ -478,7 +481,7 @@ impl From<crate::oci::LinuxDeviceCgroup> for oci::LinuxDeviceCgroup {
 
         oci::LinuxDeviceCgroup {
             allow: from.get_Allow(),
-            r#type: from.take_Type(),
+            r#type: Some(from.take_Type()),
             major,
             minor,
             access: from.take_Access(),
@@ -970,20 +973,34 @@ impl From<crate::oci::Hook> for oci::Hook {
 
 impl From<crate::oci::Hooks> for oci::Hooks {
     fn from(mut from: crate::oci::Hooks) -> Self {
-        let mut prestart = Vec::new();
-        for hook in from.take_Prestart().to_vec() {
-            prestart.push(hook.into())
-        }
-        let mut poststart = Vec::new();
-        for hook in from.take_Poststart().to_vec() {
-            poststart.push(hook.into());
-        }
-        let mut poststop = Vec::new();
-        for hook in from.take_Poststop().to_vec() {
-            poststop.push(hook.into());
-        }
+        let prestart = from.take_Prestart().into_iter().map(|i| i.into()).collect();
+        let create_runtime = from
+            .take_CreateRuntime()
+            .into_iter()
+            .map(|i| i.into())
+            .collect();
+        let create_container = from
+            .take_CreateContainer()
+            .into_iter()
+            .map(|i| i.into())
+            .collect();
+        let start_container = from
+            .take_StartContainer()
+            .into_iter()
+            .map(|i| i.into())
+            .collect();
+        let poststart = from
+            .take_Poststart()
+            .into_iter()
+            .map(|i| i.into())
+            .collect();
+        let poststop = from.take_Poststop().into_iter().map(|i| i.into()).collect();
+
         oci::Hooks {
             prestart,
+            create_runtime,
+            create_container,
+            start_container,
             poststart,
             poststop,
         }

src/libs/shim-interface/src/shim_mgmt/client.rs

@@ -53,7 +53,7 @@ impl MgmtClient {
             .method(Method::GET)
             .uri(url)
             .body(Body::empty())?;
-        return self.send_request(req).await;
+        self.send_request(req).await
     }
 
     /// The HTTP Post method for client
@@ -72,7 +72,7 @@ impl MgmtClient {
             .uri(url)
             .header("content-type", content_type)
             .body(body)?;
-        return self.send_request(req).await;
+        self.send_request(req).await
     }
 
     /// The http PUT method for client
@@ -82,7 +82,7 @@ impl MgmtClient {
             .method(Method::PUT)
             .uri(url)
             .body(Body::from(data))?;
-        return self.send_request(req).await;
+        self.send_request(req).await
     }
 
     async fn send_request(&self, req: Request<Body>) -> Result<Response<Body>> {

src/runtime-rs/Makefile

@@ -203,7 +203,7 @@ ifneq (,$(DBCMD))
     DEFNETWORKMODEL_DB := tcfilter
     KERNELPARAMS = console=ttyS1 agent.log_vport=1025
 	KERNELTYPE_DB = uncompressed
-    KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_DB))
+    KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME_DB,$(KERNELTYPE_DB))
     KERNELPATH_DB = $(KERNELDIR)/$(KERNEL_NAME_DB)
     DEFSANDBOXCGROUPONLY = true
     RUNTIMENAME := virt_container
@@ -371,8 +371,8 @@ endef
 
 # Returns the name of the kernel file to use based on the provided KERNELTYPE.
 # $1 : KERNELTYPE (compressed or uncompressed)
-define MAKE_KERNEL_NAME
-$(if $(findstring uncompressed,$1),vmlinux.container,vmlinuz.container)
+define MAKE_KERNEL_NAME_DB
+$(if $(findstring uncompressed,$1),vmlinux-dragonball-experimental.container,vmlinuz-dragonball-experimental.container)
 endef
 
 .DEFAULT_GOAL := default

src/runtime-rs/config/configuration-dragonball.toml.in

@@ -293,5 +293,12 @@ experimental=@DEFAULTEXPFEATURES@
 # (default: false)
 # enable_pprof = true
 
-static_resource_mgmt=@DEFSTATICRESOURCEMGMT_DB@
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_DB@
 

src/runtime-rs/crates/agent/Cargo.toml

@@ -18,7 +18,7 @@ serde_json = ">=1.0.9"
 slog = "2.5.2"
 slog-scope = "4.4.0"
 ttrpc = { version = "0.6.1" }
-tokio = { version = "1.28.1", features = ["fs", "rt"] }
+tokio = { version = "1.8.0", features = ["fs", "rt"] }
 url = "2.2.2"
 nix = "0.24.2"
 

src/runtime-rs/crates/agent/src/kata/agent.rs

@@ -115,5 +115,7 @@ impl_agent!(
     copy_file | crate::CopyFileRequest | crate::Empty | None,
     get_oom_event | crate::Empty | crate::OomEventResponse | Some(0),
     get_ip_tables | crate::GetIPTablesRequest | crate::GetIPTablesResponse | None,
-    set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None
+    set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None,
+    get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None,
+    resize_volume | crate::ResizeVolumeRequest | crate::Empty | None
 );

src/runtime-rs/crates/agent/src/kata/trans.rs

@@ -8,7 +8,7 @@ use std::convert::Into;
 
 use protocols::{
     agent::{self, OOMEvent},
-    empty, health, types,
+    csi, empty, health, types,
 };
 
 use crate::{
@@ -20,11 +20,12 @@ use crate::{
         GetIPTablesResponse, GuestDetailsResponse, HealthCheckResponse, HugetlbStats, IPAddress,
         IPFamily, Interface, Interfaces, KernelModule, MemHotplugByProbeRequest, MemoryData,
         MemoryStats, NetworkStats, OnlineCPUMemRequest, PidsStats, ReadStreamRequest,
-        ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, Route, Routes,
-        SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse, SignalProcessRequest,
-        StatsContainerResponse, Storage, StringUser, ThrottlingData, TtyWinResizeRequest,
-        UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, VersionCheckResponse,
-        WaitProcessRequest, WriteStreamRequest,
+        ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, ResizeVolumeRequest,
+        Route, Routes, SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse,
+        SignalProcessRequest, StatsContainerResponse, Storage, StringUser, ThrottlingData,
+        TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest,
+        VersionCheckResponse, VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest,
+        WriteStreamRequest,
     },
     OomEventResponse, WaitProcessResponse, WriteStreamResponse,
 };
@@ -846,3 +847,35 @@ impl From<agent::OOMEvent> for OomEventResponse {
         }
     }
 }
+
+impl From<VolumeStatsRequest> for agent::VolumeStatsRequest {
+    fn from(from: VolumeStatsRequest) -> Self {
+        Self {
+            volume_guest_path: from.volume_guest_path,
+            unknown_fields: Default::default(),
+            cached_size: Default::default(),
+        }
+    }
+}
+
+impl From<csi::VolumeStatsResponse> for VolumeStatsResponse {
+    fn from(from: csi::VolumeStatsResponse) -> Self {
+        let result: String = format!(
+            "Usage: {:?} Volume Condition: {:?}",
+            from.get_usage(),
+            from.get_volume_condition()
+        );
+        Self { data: result }
+    }
+}
+
+impl From<ResizeVolumeRequest> for agent::ResizeVolumeRequest {
+    fn from(from: ResizeVolumeRequest) -> Self {
+        Self {
+            volume_guest_path: from.volume_guest_path,
+            size: from.size,
+            unknown_fields: Default::default(),
+            cached_size: Default::default(),
+        }
+    }
+}

src/runtime-rs/crates/agent/src/lib.rs

@@ -23,8 +23,8 @@ pub use types::{
     ReseedRandomDevRequest, ResizeVolumeRequest, Route, Routes, SetGuestDateTimeRequest,
     SetIPTablesRequest, SetIPTablesResponse, SignalProcessRequest, StatsContainerResponse, Storage,
     TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest,
-    VersionCheckResponse, WaitProcessRequest, WaitProcessResponse, WriteStreamRequest,
-    WriteStreamResponse,
+    VersionCheckResponse, VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest,
+    WaitProcessResponse, WriteStreamRequest, WriteStreamResponse,
 };
 
 use anyhow::Result;
@@ -88,4 +88,6 @@ pub trait Agent: AgentManager + HealthService + Send + Sync {
     async fn get_oom_event(&self, req: Empty) -> Result<OomEventResponse>;
     async fn get_ip_tables(&self, req: GetIPTablesRequest) -> Result<GetIPTablesResponse>;
     async fn set_ip_tables(&self, req: SetIPTablesRequest) -> Result<SetIPTablesResponse>;
+    async fn get_volume_stats(&self, req: VolumeStatsRequest) -> Result<VolumeStatsResponse>;
+    async fn resize_volume(&self, req: ResizeVolumeRequest) -> Result<Empty>;
 }

src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs

@@ -15,9 +15,6 @@ use tokio::{
 
 use super::{ConnectConfig, Sock, Stream};
 
-unsafe impl Send for HybridVsock {}
-unsafe impl Sync for HybridVsock {}
-
 #[derive(Debug, PartialEq)]
 pub struct HybridVsock {
     uds: String,

src/runtime-rs/crates/agent/src/sock/vsock.rs

@@ -16,9 +16,6 @@ use tokio::net::UnixStream;
 
 use super::{ConnectConfig, Sock, Stream};
 
-unsafe impl Send for Vsock {}
-unsafe impl Sync for Vsock {}
-
 #[derive(Debug, PartialEq)]
 pub struct Vsock {
     vsock_cid: u32,

src/runtime-rs/crates/agent/src/types.rs

@@ -9,6 +9,8 @@ use std::convert::TryFrom;
 
 use serde::{Deserialize, Serialize};
 
+pub const DEFAULT_REMOVE_CONTAINER_REQUEST_TIMEOUT: u32 = 10;
+
 #[derive(PartialEq, Clone, Default)]
 pub struct Empty {}
 
@@ -124,7 +126,6 @@ pub struct CreateContainerRequest {
     pub devices: Vec<Device>,
     pub storages: Vec<Storage>,
     pub oci: Option<oci::Spec>,
-    pub guest_hooks: Option<oci::Hooks>,
     pub sandbox_pidns: bool,
     pub rootfs_mounts: Vec<oci::Mount>,
 }
@@ -165,7 +166,7 @@ impl ContainerProcessID {
     }
 }
 
-#[derive(PartialEq, Clone, Debug, Default)]
+#[derive(PartialEq, Clone, Debug)]
 pub struct RemoveContainerRequest {
     pub container_id: String,
     pub timeout: u32,
@@ -180,6 +181,15 @@ impl RemoveContainerRequest {
     }
 }
 
+impl std::default::Default for RemoveContainerRequest {
+    fn default() -> Self {
+        Self {
+            container_id: "".to_string(),
+            timeout: DEFAULT_REMOVE_CONTAINER_REQUEST_TIMEOUT,
+        }
+    }
+}
+
 #[derive(PartialEq, Clone, Default)]
 pub struct SignalProcessRequest {
     pub process_id: ContainerProcessID,
@@ -569,6 +579,16 @@ pub struct ResizeVolumeRequest {
     pub size: u64,
 }
 
+#[derive(PartialEq, Clone, Default, Debug)]
+pub struct VolumeStatsRequest {
+    pub volume_guest_path: String,
+}
+
+#[derive(PartialEq, Clone, Default, Debug)]
+pub struct VolumeStatsResponse {
+    pub data: String,
+}
+
 #[cfg(test)]
 mod test {
     use std::convert::TryFrom;

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -21,7 +21,7 @@ serde_json = ">=1.0.9"
 slog = "2.5.2"
 slog-scope = "4.4.0"
 thiserror = "1.0"
-tokio = { version = "1.28.1", features = ["sync", "fs"] }
+tokio = { version = "1.8.0", features = ["sync", "fs"] }
 vmm-sys-util = "0.11.0"
 rand = "0.8.4"
 
@@ -32,4 +32,15 @@ shim-interface = { path = "../../../libs/shim-interface" }
 
 dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs","dbs-upcall"] }
 
+ch-config = { path = "ch-config", optional = true }
+
+futures = "0.3.25"
+safe-path = "0.1.0"
+crossbeam-channel = "0.5.6"
+
 [features]
+default = []
+
+# Feature is not yet complete, so not enabled by default.
+# See https://github.com/kata-containers/kata-containers/issues/6264.
+cloud-hypervisor = ["ch-config"]

src/runtime-rs/crates/hypervisor/README.md

@@ -1,4 +1,26 @@
 # Multi-vmm support for runtime-rs
+
+## 0. Status
+
+External hypervisor support is currently being developed.
+
+See [the main tracking issue](https://github.com/kata-containers/kata-containers/issues/4634)
+for further details.
+
+### Cloud Hypervisor
+
+A basic implementation currently exists for Cloud Hypervisor. However,
+since it is not yet fully functional, the feature is disabled by
+default. When the implementation matures, the feature will be enabled
+by default.
+
+> **Note:**
+>
+> To enable the feature, follow the instructions on https://github.com/kata-containers/kata-containers/pull/6201.
+
+See the [Cloud Hypervisor tracking issue](https://github.com/kata-containers/kata-containers/issues/6263)
+for further details.
+
 Some key points for supporting multi-vmm in rust runtime.
 ## 1. Hypervisor Config
 

src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml

@@ -0,0 +1,25 @@
+# Copyright (c) 2022-2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "ch-config"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.68"
+serde = { version = "1.0.145", features = ["rc", "derive"] }
+serde_json = "1.0.91"
+tokio = { version = "1.25.0", features = ["sync", "rt"] }
+
+# Cloud Hypervisor public HTTP API functions
+# Note that the version specified is not necessarily the version of CH
+# being used. This version is used to pin the CH config structure
+# which is relatively static.
+api_client = { git = "https://github.com/cloud-hypervisor/cloud-hypervisor", crate = "api_client", tag = "v27.0" }
+
+kata-types = { path = "../../../../libs/kata-types"}
+nix = "0.26.2"

src/runtime-rs/crates/hypervisor/ch-config/src/ch_api.rs

@@ -0,0 +1,108 @@
+// Copyright (c) 2022-2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{DeviceConfig, FsConfig, VmConfig};
+use anyhow::{anyhow, Result};
+use api_client::simple_api_full_command_and_response;
+
+use std::os::unix::net::UnixStream;
+use tokio::task;
+
+pub async fn cloud_hypervisor_vmm_ping(mut socket: UnixStream) -> Result<Option<String>> {
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let response = simple_api_full_command_and_response(&mut socket, "GET", "vmm.ping", None)
+            .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+pub async fn cloud_hypervisor_vmm_shutdown(mut socket: UnixStream) -> Result<Option<String>> {
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let response =
+            simple_api_full_command_and_response(&mut socket, "PUT", "vmm.shutdown", None)
+                .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+pub async fn cloud_hypervisor_vm_create(
+    mut socket: UnixStream,
+    cfg: VmConfig,
+) -> Result<Option<String>> {
+    let serialised = serde_json::to_string_pretty(&cfg)?;
+
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let data = Some(serialised.as_str());
+
+        let response = simple_api_full_command_and_response(&mut socket, "PUT", "vm.create", data)
+            .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+pub async fn cloud_hypervisor_vm_start(mut socket: UnixStream) -> Result<Option<String>> {
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let response = simple_api_full_command_and_response(&mut socket, "PUT", "vm.boot", None)
+            .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+#[allow(dead_code)]
+pub async fn cloud_hypervisor_vm_stop(mut socket: UnixStream) -> Result<Option<String>> {
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let response =
+            simple_api_full_command_and_response(&mut socket, "PUT", "vm.shutdown", None)
+                .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+#[allow(dead_code)]
+pub async fn cloud_hypervisor_vm_device_add(mut socket: UnixStream) -> Result<Option<String>> {
+    let device_config = DeviceConfig::default();
+
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let response = simple_api_full_command_and_response(
+            &mut socket,
+            "PUT",
+            "vm.add-device",
+            Some(&serde_json::to_string(&device_config)?),
+        )
+        .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+pub async fn cloud_hypervisor_vm_fs_add(
+    mut socket: UnixStream,
+    fs_config: FsConfig,
+) -> Result<Option<String>> {
+    let result = task::spawn_blocking(move || -> Result<Option<String>> {
+        let response = simple_api_full_command_and_response(
+            &mut socket,
+            "PUT",
+            "vm.add-fs",
+            Some(&serde_json::to_string(&fs_config)?),
+        )
+        .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?;
+
+    result
+}