Merge pull request #8400 from littlejawa/fix_pullimage_in_guest_for_crio

CC | agent: fix annotation name to allow cri-o support of CoCo pullimage in guest

.github/workflows/cc-payload-after-push-amd64.yaml

@@ -0,0 +1,170 @@
+name: CI | Publish CC runtime payload for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        measured_rootfs:
+          - no
+        asset:
+          - cloud-hypervisor
+          - qemu
+          - virtiofsd
+          - kernel-sev
+          - ovmf-sev
+          - ovmf
+          - qemu-snp-experimental
+          - qemu-tdx-experimental
+          - rootfs-initrd-sev
+          - cc-tdx-td-shim
+          - tdvf
+        include:
+          - measured_rootfs: yes
+            asset: kernel
+          - measured_rootfs: yes
+            asset: kernel-tdx-experimental
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
+          - measured_rootfs: yes
+            asset: rootfs-image-tdx
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          USE_CACHE="no" make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_tdx.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/root_hash_tdx.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_tdx.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          USE_CACHE="no" make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push-s390x.yaml

@@ -0,0 +1,207 @@
+name: CI | Publish CC runtime payload for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        measured_rootfs:
+          - no
+        asset:
+          - qemu
+          - cc-rootfs-initrd
+          - virtiofsd
+        include:
+          - measured_rootfs: yes
+            asset: kernel
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          USE_CACHE="no" make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+          sudo chown -R $(id -u):$(id -g) "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          USE_CACHE="no" make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  build-asset-cc-se-image:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+
+      - name: Place a host key document
+        run: |
+          mkdir -p "host-key-document"
+          cp "${CI_HKD_PATH}" "host-key-document"
+        env:
+          CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
+
+      - name: Build cc-se-image
+        run: |
+          base_dir=tools/packaging/kata-deploy/local-build/
+          cp -r kata-artifacts ${base_dir}/build
+          # Skip building dependant artifacts of cc-se-image-tarball
+          # because we already have them from the previous build
+          sed -i 's/\(^cc-se-image-tarball:\).*/\1/g' ${base_dir}/Makefile
+          USE_CACHE="no" make cc-se-image-tarball
+          build_dir=$(readlink -f build)
+          sudo cp -r "${build_dir}" "kata-build"
+          sudo chown -R $(id -u):$(id -g) "kata-build"
+        env:
+          HKD_PATH: "host-key-document"
+
+      - name: store-artifact cc-se-image
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-cc-se-image.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: [build-asset, build-asset-cc-shim-v2, build-asset-cc-se-image]
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push.yaml

@@ -0,0 +1,47 @@
+name: CI | Publish Kata Containers payload for Confidential Containers
+on:
+  push:
+    branches:
+      - CCv0
+  workflow_dispatch:
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/cc-payload-after-push-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/cc-payload-after-push-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-assets-amd64, build-assets-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push commit multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA} \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}
+
+      - name: Push latest multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest

.github/workflows/cc-payload-amd64.yaml

@@ -0,0 +1,155 @@
+name: Publish Kata Containers payload for Confidential Containers (amd64)
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        measured_rootfs:
+          - no
+        asset:
+          - cloud-hypervisor
+          - qemu
+          - virtiofsd
+          - kernel-sev
+          - kernel-snp-experimental
+          - ovmf-sev
+          - ovmf
+          - qemu-snp-experimental
+          - qemu-tdx-experimental
+          - rootfs-initrd-sev
+          - cc-tdx-td-shim
+          - tdvf
+        include:
+          - measured_rootfs: yes
+            asset: kernel
+          - measured_rootfs: yes
+            asset: kernel-tdx-experimental
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
+          - measured_rootfs: yes
+            asset: rootfs-image-tdx
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_tdx.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/root_hash_tdx.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_tdx.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          "quay.io/confidential-containers/runtime-payload" \
+          "kata-containers-${{ inputs.target-arch }}"
+

.github/workflows/cc-payload-s390x.yaml

@@ -0,0 +1,142 @@
+name: Publish Kata Containers payload for Confidential Containers (s390x)
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        measured_rootfs:
+          - no
+        asset:
+          - qemu
+          - virtiofsd
+        include:
+          - measured_rootfs: yes
+            asset: kernel
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          "quay.io/confidential-containers/runtime-payload" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload.yaml

@@ -0,0 +1,46 @@
+name: Publish Kata Containers payload for Confidential Containers
+on:
+  push:
+    tags:
+      - 'CC\-[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/cc-payload-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/cc-payload-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-assets-amd64, build-assets-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push commit multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA} \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}
+
+      - name: Push latest multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-latest \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-latest

.github/workflows/commit-message-check.yaml

@@ -51,7 +51,7 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^.{0,75}(\n.*)*$'
+        pattern: '^.{0,75}(\n.*)*$|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
         error: 'Subject too long (max 75)'
         post_error: ${{ env.error_msg }}
 
@@ -102,6 +102,6 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:'
+        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
         error: 'Failed to find subsystem in subject'
         post_error: ${{ env.error_msg }}

.github/workflows/deploy-ccv0-demo.yaml

@@ -0,0 +1,124 @@
+on:
+  issue_comment:
+    types: [created, edited]
+
+name: deploy-ccv0-demo
+
+jobs:
+  check-comment-and-membership:
+    runs-on: ubuntu-latest
+    if: |
+      github.event.issue.pull_request
+      && github.event_name == 'issue_comment'
+      && github.event.action == 'created'
+      && startsWith(github.event.comment.body, '/deploy-ccv0-demo')
+    steps:
+      - name: Check membership
+        uses: kata-containers/is-organization-member@1.0.1
+        id: is_organization_member
+        with:
+          organization: kata-containers
+          username: ${{ github.event.comment.user.login }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Fail if not member
+        run: |
+          result=${{ steps.is_organization_member.outputs.result }}
+          if [ $result == false ]; then
+              user=${{ github.event.comment.user.login }}
+              echo Either ${user} is not part of the kata-containers organization
+              echo or ${user} has its Organization Visibility set to Private at
+              echo https://github.com/orgs/kata-containers/people?query=${user}
+              echo 
+              echo Ensure you change your Organization Visibility to Public and
+              echo trigger the test again.
+              exit 1
+          fi
+
+  build-asset:
+    runs-on: ubuntu-latest
+    needs: check-comment-and-membership
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
+    steps:
+      - uses: actions/checkout@v2
+      - name: Prepare confidential container rootfs
+        if: ${{ matrix.asset == 'rootfs-initrd' }}
+        run: |
+          pushd include_rootfs/etc
+          curl -LO https://raw.githubusercontent.com/confidential-containers/documentation/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json
+          mkdir kata-containers
+          envsubst < docs/how-to/data/confidential-agent-config.toml.in > kata-containers/agent.toml
+          popd
+        env:
+          AA_KBC_PARAMS: offline_fs_kbc::null
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          AA_KBC: offline_fs_kbc
+          INCLUDE_ROOTFS: include_rootfs
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+
+  kata-deploy:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-static-tarball
+      - name: build-and-push-kata-deploy-ci
+        id: build-and-push-kata-deploy-ci
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/confidential-containers/runtime-payload:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/confidential-containers/runtime-payload:$pkg_sha
+          mkdir -p packaging/kata-deploy
+          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
+          echo "::set-output name=PKG_SHA::${pkg_sha}"

.github/workflows/static-checks.yaml

@@ -39,7 +39,11 @@ jobs:
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install -y --no-install-recommends build-essential haveged
+        sudo apt-get install -y --no-install-recommends \
+          build-essential \
+          haveged \
+          libdevmapper-dev \
+          clang
     - name: Install Go
       uses: actions/setup-go@v3
       with:
@@ -49,8 +53,8 @@ jobs:
         cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
         kernel_dir="tools/packaging/kernel/"
         kernel_version_file="${kernel_dir}kata_config_version"
-        modified_files=$(git diff --name-only origin/main..HEAD)
-        if git diff --name-only origin/main..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+        modified_files=$(git diff --name-only origin/CCv0..HEAD)
+        if git diff --name-only origin/CCv0..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
           echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
           if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
             echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)

ci/install_libseccomp.sh

@@ -72,7 +72,8 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    # Unset $CC for configure, we will always use native for gperf
+    # gperf is a build time dependency of libseccomp and not to be used in the target.
+    # Unset $CC since that might point to a cross compiler.
     CC= ./configure --prefix="${gperf_install_dir}"
     make
     make install

ci/lib.sh

@@ -64,3 +64,86 @@ run_get_pr_changed_file_details()
 	source "$tests_repo_dir/.ci/lib.sh"
 	get_pr_changed_file_details
 }
+
+# Check if the 1st argument version is greater than and equal to 2nd one
+# Version format: [0-9]+ separated by period (e.g. 2.4.6, 1.11.3 and etc.)
+#
+# Parameters:
+#	$1	- a version to be tested
+#	$2	- a target version
+#
+# Return:
+# 	0 if $1 is greater than and equal to $2
+#	1 otherwise
+version_greater_than_equal() {
+	local current_version=$1
+	local target_version=$2
+	smaller_version=$(echo -e "$current_version\n$target_version" | sort -V | head -1)
+	if [ "${smaller_version}" = "${target_version}" ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
+# Build a IBM zSystem secure execution (SE) image
+#
+# Parameters:
+#	$1	- kernel_parameters
+#	$2	- a source directory where kernel and initrd are located
+#	$3	- a destination directory where a SE image is built
+#
+# Return:
+# 	0 if the image is successfully built
+#	1 otherwise
+build_secure_image() {
+	kernel_params="${1:-}"
+	install_src_dir="${2:-}"
+	install_dest_dir="${3:-}"
+
+	if [ ! -f "${install_src_dir}/vmlinuz.container" ] ||
+		[ ! -f "${install_src_dir}/kata-containers-initrd.img" ]; then
+		cat << EOF >&2
+Either kernel or initrd does not exist or is mistakenly named
+A file name for kernel must be vmlinuz.container (raw binary)
+A file name for initrd must be kata-containers-initrd.img
+EOF
+		return 1
+	fi
+
+	cmdline="${kernel_params} panic=1 scsi_mod.scan=none swiotlb=262144"
+	parmfile="$(mktemp --suffix=-cmdline)"
+	echo "${cmdline}" > "${parmfile}"
+	chmod 600 "${parmfile}"
+
+	[ -n "${HKD_PATH:-}" ] || (echo >&2 "No host key document specified." && return 1)
+	cert_list=($(ls -1 $HKD_PATH))
+	declare hkd_options
+	eval "for cert in ${cert_list[*]}; do
+		hkd_options+=\"--host-key-document=\\\"\$HKD_PATH/\$cert\\\" \"
+	done"
+
+	command -v genprotimg > /dev/null 2>&1 || { apt update; apt install -y s390-tools; }
+	extra_arguments=""
+	genprotimg_version=$(genprotimg --version | grep -Po '(?<=version )[^-]+')
+	if ! version_greater_than_equal "${genprotimg_version}" "2.17.0"; then
+		extra_arguments="--x-pcf '0xe0'"
+	fi
+
+	eval genprotimg \
+		"${extra_arguments}" \
+		"${hkd_options}" \
+		--output="${install_dest_dir}/kata-containers-secure.img" \
+		--image="${install_src_dir}/vmlinuz.container" \
+		--ramdisk="${install_src_dir}/kata-containers-initrd.img" \
+		--parmfile="${parmfile}" \
+		--no-verify # no verification for CI testing purposes
+
+	build_result=$?
+	rm -f "${parmfile}"
+	if [ $build_result -eq 0 ]; then
+		return 0
+	else
+		return 1
+	fi
+}

docs/how-to/README.md

@@ -45,4 +45,8 @@
 - [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
 - [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
 - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
-- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)
+- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)
+
+## Confidential Containers
+- [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
+- [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md)

docs/how-to/ccv0.sh

@@ -0,0 +1,635 @@
+#!/bin/bash -e
+#
+# Copyright (c) 2021, 2023 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# Disclaimer: This script is work in progress for supporting the CCv0 prototype
+# It shouldn't be considered supported by the Kata Containers community, or anyone else
+
+# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md,
+# but with elements of the tests/.ci scripts used
+
+readonly script_name="$(basename "${BASH_SOURCE[0]}")"
+
+# By default in Golang >= 1.16 GO111MODULE is set to "on", but not all modules support it, so overwrite to "auto"
+export GO111MODULE="auto"
+
+# Setup kata containers environments if not set - we default to use containerd
+export CRI_CONTAINERD=${CRI_CONTAINERD:-"yes"}
+export CRI_RUNTIME=${CRI_RUNTIME:-"containerd"}
+export CRIO=${CRIO:-"no"}
+export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
+export KUBERNETES=${KUBERNETES:-"no"}
+export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}"
+export AA_KBC="${AA_KBC:-offline_fs_kbc}"
+export KATA_BUILD_CC=${KATA_BUILD_CC:-"yes"}
+export TEE_TYPE=${TEE_TYPE:-}
+export PREFIX="${PREFIX:-/opt/confidential-containers}"
+export RUNTIME_CONFIG_PATH="${RUNTIME_CONFIG_PATH:-${PREFIX}/share/defaults/kata-containers/configuration.toml}"
+
+# Allow the user to overwrite the default repo and branch names if they want to build from a fork
+export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}"
+export katacontainers_branch="${katacontainers_branch:-CCv0}" 
+export kata_default_branch=${katacontainers_branch}
+export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
+export tests_branch="${tests_branch:-CCv0}"
+export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
+
+# if .bash_profile exists then use it, otherwise fall back to .profile
+export PROFILE="${HOME}/.profile"
+if [ -r "${HOME}/.bash_profile" ]; then
+    export PROFILE="${HOME}/.bash_profile"
+fi
+# Stop PS1: unbound variable error happening
+export PS1=${PS1:-}
+
+# Create a bunch of common, derived values up front so we don't need to create them in all the different functions
+. ${PROFILE}
+if [ -z ${GOPATH} ]; then
+    export GOPATH=${HOME}/go
+fi
+export tests_repo_dir="${GOPATH}/src/${tests_repo}"
+export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
+export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
+export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication
+export CONTAINER_ID="${CONTAINER_ID:-0123456789}"
+source /etc/os-release || source /usr/lib/os-release
+grep -Eq "\<fedora\>" /etc/os-release 2> /dev/null && export USE_PODMAN=true
+
+
+# If we've already checked out the test repo then source the confidential scripts
+if [ "${KUBERNETES}" == "yes" ]; then
+    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/kubernetes/confidential"
+    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
+else
+    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/containerd/confidential"
+    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
+fi
+
+[ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+
+usage() {
+    exit_code="$1"
+    cat <<EOF
+Overview:
+    Build and test kata containers from source
+    Optionally set kata-containers and tests repo and branch as exported variables before running
+    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all
+Usage:
+    ${script_name} [options] <command>
+Commands:
+- agent_create_container:           Run CreateContainer command against the agent with agent-ctl
+- agent_pull_image:                 Run PullImage command against the agent with agent-ctl
+- all:                              Build and install everything, test kata with containerd and capture the logs
+- build_and_add_agent_to_rootfs:    Builds the kata-agent and adds it to the rootfs
+- build_and_install_all:            Build and install everything
+- build_and_install_rootfs:         Builds and installs the rootfs image
+- build_kata_runtime:               Build and install the kata runtime
+- build_cloud_hypervisor            Checkout, patch, build and install Cloud Hypervisor
+- build_qemu:                       Checkout, patch, build and install QEMU
+- configure:                        Configure Kata to use rootfs and enable debug
+- connect_to_ssh_demo_pod:          Ssh into the ssh demo pod, showing that the decryption succeeded
+- copy_signature_files_to_guest     Copies signature verification files to guest
+- create_rootfs:                    Create a local rootfs
+- crictl_create_cc_container        Use crictl to create a new busybox container in the kata cc pod
+- crictl_create_cc_pod              Use crictl to create a new kata cc pod
+- crictl_delete_cc                  Use crictl to delete the kata cc pod sandbox and container in it
+- help:                             Display this help
+- init_kubernetes:                  initialize a Kubernetes cluster on this system
+- initialize:                       Install dependencies and check out kata-containers source
+- install_guest_kernel:             Setup, build and install the guest kernel
+- kubernetes_create_cc_pod:         Create a Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_create_ssh_demo_pod:   Create a Kata CC runtime pod based on the ssh demo
+- kubernetes_delete_cc_pod:         Delete the Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_delete_ssh_demo_pod:   Delete the Kata CC runtime pod based on the ssh demo
+- open_kata_shell:                  Open a shell into the kata runtime
+- rebuild_and_install_kata:         Rebuild the kata runtime and agent and build and install the image
+- shim_pull_image:                  Run PullImage command against the shim with ctr
+- test_capture_logs:                Test using kata with containerd and capture the logs in the user's home directory
+- test:                             Test using kata with containerd
+
+Options:
+    -d: Enable debug
+    -h: Display this help
+EOF
+    # if script sourced don't exit as this will exit the main shell, just return instead
+    [[ $_ != $0 ]] && return "$exit_code" || exit "$exit_code"
+}
+
+build_and_install_all() {
+    initialize
+    build_and_install_kata_runtime
+    configure
+    create_a_local_rootfs
+    build_and_install_rootfs
+    install_guest_kernel_image
+    case "$KATA_HYPERVISOR" in
+        "qemu") 
+            build_qemu
+            ;;
+        "cloud-hypervisor") 
+            build_cloud_hypervisor
+            ;;
+        *)
+            echo "Invalid option: $KATA_HYPERVISOR is not supported." >&2
+            ;;
+    esac
+
+    check_kata_runtime
+    if [ "${KUBERNETES}" == "yes" ]; then
+        init_kubernetes
+    fi
+}
+
+rebuild_and_install_kata() {
+    checkout_tests_repo
+    checkout_kata_containers_repo
+    build_and_install_kata_runtime
+    build_and_add_agent_to_rootfs
+    build_and_install_rootfs
+    check_kata_runtime
+}
+
+# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
+initialize() {
+    # We need git to checkout and bootstrap the ci scripts and some other packages used in testing
+    sudo apt-get update && sudo apt-get install -y curl git qemu-utils
+    
+    grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
+    grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
+    grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}"
+    
+    # Load the new go and PATH parameters from the profile
+    . ${PROFILE}
+    mkdir -p "${GOPATH}"
+
+    checkout_tests_repo
+
+    pushd "${tests_repo_dir}"
+    local ci_dir_name=".ci"
+    sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f
+    sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh"
+    # Need to change ownership of rustup so later process can create temp files there
+    sudo chown -R ${USER}:${USER} "${HOME}/.rustup"
+
+    checkout_kata_containers_repo
+
+    # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
+    export INSTALL_KATA="no"
+    sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh
+    # Reload the profile to pick up installed dependencies
+    . ${PROFILE}
+    popd
+}
+
+checkout_tests_repo() {
+    echo "Creating repo: ${tests_repo} and branch ${tests_branch} into ${tests_repo_dir}..."
+    # Due to git https://github.blog/2022-04-12-git-security-vulnerability-announced/ the tests repo needs
+    # to be owned by root as it is re-checked out in rootfs.sh
+    mkdir -p $(dirname "${tests_repo_dir}")
+    [ -d "${tests_repo_dir}" ] || sudo -E git clone "https://${tests_repo}.git" "${tests_repo_dir}"
+    sudo -E chown -R root:root "${tests_repo_dir}"
+    pushd "${tests_repo_dir}"
+    sudo -E git fetch
+    if [ -n "${tests_branch}" ]; then
+        sudo -E git checkout ${tests_branch}
+    fi
+    sudo -E git reset --hard origin/${tests_branch}
+    popd
+
+    source "${BATS_TEST_DIRNAME}/lib.sh"
+    source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+}
+
+# Note: clone_katacontainers_repo using go, so that needs to be installed first
+checkout_kata_containers_repo() {
+    source "${tests_repo_dir}/.ci/lib.sh"
+    echo "Creating repo: ${katacontainers_repo} and branch ${kata_default_branch} into ${katacontainers_repo_dir}..."
+    clone_katacontainers_repo
+    sudo -E chown -R ${USER}:${USER} "${katacontainers_repo_dir}"
+}
+
+build_and_install_kata_runtime() {
+    export DEFAULT_HYPERVISOR=${KATA_HYPERVISOR}
+    ${tests_repo_dir}/.ci/install_runtime.sh
+}
+
+configure() {
+    # configure kata to use rootfs, not initrd
+    sudo sed -i 's/^\(initrd =.*\)/# \1/g' ${RUNTIME_CONFIG_PATH}
+
+    enable_full_debug
+    enable_agent_console
+
+    # Switch image offload to true in kata config
+    switch_image_service_offload "on"
+
+    configure_cc_containerd
+    # From crictl v1.24.1 the default timoout leads to the pod creation failing, so update it
+    sudo crictl config --set timeout=10
+
+    # Verity checks aren't working locally, as we aren't re-genning the hash maybe? so remove it from the kernel parameters
+    remove_kernel_param "cc_rootfs_verity.scheme"
+}
+
+build_and_add_agent_to_rootfs() {
+    build_a_custom_kata_agent
+    add_custom_agent_to_rootfs
+}
+
+build_a_custom_kata_agent() {
+    # Install libseccomp for static linking
+    sudo -E PATH=$PATH GOPATH=$GOPATH ${katacontainers_repo_dir}/ci/install_libseccomp.sh /tmp/kata-libseccomp /tmp/kata-gperf
+    export LIBSECCOMP_LINK_TYPE=static
+    export LIBSECCOMP_LIB_PATH=/tmp/kata-libseccomp/lib
+
+    . "$HOME/.cargo/env"
+    pushd ${katacontainers_repo_dir}/src/agent
+    sudo -E PATH=$PATH make
+
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+
+    # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
+    sudo -E PATH=$PATH make install DESTDIR="${ROOTFS_DIR}"
+    popd
+}
+
+create_a_local_rootfs() {
+    sudo rm -rf "${ROOTFS_DIR}"
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
+    export distro="ubuntu"
+    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
+    sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG:-}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} AA_KBC=${AA_KBC:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
+
+     # Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now
+    pushd "${tests_repo_dir}"
+    sudo -E git checkout ${tests_branch}
+    popd
+    # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
+    sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
+
+    popd
+}
+
+add_custom_agent_to_rootfs() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
+
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+
+    sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent
+    sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
+    sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
+    popd
+}
+
+build_and_install_rootfs() {
+    build_rootfs_image
+    install_rootfs_image
+}
+
+build_rootfs_image() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
+    # Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker
+    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
+    sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR}
+    popd
+}
+
+install_rootfs_image() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
+    local commit=$(git log --format=%h -1 HEAD)
+    local date=$(date +%Y-%m-%d-%T.%N%z)
+    local image="kata-containers-${date}-${commit}"
+    sudo install -o root -g root -m 0640 -D kata-containers.img "${PREFIX}/share/kata-containers/${image}"
+    (cd ${PREFIX}/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+    echo "Built Rootfs from ${ROOTFS_DIR} to ${PREFIX}/share/kata-containers/${image}"
+    ls -al ${PREFIX}/share/kata-containers
+    popd
+}
+
+install_guest_kernel_image() {
+   ${tests_repo_dir}/.ci/install_kata_kernel.sh
+}
+
+build_qemu() {
+    ${tests_repo_dir}/.ci/install_virtiofsd.sh
+    ${tests_repo_dir}/.ci/install_qemu.sh
+}
+
+build_cloud_hypervisor() {
+    ${tests_repo_dir}/.ci/install_virtiofsd.sh
+    ${tests_repo_dir}/.ci/install_cloud_hypervisor.sh
+}
+
+check_kata_runtime() {
+    sudo kata-runtime check
+}
+
+k8s_pod_file="${HOME}/busybox-cc.yaml"
+init_kubernetes() {
+    # Check that kubeadm was installed and install it otherwise
+    if ! [ -x "$(command -v kubeadm)" ]; then
+        pushd "${tests_repo_dir}/.ci"
+        sudo -E PATH=$PATH -s install_kubernetes.sh
+        if [ "${CRI_CONTAINERD}" == "yes" ]; then
+            sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh"
+        fi
+        popd
+    fi
+
+    # If kubernetes init has previously run we need to clean it by removing the image and resetting k8s
+    local cid=$(sudo docker ps -a -q -f name=^/kata-registry$)
+    if [ -n "${cid}" ]; then
+        sudo docker stop ${cid} && sudo docker rm ${cid}
+    fi
+    local k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true)
+    if [ -n "${k8s_nodes}" ]; then
+        sudo kubeadm reset -f
+    fi
+
+    export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh
+    sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config"
+    cat << EOF > ${k8s_pod_file}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: busybox-cc
+spec:
+  runtimeClassName: kata
+  containers:
+  - name: nginx
+    image: quay.io/kata-containers/confidential-containers:signed
+    imagePullPolicy: Always  
+EOF
+}
+
+call_kubernetes_create_cc_pod() {
+    kubernetes_create_cc_pod ${k8s_pod_file}
+}
+
+call_kubernetes_delete_cc_pod() {
+    pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
+    kubernetes_delete_cc_pod $pod_name
+}
+
+call_kubernetes_create_ssh_demo_pod() {
+    setup_decryption_files_in_guest
+    kubernetes_create_ssh_demo_pod
+}
+
+call_connect_to_ssh_demo_pod() {
+    connect_to_ssh_demo_pod
+}
+
+call_kubernetes_delete_ssh_demo_pod() {
+    pod=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
+    kubernetes_delete_ssh_demo_pod $pod
+}
+
+crictl_sandbox_name=kata-cc-busybox-sandbox
+call_crictl_create_cc_pod() {
+    # Update iptables to allow forwarding to the cni0 bridge avoiding issues caused by the docker0 bridge
+    sudo iptables -P FORWARD ACCEPT
+    
+    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
+    get_pod_config
+
+    crictl_delete_cc_pod_if_exists "${crictl_sandbox_name}"
+    crictl_create_cc_pod "${pod_config}"
+    sudo crictl pods
+}
+
+call_crictl_create_cc_container() {
+    # Create container configuration yaml based on our test copy of busybox
+    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
+    get_pod_config
+
+    local container_config="${FIXTURES_DIR}/${CONTAINER_CONFIG_FILE:-container-config.yaml}"
+    local pod_name=${crictl_sandbox_name}
+    crictl_create_cc_container ${pod_name} ${pod_config} ${container_config}
+    sudo crictl ps -a
+}
+
+crictl_delete_cc() {
+    crictl_delete_cc_pod ${crictl_sandbox_name}
+}
+
+test_kata_runtime() {
+    echo "Running ctr with the kata runtime..."
+    local test_image="quay.io/kata-containers/confidential-containers:signed"
+    if [ -z $(sudo ctr images ls -q name=="${test_image}") ]; then
+        sudo ctr image pull "${test_image}"
+    fi
+    sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
+}
+
+run_kata_and_capture_logs() {
+    echo "Clearing systemd journal..."
+    sudo systemctl stop systemd-journald
+    sudo rm -f /var/log/journal/*/* /run/log/journal/*/*
+    sudo systemctl start systemd-journald
+    test_kata_runtime
+    echo "Collecting logs..."
+    sudo journalctl -q -o cat -a -t kata-runtime > ${HOME}/kata-runtime.log
+    sudo journalctl -q -o cat -a -t kata > ${HOME}/shimv2.log
+    echo "Logs output to ${HOME}/kata-runtime.log and ${HOME}/shimv2.log"
+}
+
+get_ids() {
+    guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
+    sandbox_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
+}
+
+open_kata_shell() {
+    get_ids
+    sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id}
+}
+
+build_bundle_dir_if_necessary() {
+    bundle_dir="/tmp/bundle"
+    if [ ! -d "${bundle_dir}" ]; then
+        rootfs_dir="$bundle_dir/rootfs"
+        image="quay.io/kata-containers/confidential-containers:signed"
+        mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec)
+        sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf -
+    fi
+    # There were errors in create container agent-ctl command due to /bin/ seemingly not being on the path, so hardcode it
+    sudo sed -i -e 's%^\(\t*\)"sh"$%\1"/bin/sh"%g' "${bundle_dir}/config.json"
+}
+
+build_agent_ctl() {
+    cd ${GOPATH}/src/${katacontainers_repo}/src/tools/agent-ctl/
+    if [ -e "${HOME}/.cargo/registry" ]; then
+        sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
+    fi
+    sudo -E PATH=$PATH -s  make
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+    cd "./target/${ARCH}-unknown-linux-${LIBC}/release/"
+}
+
+run_agent_ctl_command() {
+    get_ids
+    build_bundle_dir_if_necessary
+    command=$1
+    # If kata-agent-ctl pre-built in this directory, use it directly, otherwise build it first and switch to release
+    if [ ! -x kata-agent-ctl ]; then
+        build_agent_ctl
+    fi 
+     ./kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c "${command}"
+}
+
+agent_pull_image() {
+    run_agent_ctl_command "PullImage image=${PULL_IMAGE} cid=${CONTAINER_ID} source_creds=${SOURCE_CREDS}"
+}
+
+agent_create_container() {
+    run_agent_ctl_command "CreateContainer cid=${CONTAINER_ID}"
+}
+
+shim_pull_image() {
+    get_ids
+    local ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
+    echo "Issuing command '${ctr_shim_command}'"
+    ${ctr_shim_command}
+}
+
+call_copy_signature_files_to_guest() {
+    # TODO #5173 - remove this once the kernel_params aren't ignored by the agent config
+    export DEBUG_CONSOLE="true"
+    
+    if [ "${SKOPEO:-}" = "yes" ]; then
+        add_kernel_params "agent.container_policy_file=/etc/containers/quay_verification/quay_policy.json"
+        setup_skopeo_signature_files_in_guest
+    else
+        # TODO #4888 - set config to specifically enable signature verification to be on in ImageClient
+        setup_offline_fs_kbc_signature_files_in_guest
+    fi
+}
+
+main() {
+    while getopts "dh" opt; do
+        case "$opt" in
+            d) 
+                export DEBUG="-d"
+                set -x
+                ;;
+            h) 
+                usage 0
+                ;;
+            \?)
+                echo "Invalid option: -$OPTARG" >&2
+                usage 1
+                ;;
+        esac
+    done
+
+    shift $((OPTIND - 1))
+
+    subcmd="${1:-}"
+
+    [ -z "${subcmd}" ] && usage 1
+
+    case "${subcmd}" in
+        all)
+            build_and_install_all
+            run_kata_and_capture_logs
+            ;;
+        build_and_install_all)
+            build_and_install_all
+            ;;
+        rebuild_and_install_kata)
+            rebuild_and_install_kata
+            ;;
+        initialize)
+            initialize
+            ;;
+        build_kata_runtime)
+            build_and_install_kata_runtime
+            ;;
+        configure)
+            configure
+            ;;
+        create_rootfs)
+            create_a_local_rootfs
+            ;;
+        build_and_add_agent_to_rootfs)
+            build_and_add_agent_to_rootfs
+            ;;
+        build_and_install_rootfs)
+            build_and_install_rootfs
+            ;;
+        install_guest_kernel)
+            install_guest_kernel_image
+            ;;
+        build_cloud_hypervisor)
+            build_cloud_hypervisor
+            ;;
+        build_qemu)
+            build_qemu
+            ;;
+        init_kubernetes)
+            init_kubernetes
+            ;;
+        crictl_create_cc_pod)
+            call_crictl_create_cc_pod
+            ;;
+        crictl_create_cc_container)
+            call_crictl_create_cc_container
+            ;;
+        crictl_delete_cc)
+            crictl_delete_cc
+            ;;
+        kubernetes_create_cc_pod)
+            call_kubernetes_create_cc_pod
+            ;;
+        kubernetes_delete_cc_pod)
+            call_kubernetes_delete_cc_pod
+            ;;
+        kubernetes_create_ssh_demo_pod)
+            call_kubernetes_create_ssh_demo_pod
+            ;;
+        connect_to_ssh_demo_pod)
+            call_connect_to_ssh_demo_pod
+            ;;
+        kubernetes_delete_ssh_demo_pod)
+            call_kubernetes_delete_ssh_demo_pod
+            ;;
+        test)
+            test_kata_runtime
+            ;;
+        test_capture_logs)
+            run_kata_and_capture_logs
+            ;;
+        open_kata_console)
+            open_kata_console
+            ;;
+        open_kata_shell)
+            open_kata_shell
+            ;;
+        agent_pull_image)
+            agent_pull_image
+            ;;
+        shim_pull_image)
+            shim_pull_image
+            ;;
+        agent_create_container)
+            agent_create_container
+            ;;
+        copy_signature_files_to_guest)
+            call_copy_signature_files_to_guest
+            ;;
+        *)
+            usage 1
+            ;;
+    esac
+}
+
+main $@

docs/how-to/data/confidential-agent-config.toml.in

@@ -0,0 +1,45 @@
+# Copyright (c) 2021 IBM Corp.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+aa_kbc_params = "$AA_KBC_PARAMS"
+https_proxy = "$HTTPS_PROXY"
+[endpoints]
+allowed = [
+"AddARPNeighborsRequest",
+"AddSwapRequest",
+"CloseStdinRequest",
+"CopyFileRequest",
+"CreateContainerRequest",
+"CreateSandboxRequest",
+"DestroySandboxRequest",
+#"ExecProcessRequest",
+"GetMetricsRequest",
+"GetOOMEventRequest",
+"GuestDetailsRequest",
+"ListInterfacesRequest",
+"ListRoutesRequest",
+"MemHotplugByProbeRequest",
+"OnlineCPUMemRequest",
+"PauseContainerRequest",
+"PullImageRequest",
+"ReadStreamRequest",
+"RemoveContainerRequest",
+#"ReseedRandomDevRequest",
+"ResizeVolumeRequest",
+"ResumeContainerRequest",
+"SetGuestDateTimeRequest",
+"SignalProcessRequest",
+"StartContainerRequest",
+"StartTracingRequest",
+"StatsContainerRequest",
+"StopTracingRequest",
+"TtyWinResizeRequest",
+"UpdateContainerRequest",
+"UpdateInterfaceRequest",
+"UpdateRoutesRequest",
+"VolumeStatsRequest",
+"WaitProcessRequest",
+"WriteStreamRequest"
+]

docs/how-to/how-to-build-and-test-ccv0.md

@@ -0,0 +1,475 @@
+# How to build, run and test Kata CCv0
+
+## Introduction and Background
+
+In order to try and make building (locally) and demoing the Kata Containers `CCv0` code base as simple as possible I've
+shared a script [`ccv0.sh`](./ccv0.sh). This script was originally my attempt to automate the steps of the 
+[Developer Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md) so that I could do
+different sections of them repeatedly and reliably as I was playing around with make changes to different parts of the 
+Kata code base. I then tried to weave in some of the [`tests/.ci`](https://github.com/kata-containers/tests/tree/main/.ci) 
+scripts in order to have less duplicated code.
+As we're progress on the confidential containers journey I hope to add more features to demonstrate the functionality
+we have working.
+
+*Disclaimer: This script has mostly just been used and tested by me ([@stevenhorsman](https://github.com/stevenhorsman)),*
+*so there might be issues with it. I'm happy to try and help solve these if possible, but this shouldn't be considered a*
+*fully supported process by the Kata Containers community.*
+
+### Basic script set-up and optional environment variables
+
+In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take:
+- Provision a new VM
+    - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed*
+    *packages, so these will need re-working to be compatible with other platforms.*
+- Copy the script over to your VM *(I put it in the home directory)* and ensure it has execute permission by running 
+```bash
+$ chmod u+x ccv0.sh
+```
+- Optionally set up some environment variables
+    - By default the script checks out the `CCv0` branches of the `kata-containers/kata-containers` and 
+      `kata-containers/tests` repositories, but it is designed to be used to test of personal forks and branches as well. 
+      If you want to build and run these you can export the `katacontainers_repo`, `katacontainers_branch`, `tests_repo`
+      and `tests_branch` variables e.g.
+      ```bash
+      $ export katacontainers_repo=github.com/stevenhorsman/kata-containers
+      $ export katacontainers_branch=stevenh/agent-pull-image-endpoint
+      $ export tests_repo=github.com/stevenhorsman/tests
+      $ export tests_branch=stevenh/add-ccv0-changes-to-build
+      ```
+      before running the script.
+    - By default the build and configuration are using `QEMU` as the hypervisor. In order to use `Cloud Hypervisor` instead
+      set:
+      ```
+      $ export KATA_HYPERVISOR="cloud-hypervisor"
+      ```
+      before running the build.
+
+- At this point you can provision a Kata confidential containers pod and container with either
+  [`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image),
+  or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+  and then test and use it. 
+
+### Using crictl for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
+
+- Run the full build process with Kubernetes turned off, so its configuration doesn't interfere with `crictl` using:
+  ```bash
+  $ export KUBERNETES="no"
+  $ export KATA_HYPERVISOR="qemu"
+  $ ~/ccv0.sh -d build_and_install_all
+  ```
+    > **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password.
+    - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest*
+      *of the process without having to reload the session.*
+    - The steps that `build_and_install_all` takes is:
+        - Checkout the git repos for the `tests` and `kata-containers` repos as specified by the environment variables
+        (default to `CCv0` branches if they are not supplied)
+        - Use the `tests/.ci` scripts to install the build dependencies
+        - Build and install the Kata runtime
+        - Configure Kata to use containerd and for debug and confidential containers features to be enabled (including
+          enabling console access to the Kata guest shell, which should only be done in development)
+        - Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Ubuntu
+        20.04.
+        - Build the Kata guest kernel
+        - Install the hypervisor (in order to select which hypervisor will be used, the `KATA_HYPERVISOR` environment
+        variable can be used to select between `qemu` or `cloud-hypervisor`)
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull ubuntu
+  >  ```
+  >  then re-run the command.
+    - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already
+      installed and they can be further cut down by not running all the above steps 
+      [see "Additional script usage" below](#additional-script-usage)
+   
+- Create a new Kata sandbox pod using `crictl` with:
+  ```bash
+  $ ~/ccv0.sh crictl_create_cc_pod
+  ```
+    - This creates a pod configuration file, creates the pod from this using 
+    `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod
+- Create a new Kata confidential container with:
+  ```bash
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+    - This creates a container (based on `busybox:1.33.1`) in the Kata cc sandbox and prints a list of containers.
+      This will have been created based on an image pulled in the Kata pod sandbox/guest, not on the host machine.
+
+As this point you should have a `crictl` pod and container that is using the Kata confidential containers runtime.
+You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
+or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
+
+#### Clean up the `crictl` pod sandbox and container
+- When the testing is complete you can delete the container and pod by running:
+  ```bash
+  $ ~/ccv0.sh crictl_delete_cc
+  ```
+### Using Kubernetes for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
+
+- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
+  configured and created using the VM
+  as a single node cluster: 
+  ```bash
+  $ export KUBERNETES="yes"
+  $ ~/ccv0.sh build_and_install_all
+  ```
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull registry:2
+  >  $ sudo docker pull ubuntu:20.04
+  >  ```
+  >  then re-run the command.
+- Check that your Kubernetes cluster has been correctly set-up by running : 
+  ```bash
+  $ kubectl get nodes
+  ```
+  and checking that you see a single node e.g.
+  ```text
+  NAME                             STATUS   ROLES                  AGE   VERSION
+  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
+  ```
+- Create a Kata confidential containers pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_create_cc_pod
+  ```
+- Wait a few seconds for pod to start then check that the pod's status is `Running` with 
+  ```bash
+  $ kubectl get pods
+  ```
+  which should show something like:
+  ```text
+  NAME         READY   STATUS    RESTARTS   AGE
+  busybox-cc   1/1     Running   0          54s
+  ```
+
+- As this point you should have a Kubernetes pod and container running, that is using the Kata 
+confidential containers runtime.
+You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
+or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
+
+#### Clean up the Kubernetes pod sandbox and container
+- When the testing is complete you can delete the container and pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_delete_cc_pod
+  ```
+
+### Validate that the container image was pulled on the guest
+
+There are a couple of ways we can check that the container pull image action was offloaded to the guest, by checking
+the guest's file system for the unpacked bundle and checking the host's directories to ensure it wasn't also pulled
+there.
+- To check the guest's file system:
+    - Open a shell into the Kata guest with:
+      ```bash
+      $ ~/ccv0.sh open_kata_shell
+      ```
+    - List the files in the directory that the container image bundle should have been unpacked to with:
+      ```bash
+      $ ls -ltr /run/kata-containers/confidential-containers_signed/
+      ```
+    - This should give something like
+        ```
+        total 72
+        -rw-r--r--  1 root root  2977 Jan 20 10:03 config.json
+        drwxr-xr-x 12 root root   240 Jan 20 10:03 rootfs
+        ```
+        which shows how the image has been pulled and then unbundled on the guest.
+    - Leave the Kata guest shell by running:
+      ```bash
+      $ exit
+      ```
+- To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we
+  should only see a single bundle for the pause container as the `busybox` based container image should have been
+  pulled on the guest:
+    - Find all the `rootfs` directories under in the pod's shared directory with:
+      ```bash
+      $ pod_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
+      $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs
+      ```
+      which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host
+    - Looking that `rootfs` directory with
+      ```bash
+      $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs)
+      ```
+      shows something similar to
+      ```
+      total 668
+      -rwxr-xr-x 1 root root 682696 Aug 25 13:58 pause
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 proc
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 dev
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 sys
+      drwxr-xr-x 2 root root     25 Jan 20 02:01 etc
+      ```
+      which is clearly the pause container indicating that the `busybox` based container image is not exposed to the host.
+
+### Using a Kata pod sandbox for testing with `agent-ctl` or `ctr shim`
+
+Once you have a kata pod sandbox created as described above, either using 
+[`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image), or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+, you can use this to test specific components of the Kata confidential
+containers architecture. This can be useful for development and debugging to isolate and test features
+that aren't broadly supported end-to-end. Here are some examples:
+
+- In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`).
+This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to
+interact with the shim directly. The command takes the form
+`ctr --namespace k8s.io shim --id <sandbox-id> pull-image <image> <new-container-id>` and can been run directly, or through
+the `ccv0.sh` script to automatically fill in the variables:
+  - Optionally, set up some environment variables to set the image and credentials used:
+    - By default the shim pull image test in `ccv0.sh` will use the `busybox:1.33.1` based test image
+     `quay.io/kata-containers/confidential-containers:signed` which requires no authentication. To use a different
+      image, set the `PULL_IMAGE` environment variable e.g. 
+      ```bash
+      $ export PULL_IMAGE="docker.io/library/busybox:latest"
+      ```
+      Currently the containerd shim pull image
+      code doesn't support using a container registry that requires authentication, so if this is required, see the 
+      below steps to run the pull image command against the agent directly.
+  - Run the pull image agent endpoint with:
+    ```bash
+    $ ~/ccv0.sh shim_pull_image
+    ```
+    which we print the `ctr shim` command for reference
+- Alternatively you can issue the command directly to the `kata-agent` pull image endpoint, which also supports
+  credentials in order to pull from an authenticated registry:
+    - Optionally set up some environment variables to set the image and credentials used:
+        - Set the `PULL_IMAGE` environment variable e.g. `export PULL_IMAGE="docker.io/library/busybox:latest"`
+          if a specific container image is required.
+        - If the container registry for the image requires authentication then this can be set with an environment
+          variable `SOURCE_CREDS`. For example to use Docker Hub (`docker.io`) as an authenticated user first run 
+          `export SOURCE_CREDS="<dockerhub username>:<dockerhub api key>"`
+            > **Note**: the credentials support on the agent request is a tactical solution for the short-term
+              proof of concept to allow more images to be pulled and tested. Once we have support for getting
+              keys into the Kata guest image using the attestation-agent and/or KBS I'd expect container registry
+              credentials to be looked up using that mechanism.
+    - Run the pull image agent endpoint with
+      ```bash
+      $ ~/ccv0.sh agent_pull_image
+      ```
+      and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate
+      that the `PullImage` request was successful e.g.
+      ```
+      Finished release [optimized] target(s) in 0.21s
+      {"msg":"announce","level":"INFO","ts":"2021-09-15T08:40:14.189360410-07:00","subsystem":"rpc","name":"kata-agent-ctl","pid":"830920","version":"0.1.0","source":"kata-agent-ctl","config":"Config { server_address: \"vsock://1970354082:1024\", bundle_dir: \"/tmp/bundle\", timeout_nano: 0, interactive: false, ignore_errors: false }"}
+      {"msg":"client setup complete","level":"INFO","ts":"2021-09-15T08:40:14.193639057-07:00","pid":"830920","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","server-address":"vsock://1970354082:1024"}
+      {"msg":"Run command PullImage (1 of 1)","level":"INFO","ts":"2021-09-15T08:40:14.196643765-07:00","pid":"830920","source":"kata-agent-ctl","subsystem":"rpc","name":"kata-agent-ctl","version":"0.1.0"}
+      {"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""}
+      {"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"}
+      ```
+      > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built
+      which may take a few minutes.
+- To validate that the image pull was successful, you can open a shell into the Kata guest with:
+  ```bash
+  $ ~/ccv0.sh open_kata_shell
+  ```
+- Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory
+  named either `01234556789` (for the container id), or the container image name, e.g.
+  ```bash
+  $ ls -ltr /run/kata-containers/confidential-containers_signed/
+  ```
+  which should show something like
+  ```
+  total 72
+  drwxr-xr-x 10 root root   200 Jan  1  1970 rootfs
+  -rw-r--r--  1 root root  2977 Jan 20 16:45 config.json
+  ```
+- Leave the Kata shell by running:
+  ```bash
+  $ exit
+  ```
+
+## Verifying signed images
+
+For this sample demo, we use local attestation to pass through the required
+configuration to do container image signature verification. Due to this, the ability to verify images is limited
+to a pre-created selection of test images in our test
+repository [`quay.io/kata-containers/confidential-containers`](https://quay.io/repository/kata-containers/confidential-containers?tab=tags).
+For pulling images not in this test repository (called an *unprotected* registry below), we fall back to the behaviour
+of not enforcing signatures. More documentation on how to customise this to match your own containers through local,
+or remote attestation will be available in future.
+
+In our test repository there are three tagged images:
+
+| Test Image | Base Image used | Signature status | GPG key status |
+| --- | --- | --- | --- |
+| `quay.io/kata-containers/confidential-containers:signed` | `busybox:1.33.1` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs |  [public key](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/public.gpg) embedded in kata rootfs |
+| `quay.io/kata-containers/confidential-containers:unsigned` | `busybox:1.33.1` | not signed | not signed |
+| `quay.io/kata-containers/confidential-containers:other_signed` | `nginx:1.21.3` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs | GPG key not kept |
+
+Using a standard unsigned `busybox` image that can be pulled from another, *unprotected*, `quay.io` repository we can
+test a few scenarios.
+
+In this sample, with local attestation, we pass in the the public GPG key and signature files, and the [`offline_fs_kbc`
+configuration](https://github.com/confidential-containers/attestation-agent/blob/main/src/kbc_modules/offline_fs_kbc/README.md)
+into the guest image which specifies that any container image from `quay.io/kata-containers`
+must be signed with the embedded GPG key and the agent configuration needs updating to enable this. 
+With this policy set a few tests of image verification can be done to test different scenarios by attempting
+to create containers from these images using `crictl`:
+
+- If you don't already have the Kata Containers CC code built and configured for `crictl`, then follow the 
+[instructions above](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+up to the `~/ccv0.sh crictl_create_cc_pod` command.
+
+- In order to enable the guest image, you will need to setup the required configuration, policy and signature files
+needed by running 
+`~/ccv0.sh copy_signature_files_to_guest` and then run `~/ccv0.sh crictl_create_cc_pod` which will delete and recreate 
+your pod - adding in the new files.
+ 
+- To test the fallback behaviour works using an unsigned image from an *unprotected* registry we can pull the `busybox`
+image by running:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-unprotected.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This finishes showing the running container e.g.
+  ```text
+  CONTAINER           IMAGE                               CREATED                  STATE               NAME                        ATTEMPT             POD ID
+  98c70fefe997a       quay.io/prometheus/busybox:latest   Less than a second ago   Running             prometheus-busybox-signed   0                   70119e0539238
+  ```
+- To test that an unsigned image from our *protected* test container registry is rejected we can run:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-protected.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This correctly results in an error message from `crictl`:
+  `PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [Match reference failed.]" image="quay.io/kata-containers/confidential-containers:unsigned"`
+- To test that the signed image our *protected* test container registry is accepted we can run:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This finishes by showing a new `kata-cc-busybox-signed` running container e.g.
+  ```text
+  CONTAINER           IMAGE                                                    CREATED                  STATE               NAME                        ATTEMPT             POD ID
+  b4d85c2132ed9       quay.io/kata-containers/confidential-containers:signed   Less than a second ago   Running             kata-cc-busybox-signed      0                   70119e0539238
+  ...
+  ```
+- Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really
+want to protect with the attestation agent in future) fails we can run: 
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_signed-protected-other.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - Again this results in an error message from `crictl`:
+  `"PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [signature verify failed! There is no pubkey can verify the signature!]" image="quay.io/kata-containers/confidential-containers:other_signed"`
+
+### Using Kubernetes to create a Kata confidential containers pod from the encrypted ssh demo sample image
+
+The [ssh-demo](https://github.com/confidential-containers/documentation/tree/main/demos/ssh-demo) explains how to
+demonstrate creating a Kata confidential containers pod from an encrypted image with the runtime created by the
+[confidential-containers operator](https://github.com/confidential-containers/documentation/blob/main/demos/operator-demo).
+To be fully confidential, this should be run on a Trusted Execution Environment, but it can be tested on generic
+hardware as well.
+
+If you wish to build the Kata confidential containers runtime to do this yourself, then you can using the following
+steps:
+
+- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
+  configured and created using the VM as a single node cluster and with `AA_KBC` set to `offline_fs_kbc`. 
+  ```bash
+  $ export KUBERNETES="yes"
+  $ export AA_KBC=offline_fs_kbc
+  $ ~/ccv0.sh build_and_install_all
+  ```
+    - The `AA_KBC=offline_fs_kbc` mode will ensure that, when creating the rootfs of the Kata guest, the 
+      [attestation-agent](https://github.com/confidential-containers/attestation-agent) will be added along with the
+      [sample offline KBC](https://github.com/confidential-containers/documentation/blob/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json)
+      and an agent configuration file
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull registry:2
+  >  $ sudo docker pull ubuntu:20.04
+  >  ```
+  >  then re-run the command.
+- Check that your Kubernetes cluster has been correctly set-up by running : 
+  ```bash
+  $ kubectl get nodes
+  ```
+  and checking that you see a single node e.g.
+  ```text
+  NAME                             STATUS   ROLES                  AGE   VERSION
+  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
+  ```
+- Create a sample Kata confidential containers ssh pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_create_ssh_demo_pod
+  ```
+- As this point you should have a Kubernetes pod running the Kata confidential containers runtime that has pulled
+the [sample image](https://hub.docker.com/r/katadocker/ccv0-ssh) which was encrypted by the key file that we included
+in the rootfs.
+During the pod deployment the image was pulled and then decrypted using the key file, on the Kata guest image, without
+it ever being available to the host.
+
+- To validate that the container is working you, can connect to the image via SSH by running:
+  ```bash
+  $ ~/ccv0.sh connect_to_ssh_demo_pod
+  ```
+  - During this connection the host key fingerprint is shown and should match:
+    `ED25519 key fingerprint is SHA256:wK7uOpqpYQczcgV00fGCh+X97sJL3f6G1Ku4rvlwtR0.`
+  - After you are finished connecting then run:
+    ```bash
+    $ exit
+    ```
+  
+- To delete the sample SSH demo pod run:
+  ```bash
+  $ ~/ccv0.sh kubernetes_delete_ssh_demo_pod
+  ```
+
+## Additional script usage
+
+As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just
+re-build bits of it by running the script with different parameters. For example after the first build you will often
+not need to re-install the dependencies, the hypervisor or the Guest kernel, but just test code changes made to the
+runtime and agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout*
+*from git, so if your changes are only made locally it is better to do the individual steps e.g.* 
+`~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`).
+There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running
+`~/ccv0.sh help`:
+```
+$ ~/ccv0.sh help
+Overview:
+    Build and test kata containers from source
+    Optionally set kata-containers and tests repo and branch as exported variables before running
+    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/ccv0.sh build_and_install_all
+Usage:
+    ccv0.sh [options] <command>
+Commands:
+- help:                         Display this help
+- all:                          Build and install everything, test kata with containerd and capture the logs
+- build_and_install_all:        Build and install everything
+- initialize:                   Install dependencies and check out kata-containers source
+- rebuild_and_install_kata:     Rebuild the kata runtime and agent and build and install the image
+- build_kata_runtime:           Build and install the kata runtime
+- configure:                    Configure Kata to use rootfs and enable debug
+- create_rootfs:                Create a local rootfs
+- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs
+- build_and_install_rootfs:     Builds and installs the rootfs image
+- install_guest_kernel:         Setup, build and install the guest kernel
+- build_cloud_hypervisor        Checkout, patch, build and install Cloud Hypervisor
+- build_qemu:                   Checkout, patch, build and install QEMU
+- init_kubernetes:              initialize a Kubernetes cluster on this system
+- crictl_create_cc_pod          Use crictl to create a new kata cc pod
+- crictl_create_cc_container    Use crictl to create a new busybox container in the kata cc pod
+- crictl_delete_cc              Use crictl to delete the kata cc pod sandbox and container in it
+- kubernetes_create_cc_pod:     Create a Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_delete_cc_pod:     Delete the Kata CC runtime busybox-based pod in Kubernetes
+- open_kata_shell:              Open a shell into the kata runtime
+- agent_pull_image:             Run PullImage command against the agent with agent-ctl
+- shim_pull_image:              Run PullImage command against the shim with ctr
+- agent_create_container:       Run CreateContainer command against the agent with agent-ctl
+- test:                         Test using kata with containerd
+- test_capture_logs:            Test using kata with containerd and capture the logs in the user's home directory
+
+Options:
+    -d: Enable debug
+    -h: Display this help
+```

docs/how-to/how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md

@@ -0,0 +1,44 @@
+# Generating a Kata Containers payload for the Confidential Containers Operator
+
+[Confidential Containers
+Operator](https://github.com/confidential-containers/operator) consumes a Kata
+Containers payload, generated from the `CCv0` branch, and here one can find all
+the necessary info on how to build such a payload.
+
+## Requirements
+
+* `make` installed in the machine
+* Docker installed in the machine
+* `sudo` access to the machine
+
+## Process
+
+* Clone [Kata Containers](https://github.com/kata-containers/kata-containers)
+  ```sh
+  git clone --branch CCv0 https://github.com/kata-containers/kata-containers
+  ```
+  * In case you've already cloned the repo, make sure to switch to the `CCv0` branch
+    ```sh
+    git checkout CCv0
+    ```
+  * Ensure your tree is clean and in sync with upstream `CCv0`
+    ```sh
+    git clean -xfd
+    git reset --hard <upstream>/CCv0
+    ```
+* Make sure you're authenticated to `quay.io`
+    ```sh
+    sudo docker login quay.io
+    ```
+* From the top repo directory, run:
+  ```sh
+  sudo make cc-payload
+  ```
+* Make sure the image was upload to the [Confidential Containers
+  runtime-payload
+registry](https://quay.io/repository/confidential-containers/runtime-payload?tab=tags)
+
+## Notes
+
+Make sure to run it on a machine that's not the one you're hacking on, prepare a
+cup of tea, and get back to it an hour later (at least).

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -27,6 +27,8 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.runtime.internetworking_model` | string| determines how the VM should be connected to the container network interface. Valid values are `macvtap`, `tcfilter` and `none` |
 | `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup |
 | `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process |
+| `io.katacontainers.config.runtime.image_request_timeout` | `uint64` | the timeout for pulling an image within the guest in `seconds`, default is `60` |
+| `io.katacontainers.config.runtime.sealed_secret_enabled` | `boolean` | enables the sealed secret feature, default is `false` |
 
 ## Agent Options
 | Key | Value Type | Comments |
@@ -94,6 +96,16 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
 | `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
 
+## Confidential Computing Options
+| Key | Value Type | Comments |
+|-------| ----- | ----- |
+| `io.katacontainers.config.pre_attestation.enabled"` | `bool` |
+determines if SEV/-ES attestation is enabled |
+| `io.katacontainers.config.pre_attestation.uri"` | `string` |
+specify the location of the attestation server |
+| `io.katacontainers.config.sev.policy"` | `uint32` |
+specify the SEV guest policy |
+
 ## Container Options
 | Key | Value Type | Comments |
 |-------| ----- | ----- |

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -27,6 +27,8 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
+  uid: $(uuidgen)
+  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -32,6 +32,7 @@ The `nydus-sandbox.yaml` looks like below:
 metadata:
   attempt: 1
   name: nydus-sandbox
+  uid: nydus-uid
   namespace: default
 log_directory: /tmp
 linux:

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -42,6 +42,8 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
+  uid: $(uuidgen)
+  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

src/agent/Cargo.toml

@@ -23,17 +23,20 @@ regex = "1.5.6"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
+const_format = "0.2.30"
+url = "2.2.2"
+derivative = "2.2.0"
 
 # Async helpers
 async-trait = "0.1.42"
 async-recursion = "0.3.2"
-futures = "0.3.17"
+futures = "0.3.28"
 
 # Async runtime
 tokio = { version = "1.28.1", features = ["full"] }
 tokio-vsock = "0.3.1"
 
-netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
+netlink-sys = { version = "0.7.0", features = ["tokio_socket"] }
 rtnetlink = "0.8.0"
 netlink-packet-utils = "0.4.1"
 ipnetwork = "0.17.0"
@@ -59,7 +62,7 @@ cgroups = { package = "cgroups-rs", version = "0.3.2" }
 tracing = "0.1.26"
 tracing-subscriber = "0.2.18"
 tracing-opentelemetry = "0.13.0"
-opentelemetry = { version = "0.14.0", features = ["rt-tokio-current-thread"]}
+opentelemetry = { version = "0.14.0", features = ["rt-tokio-current-thread"] }
 vsock-exporter = { path = "vsock-exporter" }
 
 # Configuration
@@ -67,21 +70,35 @@ serde = { version = "1.0.129", features = ["derive"] }
 toml = "0.5.8"
 clap = { version = "3.0.1", features = ["derive"] }
 
+# "vendored" feature for openssl is required by musl build
+openssl = { version = "0.10.38", features = ["vendored"] }
+
+# Image pull/decrypt
+image-rs = { git = "https://github.com/confidential-containers/guest-components", tag = "v0.8.0", default-features = false, features = [
+    "kata-cc-native-tls",
+    "verity",
+    "signature-simple-xrss",
+] }
+
+[patch.crates-io]
+oci-distribution = { git = "https://github.com/krustlet/oci-distribution.git", rev = "f44124c" }
+
 [dev-dependencies]
 tempfile = "3.1.0"
 test-utils = { path = "../libs/test-utils" }
 which = "4.3.0"
 
 [workspace]
-members = [
-    "rustjail",
-]
+resolver = "2"
+members = ["rustjail"]
 
 [profile.release]
 lto = true
 
 [features]
+confidential-data-hub = []
 seccomp = ["rustjail/seccomp"]
+sealed-secret = ["protocols/sealed-secret", "confidential-data-hub"]
 standard-oci-runtime = ["rustjail/standard-oci-runtime"]
 
 [[bin]]

src/agent/Makefile

@@ -33,6 +33,13 @@ ifeq ($(SECCOMP),yes)
     override EXTRA_RUSTFEATURES += seccomp
 endif
 
+SEALED_SECRET ?= no
+
+# Enable sealed-secret feature of rust build
+ifeq ($(SEALED_SECRET),yes)
+    override EXTRA_RUSTFEATURES += sealed-secret
+endif
+
 include ../../utils.mk
 
 ifeq ($(ARCH), ppc64le)

src/agent/rustjail/Cargo.toml

@@ -34,7 +34,7 @@ futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"
 libseccomp = { version = "0.3.0", optional = true }
-zbus = "2.3.0"
+zbus = "3.12.0"
 bit-vec= "0.6.3"
 xattr = "0.2.3"
 

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -541,8 +541,11 @@ fn linux_device_to_cgroup_device(d: &LinuxDevice) -> Option<DeviceResource> {
 }
 
 fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> Option<DeviceResource> {
-    let dev_type = match DeviceType::from_char(d.r#type.chars().next()) {
-        Some(t) => t,
+    let dev_type = match &d.r#type {
+        Some(t_s) => match DeviceType::from_char(t_s.chars().next()) {
+            Some(t_c) => t_c,
+            None => return None,
+        },
         None => return None,
     };
 
@@ -599,7 +602,7 @@ lazy_static! {
             // all mknod to all char devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -608,7 +611,7 @@ lazy_static! {
             // all mknod to all block devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "b".to_string(),
+                r#type: Some("b".to_string()),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -617,7 +620,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/console
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(5),
                 minor: Some(1),
                 access: "rwm".to_string(),
@@ -626,7 +629,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/pts/<N>
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(136),
                 minor: Some(WILDCARD),
                 access: "rwm".to_string(),
@@ -635,7 +638,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/ptmx
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(5),
                 minor: Some(2),
                 access: "rwm".to_string(),
@@ -644,7 +647,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/net/tun
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(10),
                 minor: Some(200),
                 access: "rwm".to_string(),

src/agent/rustjail/src/cgroups/systemd/common.rs

@@ -6,7 +6,10 @@
 pub const DEFAULT_SLICE: &str = "system.slice";
 pub const SLICE_SUFFIX: &str = ".slice";
 pub const SCOPE_SUFFIX: &str = ".scope";
-pub const UNIT_MODE: &str = "replace";
+pub const WHO_ENUM_ALL: &str = "all";
+pub const SIGNAL_KILL: i32 = nix::sys::signal::SIGKILL as i32;
+pub const UNIT_MODE_REPLACE: &str = "replace";
+pub const NO_SUCH_UNIT_ERROR: &str = "org.freedesktop.systemd1.NoSuchUnit";
 
 pub type Properties<'a> = Vec<(&'a str, zbus::zvariant::Value<'a>)>;
 

src/agent/rustjail/src/cgroups/systemd/dbus_client.rs

@@ -1,56 +1,50 @@
-// Copyright 2021-2022 Kata Contributors
+// Copyright 2021-2023 Kata Contributors
 //
 // SPDX-License-Identifier: Apache-2.0
 //
 
 use std::vec;
 
-use super::common::CgroupHierarchy;
-use super::common::{Properties, SLICE_SUFFIX, UNIT_MODE};
+use super::common::{
+    CgroupHierarchy, Properties, NO_SUCH_UNIT_ERROR, SIGNAL_KILL, SLICE_SUFFIX, UNIT_MODE_REPLACE,
+    WHO_ENUM_ALL,
+};
 use super::interface::system::ManagerProxyBlocking as SystemManager;
-use anyhow::{Context, Result};
+use anyhow::{anyhow, Context, Result};
 use zbus::zvariant::Value;
 
 pub trait SystemdInterface {
-    fn start_unit(
-        &self,
-        pid: i32,
-        parent: &str,
-        unit_name: &str,
-        cg_hierarchy: &CgroupHierarchy,
-    ) -> Result<()>;
-
-    fn set_properties(&self, unit_name: &str, properties: &Properties) -> Result<()>;
-
-    fn stop_unit(&self, unit_name: &str) -> Result<()>;
-
+    fn start_unit(&self, pid: i32, parent: &str, cg_hierarchy: &CgroupHierarchy) -> Result<()>;
+    fn set_properties(&self, properties: &Properties) -> Result<()>;
+    fn kill_unit(&self) -> Result<()>;
+    fn freeze_unit(&self) -> Result<()>;
+    fn thaw_unit(&self) -> Result<()>;
+    fn add_process(&self, pid: i32) -> Result<()>;
     fn get_version(&self) -> Result<String>;
-
-    fn unit_exists(&self, unit_name: &str) -> Result<bool>;
-
-    fn add_process(&self, pid: i32, unit_name: &str) -> Result<()>;
+    fn unit_exists(&self) -> Result<bool>;
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct DBusClient {}
+pub struct DBusClient {
+    unit_name: String,
+}
 
 impl DBusClient {
+    pub fn new(unit_name: String) -> Self {
+        Self { unit_name }
+    }
+
     fn build_proxy(&self) -> Result<SystemManager<'static>> {
         let connection =
             zbus::blocking::Connection::system().context("Establishing a D-Bus connection")?;
         let proxy = SystemManager::new(&connection).context("Building a D-Bus proxy manager")?;
+
         Ok(proxy)
     }
 }
 
 impl SystemdInterface for DBusClient {
-    fn start_unit(
-        &self,
-        pid: i32,
-        parent: &str,
-        unit_name: &str,
-        cg_hierarchy: &CgroupHierarchy,
-    ) -> Result<()> {
+    fn start_unit(&self, pid: i32, parent: &str, cg_hierarchy: &CgroupHierarchy) -> Result<()> {
         let proxy = self.build_proxy()?;
 
         // enable CPUAccounting & MemoryAccounting & (Block)IOAccounting by default
@@ -68,7 +62,7 @@ impl SystemdInterface for DBusClient {
             CgroupHierarchy::Unified => properties.push(("BlockIOAccounting", Value::Bool(true))),
         }
 
-        if unit_name.ends_with(SLICE_SUFFIX) {
+        if self.unit_name.ends_with(SLICE_SUFFIX) {
             properties.push(("Wants", Value::Str(parent.into())));
         } else {
             properties.push(("Slice", Value::Str(parent.into())));
@@ -76,27 +70,57 @@ impl SystemdInterface for DBusClient {
         }
 
         proxy
-            .start_transient_unit(unit_name, UNIT_MODE, &properties, &[])
-            .with_context(|| format!("failed to start transient unit {}", unit_name))?;
-        Ok(())
-    }
-
-    fn set_properties(&self, unit_name: &str, properties: &Properties) -> Result<()> {
-        let proxy = self.build_proxy()?;
-
-        proxy
-            .set_unit_properties(unit_name, true, properties)
-            .with_context(|| format!("failed to set unit properties {}", unit_name))?;
+            .start_transient_unit(&self.unit_name, UNIT_MODE_REPLACE, &properties, &[])
+            .context(format!("failed to start transient unit {}", self.unit_name))?;
 
         Ok(())
     }
 
-    fn stop_unit(&self, unit_name: &str) -> Result<()> {
+    fn set_properties(&self, properties: &Properties) -> Result<()> {
         let proxy = self.build_proxy()?;
 
         proxy
-            .stop_unit(unit_name, UNIT_MODE)
-            .with_context(|| format!("failed to stop unit {}", unit_name))?;
+            .set_unit_properties(&self.unit_name, true, properties)
+            .context(format!("failed to set unit {} properties", self.unit_name))?;
+
+        Ok(())
+    }
+
+    fn kill_unit(&self) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .kill_unit(&self.unit_name, WHO_ENUM_ALL, SIGNAL_KILL)
+            .or_else(|e| match e {
+                zbus::Error::MethodError(error_name, _, _)
+                    if error_name.as_str() == NO_SUCH_UNIT_ERROR =>
+                {
+                    Ok(())
+                }
+                _ => Err(e),
+            })
+            .context(format!("failed to kill unit {}", self.unit_name))?;
+
+        Ok(())
+    }
+
+    fn freeze_unit(&self) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .freeze_unit(&self.unit_name)
+            .context(format!("failed to freeze unit {}", self.unit_name))?;
+
+        Ok(())
+    }
+
+    fn thaw_unit(&self) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .thaw_unit(&self.unit_name)
+            .context(format!("failed to thaw unit {}", self.unit_name))?;
+
         Ok(())
     }
 
@@ -105,24 +129,37 @@ impl SystemdInterface for DBusClient {
 
         let systemd_version = proxy
             .version()
-            .with_context(|| "failed to get systemd version".to_string())?;
+            .context("failed to get systemd version".to_string())?;
+
         Ok(systemd_version)
     }
 
-    fn unit_exists(&self, unit_name: &str) -> Result<bool> {
-        let proxy = self
-            .build_proxy()
-            .with_context(|| format!("Checking if systemd unit {} exists", unit_name))?;
+    fn unit_exists(&self) -> Result<bool> {
+        let proxy = self.build_proxy()?;
 
-        Ok(proxy.get_unit(unit_name).is_ok())
+        match proxy.get_unit(&self.unit_name) {
+            Ok(_) => Ok(true),
+            Err(zbus::Error::MethodError(error_name, _, _))
+                if error_name.as_str() == NO_SUCH_UNIT_ERROR =>
+            {
+                Ok(false)
+            }
+            Err(e) => Err(anyhow!(format!(
+                "failed to check if unit {} exists: {:?}",
+                self.unit_name, e
+            ))),
+        }
     }
 
-    fn add_process(&self, pid: i32, unit_name: &str) -> Result<()> {
+    fn add_process(&self, pid: i32) -> Result<()> {
         let proxy = self.build_proxy()?;
 
         proxy
-            .attach_processes_to_unit(unit_name, "/", &[pid as u32])
-            .with_context(|| format!("failed to add process {}", unit_name))?;
+            .attach_processes_to_unit(&self.unit_name, "/", &[pid as u32])
+            .context(format!(
+                "failed to add process into unit {}",
+                self.unit_name
+            ))?;
 
         Ok(())
     }

src/agent/rustjail/src/cgroups/systemd/interface/system.rs

@@ -1,4 +1,4 @@
-// Copyright 2021-2022 Kata Contributors
+// Copyright 2021-2023 Kata Contributors
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -8,7 +8,7 @@
 
 //! # DBus interface proxy for: `org.freedesktop.systemd1.Manager`
 //!
-//! This code was generated by `zbus-xmlgen` `2.0.1` from DBus introspection data.
+//! This code was generated by `zbus-xmlgen` `3.1.1` from DBus introspection data.
 //! Source: `Interface '/org/freedesktop/systemd1' from service 'org.freedesktop.systemd1' on system bus`.
 //!
 //! You may prefer to adapt it, instead of using it verbatim.
@@ -189,12 +189,14 @@ trait Manager {
     ) -> zbus::Result<zbus::zvariant::OwnedObjectPath>;
 
     /// GetUnitByInvocationID method
+    #[dbus_proxy(name = "GetUnitByInvocationID")]
     fn get_unit_by_invocation_id(
         &self,
         invocation_id: &[u8],
     ) -> zbus::Result<zbus::zvariant::OwnedObjectPath>;
 
     /// GetUnitByPID method
+    #[dbus_proxy(name = "GetUnitByPID")]
     fn get_unit_by_pid(&self, pid: u32) -> zbus::Result<zbus::zvariant::OwnedObjectPath>;
 
     /// GetUnitFileLinks method
@@ -210,6 +212,7 @@ trait Manager {
     fn halt(&self) -> zbus::Result<()>;
 
     /// KExec method
+    #[dbus_proxy(name = "KExec")]
     fn kexec(&self) -> zbus::Result<()>;
 
     /// KillUnit method
@@ -330,6 +333,7 @@ trait Manager {
     fn lookup_dynamic_user_by_name(&self, name: &str) -> zbus::Result<u32>;
 
     /// LookupDynamicUserByUID method
+    #[dbus_proxy(name = "LookupDynamicUserByUID")]
     fn lookup_dynamic_user_by_uid(&self, uid: u32) -> zbus::Result<String>;
 
     /// MaskUnitFiles method
@@ -571,139 +575,139 @@ trait Manager {
     fn ctrl_alt_del_burst_action(&self) -> zbus::Result<String>;
 
     /// DefaultBlockIOAccounting property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultBlockIOAccounting")]
     fn default_block_ioaccounting(&self) -> zbus::Result<bool>;
 
     /// DefaultCPUAccounting property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultCPUAccounting")]
     fn default_cpuaccounting(&self) -> zbus::Result<bool>;
 
     /// DefaultLimitAS property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitAS")]
     fn default_limit_as(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitASSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitASSoft")]
     fn default_limit_assoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitCORE property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitCORE")]
     fn default_limit_core(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitCORESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitCORESoft")]
     fn default_limit_coresoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitCPU property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitCPU")]
     fn default_limit_cpu(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitCPUSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitCPUSoft")]
     fn default_limit_cpusoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitDATA property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitDATA")]
     fn default_limit_data(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitDATASoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitDATASoft")]
     fn default_limit_datasoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitFSIZE property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitFSIZE")]
     fn default_limit_fsize(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitFSIZESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitFSIZESoft")]
     fn default_limit_fsizesoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitLOCKS property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitLOCKS")]
     fn default_limit_locks(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitLOCKSSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitLOCKSSoft")]
     fn default_limit_lockssoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitMEMLOCK property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitMEMLOCK")]
     fn default_limit_memlock(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitMEMLOCKSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitMEMLOCKSoft")]
     fn default_limit_memlocksoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitMSGQUEUE property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitMSGQUEUE")]
     fn default_limit_msgqueue(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitMSGQUEUESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitMSGQUEUESoft")]
     fn default_limit_msgqueuesoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNICE property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNICE")]
     fn default_limit_nice(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNICESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNICESoft")]
     fn default_limit_nicesoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNOFILE property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNOFILE")]
     fn default_limit_nofile(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNOFILESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNOFILESoft")]
     fn default_limit_nofilesoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNPROC property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNPROC")]
     fn default_limit_nproc(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitNPROCSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitNPROCSoft")]
     fn default_limit_nprocsoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRSS property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRSS")]
     fn default_limit_rss(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRSSSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRSSSoft")]
     fn default_limit_rsssoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRTPRIO property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRTPRIO")]
     fn default_limit_rtprio(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRTPRIOSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRTPRIOSoft")]
     fn default_limit_rtpriosoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRTTIME property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRTTIME")]
     fn default_limit_rttime(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitRTTIMESoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitRTTIMESoft")]
     fn default_limit_rttimesoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitSIGPENDING property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitSIGPENDING")]
     fn default_limit_sigpending(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitSIGPENDINGSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitSIGPENDINGSoft")]
     fn default_limit_sigpendingsoft(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitSTACK property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitSTACK")]
     fn default_limit_stack(&self) -> zbus::Result<u64>;
 
     /// DefaultLimitSTACKSoft property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultLimitSTACKSoft")]
     fn default_limit_stacksoft(&self) -> zbus::Result<u64>;
 
     /// DefaultMemoryAccounting property
@@ -711,11 +715,11 @@ trait Manager {
     fn default_memory_accounting(&self) -> zbus::Result<bool>;
 
     /// DefaultOOMPolicy property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultOOMPolicy")]
     fn default_oompolicy(&self) -> zbus::Result<String>;
 
     /// DefaultRestartUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultRestartUSec")]
     fn default_restart_usec(&self) -> zbus::Result<u64>;
 
     /// DefaultStandardError property
@@ -731,7 +735,7 @@ trait Manager {
     fn default_start_limit_burst(&self) -> zbus::Result<u32>;
 
     /// DefaultStartLimitIntervalUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultStartLimitIntervalUSec")]
     fn default_start_limit_interval_usec(&self) -> zbus::Result<u64>;
 
     /// DefaultTasksAccounting property
@@ -743,19 +747,19 @@ trait Manager {
     fn default_tasks_max(&self) -> zbus::Result<u64>;
 
     /// DefaultTimeoutAbortUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultTimeoutAbortUSec")]
     fn default_timeout_abort_usec(&self) -> zbus::Result<u64>;
 
     /// DefaultTimeoutStartUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultTimeoutStartUSec")]
     fn default_timeout_start_usec(&self) -> zbus::Result<u64>;
 
     /// DefaultTimeoutStopUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultTimeoutStopUSec")]
     fn default_timeout_stop_usec(&self) -> zbus::Result<u64>;
 
     /// DefaultTimerAccuracyUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "DefaultTimerAccuracyUSec")]
     fn default_timer_accuracy_usec(&self) -> zbus::Result<u64>;
 
     /// Environment property
@@ -803,65 +807,64 @@ trait Manager {
     fn generators_start_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDGeneratorsFinishTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDGeneratorsFinishTimestamp")]
     fn init_rdgenerators_finish_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDGeneratorsFinishTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDGeneratorsFinishTimestampMonotonic")]
     fn init_rdgenerators_finish_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDGeneratorsStartTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDGeneratorsStartTimestamp")]
     fn init_rdgenerators_start_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDGeneratorsStartTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDGeneratorsStartTimestampMonotonic")]
     fn init_rdgenerators_start_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDSecurityFinishTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDSecurityFinishTimestamp")]
     fn init_rdsecurity_finish_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDSecurityFinishTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDSecurityFinishTimestampMonotonic")]
     fn init_rdsecurity_finish_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDSecurityStartTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDSecurityStartTimestamp")]
     fn init_rdsecurity_start_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDSecurityStartTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDSecurityStartTimestampMonotonic")]
     fn init_rdsecurity_start_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDTimestamp")]
     fn init_rdtimestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDTimestampMonotonic")]
     fn init_rdtimestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDUnitsLoadFinishTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDUnitsLoadFinishTimestamp")]
     fn init_rdunits_load_finish_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDUnitsLoadFinishTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDUnitsLoadFinishTimestampMonotonic")]
     fn init_rdunits_load_finish_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// InitRDUnitsLoadStartTimestamp property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDUnitsLoadStartTimestamp")]
     fn init_rdunits_load_start_timestamp(&self) -> zbus::Result<u64>;
 
     /// InitRDUnitsLoadStartTimestampMonotonic property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "InitRDUnitsLoadStartTimestampMonotonic")]
     fn init_rdunits_load_start_timestamp_monotonic(&self) -> zbus::Result<u64>;
 
     /// KExecWatchdogUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "KExecWatchdogUSec")]
     fn kexec_watchdog_usec(&self) -> zbus::Result<u64>;
-    #[dbus_proxy(property)]
     fn set_kexec_watchdog_usec(&self, value: u64) -> zbus::Result<()>;
 
     /// KernelTimestamp property
@@ -883,33 +886,31 @@ trait Manager {
     /// LogLevel property
     #[dbus_proxy(property)]
     fn log_level(&self) -> zbus::Result<String>;
-    #[dbus_proxy(property)]
     fn set_log_level(&self, value: &str) -> zbus::Result<()>;
 
     /// LogTarget property
     #[dbus_proxy(property)]
     fn log_target(&self) -> zbus::Result<String>;
-    #[dbus_proxy(property)]
     fn set_log_target(&self, value: &str) -> zbus::Result<()>;
 
     /// NFailedJobs property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "NFailedJobs")]
     fn nfailed_jobs(&self) -> zbus::Result<u32>;
 
     /// NFailedUnits property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "NFailedUnits")]
     fn nfailed_units(&self) -> zbus::Result<u32>;
 
     /// NInstalledJobs property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "NInstalledJobs")]
     fn ninstalled_jobs(&self) -> zbus::Result<u32>;
 
     /// NJobs property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "NJobs")]
     fn njobs(&self) -> zbus::Result<u32>;
 
     /// NNames property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "NNames")]
     fn nnames(&self) -> zbus::Result<u32>;
 
     /// Progress property
@@ -917,15 +918,13 @@ trait Manager {
     fn progress(&self) -> zbus::Result<f64>;
 
     /// RebootWatchdogUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "RebootWatchdogUSec")]
     fn reboot_watchdog_usec(&self) -> zbus::Result<u64>;
-    #[dbus_proxy(property)]
     fn set_reboot_watchdog_usec(&self, value: u64) -> zbus::Result<()>;
 
     /// RuntimeWatchdogUSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "RuntimeWatchdogUSec")]
     fn runtime_watchdog_usec(&self) -> zbus::Result<u64>;
-    #[dbus_proxy(property)]
     fn set_runtime_watchdog_usec(&self, value: u64) -> zbus::Result<()>;
 
     /// SecurityFinishTimestamp property
@@ -947,7 +946,6 @@ trait Manager {
     /// ServiceWatchdogs property
     #[dbus_proxy(property)]
     fn service_watchdogs(&self) -> zbus::Result<bool>;
-    #[dbus_proxy(property)]
     fn set_service_watchdogs(&self, value: bool) -> zbus::Result<()>;
 
     /// ShowStatus property
@@ -963,7 +961,7 @@ trait Manager {
     fn tainted(&self) -> zbus::Result<String>;
 
     /// TimerSlackNSec property
-    #[dbus_proxy(property)]
+    #[dbus_proxy(property, name = "TimerSlackNSec")]
     fn timer_slack_nsec(&self) -> zbus::Result<u64>;
 
     /// UnitPath property

src/agent/rustjail/src/cgroups/systemd/manager.rs

@@ -5,7 +5,7 @@
 
 use crate::cgroups::Manager as CgroupManager;
 use crate::protocols::agent::CgroupStats;
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use cgroups::freezer::FreezerState;
 use libc::{self, pid_t};
 use oci::LinuxResources;
@@ -29,7 +29,6 @@ pub struct Manager {
     pub mounts: HashMap<String, String>,
     pub cgroups_path: CgroupsPath,
     pub cpath: String,
-    pub unit_name: String,
     // dbus client for set properties
     dbus_client: DBusClient,
     // fs manager for get properties
@@ -40,14 +39,12 @@ pub struct Manager {
 
 impl CgroupManager for Manager {
     fn apply(&self, pid: pid_t) -> Result<()> {
-        let unit_name = self.unit_name.as_str();
-        if self.dbus_client.unit_exists(unit_name)? {
-            self.dbus_client.add_process(pid, self.unit_name.as_str())?;
+        if self.dbus_client.unit_exists()? {
+            self.dbus_client.add_process(pid)?;
         } else {
             self.dbus_client.start_unit(
                 (pid as u32).try_into().unwrap(),
                 self.cgroups_path.slice.as_str(),
-                self.unit_name.as_str(),
                 &self.cg_hierarchy,
             )?;
         }
@@ -66,8 +63,7 @@ impl CgroupManager for Manager {
         Pids::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
         CpuSet::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
 
-        self.dbus_client
-            .set_properties(self.unit_name.as_str(), &properties)?;
+        self.dbus_client.set_properties(&properties)?;
 
         Ok(())
     }
@@ -77,11 +73,15 @@ impl CgroupManager for Manager {
     }
 
     fn freeze(&self, state: FreezerState) -> Result<()> {
-        self.fs_manager.freeze(state)
+        match state {
+            FreezerState::Thawed => self.dbus_client.thaw_unit(),
+            FreezerState::Frozen => self.dbus_client.freeze_unit(),
+            _ => Err(anyhow!("Invalid FreezerState")),
+        }
     }
 
     fn destroy(&mut self) -> Result<()> {
-        self.dbus_client.stop_unit(self.unit_name.as_str())?;
+        self.dbus_client.kill_unit()?;
         self.fs_manager.destroy()
     }
 
@@ -120,8 +120,7 @@ impl Manager {
             mounts: fs_manager.mounts.clone(),
             cgroups_path,
             cpath,
-            unit_name,
-            dbus_client: DBusClient {},
+            dbus_client: DBusClient::new(unit_name),
             fs_manager,
             cg_hierarchy: if cgroups::hierarchies::is_cgroup2_unified_mode() {
                 CgroupHierarchy::Unified

src/agent/rustjail/src/lib.rs

@@ -241,6 +241,12 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
     let devices = {
         let mut d = Vec::new();
         for dev in res.Devices.iter() {
+            let dev_type = if dev.Type.is_empty() {
+                None
+            } else {
+                Some(dev.Type.clone())
+            };
+
             let major = if dev.Major == -1 {
                 None
             } else {
@@ -254,7 +260,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             };
             d.push(oci::LinuxDeviceCgroup {
                 allow: dev.Allow,
-                r#type: dev.Type.clone(),
+                r#type: dev_type,
                 major,
                 minor,
                 access: dev.Access.clone(),

src/agent/src/cdh.rs

@@ -0,0 +1,289 @@
+// Copyright (c) 2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// Confidential Data Hub client wrapper.
+// Confidential Data Hub is a service running inside guest to provide resource related APIs.
+// https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub
+
+use anyhow::{anyhow, Result};
+use oci::{Mount, Spec};
+use protocols::{
+    sealed_secret, sealed_secret_ttrpc_async, sealed_secret_ttrpc_async::SealedSecretServiceClient,
+};
+use std::fs;
+use std::os::unix::fs::symlink;
+use std::path::Path;
+const CDH_ADDR: &str = "unix:///run/confidential-containers/cdh.sock";
+const SECRETS_DIR: &str = "/run/secrets/";
+const SEALED_SECRET_TIMEOUT: i64 = 50 * 1000 * 1000 * 1000;
+
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger()
+}
+
+#[derive(Clone)]
+pub struct CDHClient {
+    sealed_secret_client: Option<SealedSecretServiceClient>,
+}
+
+impl CDHClient {
+    pub fn new() -> Result<Self> {
+        let c = ttrpc::asynchronous::Client::connect(CDH_ADDR);
+        match c {
+            Ok(v) => {
+                let ssclient = sealed_secret_ttrpc_async::SealedSecretServiceClient::new(v);
+                Ok(CDHClient {
+                    sealed_secret_client: Some(ssclient),
+                })
+            }
+            Err(_) => Ok(CDHClient {
+                sealed_secret_client: None,
+            }),
+        }
+    }
+
+    pub async fn unseal_secret_async(
+        &self,
+        sealed: &str,
+    ) -> Result<sealed_secret::UnsealSecretOutput> {
+        let secret = sealed
+            .strip_prefix("sealed.")
+            .ok_or(anyhow!("strip_prefix \"sealed.\" failed"))?;
+        let mut input = sealed_secret::UnsealSecretInput::new();
+        input.set_secret(secret.into());
+        let unseal = self
+            .sealed_secret_client
+            .as_ref()
+            .ok_or(anyhow!("unwrap sealed_secret_client failed"))?
+            .unseal_secret(ttrpc::context::with_timeout(SEALED_SECRET_TIMEOUT), &input)
+            .await?;
+        Ok(unseal)
+    }
+
+    pub async fn unseal_env(&self, env: &str) -> Result<String> {
+        let (key, value) = env.split_once('=').unwrap_or(("", ""));
+        if value.starts_with("sealed.") {
+            let unsealed_value = self.unseal_secret_async(value).await;
+            match unsealed_value {
+                Ok(v) => {
+                    let plain_env = format!("{}={}", key, std::str::from_utf8(&v.plaintext)?);
+                    return Ok(plain_env);
+                }
+                Err(e) => {
+                    return Err(e);
+                }
+            };
+        }
+        Ok((*env.to_owned()).to_string())
+    }
+
+    pub async fn unseal_file(&self, sealed_source_path: &String) -> Result<()> {
+        if !Path::new(sealed_source_path).exists() {
+            info!(
+                sl(),
+                "sealed source path {:?} does not exist", sealed_source_path
+            );
+            return Ok(());
+        }
+
+        for entry in fs::read_dir(sealed_source_path)? {
+            let entry = entry?;
+
+            if !entry.file_type()?.is_symlink()
+                && !fs::metadata(entry.path())?.file_type().is_file()
+            {
+                info!(
+                    sl(),
+                    "skipping sealed source entry {:?} because its file type is {:?}",
+                    entry,
+                    entry.file_type()?
+                );
+                continue;
+            }
+
+            let target_path = fs::canonicalize(&entry.path())?;
+            info!(sl(), "sealed source entry target path: {:?}", target_path);
+            if !target_path.is_file() {
+                info!(sl(), "sealed source is not a file: {:?}", target_path);
+                continue;
+            }
+
+            let secret_name = entry.file_name();
+            let contents = fs::read_to_string(&target_path)?;
+            if contents.starts_with("sealed.") {
+                info!(sl(), "sealed source entry found: {:?}", target_path);
+                let unsealed_filename = SECRETS_DIR.to_string()
+                    + secret_name
+                        .as_os_str()
+                        .to_str()
+                        .ok_or(anyhow!("create unsealed_filename failed"))?;
+                let unsealed_value = self.unseal_secret_async(&contents).await?;
+                fs::write(&unsealed_filename, unsealed_value.plaintext)?;
+                fs::remove_file(&entry.path())?;
+                symlink(unsealed_filename, &entry.path())?;
+            }
+        }
+        Ok(())
+    }
+
+    pub fn create_sealed_secret_mounts(&self, spec: &mut Spec) -> Result<Vec<String>> {
+        let mut sealed_source_path: Vec<String> = vec![];
+        for m in spec.mounts.iter_mut() {
+            if let Some(unsealed_mount_point) = m.destination.strip_prefix("/sealed") {
+                info!(
+                    sl(),
+                    "sealed mount destination: {:?} source: {:?}", m.destination, m.source
+                );
+                sealed_source_path.push(m.source.clone());
+                m.destination = unsealed_mount_point.to_string();
+            }
+        }
+
+        if !sealed_source_path.is_empty() {
+            let sealed_mounts = Mount {
+                destination: SECRETS_DIR.to_string(),
+                r#type: "bind".to_string(),
+                source: SECRETS_DIR.to_string(),
+                options: vec!["bind".to_string()],
+            };
+            spec.mounts.push(sealed_mounts);
+        }
+        fs::create_dir_all(SECRETS_DIR)?;
+        Ok(sealed_source_path)
+    }
+} /* end of impl CDHClient */
+
+#[cfg(test)]
+#[cfg(feature = "sealed-secret")]
+mod tests {
+    use crate::cdh::CDHClient;
+    use crate::cdh::CDH_ADDR;
+    use crate::cdh::SECRETS_DIR;
+    use anyhow::anyhow;
+    use async_trait::async_trait;
+    use protocols::{sealed_secret, sealed_secret_ttrpc_async};
+    use std::fs;
+    use std::fs::File;
+    use std::io::{Read, Write};
+    use std::path::Path;
+    use std::sync::Arc;
+    use tokio::signal::unix::{signal, SignalKind};
+
+    struct TestService;
+
+    #[async_trait]
+    impl sealed_secret_ttrpc_async::SealedSecretService for TestService {
+        async fn unseal_secret(
+            &self,
+            _ctx: &::ttrpc::asynchronous::TtrpcContext,
+            _req: sealed_secret::UnsealSecretInput,
+        ) -> ttrpc::error::Result<sealed_secret::UnsealSecretOutput> {
+            let mut output = sealed_secret::UnsealSecretOutput::new();
+            output.set_plaintext("unsealed".into());
+            Ok(output)
+        }
+    }
+
+    fn remove_if_sock_exist(sock_addr: &str) -> std::io::Result<()> {
+        let path = sock_addr
+            .strip_prefix("unix://")
+            .expect("socket address does not have the expected format.");
+
+        if std::path::Path::new(path).exists() {
+            std::fs::remove_file(path)?;
+        }
+
+        Ok(())
+    }
+
+    fn start_ttrpc_server() {
+        tokio::spawn(async move {
+            let ss = Box::new(TestService {})
+                as Box<dyn sealed_secret_ttrpc_async::SealedSecretService + Send + Sync>;
+            let ss = Arc::new(ss);
+            let ss_service = sealed_secret_ttrpc_async::create_sealed_secret_service(ss);
+
+            remove_if_sock_exist(CDH_ADDR).unwrap();
+
+            let mut server = ttrpc::asynchronous::Server::new()
+                .bind(CDH_ADDR)
+                .unwrap()
+                .register_service(ss_service);
+
+            server.start().await.unwrap();
+
+            let mut interrupt = signal(SignalKind::interrupt()).unwrap();
+            tokio::select! {
+                _ = interrupt.recv() => {
+                    server.shutdown().await.unwrap();
+                }
+            };
+        });
+    }
+
+    #[tokio::test]
+    async fn test_unseal_env() {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let _guard = rt.enter();
+        start_ttrpc_server();
+        std::thread::sleep(std::time::Duration::from_secs(2));
+
+        let cc = Some(CDHClient::new().unwrap());
+        let cdh_client = cc
+            .as_ref()
+            .ok_or(anyhow!("get confidential-data-hub client failed"))
+            .unwrap();
+        let sealed_env = String::from("key=sealed.testdata");
+        let unsealed_env = cdh_client.unseal_env(&sealed_env).await.unwrap();
+        assert_eq!(unsealed_env, String::from("key=unsealed"));
+        let normal_env = String::from("key=testdata");
+        let unchanged_env = cdh_client.unseal_env(&normal_env).await.unwrap();
+        assert_eq!(unchanged_env, String::from("key=testdata"));
+
+        rt.shutdown_background();
+        std::thread::sleep(std::time::Duration::from_secs(2));
+    }
+
+    #[tokio::test]
+    async fn test_unseal_file() {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let _guard = rt.enter();
+        start_ttrpc_server();
+        std::thread::sleep(std::time::Duration::from_secs(2));
+
+        let cc = Some(CDHClient::new().unwrap());
+        let cdh_client = cc
+            .as_ref()
+            .ok_or(anyhow!("get confidential-data-hub client failed"))
+            .unwrap();
+
+        fs::create_dir_all(SECRETS_DIR).unwrap();
+
+        let sealed_filename = "passwd";
+        let mut sealed_file = File::create(sealed_filename).unwrap();
+        let dir = String::from(".");
+        sealed_file.write_all(b"sealed.passwd").unwrap();
+        cdh_client.unseal_file(&dir).await.unwrap();
+        let unsealed_filename = SECRETS_DIR.to_string() + "/passwd";
+        let mut unsealed_file = fs::File::open(unsealed_filename.clone()).unwrap();
+        let mut contents = String::new();
+        unsealed_file.read_to_string(&mut contents).unwrap();
+        assert_eq!(contents, String::from("unsealed"));
+        fs::remove_file(sealed_filename).unwrap();
+        fs::remove_file(unsealed_filename).unwrap();
+
+        let normal_filename = "passwd";
+        let mut normal_file = File::create(normal_filename).unwrap();
+        normal_file.write_all(b"passwd").unwrap();
+        cdh_client.unseal_file(&dir).await.unwrap();
+        let filename = SECRETS_DIR.to_string() + "/passwd";
+        assert!(!Path::new(&filename).exists());
+        fs::remove_file(normal_filename).unwrap();
+
+        rt.shutdown_background();
+        std::thread::sleep(std::time::Duration::from_secs(2));
+    }
+}

src/agent/src/config.rs

@@ -11,6 +11,7 @@ use std::fs;
 use std::str::FromStr;
 use std::time;
 use tracing::instrument;
+use url::Url;
 
 use kata_types::config::default::DEFAULT_AGENT_VSOCK_PORT;
 
@@ -25,6 +26,15 @@ const LOG_VPORT_OPTION: &str = "agent.log_vport";
 const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
 const CONFIG_FILE: &str = "agent.config_file";
+const AA_KBC_PARAMS: &str = "agent.aa_kbc_params";
+const REST_API_OPTION: &str = "agent.rest_api";
+const HTTPS_PROXY: &str = "agent.https_proxy";
+const NO_PROXY: &str = "agent.no_proxy";
+const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity";
+const ENABLE_SIGNATURE_VERIFICATION: &str = "agent.enable_signature_verification";
+const IMAGE_POLICY_FILE: &str = "agent.image_policy";
+const IMAGE_REGISTRY_AUTH_FILE: &str = "agent.image_registry_auth";
+const SIMPLE_SIGNING_SIGSTORE_CONFIG: &str = "agent.simple_signing_sigstore_config";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
@@ -77,6 +87,16 @@ pub struct AgentConfig {
     pub tracing: bool,
     pub endpoints: AgentEndpoints,
     pub supports_seccomp: bool,
+    pub container_policy_path: String,
+    pub aa_kbc_params: String,
+    pub rest_api: String,
+    pub https_proxy: String,
+    pub no_proxy: String,
+    pub data_integrity: bool,
+    pub enable_signature_verification: bool,
+    pub image_policy_file: String,
+    pub image_registry_auth_file: String,
+    pub simple_signing_sigstore_config: String,
 }
 
 #[derive(Debug, Deserialize)]
@@ -92,6 +112,16 @@ pub struct AgentConfigBuilder {
     pub unified_cgroup_hierarchy: Option<bool>,
     pub tracing: Option<bool>,
     pub endpoints: Option<EndpointsConfig>,
+    pub container_policy_path: Option<String>,
+    pub aa_kbc_params: Option<String>,
+    pub rest_api: Option<String>,
+    pub https_proxy: Option<String>,
+    pub no_proxy: Option<String>,
+    pub data_integrity: Option<bool>,
+    pub enable_signature_verification: Option<bool>,
+    pub image_policy_file: Option<String>,
+    pub image_registry_auth_file: Option<String>,
+    pub simple_signing_sigstore_config: Option<String>,
 }
 
 macro_rules! config_override {
@@ -153,6 +183,16 @@ impl Default for AgentConfig {
             tracing: false,
             endpoints: Default::default(),
             supports_seccomp: rpc::have_seccomp(),
+            container_policy_path: String::from(""),
+            aa_kbc_params: String::from(""),
+            rest_api: String::from(""),
+            https_proxy: String::from(""),
+            no_proxy: String::from(""),
+            data_integrity: false,
+            enable_signature_verification: true,
+            image_policy_file: String::from(""),
+            image_registry_auth_file: String::from(""),
+            simple_signing_sigstore_config: String::from(""),
         }
     }
 }
@@ -181,6 +221,24 @@ impl FromStr for AgentConfig {
         config_override!(agent_config_builder, agent_config, server_addr);
         config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
         config_override!(agent_config_builder, agent_config, tracing);
+        config_override!(agent_config_builder, agent_config, container_policy_path);
+        config_override!(agent_config_builder, agent_config, aa_kbc_params);
+        config_override!(agent_config_builder, agent_config, rest_api);
+        config_override!(agent_config_builder, agent_config, https_proxy);
+        config_override!(agent_config_builder, agent_config, no_proxy);
+        config_override!(agent_config_builder, agent_config, data_integrity);
+        config_override!(
+            agent_config_builder,
+            agent_config,
+            enable_signature_verification
+        );
+        config_override!(agent_config_builder, agent_config, image_policy_file);
+        config_override!(agent_config_builder, agent_config, image_registry_auth_file);
+        config_override!(
+            agent_config_builder,
+            agent_config,
+            simple_signing_sigstore_config
+        );
 
         // Populate the allowed endpoints hash set, if we got any from the config file.
         if let Some(endpoints) = agent_config_builder.endpoints {
@@ -195,6 +253,7 @@ impl FromStr for AgentConfig {
 
 impl AgentConfig {
     #[instrument]
+    #[allow(clippy::redundant_closure_call)]
     pub fn from_cmdline(file: &str, args: Vec<String>) -> Result<AgentConfig> {
         // If config file specified in the args, generate our config from it
         let config_position = args.iter().position(|a| a == "--config" || a == "-c");
@@ -209,6 +268,10 @@ impl AgentConfig {
         let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
+
+        let mut using_config_file = false;
+        // Check if there is config file before parsing params that might
+        // override values from the config file.
         for param in params.iter() {
             // If we get a configuration file path from the command line, we
             // generate our config from it.
@@ -216,10 +279,15 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                return AgentConfig::from_config_file(&config_file)
-                    .context("AgentConfig from kernel cmdline");
+                config = AgentConfig::from_config_file(&config_file)
+                    .context("AgentConfig from kernel cmdline")
+                    .unwrap();
+                using_config_file = true;
+                break;
             }
+        }
 
+        for param in params.iter() {
             // parse cmdline flags
             parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, config.debug_console);
             parse_cmdline_param!(param, DEV_MODE_FLAG, config.dev_mode);
@@ -279,6 +347,49 @@ impl AgentConfig {
                 config.unified_cgroup_hierarchy,
                 get_bool_value
             );
+
+            parse_cmdline_param!(param, AA_KBC_PARAMS, config.aa_kbc_params, get_string_value);
+            parse_cmdline_param!(param, REST_API_OPTION, config.rest_api, get_string_value);
+            parse_cmdline_param!(param, HTTPS_PROXY, config.https_proxy, get_url_value);
+            parse_cmdline_param!(param, NO_PROXY, config.no_proxy, get_string_value);
+            parse_cmdline_param!(
+                param,
+                ENABLE_DATA_INTEGRITY,
+                config.data_integrity,
+                get_bool_value
+            );
+
+            parse_cmdline_param!(
+                param,
+                ENABLE_SIGNATURE_VERIFICATION,
+                config.enable_signature_verification,
+                get_bool_value
+            );
+
+            // URI of the image security file
+            parse_cmdline_param!(
+                param,
+                IMAGE_POLICY_FILE,
+                config.image_policy_file,
+                get_string_value
+            );
+
+            // URI of the registry auth file
+            parse_cmdline_param!(
+                param,
+                IMAGE_REGISTRY_AUTH_FILE,
+                config.image_registry_auth_file,
+                get_string_value
+            );
+
+            // URI of the simple signing sigstore file
+            // used when simple signing verification is used
+            parse_cmdline_param!(
+                param,
+                SIMPLE_SIGNING_SIGSTORE_CONFIG,
+                config.simple_signing_sigstore_config,
+                get_string_value
+            );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
@@ -298,7 +409,9 @@ impl AgentConfig {
         }
 
         // We did not get a configuration file: allow all endpoints.
-        config.endpoints.all_allowed = true;
+        if !using_config_file {
+            config.endpoints.all_allowed = true;
+        }
 
         Ok(config)
     }
@@ -433,6 +546,12 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
     Ok(value)
 }
 
+#[instrument]
+fn get_url_value(param: &str) -> Result<String> {
+    let value = get_string_value(param)?;
+    Ok(Url::parse(&value)?.to_string())
+}
+
 #[cfg(test)]
 mod tests {
     use test_utils::assert_result;
@@ -451,6 +570,11 @@ mod tests {
         assert!(!config.dev_mode);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
+        assert_eq!(config.container_policy_path, "");
+        assert!(config.enable_signature_verification);
+        assert_eq!(config.image_policy_file, "");
+        assert_eq!(config.image_registry_auth_file, "");
+        assert_eq!(config.simple_signing_sigstore_config, "");
     }
 
     #[test]
@@ -469,6 +593,16 @@ mod tests {
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
             tracing: bool,
+            container_policy_path: &'a str,
+            aa_kbc_params: &'a str,
+            rest_api: &'a str,
+            https_proxy: &'a str,
+            no_proxy: &'a str,
+            data_integrity: bool,
+            enable_signature_verification: bool,
+            image_policy_file: &'a str,
+            image_registry_auth_file: &'a str,
+            simple_signing_sigstore_config: &'a str,
         }
 
         impl Default for TestData<'_> {
@@ -484,6 +618,16 @@ mod tests {
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
                     tracing: false,
+                    container_policy_path: "",
+                    aa_kbc_params: "",
+                    rest_api: "",
+                    https_proxy: "",
+                    no_proxy: "",
+                    data_integrity: false,
+                    enable_signature_verification: true,
+                    image_policy_file: "",
+                    image_registry_auth_file: "",
+                    simple_signing_sigstore_config: "",
                 }
             }
         }
@@ -853,6 +997,141 @@ mod tests {
                 tracing: true,
                 ..Default::default()
             },
+            TestData {
+                contents: "agent.aa_kbc_params=offline_fs_kbc::null",
+                aa_kbc_params: "offline_fs_kbc::null",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.aa_kbc_params=eaa_kbc::127.0.0.1:50000",
+                aa_kbc_params: "eaa_kbc::127.0.0.1:50000",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.rest_api=attestation",
+                rest_api: "attestation",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.rest_api=resource",
+                rest_api: "resource",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.rest_api=all",
+                rest_api: "all",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.https_proxy=http://proxy.url.com:81/",
+                https_proxy: "http://proxy.url.com:81/",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.https_proxy=http://192.168.1.100:81/",
+                https_proxy: "http://192.168.1.100:81/",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.no_proxy=*.internal.url.com",
+                no_proxy: "*.internal.url.com",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.no_proxy=192.168.1.0/24,172.16.0.0/12",
+                no_proxy: "192.168.1.0/24,172.16.0.0/12",
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=true",
+                data_integrity: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=false",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=1",
+                data_integrity: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=0",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=false",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=0",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=1",
+                enable_signature_verification: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=foo",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_policy=file:///etc/policy.json",
+                image_policy_file: "file:///etc/policy.json",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_policy=kbs:///default/security-policy/test",
+                image_policy_file: "kbs:///default/security-policy/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_policy=kbs://example.kbs.org/default/security-policy/test",
+                image_policy_file: "kbs://example.kbs.org/default/security-policy/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_registry_auth=file:///etc/auth.json",
+                image_registry_auth_file: "file:///etc/auth.json",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_registry_auth=kbs:///default/credential/test",
+                image_registry_auth_file: "kbs:///default/credential/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_registry_auth=kbs://example.kbs.org/default/credential/test",
+                image_registry_auth_file: "kbs://example.kbs.org/default/credential/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.simple_signing_sigstore_config=file:///etc/containers/signature/default.yml",
+                simple_signing_sigstore_config: "file:///etc/containers/signature/default.yml",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.simple_signing_sigstore_config=kbs:///default/sigstore-config/test",
+                simple_signing_sigstore_config: "kbs:///default/sigstore-config/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.simple_signing_sigstore_config=kbs://example.kbs.org/default/sigstore-config/test",
+                simple_signing_sigstore_config: "kbs://example.kbs.org/default/sigstore-config/test",
+                ..Default::default()
+            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -900,6 +1179,32 @@ mod tests {
             assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
             assert_eq!(d.server_addr, config.server_addr, "{}", msg);
             assert_eq!(d.tracing, config.tracing, "{}", msg);
+            assert_eq!(
+                d.container_policy_path, config.container_policy_path,
+                "{}",
+                msg
+            );
+            assert_eq!(d.aa_kbc_params, config.aa_kbc_params, "{}", msg);
+            assert_eq!(d.rest_api, config.rest_api, "{}", msg);
+            assert_eq!(d.https_proxy, config.https_proxy, "{}", msg);
+            assert_eq!(d.no_proxy, config.no_proxy, "{}", msg);
+            assert_eq!(d.data_integrity, config.data_integrity, "{}", msg);
+            assert_eq!(
+                d.enable_signature_verification, config.enable_signature_verification,
+                "{}",
+                msg
+            );
+            assert_eq!(d.image_policy_file, config.image_policy_file, "{}", msg);
+            assert_eq!(
+                d.image_registry_auth_file, config.image_registry_auth_file,
+                "{}",
+                msg
+            );
+            assert_eq!(
+                d.simple_signing_sigstore_config, config.simple_signing_sigstore_config,
+                "{}",
+                msg
+            );
 
             for v in vars_to_unset {
                 env::remove_var(v);
@@ -1392,7 +1697,7 @@ Caused by:
         assert_eq!(config.server_addr, "vsock://8:2048");
         assert_eq!(
             config.endpoints.allowed,
-            vec!["CreateContainer".to_string(), "StartContainer".to_string()]
+            ["CreateContainer".to_string(), "StartContainer".to_string()]
                 .iter()
                 .cloned()
                 .collect()
@@ -1401,4 +1706,50 @@ Caused by:
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
+
+    #[test]
+    fn test_config_from_cmdline_and_config_file() {
+        let dir = tempdir().expect("failed to create tmpdir");
+
+        let agent_config = r#"
+               dev_mode = false
+               server_addr = 'vsock://8:2048'
+
+               [endpoints]
+               allowed = ["CreateContainer", "StartContainer"]
+              "#;
+
+        let config_path = dir.path().join("agent-config.toml");
+        let config_filename = config_path.to_str().expect("failed to get config filename");
+
+        fs::write(config_filename, agent_config).expect("failed to write agen config");
+
+        let cmdline = format!("agent.devmode agent.config_file={}", config_filename);
+
+        let cmdline_path = dir.path().join("cmdline");
+        let cmdline_filename = cmdline_path
+            .to_str()
+            .expect("failed to get cmdline filename");
+
+        fs::write(cmdline_filename, cmdline).expect("failed to write agen config");
+
+        let config = AgentConfig::from_cmdline(cmdline_filename, vec![])
+            .expect("failed to parse command line");
+
+        // Should be overwritten by cmdline
+        assert!(config.dev_mode);
+
+        // Should be from agent config
+        assert_eq!(config.server_addr, "vsock://8:2048");
+
+        // Should be from agent config
+        assert_eq!(
+            config.endpoints.allowed,
+            ["CreateContainer".to_string(), "StartContainer".to_string()]
+                .iter()
+                .cloned()
+                .collect()
+        );
+        assert!(!config.endpoints.all_allowed);
+    }
 }

src/agent/src/device.rs

@@ -35,9 +35,9 @@ const VM_ROOTFS: &str = "/";
 const BLOCK: &str = "block";
 pub const DRIVER_9P_TYPE: &str = "9p";
 pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
-pub const DRIVER_BLK_TYPE: &str = "blk";
+pub const DRIVER_BLK_PCI_TYPE: &str = "blk";
 pub const DRIVER_BLK_CCW_TYPE: &str = "blk-ccw";
-pub const DRIVER_MMIO_BLK_TYPE: &str = "mmioblk";
+pub const DRIVER_BLK_MMIO_TYPE: &str = "mmioblk";
 pub const DRIVER_SCSI_TYPE: &str = "scsi";
 pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
 pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
@@ -651,13 +651,15 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
 
     if let Some(resources) = linux.resources.as_mut() {
         for r in &mut resources.devices {
-            if let (Some(host_major), Some(host_minor)) = (r.major, r.minor) {
-                if let Some(update) = res_updates.get(&(r.r#type.as_str(), host_major, host_minor))
+            if let (Some(host_type), Some(host_major), Some(host_minor)) =
+                (r.r#type.as_ref(), r.major, r.minor)
+            {
+                if let Some(update) = res_updates.get(&(host_type.as_str(), host_major, host_minor))
                 {
                     info!(
                         sl(),
                         "update_spec_devices() updating resource";
-                        "type" => &r.r#type,
+                        "type" => &host_type,
                         "host_major" => host_major,
                         "host_minor" => host_minor,
                         "guest_major" => update.guest_major,
@@ -935,9 +937,9 @@ async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<Sp
     }
 
     match device.type_.as_str() {
-        DRIVER_BLK_TYPE => virtio_blk_device_handler(device, sandbox).await,
+        DRIVER_BLK_PCI_TYPE => virtio_blk_device_handler(device, sandbox).await,
         DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, sandbox).await,
-        DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, sandbox).await,
+        DRIVER_BLK_MMIO_TYPE => virtiommio_blk_device_handler(device, sandbox).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, sandbox).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, sandbox).await,
         DRIVER_VFIO_PCI_GK_TYPE | DRIVER_VFIO_PCI_TYPE => {
@@ -969,7 +971,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
         allow: false,
         major: Some(major),
         minor: Some(minor),
-        r#type: String::from("b"),
+        r#type: Some(String::from("b")),
         access: String::from("rw"),
     });
 
@@ -1132,13 +1134,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         oci::LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major_a),
                             minor: Some(host_minor_a),
                             ..oci::LinuxDeviceCgroup::default()
                         },
                         oci::LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major_b),
                             minor: Some(host_minor_b),
                             ..oci::LinuxDeviceCgroup::default()
@@ -1231,13 +1233,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
                         },
                         LinuxDeviceCgroup {
-                            r#type: "b".to_string(),
+                            r#type: Some("b".to_string()),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()

src/agent/src/image_rpc.rs

@@ -0,0 +1,586 @@
+// Copyright (c) 2021 Alibaba Cloud
+// Copyright (c) 2021, 2023 IBM Corporation
+// Copyright (c) 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::collections::HashMap;
+use std::env;
+use std::fs;
+use std::path::Path;
+use std::sync::atomic::{AtomicU16, Ordering};
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use image_rs::image::ImageClient;
+use protocols::image;
+use tokio::sync::Mutex;
+use ttrpc::{self, error::get_rpc_status as ttrpc_error};
+
+use crate::rpc::{verify_cid, CONTAINER_BASE};
+use crate::AGENT_CONFIG;
+
+// A marker to merge container spec for images pulled inside guest.
+const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name";
+
+// kata rootfs is readonly, use tmpfs before CC storage is implemented.
+const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/";
+const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle";
+const CONFIG_JSON: &str = "config.json";
+
+#[rustfmt::skip]
+lazy_static! {
+    pub static ref IMAGE_SERVICE: Mutex<Option<ImageService>> = Mutex::new(None);
+}
+
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "cgroups"))
+}
+
+#[derive(Clone)]
+pub struct ImageService {
+    image_client: Arc<Mutex<ImageClient>>,
+    images: Arc<Mutex<HashMap<String, String>>>,
+    container_count: Arc<AtomicU16>,
+}
+
+impl ImageService {
+    pub fn new() -> Self {
+        env::set_var("CC_IMAGE_WORK_DIR", KATA_CC_IMAGE_WORK_DIR);
+
+        let mut image_client = ImageClient::default();
+        if !AGENT_CONFIG.image_policy_file.is_empty() {
+            image_client.config.file_paths.policy_path = AGENT_CONFIG.image_policy_file.clone();
+        }
+        if !AGENT_CONFIG.simple_signing_sigstore_config.is_empty() {
+            image_client.config.file_paths.sigstore_config =
+                AGENT_CONFIG.simple_signing_sigstore_config.clone();
+        }
+        if !AGENT_CONFIG.image_registry_auth_file.is_empty() {
+            image_client.config.file_paths.auth_file =
+                AGENT_CONFIG.image_registry_auth_file.clone();
+        }
+
+        Self {
+            image_client: Arc::new(Mutex::new(image_client)),
+            images: Arc::new(Mutex::new(HashMap::new())),
+            container_count: Arc::new(AtomicU16::new(0)),
+        }
+    }
+
+    /// Get the singleton instance of image service.
+    pub async fn singleton() -> Result<ImageService> {
+        IMAGE_SERVICE
+            .lock()
+            .await
+            .clone()
+            .ok_or_else(|| anyhow!("image service is uninitialized"))
+    }
+
+    // pause image is packaged in rootfs for CC
+    fn unpack_pause_image(cid: &str, target_subpath: &str) -> Result<String> {
+        let cc_pause_bundle = Path::new(KATA_CC_PAUSE_BUNDLE);
+        if !cc_pause_bundle.exists() {
+            return Err(anyhow!("Pause image not present in rootfs"));
+        }
+
+        info!(sl(), "use guest pause image cid {:?}", cid);
+        let pause_bundle = Path::new(CONTAINER_BASE).join(cid).join(target_subpath);
+        let pause_rootfs = pause_bundle.join("rootfs");
+        let pause_config = pause_bundle.join(CONFIG_JSON);
+        let pause_binary = pause_rootfs.join("pause");
+        fs::create_dir_all(&pause_rootfs)?;
+        if !pause_config.exists() {
+            fs::copy(
+                cc_pause_bundle.join(CONFIG_JSON),
+                pause_bundle.join(CONFIG_JSON),
+            )?;
+        }
+        if !pause_binary.exists() {
+            fs::copy(cc_pause_bundle.join("rootfs").join("pause"), pause_binary)?;
+        }
+
+        Ok(pause_rootfs.display().to_string())
+    }
+
+    /// Determines the container id (cid) to use for a given request.
+    ///
+    /// If the request specifies a non-empty id, use it; otherwise derive it from the image path.
+    /// In either case, verify that the chosen id is valid.
+    fn cid_from_request(&self, req: &image::PullImageRequest) -> Result<String> {
+        let req_cid = req.container_id();
+        let cid = if !req_cid.is_empty() {
+            req_cid.to_string()
+        } else if let Some(last) = req.image().rsplit('/').next() {
+            // Support multiple containers with same image
+            let index = self.container_count.fetch_add(1, Ordering::Relaxed);
+
+            // ':' not valid for container id
+            format!("{}_{}", last.replace(':', "_"), index)
+        } else {
+            return Err(anyhow!("Invalid image name. {}", req.image()));
+        };
+        verify_cid(&cid)?;
+        Ok(cid)
+    }
+
+    /// Set proxy environment from AGENT_CONFIG
+    fn set_proxy_env_vars() {
+        let https_proxy = &AGENT_CONFIG.https_proxy;
+        if !https_proxy.is_empty() {
+            env::set_var("HTTPS_PROXY", https_proxy);
+        }
+        let no_proxy = &AGENT_CONFIG.no_proxy;
+        if !no_proxy.is_empty() {
+            env::set_var("NO_PROXY", no_proxy);
+        }
+    }
+
+    /// init atestation agent and read config from AGENT_CONFIG
+    async fn get_security_config(&self) -> Result<String> {
+        let aa_kbc_params = &AGENT_CONFIG.aa_kbc_params;
+        // If the attestation-agent is being used, then enable the authenticated credentials support
+        info!(
+            sl(),
+            "image_client.config.auth set to: {}",
+            !aa_kbc_params.is_empty()
+        );
+        self.image_client.lock().await.config.auth = !aa_kbc_params.is_empty();
+        let decrypt_config = format!("provider:attestation-agent:{}", aa_kbc_params);
+
+        // Read enable signature verification from the agent config and set it in the image_client
+        let enable_signature_verification = &AGENT_CONFIG.enable_signature_verification;
+        info!(
+            sl(),
+            "enable_signature_verification set to: {}", enable_signature_verification
+        );
+        self.image_client.lock().await.config.security_validate = *enable_signature_verification;
+        Ok(decrypt_config)
+    }
+
+    /// Call image-rs to pull and unpack image.
+    async fn common_image_pull(
+        &self,
+        image: &str,
+        bundle_path: &Path,
+        decrypt_config: &str,
+        source_creds: Option<&str>,
+        cid: &str,
+    ) -> Result<()> {
+        let res = self
+            .image_client
+            .lock()
+            .await
+            .pull_image(image, bundle_path, &source_creds, &Some(decrypt_config))
+            .await;
+        match res {
+            Ok(image) => {
+                info!(
+                    sl(),
+                    "pull and unpack image {:?}, cid: {:?}, with image-rs succeed. ", image, cid
+                );
+            }
+            Err(e) => {
+                error!(
+                    sl(),
+                    "pull and unpack image {:?}, cid: {:?}, with image-rs failed with {:?}. ",
+                    image,
+                    cid,
+                    e.to_string()
+                );
+                return Err(e);
+            }
+        };
+        self.add_image(String::from(image), String::from(cid)).await;
+        Ok(())
+    }
+
+    /// Pull image when creating container and return the bundle path with rootfs.
+    pub async fn pull_image_for_container(
+        &self,
+        image: &str,
+        cid: &str,
+        image_metadata: &HashMap<String, String>,
+    ) -> Result<String> {
+        info!(sl(), "image metadata: {:?}", image_metadata);
+        Self::set_proxy_env_vars();
+        let is_sandbox = if let Some(value) = image_metadata.get("io.kubernetes.cri.container-type")
+        {
+            value == "sandbox"
+        } else if let Some(value) = image_metadata.get("io.kubernetes.cri-o.ContainerType") {
+            value == "sandbox"
+        } else {
+            false
+        };
+
+        if is_sandbox {
+            let mount_path = Self::unpack_pause_image(cid, "pause")?;
+            self.add_image(String::from(image), String::from(cid)).await;
+            return Ok(mount_path);
+        }
+        let bundle_path = Path::new(CONTAINER_BASE).join(cid).join("images");
+        fs::create_dir_all(&bundle_path)?;
+        info!(sl(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
+
+        let decrypt_config = self.get_security_config().await?;
+
+        let source_creds = None; // You need to determine how to obtain this.
+
+        self.common_image_pull(image, &bundle_path, &decrypt_config, source_creds, cid)
+            .await?;
+        Ok(format! {"{}/rootfs",bundle_path.display()})
+    }
+
+    /// Pull image when recieving the PullImageRequest and return the image digest.
+    async fn pull_image(&self, req: &image::PullImageRequest) -> Result<String> {
+        Self::set_proxy_env_vars();
+        let cid = self.cid_from_request(req)?;
+        let image = req.image();
+        if cid.starts_with("pause") {
+            Self::unpack_pause_image(&cid, "")?;
+            self.add_image(String::from(image), cid).await;
+            return Ok(image.to_owned());
+        }
+
+        // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles
+        // with rootfs and config.json will store under CONTAINER_BASE/cid.
+        let bundle_path = Path::new(CONTAINER_BASE).join(&cid);
+        fs::create_dir_all(&bundle_path)?;
+
+        let decrypt_config = self.get_security_config().await?;
+        let source_creds = (!req.source_creds().is_empty()).then(|| req.source_creds());
+
+        self.common_image_pull(
+            image,
+            &bundle_path,
+            &decrypt_config,
+            source_creds,
+            cid.clone().as_str(),
+        )
+        .await?;
+        Ok(image.to_owned())
+    }
+
+    async fn add_image(&self, image: String, cid: String) {
+        self.images.lock().await.insert(image, cid);
+    }
+
+    // When being passed an image name through a container annotation, merge its
+    // corresponding bundle OCI specification into the passed container creation one.
+    pub async fn merge_bundle_oci(&self, container_oci: &mut oci::Spec) -> Result<()> {
+        if let Some(image_name) = container_oci
+            .annotations
+            .get(&ANNO_K8S_IMAGE_NAME.to_string())
+        {
+            let images = self.images.lock().await;
+            if let Some(container_id) = images.get(image_name) {
+                let image_oci_config_path = Path::new(CONTAINER_BASE)
+                    .join(container_id)
+                    .join(CONFIG_JSON);
+                debug!(
+                    sl(),
+                    "Image bundle config path: {:?}", image_oci_config_path
+                );
+
+                let image_oci =
+                    oci::Spec::load(image_oci_config_path.to_str().ok_or_else(|| {
+                        anyhow!(
+                            "Invalid container image OCI config path {:?}",
+                            image_oci_config_path
+                        )
+                    })?)
+                    .context("load image bundle")?;
+
+                if let Some(container_root) = container_oci.root.as_mut() {
+                    if let Some(image_root) = image_oci.root.as_ref() {
+                        let root_path = Path::new(CONTAINER_BASE)
+                            .join(container_id)
+                            .join(image_root.path.clone());
+                        container_root.path =
+                            String::from(root_path.to_str().ok_or_else(|| {
+                                anyhow!("Invalid container image root path {:?}", root_path)
+                            })?);
+                    }
+                }
+
+                if let Some(container_process) = container_oci.process.as_mut() {
+                    if let Some(image_process) = image_oci.process.as_ref() {
+                        self.merge_oci_process(container_process, image_process);
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    // Partially merge an OCI process specification into another one.
+    fn merge_oci_process(&self, target: &mut oci::Process, source: &oci::Process) {
+        if target.args.is_empty() && !source.args.is_empty() {
+            target.args.append(&mut source.args.clone());
+        }
+
+        if target.cwd == "/" && source.cwd != "/" {
+            target.cwd = String::from(&source.cwd);
+        }
+
+        for source_env in &source.env {
+            let variable_name: Vec<&str> = source_env.split('=').collect();
+            if !target.env.iter().any(|i| i.contains(variable_name[0])) {
+                target.env.push(source_env.to_string());
+            }
+        }
+    }
+}
+
+#[async_trait]
+impl protocols::image_ttrpc_async::Image for ImageService {
+    async fn pull_image(
+        &self,
+        _ctx: &ttrpc::r#async::TtrpcContext,
+        req: image::PullImageRequest,
+    ) -> ttrpc::Result<image::PullImageResponse> {
+        match self.pull_image(&req).await {
+            Ok(r) => {
+                let mut resp = image::PullImageResponse::new();
+                resp.image_ref = r;
+                return Ok(resp);
+            }
+            Err(e) => {
+                return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()));
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::ImageService;
+    use protocols::image;
+
+    #[tokio::test]
+    async fn test_cid_from_request() {
+        struct Case {
+            cid: &'static str,
+            image: &'static str,
+            result: Option<&'static str>,
+        }
+
+        let cases = [
+            Case {
+                cid: "",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "..",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "..",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "abc/..",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "abc/",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "../abc",
+                result: Some("abc_4"),
+            },
+            Case {
+                cid: "",
+                image: "../9abc",
+                result: Some("9abc_5"),
+            },
+            Case {
+                cid: "some-string.1_2",
+                image: "",
+                result: Some("some-string.1_2"),
+            },
+            Case {
+                cid: "0some-string.1_2",
+                image: "",
+                result: Some("0some-string.1_2"),
+            },
+            Case {
+                cid: "a:b",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "prefix/a:b",
+                result: Some("a_b_6"),
+            },
+            Case {
+                cid: "",
+                image: "/a/b/c/d:e",
+                result: Some("d_e_7"),
+            },
+        ];
+
+        let image_service = ImageService::new();
+        for case in &cases {
+            let mut req = image::PullImageRequest::new();
+            req.set_image(case.image.to_string());
+            req.set_container_id(case.cid.to_string());
+            let ret = image_service.cid_from_request(&req);
+            match (case.result, ret) {
+                (Some(expected), Ok(actual)) => assert_eq!(expected, actual),
+                (None, Err(_)) => (),
+                (None, Ok(r)) => panic!("Expected an error, got {}", r),
+                (Some(expected), Err(e)) => {
+                    panic!("Expected {} but got an error ({})", expected, e)
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_merge_cwd() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            container_process_cwd: &'a str,
+            image_process_cwd: &'a str,
+            expected: &'a str,
+        }
+
+        let tests = &[
+            // Image cwd should override blank container cwd
+            // TODO - how can we tell the user didn't specifically set it to `/` vs not setting at all? Is that scenario valid?
+            TestData {
+                container_process_cwd: "/",
+                image_process_cwd: "/imageDir",
+                expected: "/imageDir",
+            },
+            // Container cwd should override image cwd
+            TestData {
+                container_process_cwd: "/containerDir",
+                image_process_cwd: "/imageDir",
+                expected: "/containerDir",
+            },
+            // Container cwd should override blank image cwd
+            TestData {
+                container_process_cwd: "/containerDir",
+                image_process_cwd: "/",
+                expected: "/containerDir",
+            },
+        ];
+
+        let image_service = ImageService::new();
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mut container_process = oci::Process {
+                cwd: d.container_process_cwd.to_string(),
+                ..Default::default()
+            };
+
+            let image_process = oci::Process {
+                cwd: d.image_process_cwd.to_string(),
+                ..Default::default()
+            };
+
+            image_service.merge_oci_process(&mut container_process, &image_process);
+
+            assert_eq!(d.expected, container_process.cwd, "{}", msg);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_merge_env() {
+        #[derive(Debug)]
+        struct TestData {
+            container_process_env: Vec<String>,
+            image_process_env: Vec<String>,
+            expected: Vec<String>,
+        }
+
+        let tests = &[
+            // Test that the pods environment overrides the images
+            TestData {
+                container_process_env: vec!["ISPRODUCTION=true".to_string()],
+                image_process_env: vec!["ISPRODUCTION=false".to_string()],
+                expected: vec!["ISPRODUCTION=true".to_string()],
+            },
+            // Test that multiple environment variables can be overrided
+            TestData {
+                container_process_env: vec![
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=false".to_string(),
+                ],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=false".to_string(),
+                ],
+            },
+            // Test that when none of the variables match do not override them
+            TestData {
+                container_process_env: vec!["ANOTHERENV=TEST".to_string()],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+            },
+            // Test a mix of both overriding and not
+            TestData {
+                container_process_env: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=true".to_string(),
+                ],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+            },
+        ];
+
+        let image_service = ImageService::new();
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mut container_process = oci::Process {
+                env: d.container_process_env.clone(),
+                ..Default::default()
+            };
+
+            let image_process = oci::Process {
+                env: d.image_process_env.clone(),
+                ..Default::default()
+            };
+
+            image_service.merge_oci_process(&mut container_process, &image_process);
+
+            assert_eq!(d.expected, container_process.env, "{}", msg);
+        }
+    }
+}

src/agent/src/main.rs

@@ -22,6 +22,7 @@ extern crate slog;
 use anyhow::{anyhow, Context, Result};
 use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
+use const_format::concatcp;
 use nix::fcntl::OFlag;
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
 use nix::unistd::{self, dup, Pid};
@@ -32,9 +33,12 @@ use std::os::unix::fs as unixfs;
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
 use std::process::exit;
+use std::process::Command;
 use std::sync::Arc;
 use tracing::{instrument, span};
 
+#[cfg(feature = "confidential-data-hub")]
+mod cdh;
 mod config;
 mod console;
 mod device;
@@ -48,6 +52,7 @@ mod pci;
 pub mod random;
 mod sandbox;
 mod signal;
+mod storage;
 mod uevent;
 mod util;
 mod version;
@@ -70,6 +75,7 @@ use tokio::{
     task::JoinHandle,
 };
 
+mod image_rpc;
 mod rpc;
 mod tracer;
 
@@ -82,6 +88,27 @@ cfg_if! {
 
 const NAME: &str = "kata-agent";
 
+const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+const AA_PATH: &str = "/usr/local/bin/attestation-agent";
+const AA_UNIX_SOCKET_DIR: &str = "/run/confidential-containers/attestation-agent/";
+const UNIX_SOCKET_PREFIX: &str = "unix://";
+const AA_KEYPROVIDER_URI: &str =
+    concatcp!(UNIX_SOCKET_PREFIX, AA_UNIX_SOCKET_DIR, "keyprovider.sock");
+const AA_GETRESOURCE_URI: &str =
+    concatcp!(UNIX_SOCKET_PREFIX, AA_UNIX_SOCKET_DIR, "getresource.sock");
+const AA_ATTESTATION_SOCKET: &str = concatcp!(AA_UNIX_SOCKET_DIR, "attestation-agent.sock");
+const AA_ATTESTATION_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, AA_ATTESTATION_SOCKET);
+
+const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6;
+
+cfg_if! {
+    if #[cfg(feature = "confidential-data-hub")] {
+        const CDH_PATH: &str = "/usr/local/bin/confidential-data-hub";
+        const CDH_SOCKET: &str = "/run/confidential-containers/cdh.sock";
+        const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest";
+    }
+}
+
 lazy_static! {
     static ref AGENT_CONFIG: AgentConfig =
         // Note: We can't do AgentOpts.parse() here to send through the processed arguments to AgentConfig
@@ -343,8 +370,12 @@ async fn start_sandbox(
     let (tx, rx) = tokio::sync::oneshot::channel();
     sandbox.lock().await.sender = Some(tx);
 
+    if !config.aa_kbc_params.is_empty() {
+        init_attestation_agent(logger, config)?;
+    }
+
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode)?;
+    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode).await?;
     server.start().await?;
 
     rx.await?;
@@ -353,6 +384,110 @@ async fn start_sandbox(
     Ok(())
 }
 
+// If we fail to start the AA, ocicrypt won't be able to unwrap keys
+// and container decryption will fail.
+fn init_attestation_agent(logger: &Logger, _config: &AgentConfig) -> Result<()> {
+    let config_path = OCICRYPT_CONFIG_PATH;
+
+    // The image will need to be encrypted using a keyprovider
+    // that has the same name (at least according to the config).
+    let ocicrypt_config = serde_json::json!({
+        "key-providers": {
+            "attestation-agent":{
+                "ttrpc":AA_KEYPROVIDER_URI
+            }
+        }
+    });
+
+    fs::write(config_path, ocicrypt_config.to_string().as_bytes())?;
+
+    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", config_path);
+
+    // The Attestation Agent will run for the duration of the guest.
+    launch_process(
+        logger,
+        AA_PATH,
+        &vec![
+            "--keyprovider_sock",
+            AA_KEYPROVIDER_URI,
+            "--getresource_sock",
+            AA_GETRESOURCE_URI,
+            "--attestation_sock",
+            AA_ATTESTATION_URI,
+        ],
+        AA_ATTESTATION_SOCKET,
+        DEFAULT_LAUNCH_PROCESS_TIMEOUT,
+    )
+    .map_err(|e| anyhow!("launch_process {} failed: {:?}", AA_PATH, e))?;
+
+    #[cfg(feature = "confidential-data-hub")]
+    {
+        if let Err(e) = launch_process(
+            logger,
+            CDH_PATH,
+            &vec![],
+            CDH_SOCKET,
+            DEFAULT_LAUNCH_PROCESS_TIMEOUT,
+        ) {
+            error!(logger, "launch_process {} failed: {:?}", CDH_PATH, e);
+        } else if !_config.rest_api.is_empty() {
+            if let Err(e) = launch_process(
+                logger,
+                API_SERVER_PATH,
+                &vec!["--features", &_config.rest_api],
+                "",
+                0,
+            ) {
+                error!(logger, "launch_process {} failed: {:?}", API_SERVER_PATH, e);
+            }
+        }
+    }
+
+    Ok(())
+}
+
+fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> {
+    let p = Path::new(path);
+    let mut attempts = 0;
+    loop {
+        std::thread::sleep(std::time::Duration::from_secs(1));
+        if p.exists() {
+            return Ok(());
+        }
+        if attempts >= timeout_secs {
+            break;
+        }
+        attempts += 1;
+        info!(
+            logger,
+            "waiting for {} to exist (attempts={})", path, attempts
+        );
+    }
+
+    Err(anyhow!("wait for {} to exist timeout.", path))
+}
+
+fn launch_process(
+    logger: &Logger,
+    path: &str,
+    args: &Vec<&str>,
+    unix_socket_path: &str,
+    timeout_secs: i32,
+) -> Result<()> {
+    if !Path::new(path).exists() {
+        return Err(anyhow!("path {} does not exist.", path));
+    }
+    if !unix_socket_path.is_empty() && Path::new(unix_socket_path).exists() {
+        fs::remove_file(unix_socket_path)?;
+    }
+    Command::new(path).args(args).spawn()?;
+    if !unix_socket_path.is_empty() && timeout_secs > 0 {
+        wait_for_path_to_exist(logger, unix_socket_path, timeout_secs)?;
+    }
+
+    Ok(())
+}
+
 // init_agent_as_init will do the initializations such as setting up the rootfs
 // when this agent has been run as the init process.
 fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result<()> {

src/agent/src/namespace.rs

@@ -7,14 +7,14 @@ use anyhow::{anyhow, Result};
 use nix::mount::MsFlags;
 use nix::sched::{unshare, CloneFlags};
 use nix::unistd::{getpid, gettid};
+use slog::Logger;
 use std::fmt;
 use std::fs;
 use std::fs::File;
 use std::path::{Path, PathBuf};
 use tracing::instrument;
 
-use crate::mount::{baremount, FLAGS};
-use slog::Logger;
+use crate::mount::baremount;
 
 const PERSISTENT_NS_DIR: &str = "/var/run/sandbox-ns";
 pub const NSTYPEIPC: &str = "ipc";
@@ -116,15 +116,7 @@ impl Namespace {
                 // Bind mount the new namespace from the current thread onto the mount point to persist it.
 
                 let mut flags = MsFlags::empty();
-
-                if let Some(x) = FLAGS.get("rbind") {
-                    let (clear, f) = *x;
-                    if clear {
-                        flags &= !f;
-                    } else {
-                        flags |= f;
-                    }
-                };
+                flags |= MsFlags::MS_BIND | MsFlags::MS_REC;
 
                 baremount(source, destination, "none", flags, "", &logger).map_err(|e| {
                     anyhow!(

src/agent/src/rpc.rs

@@ -37,7 +37,10 @@ use protocols::health::{
     VersionCheckResponse,
 };
 use protocols::types::Interface;
-use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc};
+use protocols::{
+    agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc,
+    image_ttrpc_async as image_ttrpc,
+};
 use rustjail::cgroups::notifier;
 use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT};
 use rustjail::mount::parse_mount_table;
@@ -53,19 +56,22 @@ use rustjail::process::ProcessOperations;
 use crate::device::{
     add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci,
 };
+use crate::image_rpc;
 use crate::linux_abi::*;
 use crate::metrics::get_metrics;
-use crate::mount::{add_storages, baremount, update_ephemeral_mounts, STORAGE_HANDLER_LIST};
+use crate::mount::baremount;
 use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS};
 use crate::network::setup_guest_dns;
 use crate::pci;
 use crate::random;
 use crate::sandbox::Sandbox;
+use crate::storage::{add_storages, update_ephemeral_mounts, STORAGE_HANDLERS};
 use crate::version::{AGENT_VERSION, API_VERSION};
 use crate::AGENT_CONFIG;
 
 use crate::trace_rpc_call;
 use crate::tracer::extract_carrier_from_ttrpc;
+use derivative::Derivative;
 use opentelemetry::global;
 use tracing::span;
 use tracing_opentelemetry::OpenTelemetrySpanExt;
@@ -84,8 +90,14 @@ use std::io::{BufRead, BufReader, Write};
 use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
 
+#[cfg(feature = "sealed-secret")]
+use crate::cdh::CDHClient;
+
 pub const CONTAINER_BASE: &str = "/run/kata-containers";
 const MODPROBE_PATH: &str = "/sbin/modprobe";
+const CONFIG_JSON: &str = "config.json";
+const INIT_TRUSTED_STORAGE: &str = "/usr/bin/kata-init-trusted-storage";
+const TRUSTED_STORAGE_DEVICE: &str = "/dev/trusted_store";
 
 /// the iptables seriers binaries could appear either in /sbin
 /// or /usr/sbin, we need to check both of them
@@ -131,10 +143,31 @@ fn is_allowed(req: &impl MessageDyn) -> ttrpc::Result<()> {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Derivative)]
+#[derivative(Clone, Debug)]
 pub struct AgentService {
     sandbox: Arc<Mutex<Sandbox>>,
     init_mode: bool,
+    #[derivative(Debug = "ignore")]
+    #[cfg(feature = "sealed-secret")]
+    cdh_client: Option<CDHClient>,
+}
+
+// A container ID must match this regex:
+//
+//     ^[a-zA-Z0-9][a-zA-Z0-9_.-]+$
+//
+pub fn verify_cid(id: &str) -> Result<()> {
+    let mut chars = id.chars();
+
+    let valid = matches!(chars.next(), Some(first) if first.is_alphanumeric()
+                && id.len() > 1
+                && chars.all(|c| c.is_alphanumeric() || ['.', '-', '_'].contains(&c)));
+
+    match valid {
+        true => Ok(()),
+        false => Err(anyhow!("invalid container ID: {:?}", id)),
+    }
 }
 
 impl AgentService {
@@ -164,6 +197,11 @@ impl AgentService {
             "receive createcontainer, storages: {:?}", &req.storages
         );
 
+        // In case of pulling image inside guest, we need to merge the image bundle OCI spec
+        // into the container creation request OCI spec.
+        let image_service = image_rpc::ImageService::singleton().await?;
+        image_service.merge_bundle_oci(&mut oci).await?;
+
         // Some devices need some extra processing (the ones invoked with
         // --device for instance), and that's what this call is doing. It
         // updates the devices listed in the OCI spec, so that they actually
@@ -171,6 +209,52 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&req.devices, &mut oci, &self.sandbox).await?;
 
+        #[cfg(feature = "sealed-secret")]
+        let mut sealed_source_path = {
+            let process = oci
+                .process
+                .as_mut()
+                .ok_or_else(|| anyhow!("Spec didn't contain process field"))?;
+
+            let client = self
+                .cdh_client
+                .as_ref()
+                .ok_or(anyhow!("get cdh_client failed"))?;
+            for env in process.env.iter_mut() {
+                let unsealed_env = client
+                    .unseal_env(env)
+                    .await
+                    .map_err(|e| anyhow!("unseal env failed: {:?}", e))?;
+                *env = unsealed_env.to_string();
+            }
+
+            client.create_sealed_secret_mounts(&mut oci)?
+        };
+
+        let linux = oci
+            .linux
+            .as_mut()
+            .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?;
+
+        for specdev in &mut linux.devices {
+            let dev_major_minor = format!("{}:{}", specdev.major, specdev.minor);
+
+            if specdev.path == TRUSTED_STORAGE_DEVICE {
+                let data_integrity = AGENT_CONFIG.data_integrity;
+                info!(
+                    sl(),
+                    "trusted_store device major:min {}, enable data integrity {}",
+                    dev_major_minor,
+                    data_integrity.to_string()
+                );
+
+                Command::new(INIT_TRUSTED_STORAGE)
+                    .args([&dev_major_minor, &data_integrity.to_string()])
+                    .output()
+                    .expect("Failed to initialize confidential storage");
+            }
+        }
+
         // Both rootfs and volumes (invoked with --volume for instance) will
         // be processed the same way. The idea is to always mount any provided
         // storage to the specified MountPoint, so that it will match what's
@@ -178,13 +262,7 @@ impl AgentService {
         // After all those storages have been processed, no matter the order
         // here, the agent will rely on rustjail (using the oci.Mounts
         // list) to bind mount all of them inside the container.
-        let m = add_storages(
-            sl(),
-            &req.storages,
-            &self.sandbox,
-            Some(req.container_id.clone()),
-        )
-        .await?;
+        let m = add_storages(sl(), req.storages, &self.sandbox, Some(req.container_id)).await?;
 
         let mut s = self.sandbox.lock().await;
         s.container_mounts.insert(cid.clone(), m);
@@ -236,6 +314,17 @@ impl AgentService {
             return Err(anyhow!(nix::Error::EINVAL));
         };
 
+        #[cfg(feature = "sealed-secret")]
+        {
+            let client = self
+                .cdh_client
+                .as_ref()
+                .ok_or(anyhow!("get cdh_client failed"))?;
+            for source_path in sealed_source_path.iter_mut() {
+                client.unseal_file(source_path).await?;
+            }
+        }
+
         // if starting container failed, we will do some rollback work
         // to ensure no resources are leaked.
         if let Err(err) = ctr.start(p).await {
@@ -243,7 +332,7 @@ impl AgentService {
             if let Err(e) = ctr.destroy().await {
                 error!(sl(), "failed to destroy container: {:?}", e);
             }
-            if let Err(e) = remove_container_resources(&mut s, &cid) {
+            if let Err(e) = remove_container_resources(&mut s, &cid).await {
                 error!(sl(), "failed to remove container resources: {:?}", e);
             }
             return Err(err);
@@ -295,7 +384,7 @@ impl AgentService {
                 .ok_or_else(|| anyhow!("Invalid container id"))?
                 .destroy()
                 .await?;
-            remove_container_resources(&mut sandbox, &cid)?;
+            remove_container_resources(&mut sandbox, &cid).await?;
             return Ok(());
         }
 
@@ -317,7 +406,7 @@ impl AgentService {
             .await
             .map_err(|_| anyhow!(nix::Error::ETIME))???;
 
-        remove_container_resources(&mut *self.sandbox.lock().await, &cid)
+        remove_container_resources(&mut *self.sandbox.lock().await, &cid).await
     }
 
     #[instrument]
@@ -1215,7 +1304,7 @@ impl agent_ttrpc::AgentService for AgentService {
                 .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e))?;
         }
 
-        match add_storages(sl(), &req.storages, &self.sandbox, None).await {
+        match add_storages(sl(), req.storages, &self.sandbox, None).await {
             Ok(m) => {
                 self.sandbox.lock().await.mounts = m;
             }
@@ -1622,7 +1711,7 @@ fn get_agent_details() -> AgentDetails {
     detail.init_daemon = unistd::getpid() == Pid::from_raw(1);
 
     detail.device_handlers = Vec::new();
-    detail.storage_handlers = STORAGE_HANDLER_LIST.iter().map(|x| x.to_string()).collect();
+    detail.storage_handlers = STORAGE_HANDLERS.get_handlers();
 
     detail
 }
@@ -1641,20 +1730,33 @@ async fn read_stream(reader: &Mutex<ReadHalf<PipeStream>>, l: usize) -> Result<V
     Ok(content)
 }
 
-pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str, init_mode: bool) -> Result<TtrpcServer> {
+pub async fn start(
+    s: Arc<Mutex<Sandbox>>,
+    server_address: &str,
+    init_mode: bool,
+) -> Result<TtrpcServer> {
     let agent_service = Box::new(AgentService {
-        sandbox: s,
+        sandbox: s.clone(),
         init_mode,
+        #[cfg(feature = "sealed-secret")]
+        cdh_client: Some(CDHClient::new()?),
     }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
     let aservice = agent_ttrpc::create_agent_service(Arc::new(agent_service));
 
     let health_service = Box::new(HealthService {}) as Box<dyn health_ttrpc::Health + Send + Sync>;
     let hservice = health_ttrpc::create_health(Arc::new(health_service));
 
+    let image_service = image_rpc::ImageService::new();
+    *image_rpc::IMAGE_SERVICE.lock().await = Some(image_service.clone());
+    let image_service =
+        Arc::new(Box::new(image_service) as Box<dyn image_ttrpc::Image + Send + Sync>);
+    let iservice = image_ttrpc::create_image(image_service);
+
     let server = TtrpcServer::new()
         .bind(server_address)?
         .register_service(aservice)
-        .register_service(hservice);
+        .register_service(hservice)
+        .register_service(iservice);
 
     info!(sl(), "ttRPC server started"; "address" => server_address);
 
@@ -1716,21 +1818,21 @@ fn update_container_namespaces(
     Ok(())
 }
 
-fn remove_container_resources(sandbox: &mut Sandbox, cid: &str) -> Result<()> {
+async fn remove_container_resources(sandbox: &mut Sandbox, cid: &str) -> Result<()> {
     let mut cmounts: Vec<String> = vec![];
 
     // Find the sandbox storage used by this container
     let mounts = sandbox.container_mounts.get(cid);
     if let Some(mounts) = mounts {
         for m in mounts.iter() {
-            if sandbox.storages.get(m).is_some() {
+            if sandbox.storages.contains_key(m) {
                 cmounts.push(m.to_string());
             }
         }
     }
 
     for m in cmounts.iter() {
-        if let Err(err) = sandbox.unset_and_remove_sandbox_storage(m) {
+        if let Err(err) = sandbox.remove_sandbox_storage(m).await {
             error!(
                 sl(),
                 "failed to unset_and_remove_sandbox_storage for container {}, error: {:?}",
@@ -1856,6 +1958,38 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
         }
     }
 
+    let sflag = stat::SFlag::from_bits_truncate(req.file_mode);
+
+    if sflag.contains(stat::SFlag::S_IFDIR) {
+        fs::create_dir(path.clone()).or_else(|e| {
+            if e.kind() != std::io::ErrorKind::AlreadyExists {
+                return Err(e);
+            }
+            Ok(())
+        })?;
+
+        std::fs::set_permissions(path.clone(), std::fs::Permissions::from_mode(req.file_mode))?;
+
+        unistd::chown(
+            &path,
+            Some(Uid::from_raw(req.uid as u32)),
+            Some(Gid::from_raw(req.gid as u32)),
+        )?;
+
+        return Ok(());
+    }
+
+    if sflag.contains(stat::SFlag::S_IFLNK) {
+        let src = PathBuf::from(String::from_utf8(req.data.clone()).unwrap());
+
+        unistd::symlinkat(&src, None, &path)?;
+        let path_str = CString::new(path.to_str().unwrap())?;
+        let ret = unsafe { libc::lchown(path_str.as_ptr(), req.uid as u32, req.gid as u32) };
+        Errno::result(ret).map(drop)?;
+
+        return Ok(());
+    }
+
     let mut tmpfile = path.clone();
     tmpfile.set_extension("tmp");
 
@@ -1921,18 +2055,26 @@ pub fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
     let spec_root_path = Path::new(&spec_root.path);
 
     let bundle_path = Path::new(CONTAINER_BASE).join(cid);
-    let config_path = bundle_path.join("config.json");
+    let config_path = bundle_path.join(CONFIG_JSON);
     let rootfs_path = bundle_path.join("rootfs");
 
-    fs::create_dir_all(&rootfs_path)?;
-    baremount(
-        spec_root_path,
-        &rootfs_path,
-        "bind",
-        MsFlags::MS_BIND,
-        "",
+    let rootfs_exists = Path::new(&rootfs_path).exists();
+    info!(
         &sl(),
-    )?;
+        "The rootfs_path is {:?} and exists: {}", rootfs_path, rootfs_exists
+    );
+
+    if !rootfs_exists {
+        fs::create_dir_all(&rootfs_path)?;
+        baremount(
+            spec_root_path,
+            &rootfs_path,
+            "bind",
+            MsFlags::MS_BIND,
+            "",
+            &sl(),
+        )?;
+    }
 
     let rootfs_path_name = rootfs_path
         .to_str()
@@ -2113,6 +2255,8 @@ mod tests {
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            #[cfg(feature = "sealed-secret")]
+            cdh_client: None,
         });
 
         let req = protocols::agent::UpdateInterfaceRequest::default();
@@ -2127,10 +2271,11 @@ mod tests {
     async fn test_update_routes() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let sandbox = Sandbox::new(&logger).unwrap();
-
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            #[cfg(feature = "sealed-secret")]
+            cdh_client: None,
         });
 
         let req = protocols::agent::UpdateRoutesRequest::default();
@@ -2145,10 +2290,11 @@ mod tests {
     async fn test_add_arp_neighbors() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let sandbox = Sandbox::new(&logger).unwrap();
-
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            #[cfg(feature = "sealed-secret")]
+            cdh_client: None,
         });
 
         let req = protocols::agent::AddARPNeighborsRequest::default();
@@ -2283,6 +2429,8 @@ mod tests {
             let agent_service = Box::new(AgentService {
                 sandbox: Arc::new(Mutex::new(sandbox)),
                 init_mode: true,
+                #[cfg(feature = "sealed-secret")]
+                cdh_client: None,
             });
 
             let result = agent_service
@@ -2764,6 +2912,8 @@ OtherField:other
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            #[cfg(feature = "sealed-secret")]
+            cdh_client: None,
         });
 
         let ctx = mk_ttrpc_context();

src/agent/src/sandbox.rs

@@ -3,16 +3,20 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use std::collections::hash_map::Entry;
 use std::collections::HashMap;
+use std::fmt::{Debug, Formatter};
 use std::fs;
 use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
 use std::str::FromStr;
+use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
 use std::{thread, time};
 
 use anyhow::{anyhow, Context, Result};
 use kata_types::cpu::CpuSet;
+use kata_types::mount::StorageDevice;
 use libc::pid_t;
 use oci::{Hook, Hooks};
 use protocols::agent::OnlineCPUMemRequest;
@@ -28,11 +32,12 @@ use tokio::sync::Mutex;
 use tracing::instrument;
 
 use crate::linux_abi::*;
-use crate::mount::{get_mount_fs_type, remove_mounts, TYPE_ROOTFS};
+use crate::mount::{get_mount_fs_type, TYPE_ROOTFS};
 use crate::namespace::Namespace;
 use crate::netlink::Handle;
 use crate::network::Network;
 use crate::pci;
+use crate::storage::StorageDeviceGeneric;
 use crate::uevent::{Uevent, UeventMatcher};
 use crate::watcher::BindWatcher;
 
@@ -40,6 +45,50 @@ pub const ERR_INVALID_CONTAINER_ID: &str = "Invalid container id";
 
 type UeventWatcher = (Box<dyn UeventMatcher>, oneshot::Sender<Uevent>);
 
+#[derive(Clone)]
+pub struct StorageState {
+    count: Arc<AtomicU32>,
+    device: Arc<dyn StorageDevice>,
+}
+
+impl Debug for StorageState {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("StorageState").finish()
+    }
+}
+
+impl StorageState {
+    fn new() -> Self {
+        StorageState {
+            count: Arc::new(AtomicU32::new(1)),
+            device: Arc::new(StorageDeviceGeneric::default()),
+        }
+    }
+
+    pub fn from_device(device: Arc<dyn StorageDevice>) -> Self {
+        Self {
+            count: Arc::new(AtomicU32::new(1)),
+            device,
+        }
+    }
+
+    pub fn path(&self) -> Option<&str> {
+        self.device.path()
+    }
+
+    pub async fn ref_count(&self) -> u32 {
+        self.count.load(Ordering::Relaxed)
+    }
+
+    async fn inc_ref_count(&self) {
+        self.count.fetch_add(1, Ordering::Acquire);
+    }
+
+    async fn dec_and_test_ref_count(&self) -> bool {
+        self.count.fetch_sub(1, Ordering::AcqRel) == 1
+    }
+}
+
 #[derive(Debug)]
 pub struct Sandbox {
     pub logger: Logger,
@@ -54,7 +103,7 @@ pub struct Sandbox {
     pub shared_utsns: Namespace,
     pub shared_ipcns: Namespace,
     pub sandbox_pidns: Option<Namespace>,
-    pub storages: HashMap<String, u32>,
+    pub storages: HashMap<String, StorageState>,
     pub running: bool,
     pub no_pivot_root: bool,
     pub sender: Option<tokio::sync::oneshot::Sender<i32>>,
@@ -100,73 +149,60 @@ impl Sandbox {
         })
     }
 
-    // set_sandbox_storage sets the sandbox level reference
-    // counter for the sandbox storage.
-    // This method also returns a boolean to let
-    // callers know if the storage already existed or not.
-    // It will return true if storage is new.
+    /// Add a new storage object or increase reference count of existing one.
+    /// The caller may detect new storage object by checking `StorageState.refcount == 1`.
     #[instrument]
-    pub fn set_sandbox_storage(&mut self, path: &str) -> bool {
-        match self.storages.get_mut(path) {
-            None => {
-                self.storages.insert(path.to_string(), 1);
-                true
+    pub async fn add_sandbox_storage(&mut self, path: &str) -> StorageState {
+        match self.storages.entry(path.to_string()) {
+            Entry::Occupied(e) => {
+                let state = e.get().clone();
+                state.inc_ref_count().await;
+                state
             }
-            Some(count) => {
-                *count += 1;
-                false
+            Entry::Vacant(e) => {
+                let state = StorageState::new();
+                e.insert(state.clone());
+                state
             }
         }
     }
 
-    // unset_sandbox_storage will decrement the sandbox storage
-    // reference counter. If there aren't any containers using
-    // that sandbox storage, this method will remove the
-    // storage reference from the sandbox and return 'true' to
-    // let the caller know that they can clean up the storage
-    // related directories by calling remove_sandbox_storage
+    /// Update the storage device associated with a path.
+    pub fn update_sandbox_storage(
+        &mut self,
+        path: &str,
+        device: Arc<dyn StorageDevice>,
+    ) -> std::result::Result<Arc<dyn StorageDevice>, Arc<dyn StorageDevice>> {
+        if !self.storages.contains_key(path) {
+            return Err(device);
+        }
+
+        let state = StorageState::from_device(device);
+        // Safe to unwrap() because we have just ensured existence of entry.
+        let state = self.storages.insert(path.to_string(), state).unwrap();
+        Ok(state.device)
+    }
+
+    /// Decrease reference count and destroy the storage object if reference count reaches zero.
+    /// Returns `Ok(true)` if the reference count has reached zero and the storage object has been
+    /// removed.
     #[instrument]
-    pub fn unset_sandbox_storage(&mut self, path: &str) -> Result<bool> {
-        match self.storages.get_mut(path) {
+    pub async fn remove_sandbox_storage(&mut self, path: &str) -> Result<bool> {
+        match self.storages.get(path) {
             None => Err(anyhow!("Sandbox storage with path {} not found", path)),
-            Some(count) => {
-                *count -= 1;
-                if *count == 0 {
-                    self.storages.remove(path);
-                    return Ok(true);
+            Some(state) => {
+                if state.dec_and_test_ref_count().await {
+                    if let Some(storage) = self.storages.remove(path) {
+                        storage.device.cleanup()?;
+                    }
+                    Ok(true)
+                } else {
+                    Ok(false)
                 }
-                Ok(false)
             }
         }
     }
 
-    // remove_sandbox_storage removes the sandbox storage if no
-    // containers are using that storage.
-    #[instrument]
-    pub fn remove_sandbox_storage(&mut self, path: &str) -> Result<()> {
-        let mounts = vec![path.to_string()];
-        remove_mounts(&mounts)?;
-        // "remove_dir" will fail if the mount point is backed by a read-only filesystem.
-        // This is the case with the device mapper snapshotter, where we mount the block device directly
-        // at the underlying sandbox path which was provided from the base RO kataShared path from the host.
-        if let Err(err) = fs::remove_dir(path) {
-            warn!(self.logger, "failed to remove dir {}, {:?}", path, err);
-        }
-        Ok(())
-    }
-
-    // unset_and_remove_sandbox_storage unsets the storage from sandbox
-    // and if there are no containers using this storage it will
-    // remove it from the sandbox.
-    #[instrument]
-    pub fn unset_and_remove_sandbox_storage(&mut self, path: &str) -> Result<()> {
-        if self.unset_sandbox_storage(path)? {
-            return self.remove_sandbox_storage(path);
-        }
-
-        Ok(())
-    }
-
     #[instrument]
     pub async fn setup_shared_namespaces(&mut self) -> Result<bool> {
         // Set up shared IPC namespace
@@ -493,24 +529,22 @@ mod tests {
         let tmpdir_path = tmpdir.path().to_str().unwrap();
 
         // Add a new sandbox storage
-        let new_storage = s.set_sandbox_storage(tmpdir_path);
+        let new_storage = s.add_sandbox_storage(tmpdir_path).await;
 
         // Check the reference counter
-        let ref_count = s.storages[tmpdir_path];
+        let ref_count = new_storage.ref_count().await;
         assert_eq!(
             ref_count, 1,
             "Invalid refcount, got {} expected 1.",
             ref_count
         );
-        assert!(new_storage);
 
         // Use the existing sandbox storage
-        let new_storage = s.set_sandbox_storage(tmpdir_path);
-        assert!(!new_storage, "Should be false as already exists.");
+        let new_storage = s.add_sandbox_storage(tmpdir_path).await;
 
         // Since we are using existing storage, the reference counter
         // should be 2 by now.
-        let ref_count = s.storages[tmpdir_path];
+        let ref_count = new_storage.ref_count().await;
         assert_eq!(
             ref_count, 2,
             "Invalid refcount, got {} expected 2.",
@@ -518,52 +552,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    #[serial]
-    async fn remove_sandbox_storage() {
-        skip_if_not_root!();
-
-        let logger = slog::Logger::root(slog::Discard, o!());
-        let mut s = Sandbox::new(&logger).unwrap();
-
-        let tmpdir = Builder::new().tempdir().unwrap();
-        let tmpdir_path = tmpdir.path().to_str().unwrap();
-
-        let srcdir = Builder::new()
-            .prefix("src")
-            .tempdir_in(tmpdir_path)
-            .unwrap();
-        let srcdir_path = srcdir.path().to_str().unwrap();
-
-        let destdir = Builder::new()
-            .prefix("dest")
-            .tempdir_in(tmpdir_path)
-            .unwrap();
-        let destdir_path = destdir.path().to_str().unwrap();
-
-        let emptydir = Builder::new()
-            .prefix("empty")
-            .tempdir_in(tmpdir_path)
-            .unwrap();
-
-        assert!(
-            s.remove_sandbox_storage(srcdir_path).is_err(),
-            "Expect Err as the directory is not a mountpoint"
-        );
-
-        assert!(s.remove_sandbox_storage("").is_err());
-
-        let invalid_dir = emptydir.path().join("invalid");
-
-        assert!(s
-            .remove_sandbox_storage(invalid_dir.to_str().unwrap())
-            .is_err());
-
-        assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
-
-        assert!(s.remove_sandbox_storage(destdir_path).is_ok());
-    }
-
     #[tokio::test]
     #[serial]
     async fn unset_and_remove_sandbox_storage() {
@@ -573,8 +561,7 @@ mod tests {
         let mut s = Sandbox::new(&logger).unwrap();
 
         assert!(
-            s.unset_and_remove_sandbox_storage("/tmp/testEphePath")
-                .is_err(),
+            s.remove_sandbox_storage("/tmp/testEphePath").await.is_err(),
             "Should fail because sandbox storage doesn't exist"
         );
 
@@ -595,8 +582,12 @@ mod tests {
 
         assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
 
-        assert!(s.set_sandbox_storage(destdir_path));
-        assert!(s.unset_and_remove_sandbox_storage(destdir_path).is_ok());
+        s.add_sandbox_storage(destdir_path).await;
+        let storage = StorageDeviceGeneric::new(destdir_path.to_string());
+        assert!(s
+            .update_sandbox_storage(destdir_path, Arc::new(storage))
+            .is_ok());
+        assert!(s.remove_sandbox_storage(destdir_path).await.is_ok());
 
         let other_dir_str;
         {
@@ -609,10 +600,14 @@ mod tests {
             let other_dir_path = other_dir.path().to_str().unwrap();
             other_dir_str = other_dir_path.to_string();
 
-            assert!(s.set_sandbox_storage(other_dir_path));
+            s.add_sandbox_storage(other_dir_path).await;
+            let storage = StorageDeviceGeneric::new(other_dir_path.to_string());
+            assert!(s
+                .update_sandbox_storage(other_dir_path, Arc::new(storage))
+                .is_ok());
         }
 
-        assert!(s.unset_and_remove_sandbox_storage(&other_dir_str).is_err());
+        assert!(s.remove_sandbox_storage(&other_dir_str).await.is_ok());
     }
 
     #[tokio::test]
@@ -624,28 +619,30 @@ mod tests {
         let storage_path = "/tmp/testEphe";
 
         // Add a new sandbox storage
-        assert!(s.set_sandbox_storage(storage_path));
+        s.add_sandbox_storage(storage_path).await;
         // Use the existing sandbox storage
+        let state = s.add_sandbox_storage(storage_path).await;
         assert!(
-            !s.set_sandbox_storage(storage_path),
+            state.ref_count().await > 1,
             "Expects false as the storage is not new."
         );
 
         assert!(
-            !s.unset_sandbox_storage(storage_path).unwrap(),
+            !s.remove_sandbox_storage(storage_path).await.unwrap(),
             "Expects false as there is still a storage."
         );
 
         // Reference counter should decrement to 1.
-        let ref_count = s.storages[storage_path];
+        let storage = &s.storages[storage_path];
+        let refcount = storage.ref_count().await;
         assert_eq!(
-            ref_count, 1,
+            refcount, 1,
             "Invalid refcount, got {} expected 1.",
-            ref_count
+            refcount
         );
 
         assert!(
-            s.unset_sandbox_storage(storage_path).unwrap(),
+            s.remove_sandbox_storage(storage_path).await.unwrap(),
             "Expects true as there is still a storage."
         );
 
@@ -661,7 +658,7 @@ mod tests {
         // If no container is using the sandbox storage, the reference
         // counter for it should not exist.
         assert!(
-            s.unset_sandbox_storage(storage_path).is_err(),
+            s.remove_sandbox_storage(storage_path).await.is_err(),
             "Expects false as the reference counter should no exist."
         );
     }

src/agent/src/storage/bind_watcher_handler.rs

@@ -0,0 +1,37 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::Result;
+use kata_types::mount::StorageDevice;
+use protocols::agent::Storage;
+use std::iter;
+use std::sync::Arc;
+use tracing::instrument;
+
+use crate::storage::{new_device, StorageContext, StorageHandler};
+
+#[derive(Debug)]
+pub struct BindWatcherHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for BindWatcherHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        if let Some(cid) = ctx.cid {
+            ctx.sandbox
+                .lock()
+                .await
+                .bind_watcher
+                .add_container(cid.to_string(), iter::once(storage.clone()), ctx.logger)
+                .await?;
+        }
+        new_device("".to_string())
+    }
+}

src/agent/src/storage/block_handler.rs

@@ -0,0 +1,197 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fs;
+use std::os::unix::fs::PermissionsExt;
+use std::path::Path;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use kata_types::mount::StorageDevice;
+use protocols::agent::Storage;
+use tracing::instrument;
+
+use crate::device::{
+    get_scsi_device_name, get_virtio_blk_pci_device_name, get_virtio_mmio_device_name,
+    wait_for_pmem_device,
+};
+use crate::pci;
+use crate::storage::{common_storage_handler, new_device, StorageContext, StorageHandler};
+#[cfg(target_arch = "s390x")]
+use crate::{ccw, device::get_virtio_blk_ccw_device_name};
+
+#[derive(Debug)]
+pub struct VirtioBlkMmioHandler {}
+
+impl VirtioBlkMmioHandler {
+    pub async fn update_device_path(
+        storage: &mut Storage,
+        ctx: &mut StorageContext<'_>,
+    ) -> Result<()> {
+        if !Path::new(&storage.source).exists() {
+            get_virtio_mmio_device_name(ctx.sandbox, &storage.source)
+                .await
+                .context("failed to get mmio device name")?;
+        }
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for VirtioBlkMmioHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Self::update_device_path(&mut storage, ctx).await?;
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}
+
+#[derive(Debug)]
+pub struct VirtioBlkPciHandler {}
+
+impl VirtioBlkPciHandler {
+    pub async fn update_device_path(
+        storage: &mut Storage,
+        ctx: &mut StorageContext<'_>,
+    ) -> Result<()> {
+        // If hot-plugged, get the device node path based on the PCI path
+        // otherwise use the virt path provided in Storage Source
+        if storage.source.starts_with("/dev") {
+            let metadata = fs::metadata(&storage.source)
+                .context(format!("get metadata on file {:?}", &storage.source))?;
+            let mode = metadata.permissions().mode();
+            if mode & libc::S_IFBLK == 0 {
+                return Err(anyhow!("Invalid device {}", &storage.source));
+            }
+        } else {
+            let pcipath = pci::Path::from_str(&storage.source)?;
+            let dev_path = get_virtio_blk_pci_device_name(ctx.sandbox, &pcipath).await?;
+            storage.source = dev_path;
+        }
+
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for VirtioBlkPciHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Self::update_device_path(&mut storage, ctx).await?;
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}
+
+#[derive(Debug)]
+pub struct VirtioBlkCcwHandler {}
+
+impl VirtioBlkCcwHandler {
+    pub async fn update_device_path(
+        _storage: &mut Storage,
+        _ctx: &mut StorageContext<'_>,
+    ) -> Result<()> {
+        #[cfg(target_arch = "s390x")]
+        {
+            let ccw_device = ccw::Device::from_str(&_storage.source)?;
+            let dev_path = get_virtio_blk_ccw_device_name(_ctx.sandbox, &ccw_device).await?;
+            _storage.source = dev_path;
+        }
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for VirtioBlkCcwHandler {
+    #[cfg(target_arch = "s390x")]
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Self::update_device_path(&mut storage, ctx).await?;
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+
+    #[cfg(not(target_arch = "s390x"))]
+    #[instrument]
+    async fn create_device(
+        &self,
+        _storage: Storage,
+        _ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Err(anyhow!("CCW is only supported on s390x"))
+    }
+}
+
+#[derive(Debug)]
+pub struct ScsiHandler {}
+
+impl ScsiHandler {
+    pub async fn update_device_path(
+        storage: &mut Storage,
+        ctx: &mut StorageContext<'_>,
+    ) -> Result<()> {
+        // Retrieve the device path from SCSI address.
+        let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?;
+        storage.source = dev_path;
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for ScsiHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Self::update_device_path(&mut storage, ctx).await?;
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}
+
+#[derive(Debug)]
+pub struct PmemHandler {}
+
+impl PmemHandler {
+    pub async fn update_device_path(
+        storage: &mut Storage,
+        ctx: &mut StorageContext<'_>,
+    ) -> Result<()> {
+        // Retrieve the device for pmem storage
+        wait_for_pmem_device(ctx.sandbox, &storage.source).await?;
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for PmemHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Self::update_device_path(&mut storage, ctx).await?;
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}

src/agent/src/storage/dm_verity.rs

@@ -0,0 +1,165 @@
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use image_rs::verity::{create_dmverity_device, destroy_dmverity_device};
+use kata_sys_util::mount::create_mount_destination;
+use kata_types::mount::{DmVerityInfo, StorageDevice};
+use kata_types::volume::{
+    KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE, KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO,
+    KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM, KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI,
+    KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW, KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO,
+    KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI,
+};
+use protocols::agent::Storage;
+use slog::Logger;
+use tracing::instrument;
+
+use crate::storage::block_handler::{
+    PmemHandler, ScsiHandler, VirtioBlkCcwHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler,
+};
+use crate::storage::{common_storage_handler, StorageContext, StorageHandler};
+
+use super::StorageDeviceGeneric;
+
+#[derive(Debug)]
+pub struct DmVerityHandler {}
+
+impl DmVerityHandler {
+    fn get_dm_verity_info(storage: &Storage) -> Result<DmVerityInfo> {
+        for option in storage.driver_options.iter() {
+            if let Some((key, value)) = option.split_once('=') {
+                if key == KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO {
+                    let verity_info: DmVerityInfo = serde_json::from_str(value)?;
+                    return Ok(verity_info);
+                }
+            }
+        }
+
+        Err(anyhow!("missing DmVerity information for DmVerity volume"))
+    }
+
+    async fn update_source_device(
+        storage: &mut Storage,
+        ctx: &mut StorageContext<'_>,
+    ) -> Result<()> {
+        for option in storage.driver_options.clone() {
+            if let Some((key, value)) = option.split_once('=') {
+                if key == KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE {
+                    match value {
+                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI => {
+                            VirtioBlkPciHandler::update_device_path(storage, ctx).await?;
+                        }
+                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO => {
+                            VirtioBlkMmioHandler::update_device_path(storage, ctx).await?;
+                        }
+                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW => {
+                            VirtioBlkCcwHandler::update_device_path(storage, ctx).await?;
+                        }
+                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI => {
+                            ScsiHandler::update_device_path(storage, ctx).await?;
+                        }
+                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM => {
+                            PmemHandler::update_device_path(storage, ctx).await?;
+                        }
+                        _ => {}
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for DmVerityHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Self::update_source_device(&mut storage, ctx).await?;
+        create_mount_destination(&storage.source, &storage.mount_point, "", &storage.fstype)
+            .context("Could not create mountpoint")?;
+
+        let verity_info = Self::get_dm_verity_info(&storage)?;
+        let verity_info = serde_json::to_string(&verity_info)
+            .map_err(|e| anyhow!("failed to serialize dm_verity info, {}", e))?;
+        let verity_device_path = create_dmverity_device(&verity_info, Path::new(storage.source()))
+            .context("create device with dm-verity enabled")?;
+        storage.source = verity_device_path;
+        common_storage_handler(ctx.logger, &storage)?;
+
+        Ok(Arc::new(DmVerityDevice {
+            common: StorageDeviceGeneric::new(storage.mount_point),
+            verity_device_path: storage.source,
+            logger: ctx.logger.clone(),
+        }))
+    }
+}
+
+struct DmVerityDevice {
+    common: StorageDeviceGeneric,
+    verity_device_path: String,
+    logger: Logger,
+}
+
+impl StorageDevice for DmVerityDevice {
+    fn path(&self) -> Option<&str> {
+        self.common.path()
+    }
+
+    fn cleanup(&self) -> Result<()> {
+        self.common.cleanup().context("clean up dm-verity volume")?;
+        let device_path = &self.verity_device_path;
+        debug!(
+            self.logger,
+            "destroy verity device path = {:?}", device_path
+        );
+        destroy_dmverity_device(device_path)?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use kata_types::{mount::DmVerityInfo, volume::KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO};
+    use protocols::agent::Storage;
+
+    use crate::storage::dm_verity::DmVerityHandler;
+
+    #[test]
+    fn test_get_dm_verity_info() {
+        let verity_info = DmVerityInfo {
+            hashtype: "sha256".to_string(),
+            hash: "d86104eee715a1b59b62148641f4ca73edf1be3006c4d481f03f55ac05640570".to_string(),
+            blocknum: 2361,
+            blocksize: 512,
+            hashsize: 4096,
+            offset: 1212416,
+        };
+
+        let verity_info_str = serde_json::to_string(&verity_info);
+        assert!(verity_info_str.is_ok());
+
+        let storage = Storage {
+            driver: KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO.to_string(),
+            driver_options: vec![format!("verity_info={}", verity_info_str.ok().unwrap())],
+            ..Default::default()
+        };
+
+        match DmVerityHandler::get_dm_verity_info(&storage) {
+            Ok(result) => {
+                assert_eq!(verity_info, result);
+            }
+            Err(e) => panic!("err = {}", e),
+        }
+    }
+}

src/agent/src/storage/ephemeral_handler.rs

@@ -0,0 +1,293 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fs;
+use std::fs::OpenOptions;
+use std::io::Write;
+use std::os::unix::fs::{MetadataExt, PermissionsExt};
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use kata_sys_util::mount::parse_mount_options;
+use kata_types::mount::{StorageDevice, KATA_MOUNT_OPTION_FS_GID};
+use nix::unistd::Gid;
+use protocols::agent::Storage;
+use slog::Logger;
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+use crate::device::{DRIVER_EPHEMERAL_TYPE, FS_TYPE_HUGETLB};
+use crate::mount::baremount;
+use crate::sandbox::Sandbox;
+use crate::storage::{
+    common_storage_handler, new_device, parse_options, StorageContext, StorageHandler, MODE_SETGID,
+};
+
+const FS_GID_EQ: &str = "fsgid=";
+const SYS_FS_HUGEPAGES_PREFIX: &str = "/sys/kernel/mm/hugepages";
+
+#[derive(Debug)]
+pub struct EphemeralHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for EphemeralHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        // hugetlbfs
+        if storage.fstype == FS_TYPE_HUGETLB {
+            info!(ctx.logger, "handle hugetlbfs storage");
+            // Allocate hugepages before mount
+            // /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
+            // /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+            // options eg "pagesize=2097152,size=524288000"(2M, 500M)
+            Self::allocate_hugepages(ctx.logger, &storage.options.to_vec())
+                .context("allocate hugepages")?;
+            common_storage_handler(ctx.logger, &storage)?;
+        } else if !storage.options.is_empty() {
+            // By now we only support one option field: "fsGroup" which
+            // isn't an valid mount option, thus we should remove it when
+            // do mount.
+            let opts = parse_options(&storage.options);
+            storage.options = Default::default();
+            common_storage_handler(ctx.logger, &storage)?;
+
+            // ephemeral_storage didn't support mount options except fsGroup.
+            if let Some(fsgid) = opts.get(KATA_MOUNT_OPTION_FS_GID) {
+                let gid = fsgid.parse::<u32>()?;
+
+                nix::unistd::chown(storage.mount_point.as_str(), None, Some(Gid::from_raw(gid)))?;
+
+                let meta = fs::metadata(&storage.mount_point)?;
+                let mut permission = meta.permissions();
+
+                let o_mode = meta.mode() | MODE_SETGID;
+                permission.set_mode(o_mode);
+                fs::set_permissions(&storage.mount_point, permission)?;
+            }
+        } else {
+            common_storage_handler(ctx.logger, &storage)?;
+        }
+
+        new_device("".to_string())
+    }
+}
+
+impl EphemeralHandler {
+    // Allocate hugepages by writing to sysfs
+    fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> {
+        info!(logger, "mounting hugePages storage options: {:?}", options);
+
+        let (pagesize, size) = Self::get_pagesize_and_size_from_option(options)
+            .context(format!("parse mount options: {:?}", &options))?;
+
+        info!(
+            logger,
+            "allocate hugepages. pageSize: {}, size: {}", pagesize, size
+        );
+
+        // sysfs entry is always of the form hugepages-${pagesize}kB
+        // Ref: https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
+        let path = Path::new(SYS_FS_HUGEPAGES_PREFIX)
+            .join(format!("hugepages-{}kB", pagesize / 1024))
+            .join("nr_hugepages");
+
+        // write numpages to nr_hugepages file.
+        let numpages = format!("{}", size / pagesize);
+        info!(logger, "write {} pages to {:?}", &numpages, &path);
+
+        let mut file = OpenOptions::new()
+            .write(true)
+            .open(&path)
+            .context(format!("open nr_hugepages directory {:?}", &path))?;
+
+        file.write_all(numpages.as_bytes())
+            .context(format!("write nr_hugepages failed: {:?}", &path))?;
+
+        // Even if the write succeeds, the kernel isn't guaranteed to be
+        // able to allocate all the pages we requested.  Verify that it
+        // did.
+        let verify = fs::read_to_string(&path).context(format!("reading {:?}", &path))?;
+        let allocated = verify
+            .trim_end()
+            .parse::<u64>()
+            .map_err(|_| anyhow!("Unexpected text {:?} in {:?}", &verify, &path))?;
+        if allocated != size / pagesize {
+            return Err(anyhow!(
+                "Only allocated {} of {} hugepages of size {}",
+                allocated,
+                numpages,
+                pagesize
+            ));
+        }
+
+        Ok(())
+    }
+
+    // Parse filesystem options string to retrieve hugepage details
+    // options eg "pagesize=2048,size=107374182"
+    fn get_pagesize_and_size_from_option(options: &[String]) -> Result<(u64, u64)> {
+        let mut pagesize_str: Option<&str> = None;
+        let mut size_str: Option<&str> = None;
+
+        for option in options {
+            let vars: Vec<&str> = option.trim().split(',').collect();
+
+            for var in vars {
+                if let Some(stripped) = var.strip_prefix("pagesize=") {
+                    pagesize_str = Some(stripped);
+                } else if let Some(stripped) = var.strip_prefix("size=") {
+                    size_str = Some(stripped);
+                }
+
+                if pagesize_str.is_some() && size_str.is_some() {
+                    break;
+                }
+            }
+        }
+
+        if pagesize_str.is_none() || size_str.is_none() {
+            return Err(anyhow!("no pagesize/size options found"));
+        }
+
+        let pagesize = pagesize_str
+            .unwrap()
+            .parse::<u64>()
+            .context(format!("parse pagesize: {:?}", &pagesize_str))?;
+        let size = size_str
+            .unwrap()
+            .parse::<u64>()
+            .context(format!("parse size: {:?}", &pagesize_str))?;
+
+        Ok((pagesize, size))
+    }
+}
+
+// update_ephemeral_mounts takes a list of ephemeral mounts and remounts them
+// with mount options passed by the caller
+#[instrument]
+pub async fn update_ephemeral_mounts(
+    logger: Logger,
+    storages: &[Storage],
+    _sandbox: &Arc<Mutex<Sandbox>>,
+) -> Result<()> {
+    for storage in storages {
+        let handler_name = &storage.driver;
+        let logger = logger.new(o!(
+            "msg" => "updating tmpfs storage",
+            "subsystem" => "storage",
+            "storage-type" => handler_name.to_owned()));
+
+        match handler_name.as_str() {
+            DRIVER_EPHEMERAL_TYPE => {
+                fs::create_dir_all(&storage.mount_point)?;
+
+                if storage.options.is_empty() {
+                    continue;
+                } else {
+                    // assume that fsGid has already been set
+                    let mount_path = Path::new(&storage.mount_point);
+                    let src_path = Path::new(&storage.source);
+                    let opts: Vec<&String> = storage
+                        .options
+                        .iter()
+                        .filter(|&opt| !opt.starts_with(FS_GID_EQ))
+                        .collect();
+                    let (flags, options) = parse_mount_options(&opts)?;
+
+                    info!(logger, "mounting storage";
+                        "mount-source" => src_path.display(),
+                        "mount-destination" => mount_path.display(),
+                        "mount-fstype"  => storage.fstype.as_str(),
+                        "mount-options" => options.as_str(),
+                    );
+
+                    baremount(
+                        src_path,
+                        mount_path,
+                        storage.fstype.as_str(),
+                        flags,
+                        options.as_str(),
+                        &logger,
+                    )?;
+                }
+            }
+            _ => {
+                return Err(anyhow!(
+                    "Unsupported storage type for syncing mounts {}. Only ephemeral storage update is supported",
+                    storage.driver
+                ));
+            }
+        };
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_pagesize_and_size_from_option() {
+        let expected_pagesize = 2048;
+        let expected_size = 107374182;
+        let expected = (expected_pagesize, expected_size);
+
+        let data = vec![
+            // (input, expected, is_ok)
+            ("size-1=107374182,pagesize-1=2048", expected, false),
+            ("size-1=107374182,pagesize=2048", expected, false),
+            ("size=107374182,pagesize-1=2048", expected, false),
+            ("size=107374182,pagesize=abc", expected, false),
+            ("size=abc,pagesize=2048", expected, false),
+            ("size=,pagesize=2048", expected, false),
+            ("size=107374182,pagesize=", expected, false),
+            ("size=107374182,pagesize=2048", expected, true),
+            ("pagesize=2048,size=107374182", expected, true),
+            ("foo=bar,pagesize=2048,size=107374182", expected, true),
+            (
+                "foo=bar,pagesize=2048,foo1=bar1,size=107374182",
+                expected,
+                true,
+            ),
+            (
+                "pagesize=2048,foo1=bar1,foo=bar,size=107374182",
+                expected,
+                true,
+            ),
+            (
+                "foo=bar,pagesize=2048,foo1=bar1,size=107374182,foo2=bar2",
+                expected,
+                true,
+            ),
+            (
+                "foo=bar,size=107374182,foo1=bar1,pagesize=2048",
+                expected,
+                true,
+            ),
+        ];
+
+        for case in data {
+            let input = case.0;
+            let r = EphemeralHandler::get_pagesize_and_size_from_option(&[input.to_string()]);
+
+            let is_ok = case.2;
+            if is_ok {
+                let expected = case.1;
+                let (pagesize, size) = r.unwrap();
+                assert_eq!(expected.0, pagesize);
+                assert_eq!(expected.1, size);
+            } else {
+                assert!(r.is_err());
+            }
+        }
+    }
+}

src/agent/src/storage/fs_handler.rs

@@ -0,0 +1,99 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fs;
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use kata_types::mount::StorageDevice;
+use kata_types::volume::KATA_VOLUME_OVERLAYFS_CREATE_DIR;
+use protocols::agent::Storage;
+use tracing::instrument;
+
+use crate::storage::{common_storage_handler, new_device, StorageContext, StorageHandler};
+
+#[derive(Debug)]
+pub struct OverlayfsHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for OverlayfsHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        if storage
+            .options
+            .iter()
+            .any(|e| e == "io.katacontainers.fs-opt.overlay-rw")
+        {
+            let cid = ctx
+                .cid
+                .clone()
+                .ok_or_else(|| anyhow!("No container id in rw overlay"))?;
+            let cpath = Path::new(crate::rpc::CONTAINER_BASE).join(cid);
+            let work = cpath.join("work");
+            let upper = cpath.join("upper");
+
+            fs::create_dir_all(&work).context("Creating overlay work directory")?;
+            fs::create_dir_all(&upper).context("Creating overlay upper directory")?;
+
+            storage.fstype = "overlay".into();
+            storage
+                .options
+                .push(format!("upperdir={}", upper.to_string_lossy()));
+            storage
+                .options
+                .push(format!("workdir={}", work.to_string_lossy()));
+        }
+        let overlay_create_dir_prefix = &(KATA_VOLUME_OVERLAYFS_CREATE_DIR.to_string() + "=");
+        for driver_option in &storage.driver_options {
+            if let Some(dir) = driver_option
+                .as_str()
+                .strip_prefix(overlay_create_dir_prefix)
+            {
+                fs::create_dir_all(dir).context("Failed to create directory")?;
+            }
+        }
+
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}
+
+#[derive(Debug)]
+pub struct Virtio9pHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for Virtio9pHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}
+
+#[derive(Debug)]
+pub struct VirtioFsHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for VirtioFsHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        let path = common_storage_handler(ctx.logger, &storage)?;
+        new_device(path)
+    }
+}

src/agent/src/storage/image_pull_handler.rs

@@ -0,0 +1,102 @@
+// Copyright (c) 2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Result};
+use kata_types::mount::KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL;
+use kata_types::mount::{ImagePullVolume, StorageDevice};
+use protocols::agent::Storage;
+use std::sync::Arc;
+use tracing::instrument;
+
+use crate::image_rpc;
+use crate::storage::{StorageContext, StorageHandler};
+
+use super::{common_storage_handler, new_device};
+
+#[derive(Debug)]
+pub struct ImagePullHandler {}
+
+impl ImagePullHandler {
+    fn get_image_info(storage: &Storage) -> Result<ImagePullVolume> {
+        for option in storage.driver_options.iter() {
+            if let Some((key, value)) = option.split_once('=') {
+                if key == KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL {
+                    let imagepull_volume: ImagePullVolume = serde_json::from_str(value)?;
+                    return Ok(imagepull_volume);
+                }
+            }
+        }
+        Err(anyhow!("missing Image information for ImagePull volume"))
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for ImagePullHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        //Currently the image metadata is not used to pulling image in the guest.
+        let image_pull_volume = Self::get_image_info(&storage)?;
+        debug!(ctx.logger, "image_pull_volume = {:?}", image_pull_volume);
+        let image_name = storage.source();
+        debug!(ctx.logger, "image_name = {:?}", image_name);
+
+        let cid = ctx
+            .cid
+            .clone()
+            .ok_or_else(|| anyhow!("failed to get container id"))?;
+        let image_service = image_rpc::ImageService::singleton().await?;
+        let bundle_path = image_service
+            .pull_image_for_container(image_name, &cid, &image_pull_volume.metadata)
+            .await?;
+
+        storage.source = bundle_path;
+        storage.options = vec!["bind".to_string(), "ro".to_string()];
+
+        common_storage_handler(ctx.logger, &storage)?;
+
+        new_device(storage.mount_point)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use kata_types::mount::{ImagePullVolume, KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL};
+    use protocols::agent::Storage;
+
+    use crate::storage::image_pull_handler::ImagePullHandler;
+
+    #[test]
+    fn test_get_image_info() {
+        let mut res = HashMap::new();
+        res.insert("key1".to_string(), "value1".to_string());
+        res.insert("key2".to_string(), "value2".to_string());
+
+        let image_pull = ImagePullVolume {
+            metadata: res.clone(),
+        };
+
+        let image_pull_str = serde_json::to_string(&image_pull);
+        assert!(image_pull_str.is_ok());
+
+        let storage = Storage {
+            driver: KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL.to_string(),
+            driver_options: vec![format!("image_guest_pull={}", image_pull_str.ok().unwrap())],
+            ..Default::default()
+        };
+
+        match ImagePullHandler::get_image_info(&storage) {
+            Ok(image_info) => {
+                assert_eq!(image_info.metadata, res);
+            }
+            Err(e) => panic!("err = {}", e),
+        }
+    }
+}

src/agent/src/storage/local_handler.rs

@@ -0,0 +1,61 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fs;
+use std::os::unix::fs::PermissionsExt;
+use std::sync::Arc;
+
+use anyhow::{Context, Result};
+use kata_types::mount::{StorageDevice, KATA_MOUNT_OPTION_FS_GID};
+use nix::unistd::Gid;
+use protocols::agent::Storage;
+use tracing::instrument;
+
+use crate::storage::{new_device, parse_options, StorageContext, StorageHandler, MODE_SETGID};
+
+#[derive(Debug)]
+pub struct LocalHandler {}
+
+#[async_trait::async_trait]
+impl StorageHandler for LocalHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        storage: Storage,
+        _ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        fs::create_dir_all(&storage.mount_point).context(format!(
+            "failed to create dir all {:?}",
+            &storage.mount_point
+        ))?;
+
+        let opts = parse_options(&storage.options);
+
+        let mut need_set_fsgid = false;
+        if let Some(fsgid) = opts.get(KATA_MOUNT_OPTION_FS_GID) {
+            let gid = fsgid.parse::<u32>()?;
+
+            nix::unistd::chown(storage.mount_point.as_str(), None, Some(Gid::from_raw(gid)))?;
+            need_set_fsgid = true;
+        }
+
+        if let Some(mode) = opts.get("mode") {
+            let mut permission = fs::metadata(&storage.mount_point)?.permissions();
+
+            let mut o_mode = u32::from_str_radix(mode, 8)?;
+
+            if need_set_fsgid {
+                // set SetGid mode mask.
+                o_mode |= MODE_SETGID;
+            }
+            permission.set_mode(o_mode);
+
+            fs::set_permissions(&storage.mount_point, permission)?;
+        }
+
+        new_device("".to_string())
+    }
+}

src/agent/src/storage/mod.rs

@@ -0,0 +1,800 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::collections::HashMap;
+use std::fs;
+use std::os::unix::fs::{MetadataExt, PermissionsExt};
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use kata_sys_util::mount::{create_mount_destination, parse_mount_options};
+use kata_types::mount::{
+    StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG,
+    KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL,
+};
+use kata_types::volume::KATA_VOLUME_TYPE_DMVERITY;
+use nix::unistd::{Gid, Uid};
+use protocols::agent::Storage;
+use protocols::types::FSGroupChangePolicy;
+use slog::Logger;
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+use self::bind_watcher_handler::BindWatcherHandler;
+use self::block_handler::{PmemHandler, ScsiHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler};
+use self::dm_verity::DmVerityHandler;
+use self::ephemeral_handler::EphemeralHandler;
+use self::fs_handler::{OverlayfsHandler, Virtio9pHandler, VirtioFsHandler};
+use self::image_pull_handler::ImagePullHandler;
+use self::local_handler::LocalHandler;
+
+use crate::device::{
+    DRIVER_9P_TYPE, DRIVER_BLK_MMIO_TYPE, DRIVER_BLK_PCI_TYPE, DRIVER_EPHEMERAL_TYPE,
+    DRIVER_LOCAL_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE,
+    DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE,
+};
+use crate::mount::{baremount, is_mounted, remove_mounts};
+use crate::sandbox::Sandbox;
+
+pub use self::ephemeral_handler::update_ephemeral_mounts;
+
+mod bind_watcher_handler;
+mod block_handler;
+mod dm_verity;
+mod ephemeral_handler;
+mod fs_handler;
+mod image_pull_handler;
+mod local_handler;
+
+const RW_MASK: u32 = 0o660;
+const RO_MASK: u32 = 0o440;
+const EXEC_MASK: u32 = 0o110;
+const MODE_SETGID: u32 = 0o2000;
+
+#[derive(Debug)]
+pub struct StorageContext<'a> {
+    cid: &'a Option<String>,
+    logger: &'a Logger,
+    sandbox: &'a Arc<Mutex<Sandbox>>,
+}
+
+/// An implementation of generic storage device.
+#[derive(Default, Debug)]
+pub struct StorageDeviceGeneric {
+    path: Option<String>,
+}
+
+impl StorageDeviceGeneric {
+    /// Create a new instance of `StorageStateCommon`.
+    pub fn new(path: String) -> Self {
+        StorageDeviceGeneric { path: Some(path) }
+    }
+}
+
+impl StorageDevice for StorageDeviceGeneric {
+    fn path(&self) -> Option<&str> {
+        self.path.as_deref()
+    }
+
+    fn cleanup(&self) -> Result<()> {
+        let path = match self.path() {
+            None => return Ok(()),
+            Some(v) => {
+                if v.is_empty() {
+                    // TODO: Bind watch, local, ephemeral volume has empty path, which will get leaked.
+                    return Ok(());
+                } else {
+                    v
+                }
+            }
+        };
+        if !Path::new(path).exists() {
+            return Ok(());
+        }
+
+        if matches!(is_mounted(path), Ok(true)) {
+            let mounts = vec![path.to_string()];
+            remove_mounts(&mounts)?;
+        }
+        if matches!(is_mounted(path), Ok(true)) {
+            return Err(anyhow!("failed to umount mountpoint {}", path));
+        }
+
+        let p = Path::new(path);
+        if p.is_dir() {
+            let is_empty = p.read_dir()?.next().is_none();
+            if !is_empty {
+                return Err(anyhow!("directory is not empty when clean up storage"));
+            }
+            // "remove_dir" will fail if the mount point is backed by a read-only filesystem.
+            // This is the case with the device mapper snapshotter, where we mount the block device
+            // directly at the underlying sandbox path which was provided from the base RO kataShared
+            // path from the host.
+            let _ = fs::remove_dir(p);
+        } else if !p.is_file() {
+            // TODO: should we remove the file for bind mount?
+            return Err(anyhow!(
+                "storage path {} is neither directory nor file",
+                path
+            ));
+        }
+
+        Ok(())
+    }
+}
+
+/// Trait object to handle storage device.
+#[async_trait::async_trait]
+pub trait StorageHandler: Send + Sync {
+    /// Create a new storage device.
+    async fn create_device(
+        &self,
+        storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>>;
+}
+
+#[rustfmt::skip]
+lazy_static! {
+    pub static ref STORAGE_HANDLERS: StorageHandlerManager<Arc<dyn StorageHandler>> = {
+        let mut manager: StorageHandlerManager<Arc<dyn StorageHandler>> = StorageHandlerManager::new();
+        manager.add_handler(DRIVER_9P_TYPE, Arc::new(Virtio9pHandler{})).unwrap();
+        #[cfg(target_arch = "s390x")]
+        manager.add_handler(crate::device::DRIVER_BLK_CCW_TYPE, Arc::new(self::block_handler::VirtioBlkCcwHandler{})).unwrap();
+        manager.add_handler(DRIVER_BLK_MMIO_TYPE, Arc::new(VirtioBlkMmioHandler{})).unwrap();
+        manager.add_handler(DRIVER_BLK_PCI_TYPE, Arc::new(VirtioBlkPciHandler{})).unwrap();
+        manager.add_handler(DRIVER_EPHEMERAL_TYPE, Arc::new(EphemeralHandler{})).unwrap();
+        manager.add_handler(DRIVER_LOCAL_TYPE, Arc::new(LocalHandler{})).unwrap();
+        manager.add_handler(DRIVER_NVDIMM_TYPE, Arc::new(PmemHandler{})).unwrap();
+        manager.add_handler(DRIVER_OVERLAYFS_TYPE, Arc::new(OverlayfsHandler{})).unwrap();
+        manager.add_handler(DRIVER_SCSI_TYPE, Arc::new(ScsiHandler{})).unwrap();
+        manager.add_handler(DRIVER_VIRTIOFS_TYPE, Arc::new(VirtioFsHandler{})).unwrap();
+        manager.add_handler(DRIVER_WATCHABLE_BIND_TYPE, Arc::new(BindWatcherHandler{})).unwrap();
+        manager.add_handler(KATA_VOLUME_TYPE_DMVERITY, Arc::new(DmVerityHandler{})).unwrap();
+        manager.add_handler(KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL, Arc::new(ImagePullHandler{})).unwrap();
+        manager
+    };
+}
+
+// add_storages takes a list of storages passed by the caller, and perform the
+// associated operations such as waiting for the device to show up, and mount
+// it to a specific location, according to the type of handler chosen, and for
+// each storage.
+#[instrument]
+pub async fn add_storages(
+    logger: Logger,
+    storages: Vec<Storage>,
+    sandbox: &Arc<Mutex<Sandbox>>,
+    cid: Option<String>,
+) -> Result<Vec<String>> {
+    let mut mount_list = Vec::new();
+
+    for storage in storages {
+        let path = storage.mount_point.clone();
+        let state = sandbox.lock().await.add_sandbox_storage(&path).await;
+        if state.ref_count().await > 1 {
+            if let Some(path) = state.path() {
+                if !path.is_empty() {
+                    mount_list.push(path.to_string());
+                }
+            }
+            // The device already exists.
+            continue;
+        }
+
+        if let Some(handler) = STORAGE_HANDLERS.handler(&storage.driver) {
+            let logger =
+                logger.new(o!( "subsystem" => "storage", "storage-type" => storage.driver.clone()));
+            let mut ctx = StorageContext {
+                cid: &cid,
+                logger: &logger,
+                sandbox,
+            };
+
+            match handler.create_device(storage, &mut ctx).await {
+                Ok(device) => {
+                    match sandbox
+                        .lock()
+                        .await
+                        .update_sandbox_storage(&path, device.clone())
+                    {
+                        Ok(d) => {
+                            if let Some(path) = device.path() {
+                                if !path.is_empty() {
+                                    mount_list.push(path.to_string());
+                                }
+                            }
+                            drop(d);
+                        }
+                        Err(device) => {
+                            error!(logger, "failed to update device for storage");
+                            if let Err(e) = sandbox.lock().await.remove_sandbox_storage(&path).await
+                            {
+                                warn!(logger, "failed to remove dummy sandbox storage {:?}", e);
+                            }
+                            if let Err(e) = device.cleanup() {
+                                error!(
+                                    logger,
+                                    "failed to clean state for storage device {}, {}", path, e
+                                );
+                            }
+                            return Err(anyhow!("failed to update device for storage"));
+                        }
+                    }
+                }
+                Err(e) => {
+                    error!(logger, "failed to create device for storage, error: {e:?}");
+                    if let Err(e) = sandbox.lock().await.remove_sandbox_storage(&path).await {
+                        warn!(logger, "failed to remove dummy sandbox storage {e:?}");
+                    }
+                    return Err(e);
+                }
+            }
+        } else {
+            return Err(anyhow!(
+                "Failed to find the storage handler {}",
+                storage.driver
+            ));
+        }
+    }
+
+    Ok(mount_list)
+}
+
+pub(crate) fn new_device(path: String) -> Result<Arc<dyn StorageDevice>> {
+    let device = StorageDeviceGeneric::new(path);
+    Ok(Arc::new(device))
+}
+
+#[instrument]
+pub(crate) fn common_storage_handler(logger: &Logger, storage: &Storage) -> Result<String> {
+    mount_storage(logger, storage)?;
+    set_ownership(logger, storage)?;
+    Ok(storage.mount_point.clone())
+}
+
+// mount_storage performs the mount described by the storage structure.
+#[instrument]
+fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "mount"));
+
+    // There's a special mechanism to create mountpoint from a `sharedfs` instance before
+    // starting the kata-agent. Check for such cases.
+    if storage.source == KATA_SHAREDFS_GUEST_PREMOUNT_TAG && is_mounted(&storage.mount_point)? {
+        warn!(
+            logger,
+            "{} already mounted on {}, ignoring...",
+            KATA_SHAREDFS_GUEST_PREMOUNT_TAG,
+            &storage.mount_point
+        );
+        return Ok(());
+    }
+
+    let (flags, options) = parse_mount_options(&storage.options)?;
+    let mount_path = Path::new(&storage.mount_point);
+    let src_path = Path::new(&storage.source);
+    create_mount_destination(src_path, mount_path, "", &storage.fstype)
+        .context("Could not create mountpoint")?;
+
+    info!(logger, "mounting storage";
+        "mount-source" => src_path.display(),
+        "mount-destination" => mount_path.display(),
+        "mount-fstype"  => storage.fstype.as_str(),
+        "mount-options" => options.as_str(),
+    );
+
+    baremount(
+        src_path,
+        mount_path,
+        storage.fstype.as_str(),
+        flags,
+        options.as_str(),
+        &logger,
+    )
+}
+
+#[instrument]
+pub(crate) fn parse_options(option_list: &[String]) -> HashMap<String, String> {
+    let mut options = HashMap::new();
+    for opt in option_list {
+        let fields: Vec<&str> = opt.split('=').collect();
+        if fields.len() == 2 {
+            options.insert(fields[0].to_string(), fields[1].to_string());
+        }
+    }
+    options
+}
+
+#[instrument]
+pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "mount", "fn" => "set_ownership"));
+
+    // If fsGroup is not set, skip performing ownership change
+    if storage.fs_group.is_none() {
+        return Ok(());
+    }
+
+    let fs_group = storage.fs_group();
+    let read_only = storage.options.contains(&String::from("ro"));
+    let mount_path = Path::new(&storage.mount_point);
+    let metadata = mount_path.metadata().map_err(|err| {
+        error!(logger, "failed to obtain metadata for mount path";
+            "mount-path" => mount_path.to_str(),
+            "error" => err.to_string(),
+        );
+        err
+    })?;
+
+    if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch.into()
+        && metadata.gid() == fs_group.group_id
+    {
+        let mut mask = if read_only { RO_MASK } else { RW_MASK };
+        mask |= EXEC_MASK;
+
+        // With fsGroup change policy to OnRootMismatch, if the current
+        // gid of the mount path root directory matches the desired gid
+        // and the current permission of mount path root directory is correct,
+        // then ownership change will be skipped.
+        let current_mode = metadata.permissions().mode();
+        if (mask & current_mode == mask) && (current_mode & MODE_SETGID != 0) {
+            info!(logger, "skipping ownership change for volume";
+                "mount-path" => mount_path.to_str(),
+                "fs-group" => fs_group.group_id.to_string(),
+            );
+            return Ok(());
+        }
+    }
+
+    info!(logger, "performing recursive ownership change";
+        "mount-path" => mount_path.to_str(),
+        "fs-group" => fs_group.group_id.to_string(),
+    );
+    recursive_ownership_change(
+        mount_path,
+        None,
+        Some(Gid::from_raw(fs_group.group_id)),
+        read_only,
+    )
+}
+
+#[instrument]
+pub fn recursive_ownership_change(
+    path: &Path,
+    uid: Option<Uid>,
+    gid: Option<Gid>,
+    read_only: bool,
+) -> Result<()> {
+    let mut mask = if read_only { RO_MASK } else { RW_MASK };
+    if path.is_dir() {
+        for entry in fs::read_dir(path)? {
+            recursive_ownership_change(entry?.path().as_path(), uid, gid, read_only)?;
+        }
+        mask |= EXEC_MASK;
+        mask |= MODE_SETGID;
+    }
+
+    // We do not want to change the permission of the underlying file
+    // using symlink. Hence we skip symlinks from recursive ownership
+    // and permission changes.
+    if path.is_symlink() {
+        return Ok(());
+    }
+
+    nix::unistd::chown(path, uid, gid)?;
+
+    if gid.is_some() {
+        let metadata = path.metadata()?;
+        let mut permission = metadata.permissions();
+        let target_mode = metadata.mode() | mask;
+        permission.set_mode(target_mode);
+        fs::set_permissions(path, permission)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use anyhow::Error;
+    use nix::mount::MsFlags;
+    use protocols::agent::FSGroup;
+    use std::fs::File;
+    use tempfile::{tempdir, Builder};
+    use test_utils::{
+        skip_if_not_root, skip_loop_by_user, skip_loop_if_not_root, skip_loop_if_root, TestUserType,
+    };
+
+    #[test]
+    fn test_mount_storage() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            test_user: TestUserType,
+            storage: Storage,
+            error_contains: &'a str,
+
+            make_source_dir: bool,
+            make_mount_dir: bool,
+            deny_mount_permission: bool,
+        }
+
+        impl Default for TestData<'_> {
+            fn default() -> Self {
+                TestData {
+                    test_user: TestUserType::Any,
+                    storage: Storage {
+                        mount_point: "mnt".to_string(),
+                        source: "src".to_string(),
+                        fstype: "tmpfs".to_string(),
+                        ..Default::default()
+                    },
+                    make_source_dir: true,
+                    make_mount_dir: false,
+                    deny_mount_permission: false,
+                    error_contains: "",
+                }
+            }
+        }
+
+        let tests = &[
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                error_contains: "EPERM: Operation not permitted",
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::RootOnly,
+                ..Default::default()
+            },
+            TestData {
+                storage: Storage {
+                    mount_point: "mnt".to_string(),
+                    source: "src".to_string(),
+                    fstype: "bind".to_string(),
+                    ..Default::default()
+                },
+                make_source_dir: false,
+                make_mount_dir: true,
+                error_contains: "Could not create mountpoint",
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                deny_mount_permission: true,
+                error_contains: "Could not create mountpoint",
+                ..Default::default()
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            skip_loop_by_user!(msg, d.test_user);
+
+            let drain = slog::Discard;
+            let logger = slog::Logger::root(drain, o!());
+
+            let tempdir = tempdir().unwrap();
+
+            let source = tempdir.path().join(&d.storage.source);
+            let mount_point = tempdir.path().join(&d.storage.mount_point);
+
+            let storage = Storage {
+                source: source.to_str().unwrap().to_string(),
+                mount_point: mount_point.to_str().unwrap().to_string(),
+                ..d.storage.clone()
+            };
+
+            if d.make_source_dir {
+                fs::create_dir_all(&storage.source).unwrap();
+            }
+            if d.make_mount_dir {
+                fs::create_dir_all(&storage.mount_point).unwrap();
+            }
+
+            if d.deny_mount_permission {
+                fs::set_permissions(
+                    mount_point.parent().unwrap(),
+                    fs::Permissions::from_mode(0o000),
+                )
+                .unwrap();
+            }
+
+            let result = mount_storage(&logger, &storage);
+
+            // restore permissions so tempdir can be cleaned up
+            if d.deny_mount_permission {
+                fs::set_permissions(
+                    mount_point.parent().unwrap(),
+                    fs::Permissions::from_mode(0o755),
+                )
+                .unwrap();
+            }
+
+            if result.is_ok() {
+                nix::mount::umount(&mount_point).unwrap();
+            }
+
+            let msg = format!("{}: result: {:?}", msg, result);
+            if d.error_contains.is_empty() {
+                assert!(result.is_ok(), "{}", msg);
+            } else {
+                assert!(result.is_err(), "{}", msg);
+                let error_msg = format!("{}", result.unwrap_err());
+                assert!(error_msg.contains(d.error_contains), "{}", msg);
+            }
+        }
+    }
+
+    #[test]
+    fn test_set_ownership() {
+        skip_if_not_root!();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        #[derive(Debug)]
+        struct TestData<'a> {
+            mount_path: &'a str,
+            fs_group: Option<FSGroup>,
+            read_only: bool,
+            expected_group_id: u32,
+            expected_permission: u32,
+        }
+
+        let tests = &[
+            TestData {
+                mount_path: "foo",
+                fs_group: None,
+                read_only: false,
+                expected_group_id: 0,
+                expected_permission: 0,
+            },
+            TestData {
+                mount_path: "rw_mount",
+                fs_group: Some(FSGroup {
+                    group_id: 3000,
+                    group_change_policy: FSGroupChangePolicy::Always.into(),
+                    ..Default::default()
+                }),
+                read_only: false,
+                expected_group_id: 3000,
+                expected_permission: RW_MASK | EXEC_MASK | MODE_SETGID,
+            },
+            TestData {
+                mount_path: "ro_mount",
+                fs_group: Some(FSGroup {
+                    group_id: 3000,
+                    group_change_policy: FSGroupChangePolicy::OnRootMismatch.into(),
+                    ..Default::default()
+                }),
+                read_only: true,
+                expected_group_id: 3000,
+                expected_permission: RO_MASK | EXEC_MASK | MODE_SETGID,
+            },
+        ];
+
+        let tempdir = tempdir().expect("failed to create tmpdir");
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mount_dir = tempdir.path().join(d.mount_path);
+            fs::create_dir(&mount_dir)
+                .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg));
+
+            let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
+            let mut storage_data = Storage::new();
+            if d.read_only {
+                storage_data.set_options(vec!["foo".to_string(), "ro".to_string()]);
+            }
+            if let Some(fs_group) = d.fs_group.clone() {
+                storage_data.set_fs_group(fs_group);
+            }
+            storage_data.mount_point = mount_dir.clone().into_os_string().into_string().unwrap();
+
+            let result = set_ownership(&logger, &storage_data);
+            assert!(result.is_ok());
+
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().gid(),
+                d.expected_group_id
+            );
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().permissions().mode(),
+                (directory_mode | d.expected_permission)
+            );
+        }
+    }
+
+    #[test]
+    fn test_recursive_ownership_change() {
+        skip_if_not_root!();
+
+        const COUNT: usize = 5;
+
+        #[derive(Debug)]
+        struct TestData<'a> {
+            // Directory where the recursive ownership change should be performed on
+            path: &'a str,
+
+            // User ID for ownership change
+            uid: u32,
+
+            // Group ID for ownership change
+            gid: u32,
+
+            // Set when the permission should be read-only
+            read_only: bool,
+
+            // The expected permission of all directories after ownership change
+            expected_permission_directory: u32,
+
+            // The expected permission of all files after ownership change
+            expected_permission_file: u32,
+        }
+
+        let tests = &[
+            TestData {
+                path: "no_gid_change",
+                uid: 0,
+                gid: 0,
+                read_only: false,
+                expected_permission_directory: 0,
+                expected_permission_file: 0,
+            },
+            TestData {
+                path: "rw_gid_change",
+                uid: 0,
+                gid: 3000,
+                read_only: false,
+                expected_permission_directory: RW_MASK | EXEC_MASK | MODE_SETGID,
+                expected_permission_file: RW_MASK,
+            },
+            TestData {
+                path: "ro_gid_change",
+                uid: 0,
+                gid: 3000,
+                read_only: true,
+                expected_permission_directory: RO_MASK | EXEC_MASK | MODE_SETGID,
+                expected_permission_file: RO_MASK,
+            },
+        ];
+
+        let tempdir = tempdir().expect("failed to create tmpdir");
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mount_dir = tempdir.path().join(d.path);
+            fs::create_dir(&mount_dir)
+                .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg));
+
+            let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
+            let mut file_mode: u32 = 0;
+
+            // create testing directories and files
+            for n in 1..COUNT {
+                let nest_dir = mount_dir.join(format!("nested{}", n));
+                fs::create_dir(&nest_dir)
+                    .unwrap_or_else(|_| panic!("{}: failed to create nest directory", msg));
+
+                for f in 1..COUNT {
+                    let filename = nest_dir.join(format!("file{}", f));
+                    File::create(&filename)
+                        .unwrap_or_else(|_| panic!("{}: failed to create file", msg));
+                    file_mode = filename.as_path().metadata().unwrap().permissions().mode();
+                }
+            }
+
+            let uid = if d.uid > 0 {
+                Some(Uid::from_raw(d.uid))
+            } else {
+                None
+            };
+            let gid = if d.gid > 0 {
+                Some(Gid::from_raw(d.gid))
+            } else {
+                None
+            };
+            let result = recursive_ownership_change(&mount_dir, uid, gid, d.read_only);
+
+            assert!(result.is_ok());
+
+            assert_eq!(mount_dir.as_path().metadata().unwrap().gid(), d.gid);
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().permissions().mode(),
+                (directory_mode | d.expected_permission_directory)
+            );
+
+            for n in 1..COUNT {
+                let nest_dir = mount_dir.join(format!("nested{}", n));
+                for f in 1..COUNT {
+                    let filename = nest_dir.join(format!("file{}", f));
+                    let file = Path::new(&filename);
+
+                    assert_eq!(file.metadata().unwrap().gid(), d.gid);
+                    assert_eq!(
+                        file.metadata().unwrap().permissions().mode(),
+                        (file_mode | d.expected_permission_file)
+                    );
+                }
+
+                let dir = Path::new(&nest_dir);
+                assert_eq!(dir.metadata().unwrap().gid(), d.gid);
+                assert_eq!(
+                    dir.metadata().unwrap().permissions().mode(),
+                    (directory_mode | d.expected_permission_directory)
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn cleanup_storage() {
+        skip_if_not_root!();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        let tmpdir = Builder::new().tempdir().unwrap();
+        let tmpdir_path = tmpdir.path().to_str().unwrap();
+
+        let srcdir = Builder::new()
+            .prefix("src")
+            .tempdir_in(tmpdir_path)
+            .unwrap();
+        let srcdir_path = srcdir.path().to_str().unwrap();
+        let empty_file = Path::new(srcdir_path).join("emptyfile");
+        fs::write(&empty_file, "test").unwrap();
+
+        let destdir = Builder::new()
+            .prefix("dest")
+            .tempdir_in(tmpdir_path)
+            .unwrap();
+        let destdir_path = destdir.path().to_str().unwrap();
+
+        let emptydir = Builder::new()
+            .prefix("empty")
+            .tempdir_in(tmpdir_path)
+            .unwrap();
+
+        let s = StorageDeviceGeneric::default();
+        assert!(s.cleanup().is_ok());
+
+        let s = StorageDeviceGeneric::new("".to_string());
+        assert!(s.cleanup().is_ok());
+
+        let invalid_dir = emptydir
+            .path()
+            .join("invalid")
+            .to_str()
+            .unwrap()
+            .to_string();
+        let s = StorageDeviceGeneric::new(invalid_dir);
+        assert!(s.cleanup().is_ok());
+
+        assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
+
+        let s = StorageDeviceGeneric::new(destdir_path.to_string());
+        assert!(s.cleanup().is_ok());
+
+        // fail to remove non-empty directory
+        let s = StorageDeviceGeneric::new(srcdir_path.to_string());
+        s.cleanup().unwrap_err();
+
+        // remove a directory without umount
+        fs::remove_file(&empty_file).unwrap();
+        s.cleanup().unwrap();
+    }
+
+    fn bind_mount(src: &str, dst: &str, logger: &Logger) -> Result<(), Error> {
+        let src_path = Path::new(src);
+        let dst_path = Path::new(dst);
+
+        baremount(src_path, dst_path, "bind", MsFlags::MS_BIND, "", logger)
+    }
+}

src/dragonball/src/api/v1/instance_info.rs

@@ -5,6 +5,15 @@
 
 use serde_derive::{Deserialize, Serialize};
 
+/// This struct represents the strongly typed equivalent of the json body
+/// from confidential container related requests.
+#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub enum ConfidentialVmType {
+    /// Intel Trusted Domain
+    TDX = 2,
+}
+
 /// The microvm state.
 ///
 /// When Dragonball starts, the instance state is Uninitialized. Once start_microvm method is
@@ -58,10 +67,12 @@ pub struct InstanceInfo {
     pub tids: Vec<(u8, u32)>,
     /// Last instance downtime
     pub last_instance_downtime: u64,
+    /// confidential vm type
+    pub confidential_vm_type: Option<ConfidentialVmType>,
 }
 
 impl InstanceInfo {
-    /// create instance info object with given id, version, and platform type
+    /// create instance info object with given id, version, platform type and confidential vm type.
     pub fn new(id: String, vmm_version: String) -> Self {
         InstanceInfo {
             id,
@@ -72,8 +83,14 @@ impl InstanceInfo {
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
+            confidential_vm_type: None,
         }
     }
+
+    /// return true if VM confidential type is TDX
+    pub fn is_tdx_enabled(&self) -> bool {
+        matches!(self.confidential_vm_type, Some(ConfidentialVmType::TDX))
+    }
 }
 
 impl Default for InstanceInfo {
@@ -87,6 +104,7 @@ impl Default for InstanceInfo {
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
+            confidential_vm_type: None,
         }
     }
 }

src/dragonball/src/api/v1/mod.rs

@@ -12,7 +12,7 @@ pub use self::boot_source::{BootSourceConfig, BootSourceConfigError, DEFAULT_KER
 
 /// Wrapper over the microVM general information.
 mod instance_info;
-pub use self::instance_info::{InstanceInfo, InstanceState};
+pub use self::instance_info::{ConfidentialVmType, InstanceInfo, InstanceState};
 
 /// Wrapper for configuring the memory and CPU of the microVM.
 mod machine_config;

src/dragonball/src/dbs_utils/src/net/net_gen/mod.rs

@@ -10,6 +10,7 @@
 #![allow(non_snake_case)]
 #![allow(missing_docs)]
 #![allow(deref_nullptr)]
+#![allow(ambiguous_glob_reexports)]
 
 // generated with bindgen /usr/include/linux/if.h --no-unstable-rust
 // --constified-enum '*' --with-derive-default -- -D __UAPI_DEF_IF_IFNAMSIZ -D

src/dragonball/src/error.rs

@@ -75,6 +75,10 @@ pub enum Error {
     /// Cannot open the VM file descriptor.
     #[error(transparent)]
     Vm(vm::VmError),
+
+    /// confidential vm type Error
+    #[error("confidential-vm-type can only be used in x86_64 now")]
+    ConfidentialVmType,
 }
 
 /// Errors associated with starting the instance.

src/dragonball/src/kvm_context.rs

@@ -10,7 +10,7 @@ use kvm_bindings::KVM_API_VERSION;
 use kvm_ioctls::{Cap, Kvm, VmFd};
 use std::os::unix::io::{FromRawFd, RawFd};
 
-use crate::error::{Error, Result};
+use crate::error::{Error as VmError, Result};
 
 /// Describes a KVM context that gets attached to the micro VM instance.
 /// It gives access to the functionality of the KVM wrapper as long as every required
@@ -29,11 +29,11 @@ impl KvmContext {
             // Safe because we expect kvm_fd to contain a valid fd number when is_some() == true.
             unsafe { Kvm::from_raw_fd(fd) }
         } else {
-            Kvm::new().map_err(Error::Kvm)?
+            Kvm::new().map_err(VmError::Kvm)?
         };
 
         if kvm.get_api_version() != KVM_API_VERSION as i32 {
-            return Err(Error::KvmApiVersion(kvm.get_api_version()));
+            return Err(VmError::KvmApiVersion(kvm.get_api_version()));
         }
 
         Self::check_cap(&kvm, Cap::Irqchip)?;
@@ -44,7 +44,8 @@ impl KvmContext {
         Self::check_cap(&kvm, Cap::SetTssAddr)?;
 
         #[cfg(target_arch = "x86_64")]
-        let supported_msrs = dbs_arch::msr::supported_guest_msrs(&kvm).map_err(Error::GuestMSRs)?;
+        let supported_msrs =
+            dbs_arch::msr::supported_guest_msrs(&kvm).map_err(VmError::GuestMSRs)?;
         let max_memslots = kvm.get_nr_memslots();
 
         Ok(KvmContext {
@@ -67,7 +68,7 @@ impl KvmContext {
 
     /// Create a virtual machine object.
     pub fn create_vm(&self) -> Result<VmFd> {
-        self.kvm.create_vm().map_err(Error::Kvm)
+        self.kvm.create_vm().map_err(VmError::Kvm)
     }
 
     /// Get the max vcpu count supported by kvm
@@ -75,9 +76,9 @@ impl KvmContext {
         self.kvm.get_max_vcpus()
     }
 
-    fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), Error> {
+    fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), VmError> {
         if !kvm.check_extension(cap) {
-            return Err(Error::KvmCap(cap));
+            return Err(VmError::KvmCap(cap));
         }
         Ok(())
     }
@@ -91,6 +92,18 @@ mod x86_64 {
     use std::collections::HashSet;
 
     impl KvmContext {
+        /// Create a virtual machine object with specific type.
+        /// vm_type: u64
+        ///      0: legacy vm
+        ///      2: tdx vm
+        pub fn create_vm_with_type(&self, vm_type: u64) -> Result<VmFd> {
+            let fd = self
+                .kvm
+                .create_vm_with_type(vm_type)
+                .map_err(VmError::Kvm)?;
+            Ok(fd)
+        }
+
         /// Get information about supported CPUID of x86 processor.
         pub fn supported_cpuid(
             &self,
@@ -110,7 +123,7 @@ mod x86_64 {
         // It's very sensible to manipulate MSRs, so please be careful to change code below.
         fn build_msrs_list(kvm: &Kvm) -> Result<Msrs> {
             let mut mset: HashSet<u32> = HashSet::new();
-            let supported_msr_list = kvm.get_msr_index_list().map_err(super::Error::Kvm)?;
+            let supported_msr_list = kvm.get_msr_index_list().map_err(VmError::Kvm)?;
             for msr in supported_msr_list.as_slice() {
                 mset.insert(*msr);
             }
@@ -203,7 +216,7 @@ mod x86_64 {
                 })
                 .collect();
 
-            Msrs::from_entries(&msrs).map_err(super::Error::Msr)
+            Msrs::from_entries(&msrs).map_err(VmError::Msr)
         }
     }
 }
@@ -257,4 +270,20 @@ mod tests {
 
         let _ = c.create_vm().unwrap();
     }
+
+    #[test]
+    fn test_create_vm_with_type() {
+        let c = KvmContext::new(None).unwrap();
+        #[cfg(not(target_arch = "aarch64"))]
+        let _ = c.create_vm_with_type(0_u64).unwrap();
+        #[cfg(target_arch = "aarch64")]
+        {
+            /// aarch64 is using ipa_size to create vm
+            let mut ipa_size = 0; // Create using default VM type
+            if c.check_extension(kvm_ioctls::Cap::ArmVmIPASize) {
+                ipa_size = c.kvm.get_host_ipa_limit();
+            }
+            let _ = c.create_vm_with_type(ipa_size as u64).unwrap();
+        }
+    }
 }

src/dragonball/src/vcpu/vcpu_manager.rs

@@ -832,7 +832,7 @@ mod hotplug {
 
     #[cfg(all(target_arch = "x86_64", feature = "dbs-upcall"))]
     use dbs_boot::mptable::APIC_VERSION;
-    #[cfg(all(target_arch = "aarch64"))]
+    #[cfg(target_arch = "aarch64")]
     const APIC_VERSION: u8 = 0;
 
     #[cfg(feature = "dbs-upcall")]

src/dragonball/src/vm/mod.rs

@@ -215,7 +215,26 @@ impl Vm {
         let id = api_shared_info.read().unwrap().id.clone();
         let logger = slog_scope::logger().new(slog::o!("id" => id));
         let kvm = KvmContext::new(kvm_fd)?;
-        let vm_fd = Arc::new(kvm.create_vm()?);
+        let vm_fd = match api_shared_info
+            .as_ref()
+            .read()
+            .unwrap()
+            .confidential_vm_type
+        {
+            None => Arc::new(kvm.create_vm()?),
+            Some(confidential_vm_type) => {
+                #[cfg(not(any(target_arch = "x86_64")))]
+                {
+                    error!(
+                        "confidential-vm-type {} only can be used in x86_64",
+                        confidential_vm_type as u64
+                    );
+                    return Err(Error::ConfidentialVmType);
+                }
+                #[cfg(target_arch = "x86_64")]
+                Arc::new(kvm.create_vm_with_type(confidential_vm_type as u64)?)
+            }
+        };
         let resource_manager = Arc::new(ResourceManager::new(Some(kvm.max_memslots())));
         let device_manager = DeviceManager::new(
             vm_fd.clone(),
@@ -344,6 +363,15 @@ impl Vm {
         instance_state == InstanceState::Running
     }
 
+    /// return true if VM confidential type is TDX
+    pub fn is_tdx_enabled(&self) -> bool {
+        let shared_info = self
+            .shared_info()
+            .read()
+            .expect("failed to get instance state, because shared info is poisoned lock");
+        shared_info.is_tdx_enabled()
+    }
+
     /// Save VM instance exit state
     pub fn vm_exit(&self, exit_code: i32) {
         if let Ok(mut info) = self.shared_info.write() {

src/kata-opa/coco-default.rego

@@ -0,0 +1,40 @@
+package agent_policy
+
+default AddARPNeighborsRequest := true
+default AddSwapRequest := true
+default CloseStdinRequest := true
+default CopyFileRequest := true
+default CreateContainerRequest := true
+default CreateSandboxRequest := true
+default DestroySandboxRequest := true
+default GetMetricsRequest := true
+default GetOOMEventRequest := true
+default GuestDetailsRequest := true
+default ListInterfacesRequest := true
+default ListRoutesRequest := true
+default MemHotplugByProbeRequest := true
+default OnlineCPUMemRequest := true
+default PauseContainerRequest := true
+default PullImageRequest := true
+default ReadStreamRequest := true
+default RemoveContainerRequest := true
+default RemoveStaleVirtiofsShareMountsRequest := true
+default ResumeContainerRequest := true
+default SetGuestDateTimeRequest := true
+default SetPolicyRequest := true
+default SignalProcessRequest := true
+default StartContainerRequest := true
+default StartTracingRequest := true
+default StatsContainerRequest := true
+default StopTracingRequest := true
+default TtyWinResizeRequest := true
+default UpdateContainerRequest := true
+default UpdateEphemeralMountsRequest := true
+default UpdateInterfaceRequest := true
+default UpdateRoutesRequest := true
+default WaitProcessRequest := true
+default WriteStreamRequest := true
+
+default ExecProcessRequest := false
+default ReseedRandomDevRequest := false
+default SetPolicyRequest := false

src/libs/kata-sys-util/src/mount.rs

@@ -58,7 +58,8 @@ use crate::fs::is_symlink;
 use crate::sl;
 
 /// Default permission for directories created for mountpoint.
-const MOUNT_PERM: u32 = 0o755;
+const MOUNT_DIR_PERM: u32 = 0o755;
+const MOUNT_FILE_PERM: u32 = 0o644;
 
 pub const PROC_MOUNTS_FILE: &str = "/proc/mounts";
 const PROC_FIELDS_PER_LINE: usize = 6;
@@ -187,13 +188,16 @@ pub fn create_mount_destination<S: AsRef<Path>, D: AsRef<Path>, R: AsRef<Path>>(
         .parent()
         .ok_or_else(|| Error::InvalidPath(dst.to_path_buf()))?;
     let mut builder = fs::DirBuilder::new();
-    builder.mode(MOUNT_PERM).recursive(true).create(parent)?;
+    builder
+        .mode(MOUNT_DIR_PERM)
+        .recursive(true)
+        .create(parent)?;
 
     if fs_type == "bind" {
         // The source and destination for bind mounting must be the same type: file or directory.
         if !src.as_ref().is_dir() {
             fs::OpenOptions::new()
-                .mode(MOUNT_PERM)
+                .mode(MOUNT_FILE_PERM)
                 .write(true)
                 .create(true)
                 .open(dst)?;
@@ -390,19 +394,20 @@ fn do_rebind_mount<P: AsRef<Path>>(path: P, readonly: bool, flags: MsFlags) -> R
 }
 
 /// Take fstab style mount options and parses them for use with a standard mount() syscall.
-fn parse_mount_options(options: &[String]) -> Result<(MsFlags, String)> {
+pub fn parse_mount_options<T: AsRef<str>>(options: &[T]) -> Result<(MsFlags, String)> {
     let mut flags: MsFlags = MsFlags::empty();
     let mut data: Vec<String> = Vec::new();
 
     for opt in options.iter() {
-        if opt == "defaults" {
-            continue;
-        } else if opt == "loop" {
+        let opt_str = opt.as_ref();
+        if matches!(opt_str, "loop") {
             return Err(Error::InvalidMountOption("loop".to_string()));
-        } else if let Some(v) = parse_mount_flags(flags, opt) {
+        } else if let Some(v) = parse_mount_flags(flags, opt_str) {
             flags = v;
+        } else if opt_str.starts_with("io.katacontainers.") {
+            continue;
         } else {
-            data.push(opt.clone());
+            data.push(opt_str.to_string());
         }
     }
 
@@ -441,6 +446,7 @@ fn parse_mount_flags(mut flags: MsFlags, flag_str: &str) -> Option<MsFlags> {
     //   overridden by subsequent options, as in the option line users,exec,dev,suid).
     match flag_str {
         // Clear flags
+        "defaults" => {}
         "async" => flags &= !MsFlags::MS_SYNCHRONOUS,
         "atime" => flags &= !MsFlags::MS_NOATIME,
         "dev" => flags &= !MsFlags::MS_NODEV,
@@ -464,6 +470,14 @@ fn parse_mount_flags(mut flags: MsFlags, flag_str: &str) -> Option<MsFlags> {
         "noexec" => flags |= MsFlags::MS_NOEXEC,
         "nosuid" => flags |= MsFlags::MS_NOSUID,
         "rbind" => flags |= MsFlags::MS_BIND | MsFlags::MS_REC,
+        "unbindable" => flags |= MsFlags::MS_UNBINDABLE,
+        "runbindable" => flags |= MsFlags::MS_UNBINDABLE | MsFlags::MS_REC,
+        "private" => flags |= MsFlags::MS_PRIVATE,
+        "rprivate" => flags |= MsFlags::MS_PRIVATE | MsFlags::MS_REC,
+        "shared" => flags |= MsFlags::MS_SHARED,
+        "rshared" => flags |= MsFlags::MS_SHARED | MsFlags::MS_REC,
+        "slave" => flags |= MsFlags::MS_SLAVE,
+        "rslave" => flags |= MsFlags::MS_SLAVE | MsFlags::MS_REC,
         "relatime" => flags |= MsFlags::MS_RELATIME,
         "remount" => flags |= MsFlags::MS_REMOUNT,
         "ro" => flags |= MsFlags::MS_RDONLY,
@@ -1030,7 +1044,7 @@ mod tests {
 
     #[test]
     fn test_parse_mount_options() {
-        let options = vec![];
+        let options: Vec<&str> = vec![];
         let (flags, data) = parse_mount_options(&options).unwrap();
         assert!(flags.is_empty());
         assert!(data.is_empty());

src/libs/kata-types/src/lib.rs

@@ -29,6 +29,9 @@ pub mod k8s;
 /// Constants and data types related to mount point.
 pub mod mount;
 
+/// Constants and data types related to data volumes.
+pub mod volume;
+
 pub(crate) mod utils;
 
 /// hypervisor capabilities

src/libs/kata-types/src/mount.rs

@@ -4,7 +4,9 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, Context, Error, Result};
+use std::collections::hash_map::Entry;
+use std::convert::TryFrom;
 use std::{collections::HashMap, fs, path::PathBuf};
 
 /// Prefix to mark a volume as Kata special.
@@ -13,6 +15,9 @@ pub const KATA_VOLUME_TYPE_PREFIX: &str = "kata:";
 /// The Mount should be ignored by the host and handled by the guest.
 pub const KATA_GUEST_MOUNT_PREFIX: &str = "kata:guest-mount:";
 
+/// The sharedfs volume is mounted by guest OS before starting the kata-agent.
+pub const KATA_SHAREDFS_GUEST_PREMOUNT_TAG: &str = "kataShared";
+
 /// KATA_EPHEMERAL_DEV_TYPE creates a tmpfs backed volume for sharing files between containers.
 pub const KATA_EPHEMERAL_VOLUME_TYPE: &str = "ephemeral";
 
@@ -22,6 +27,9 @@ pub const KATA_HOST_DIR_VOLUME_TYPE: &str = "kata:hostdir";
 /// KATA_MOUNT_INFO_FILE_NAME is used for the file that holds direct-volume mount info
 pub const KATA_MOUNT_INFO_FILE_NAME: &str = "mountInfo.json";
 
+/// Specify `fsgid` for a volume or mount, `fsgid=1`.
+pub const KATA_MOUNT_OPTION_FS_GID: &str = "fsgid";
+
 /// KATA_DIRECT_VOLUME_ROOT_PATH is the root path used for concatenating with the direct-volume mount info file path
 pub const KATA_DIRECT_VOLUME_ROOT_PATH: &str = "/run/kata-containers/shared/direct-volumes";
 
@@ -34,6 +42,23 @@ pub const SANDBOX_BIND_MOUNTS_RO: &str = ":ro";
 /// SANDBOX_BIND_MOUNTS_RO is for sandbox bindmounts with readwrite
 pub const SANDBOX_BIND_MOUNTS_RW: &str = ":rw";
 
+/// Directly assign a block volume to vm and mount it inside guest.
+pub const KATA_VIRTUAL_VOLUME_DIRECT_BLOCK: &str = "direct_block";
+/// Present a container image as a generic block device.
+pub const KATA_VIRTUAL_VOLUME_IMAGE_RAW_BLOCK: &str = "image_raw_block";
+/// Present each container image layer as a generic block device.
+pub const KATA_VIRTUAL_VOLUME_LAYER_RAW_BLOCK: &str = "layer_raw_block";
+/// Present a container image as a nydus block device.
+pub const KATA_VIRTUAL_VOLUME_IMAGE_NYDUS_BLOCK: &str = "image_nydus_block";
+/// Present each container image layer as a nydus block device.
+pub const KATA_VIRTUAL_VOLUME_LAYER_NYDUS_BLOCK: &str = "layer_nydus_block";
+/// Present a container image as a nydus filesystem.
+pub const KATA_VIRTUAL_VOLUME_IMAGE_NYDUS_FS: &str = "image_nydus_fs";
+/// Present each container image layer as a nydus filesystem.
+pub const KATA_VIRTUAL_VOLUME_LAYER_NYDUS_FS: &str = "layer_nydus_fs";
+/// Download and extra container image inside guest vm.
+pub const KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL: &str = "image_guest_pull";
+
 /// Information about a mount.
 #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
 pub struct Mount {
@@ -66,7 +91,7 @@ impl Mount {
 
 /// DirectVolumeMountInfo contains the information needed by Kata
 /// to consume a host block device and mount it as a filesystem inside the guest VM.
-#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)]
 pub struct DirectVolumeMountInfo {
     /// The type of the volume (ie. block)
     pub volume_type: String,
@@ -80,47 +105,6 @@ pub struct DirectVolumeMountInfo {
     pub options: Vec<String>,
 }
 
-/// join_path joins user provided volumepath with kata direct-volume root path
-/// the volume_path is base64-encoded and then safely joined to the end of path prefix
-pub fn join_path(prefix: &str, volume_path: &str) -> Result<PathBuf> {
-    if volume_path.is_empty() {
-        return Err(anyhow!("volume path must not be empty"));
-    }
-    let b64_encoded_path = base64::encode(volume_path.as_bytes());
-
-    Ok(safe_path::scoped_join(prefix, b64_encoded_path)?)
-}
-
-/// get DirectVolume mountInfo from mountinfo.json.
-pub fn get_volume_mount_info(volume_path: &str) -> Result<DirectVolumeMountInfo> {
-    let mount_info_file_path =
-        join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path)?.join(KATA_MOUNT_INFO_FILE_NAME);
-    let mount_info_file = fs::read_to_string(mount_info_file_path)?;
-    let mount_info: DirectVolumeMountInfo = serde_json::from_str(&mount_info_file)?;
-
-    Ok(mount_info)
-}
-
-/// Check whether a mount type is a marker for Kata specific volume.
-pub fn is_kata_special_volume(ty: &str) -> bool {
-    ty.len() > KATA_VOLUME_TYPE_PREFIX.len() && ty.starts_with(KATA_VOLUME_TYPE_PREFIX)
-}
-
-/// Check whether a mount type is a marker for Kata guest mount volume.
-pub fn is_kata_guest_mount_volume(ty: &str) -> bool {
-    ty.len() > KATA_GUEST_MOUNT_PREFIX.len() && ty.starts_with(KATA_GUEST_MOUNT_PREFIX)
-}
-
-/// Check whether a mount type is a marker for Kata ephemeral volume.
-pub fn is_kata_ephemeral_volume(ty: &str) -> bool {
-    ty == KATA_EPHEMERAL_VOLUME_TYPE
-}
-
-/// Check whether a mount type is a marker for Kata hostdir volume.
-pub fn is_kata_host_dir_volume(ty: &str) -> bool {
-    ty == KATA_HOST_DIR_VOLUME_TYPE
-}
-
 /// Nydus extra options
 #[derive(Debug, serde::Deserialize)]
 pub struct NydusExtraOptions {
@@ -159,6 +143,383 @@ impl NydusExtraOptions {
     }
 }
 
+/// Configuration information for DmVerity device.
+#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)]
+pub struct DmVerityInfo {
+    /// Hash algorithm for dm-verity.
+    pub hashtype: String,
+    /// Root hash for device verification or activation.
+    pub hash: String,
+    /// Size of data device used in verification.
+    pub blocknum: u64,
+    /// Used block size for the data device.
+    pub blocksize: u64,
+    /// Used block size for the hash device.
+    pub hashsize: u64,
+    /// Offset of hash area/superblock on hash_device.
+    pub offset: u64,
+}
+
+/// Information about directly assigned volume.
+#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)]
+pub struct DirectAssignedVolume {
+    /// Meta information for directly assigned volume.
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub metadata: HashMap<String, String>,
+}
+
+/// Information about pulling image inside guest.
+#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)]
+pub struct ImagePullVolume {
+    /// Meta information for pulling image inside guest.
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub metadata: HashMap<String, String>,
+}
+
+/// Information about nydus image volume.
+#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)]
+pub struct NydusImageVolume {
+    /// Nydus configuration information.
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    pub config: String,
+
+    /// Nydus snapshot directory
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    pub snapshot_dir: String,
+}
+
+/// Kata virtual volume to encapsulate information for extra mount options and direct volumes.
+///
+/// It's very expensive to build direct communication channels to pass information:
+/// - between snapshotters and kata-runtime/kata-agent/image-rs
+/// - between CSI drivers and kata-runtime/kata-agent
+///
+/// So `KataVirtualVolume` is introduced to encapsulate extra mount options and direct volume
+/// information, so we can build a common infrastructure to handle them.
+/// `KataVirtualVolume` is a superset of `NydusExtraOptions` and `DirectVolumeMountInfo`.
+///
+/// Value of `volume_type` determines how to interpret other fields in the structure.
+///
+/// - `KATA_VIRTUAL_VOLUME_IGNORE`
+/// -- all other fields should be ignored/unused.
+///
+/// - `KATA_VIRTUAL_VOLUME_DIRECT_BLOCK`
+/// -- `source`: the directly assigned block device
+/// -- `fs_type`: filesystem type
+/// -- `options`: mount options
+/// -- `direct_volume`: additional metadata to pass to the agent regarding this volume.
+///
+/// - `KATA_VIRTUAL_VOLUME_IMAGE_RAW_BLOCK` or `KATA_VIRTUAL_VOLUME_LAYER_RAW_BLOCK`
+/// -- `source`: path to the raw block image for the container image or layer.
+/// -- `fs_type`: filesystem type
+/// -- `options`: mount options
+/// -- `dm_verity`: disk dm-verity information
+///
+/// - `KATA_VIRTUAL_VOLUME_IMAGE_NYDUS_BLOCK` or `KATA_VIRTUAL_VOLUME_LAYER_NYDUS_BLOCK`
+/// -- `source`: path to nydus meta blob
+/// -- `fs_type`: filesystem type
+/// -- `nydus_image`: configuration information for nydus image.
+/// -- `dm_verity`: disk dm-verity information
+///
+/// - `KATA_VIRTUAL_VOLUME_IMAGE_NYDUS_FS` or `KATA_VIRTUAL_VOLUME_LAYER_NYDUS_FS`
+/// -- `source`: path to nydus meta blob
+/// -- `fs_type`: filesystem type
+/// -- `nydus_image`: configuration information for nydus image.
+///
+/// - `KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL`
+/// -- `source`: image reference
+/// -- `image_pull`: metadata for image pulling
+#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)]
+pub struct KataVirtualVolume {
+    /// Type of virtual volume.
+    pub volume_type: String,
+    /// Source/device path for the virtual volume.
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    pub source: String,
+    /// Filesystem type.
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    pub fs_type: String,
+    /// Mount options.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub options: Vec<String>,
+
+    /// Information about directly assigned volume.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub direct_volume: Option<DirectAssignedVolume>,
+    /// Information about pulling image inside guest.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub image_pull: Option<ImagePullVolume>,
+    /// Information about nydus image volume.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub nydus_image: Option<NydusImageVolume>,
+    /// DmVerity: configuration information
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub dm_verity: Option<DmVerityInfo>,
+}
+
+impl KataVirtualVolume {
+    /// Create a new instance of `KataVirtualVolume` with specified type.
+    pub fn new(volume_type: String) -> Self {
+        Self {
+            volume_type,
+            ..Default::default()
+        }
+    }
+
+    /// Validate virtual volume object.
+    pub fn validate(&self) -> Result<()> {
+        match self.volume_type.as_str() {
+            KATA_VIRTUAL_VOLUME_DIRECT_BLOCK => {
+                if self.source.is_empty() {
+                    return Err(anyhow!(
+                        "missing source device for directly assigned block volume"
+                    ));
+                } else if self.fs_type.is_empty() {
+                    return Err(anyhow!(
+                        "missing filesystem for directly assigned block volume"
+                    ));
+                }
+            }
+            KATA_VIRTUAL_VOLUME_IMAGE_RAW_BLOCK | KATA_VIRTUAL_VOLUME_LAYER_RAW_BLOCK => {
+                if self.source.is_empty() {
+                    return Err(anyhow!("missing source device for raw block volume"));
+                } else if self.fs_type.is_empty() {
+                    return Err(anyhow!("missing filesystem for raw block volume"));
+                }
+            }
+            KATA_VIRTUAL_VOLUME_IMAGE_NYDUS_BLOCK | KATA_VIRTUAL_VOLUME_LAYER_NYDUS_BLOCK => {
+                if self.source.is_empty() {
+                    return Err(anyhow!("missing meta blob for nydus block volume"));
+                } else if self.fs_type.as_str() != "rafsv6" {
+                    return Err(anyhow!("invalid filesystem for nydus block volume"));
+                }
+                match self.nydus_image.as_ref() {
+                    None => {
+                        return Err(anyhow!(
+                            "missing nydus configuration info for nydus block volume"
+                        ))
+                    }
+                    Some(nydus) => {
+                        if nydus.config.is_empty() {
+                            return Err(anyhow!(
+                                "missing configuration info for nydus block volume"
+                            ));
+                        } else if nydus.snapshot_dir.is_empty() {
+                            return Err(anyhow!(
+                                "missing snapshot directory for nydus block volume"
+                            ));
+                        }
+                    }
+                }
+            }
+            KATA_VIRTUAL_VOLUME_IMAGE_NYDUS_FS | KATA_VIRTUAL_VOLUME_LAYER_NYDUS_FS => {
+                if self.source.is_empty() {
+                    return Err(anyhow!("missing meta blob for nydus fs volume"));
+                } else if self.fs_type.as_str() != "rafsv6" && self.fs_type.as_str() != "rafsv5" {
+                    return Err(anyhow!("invalid filesystem for nydus fs volume"));
+                }
+                match self.nydus_image.as_ref() {
+                    None => {
+                        return Err(anyhow!(
+                            "missing nydus configuration info for nydus block volume"
+                        ))
+                    }
+                    Some(nydus) => {
+                        if nydus.config.is_empty() {
+                            return Err(anyhow!(
+                                "missing configuration info for nydus block volume"
+                            ));
+                        } else if nydus.snapshot_dir.is_empty() {
+                            return Err(anyhow!(
+                                "missing snapshot directory for nydus block volume"
+                            ));
+                        }
+                    }
+                }
+            }
+            KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL => {
+                if self.source.is_empty() {
+                    return Err(anyhow!("missing image reference for guest pulling volume"));
+                }
+            }
+            _ => {}
+        }
+
+        Ok(())
+    }
+
+    /// Serialize the virtual volume object to json.
+    pub fn to_json(&self) -> Result<String> {
+        Ok(serde_json::to_string(self)?)
+    }
+
+    /// Deserialize a virtual volume object from json string.
+    pub fn from_json(value: &str) -> Result<Self> {
+        let volume: KataVirtualVolume = serde_json::from_str(value)?;
+        volume.validate()?;
+        Ok(volume)
+    }
+
+    /// Serialize the virtual volume object to json and encode the string with base64.
+    pub fn to_base64(&self) -> Result<String> {
+        let json = self.to_json()?;
+        Ok(base64::encode(json))
+    }
+
+    /// Decode and deserialize a virtual volume object from base64 encoded json string.
+    pub fn from_base64(value: &str) -> Result<Self> {
+        let json = base64::decode(value)?;
+        let volume: KataVirtualVolume = serde_json::from_slice(&json)?;
+        volume.validate()?;
+        Ok(volume)
+    }
+}
+
+impl TryFrom<&DirectVolumeMountInfo> for KataVirtualVolume {
+    type Error = Error;
+
+    fn try_from(value: &DirectVolumeMountInfo) -> std::result::Result<Self, Self::Error> {
+        let volume_type = match value.volume_type.as_str() {
+            "block" => KATA_VIRTUAL_VOLUME_DIRECT_BLOCK.to_string(),
+            _ => {
+                return Err(anyhow!(
+                    "unknown directly assigned volume type: {}",
+                    value.volume_type
+                ))
+            }
+        };
+
+        Ok(KataVirtualVolume {
+            volume_type,
+            source: value.device.clone(),
+            fs_type: value.fs_type.clone(),
+            options: value.options.clone(),
+            direct_volume: Some(DirectAssignedVolume {
+                metadata: value.metadata.clone(),
+            }),
+            ..Default::default()
+        })
+    }
+}
+
+impl TryFrom<&NydusExtraOptions> for KataVirtualVolume {
+    type Error = Error;
+
+    fn try_from(value: &NydusExtraOptions) -> std::result::Result<Self, Self::Error> {
+        let fs_type = match value.fs_version.as_str() {
+            "v6" => "rafsv6".to_string(),
+            "rafsv6" => "rafsv6".to_string(),
+            "v5" => "rafsv5".to_string(),
+            "rafsv5" => "rafsv5".to_string(),
+            _ => return Err(anyhow!("unknown RAFS version: {}", value.fs_version)),
+        };
+
+        Ok(KataVirtualVolume {
+            volume_type: KATA_VIRTUAL_VOLUME_IMAGE_NYDUS_FS.to_string(),
+            source: value.source.clone(),
+            fs_type,
+            options: vec![],
+            nydus_image: Some(NydusImageVolume {
+                config: value.config.clone(),
+                snapshot_dir: value.snapshot_dir.clone(),
+            }),
+            ..Default::default()
+        })
+    }
+}
+
+/// Trait object for storage device.
+pub trait StorageDevice: Send + Sync {
+    /// Path
+    fn path(&self) -> Option<&str>;
+
+    /// Clean up resources related to the storage device.
+    fn cleanup(&self) -> Result<()>;
+}
+
+/// Manager to manage registered storage device handlers.
+pub struct StorageHandlerManager<H> {
+    handlers: HashMap<String, H>,
+}
+
+impl<H> Default for StorageHandlerManager<H> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<H> StorageHandlerManager<H> {
+    /// Create a new instance of `StorageHandlerManager`.
+    pub fn new() -> Self {
+        Self {
+            handlers: HashMap::new(),
+        }
+    }
+
+    /// Register a storage device handler.
+    pub fn add_handler(&mut self, id: &str, handler: H) -> Result<()> {
+        match self.handlers.entry(id.to_string()) {
+            Entry::Occupied(_) => Err(anyhow!("storage handler for {} already exists", id)),
+            Entry::Vacant(entry) => {
+                entry.insert(handler);
+                Ok(())
+            }
+        }
+    }
+
+    /// Get storage handler with specified `id`.
+    pub fn handler(&self, id: &str) -> Option<&H> {
+        self.handlers.get(id)
+    }
+
+    /// Get names of registered handlers.
+    pub fn get_handlers(&self) -> Vec<String> {
+        self.handlers.keys().map(|v| v.to_string()).collect()
+    }
+}
+
+/// Join user provided volume path with kata direct-volume root path.
+///
+/// The `volume_path` is base64-encoded and then safely joined to the `prefix`
+pub fn join_path(prefix: &str, volume_path: &str) -> Result<PathBuf> {
+    if volume_path.is_empty() {
+        return Err(anyhow!("volume path must not be empty"));
+    }
+    let b64_encoded_path = base64::encode(volume_path.as_bytes());
+
+    Ok(safe_path::scoped_join(prefix, b64_encoded_path)?)
+}
+
+/// get DirectVolume mountInfo from mountinfo.json.
+pub fn get_volume_mount_info(volume_path: &str) -> Result<DirectVolumeMountInfo> {
+    let volume_path = join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path)?;
+    let mount_info_file_path = volume_path.join(KATA_MOUNT_INFO_FILE_NAME);
+    let mount_info_file = fs::read_to_string(mount_info_file_path)?;
+    let mount_info: DirectVolumeMountInfo = serde_json::from_str(&mount_info_file)?;
+
+    Ok(mount_info)
+}
+
+/// Check whether a mount type is a marker for Kata specific volume.
+pub fn is_kata_special_volume(ty: &str) -> bool {
+    ty.len() > KATA_VOLUME_TYPE_PREFIX.len() && ty.starts_with(KATA_VOLUME_TYPE_PREFIX)
+}
+
+/// Check whether a mount type is a marker for Kata guest mount volume.
+pub fn is_kata_guest_mount_volume(ty: &str) -> bool {
+    ty.len() > KATA_GUEST_MOUNT_PREFIX.len() && ty.starts_with(KATA_GUEST_MOUNT_PREFIX)
+}
+
+/// Check whether a mount type is a marker for Kata ephemeral volume.
+pub fn is_kata_ephemeral_volume(ty: &str) -> bool {
+    ty == KATA_EPHEMERAL_VOLUME_TYPE
+}
+
+/// Check whether a mount type is a marker for Kata hostdir volume.
+pub fn is_kata_host_dir_volume(ty: &str) -> bool {
+    ty == KATA_HOST_DIR_VOLUME_TYPE
+}
+
 /// sandbox bindmount format:  /path/to/dir, or /path/to/dir:ro[:rw]
 /// the real path is without suffix ":ro" or ":rw".
 pub fn split_bind_mounts(bindmount: &str) -> (&str, &str) {
@@ -242,4 +603,111 @@ mod tests {
         );
         assert_eq!(extra_option.fs_version, "v6");
     }
+
+    #[test]
+    fn test_kata_virtual_volume() {
+        let mut volume = KataVirtualVolume::new(KATA_VIRTUAL_VOLUME_DIRECT_BLOCK.to_string());
+        assert_eq!(
+            volume.volume_type.as_str(),
+            KATA_VIRTUAL_VOLUME_DIRECT_BLOCK
+        );
+        assert!(volume.fs_type.is_empty());
+
+        let value = serde_json::to_string(&volume).unwrap();
+        assert_eq!(&value, "{\"volume_type\":\"direct_block\"}");
+
+        volume.source = "/tmp".to_string();
+        volume.fs_type = "ext4".to_string();
+        volume.options = vec!["rw".to_string()];
+        volume.nydus_image = Some(NydusImageVolume {
+            config: "test".to_string(),
+            snapshot_dir: "/var/lib/nydus.dir".to_string(),
+        });
+        let mut metadata = HashMap::new();
+        metadata.insert("mode".to_string(), "rw".to_string());
+        volume.direct_volume = Some(DirectAssignedVolume { metadata });
+
+        let value = serde_json::to_string(&volume).unwrap();
+        let volume2: KataVirtualVolume = serde_json::from_str(&value).unwrap();
+        assert_eq!(volume.volume_type, volume2.volume_type);
+        assert_eq!(volume.source, volume2.source);
+        assert_eq!(volume.fs_type, volume2.fs_type);
+        assert_eq!(volume.nydus_image, volume2.nydus_image);
+        assert_eq!(volume.direct_volume, volume2.direct_volume);
+    }
+
+    #[test]
+    fn test_kata_virtual_volume_serde() {
+        let mut volume = KataVirtualVolume::new(KATA_VIRTUAL_VOLUME_DIRECT_BLOCK.to_string());
+        volume.source = "/tmp".to_string();
+        volume.fs_type = "ext4".to_string();
+        volume.options = vec!["rw".to_string()];
+        volume.nydus_image = Some(NydusImageVolume {
+            config: "test".to_string(),
+            snapshot_dir: "/var/lib/nydus.dir".to_string(),
+        });
+        let mut metadata = HashMap::new();
+        metadata.insert("mode".to_string(), "rw".to_string());
+        volume.direct_volume = Some(DirectAssignedVolume { metadata });
+
+        let value = volume.to_base64().unwrap();
+        let volume2: KataVirtualVolume = KataVirtualVolume::from_base64(value.as_str()).unwrap();
+        assert_eq!(volume.volume_type, volume2.volume_type);
+        assert_eq!(volume.source, volume2.source);
+        assert_eq!(volume.fs_type, volume2.fs_type);
+        assert_eq!(volume.nydus_image, volume2.nydus_image);
+        assert_eq!(volume.direct_volume, volume2.direct_volume);
+    }
+
+    #[test]
+    fn test_try_from_direct_volume() {
+        let mut metadata = HashMap::new();
+        metadata.insert("mode".to_string(), "rw".to_string());
+        let mut direct = DirectVolumeMountInfo {
+            volume_type: "unknown".to_string(),
+            device: "/dev/vda".to_string(),
+            fs_type: "ext4".to_string(),
+            metadata,
+            options: vec!["ro".to_string()],
+        };
+        KataVirtualVolume::try_from(&direct).unwrap_err();
+
+        direct.volume_type = "block".to_string();
+        let volume = KataVirtualVolume::try_from(&direct).unwrap();
+        assert_eq!(
+            volume.volume_type.as_str(),
+            KATA_VIRTUAL_VOLUME_DIRECT_BLOCK
+        );
+        assert_eq!(volume.source, direct.device);
+        assert_eq!(volume.fs_type, direct.fs_type);
+        assert_eq!(
+            volume.direct_volume.as_ref().unwrap().metadata,
+            direct.metadata
+        );
+        assert_eq!(volume.options, direct.options);
+    }
+
+    #[test]
+    fn test_try_from_nydus_extra_options() {
+        let mut nydus = NydusExtraOptions {
+            source: "/test/nydus".to_string(),
+            config: "test".to_string(),
+            snapshot_dir: "/var/lib/nydus".to_string(),
+            fs_version: "rafsvx".to_string(),
+        };
+        KataVirtualVolume::try_from(&nydus).unwrap_err();
+
+        nydus.fs_version = "v6".to_string();
+        let volume = KataVirtualVolume::try_from(&nydus).unwrap();
+        assert_eq!(
+            volume.volume_type.as_str(),
+            KATA_VIRTUAL_VOLUME_IMAGE_NYDUS_FS
+        );
+        assert_eq!(volume.nydus_image.as_ref().unwrap().config, nydus.config);
+        assert_eq!(
+            volume.nydus_image.as_ref().unwrap().snapshot_dir,
+            nydus.snapshot_dir
+        );
+        assert_eq!(volume.fs_type.as_str(), "rafsv6")
+    }
 }

src/libs/kata-types/src/volume.rs

@@ -0,0 +1,26 @@
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/// Volume to support dm-verity over block devices.
+pub const KATA_VOLUME_TYPE_DMVERITY: &str = "dmverity";
+
+/// Key to identify dmverity information in `Storage.driver_options`
+pub const KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO: &str = "verity_info";
+/// Key to identify type of source device in `Storage.driver_options`
+pub const KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE: &str = "source_type";
+/// Source device of dmverity volume is a Virtio PCI device
+pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI: &str = "virtio_pci";
+/// Source device of dmverity volume is a Virtio MMIO device
+pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO: &str = "virtio_mmio";
+/// Source device of dmverity volume is a Virtio CCW device
+pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW: &str = "virtio_ccw";
+/// Source device of dmverity volume is a SCSI disk.
+pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI: &str = "scsi";
+/// Source device of dmverity volume is a pmem disk.
+pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM: &str = "pmem";
+
+/// Key to indentify directory creation in `Storage.driver_options`.
+pub const KATA_VOLUME_OVERLAYFS_CREATE_DIR: &str =
+    "io.katacontainers.volume.overlayfs.create_directory";

src/libs/oci/src/lib.rs

@@ -512,8 +512,8 @@ pub struct LinuxDevice {
 pub struct LinuxDeviceCgroup {
     #[serde(default)]
     pub allow: bool,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
-    pub r#type: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub r#type: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub major: Option<i64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -1463,21 +1463,21 @@ mod tests {
                     devices: vec![
                         crate::LinuxDeviceCgroup {
                             allow: false,
-                            r#type: "".to_string(),
+                            r#type: None,
                             major: None,
                             minor: None,
                             access: "rwm".to_string(),
                         },
                         crate::LinuxDeviceCgroup {
                             allow: true,
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(10),
                             minor: Some(229),
                             access: "rw".to_string(),
                         },
                         crate::LinuxDeviceCgroup {
                             allow: true,
-                            r#type: "b".to_string(),
+                            r#type: Some("b".to_string()),
                             major: Some(8),
                             minor: Some(0),
                             access: "r".to_string(),

src/libs/protocols/Cargo.toml

@@ -9,6 +9,7 @@ license = "Apache-2.0"
 default = []
 with-serde = [ "serde", "serde_json" ]
 async = ["ttrpc/async", "async-trait"]
+sealed-secret = []
 
 [dependencies]
 ttrpc = { version = "0.7.1" }

src/libs/protocols/build.rs

@@ -198,13 +198,39 @@ fn real_main() -> Result<(), std::io::Error> {
     // generate async
     #[cfg(feature = "async")]
     {
-        codegen("src", &["protos/agent.proto", "protos/health.proto"], true)?;
+        codegen(
+            "src",
+            &[
+                "protos/agent.proto",
+                "protos/health.proto",
+                "protos/image.proto",
+                #[cfg(feature = "sealed-secret")]
+                "protos/sealed_secret.proto",
+            ],
+            true,
+        )?;
 
         fs::rename("src/agent_ttrpc.rs", "src/agent_ttrpc_async.rs")?;
         fs::rename("src/health_ttrpc.rs", "src/health_ttrpc_async.rs")?;
+        fs::rename("src/image_ttrpc.rs", "src/image_ttrpc_async.rs")?;
+        #[cfg(feature = "sealed-secret")]
+        fs::rename(
+            "src/sealed_secret_ttrpc.rs",
+            "src/sealed_secret_ttrpc_async.rs",
+        )?;
     }
 
-    codegen("src", &["protos/agent.proto", "protos/health.proto"], false)?;
+    codegen(
+        "src",
+        &[
+            "protos/agent.proto",
+            "protos/health.proto",
+            "protos/image.proto",
+            #[cfg(feature = "sealed-secret")]
+            "protos/sealed_secret.proto",
+        ],
+        false,
+    )?;
 
     // There is a message named 'Box' in oci.proto
     // so there is a struct named 'Box', we should replace Box<Self> to ::std::boxed::Box<Self>

src/libs/protocols/protos/image.proto

@@ -0,0 +1,31 @@
+//
+// Copyright (c) 2021 Alibaba Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+syntax = "proto3";
+
+option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
+
+package grpc;
+
+// Image defines the public APIs for managing images.
+service Image {
+    // PullImage pulls an image with authentication config.
+    rpc PullImage(PullImageRequest) returns (PullImageResponse) {}
+}
+
+message PullImageRequest {
+    // Image name (e.g. docker.io/library/busybox:latest).
+    string image = 1;
+    // Unique image identifier, used to avoid duplication when unpacking the image layers.
+    string container_id = 2;
+    // Use USERNAME[:PASSWORD] for accessing the registry
+    string source_creds = 3;
+}
+
+message PullImageResponse {
+    // Reference to the image in use. For most runtimes, this should be an
+    // image ID or digest.
+    string image_ref = 1;
+}

src/libs/protocols/protos/sealed_secret.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+package api;
+
+message UnsealSecretInput {
+    bytes secret = 1;
+}
+
+message UnsealSecretOutput {
+    bytes plaintext = 1;
+}
+
+message GetResourceRequest {
+    string ResourcePath = 1;
+}
+
+message GetResourceResponse {
+    bytes Resource = 1;
+}
+
+service SealedSecretService {
+    rpc UnsealSecret(UnsealSecretInput) returns (UnsealSecretOutput) {};
+}
+
+service GetResourceService {
+    rpc GetResource(GetResourceRequest) returns (GetResourceResponse) {};
+}

src/libs/protocols/src/lib.rs

@@ -16,6 +16,10 @@ pub mod health;
 pub mod health_ttrpc;
 #[cfg(feature = "async")]
 pub mod health_ttrpc_async;
+pub mod image;
+pub mod image_ttrpc;
+#[cfg(feature = "async")]
+pub mod image_ttrpc_async;
 pub mod oci;
 #[cfg(feature = "with-serde")]
 mod serde_config;
@@ -27,3 +31,10 @@ pub use serde_config::{
     deserialize_enum_or_unknown, deserialize_message_field, serialize_enum_or_unknown,
     serialize_message_field,
 };
+
+#[cfg(feature = "sealed-secret")]
+pub mod sealed_secret;
+#[cfg(feature = "sealed-secret")]
+pub mod sealed_secret_ttrpc;
+#[cfg(all(feature = "sealed-secret", feature = "async"))]
+pub mod sealed_secret_ttrpc_async;

src/libs/protocols/src/trans.rs

@@ -100,7 +100,7 @@ impl From<oci::LinuxDeviceCgroup> for crate::oci::LinuxDeviceCgroup {
     fn from(from: oci::LinuxDeviceCgroup) -> Self {
         crate::oci::LinuxDeviceCgroup {
             Allow: from.allow,
-            Type: from.r#type,
+            Type: from.r#type.map_or("".to_string(), |t| t as String),
             Major: from.major.map_or(0, |t| t),
             Minor: from.minor.map_or(0, |t| t),
             Access: from.access,
@@ -452,7 +452,7 @@ impl From<crate::oci::LinuxDeviceCgroup> for oci::LinuxDeviceCgroup {
 
         oci::LinuxDeviceCgroup {
             allow: from.Allow(),
-            r#type: from.take_Type(),
+            r#type: Some(from.take_Type()),
             major,
             minor,
             access: from.take_Access(),

src/libs/safe-path/src/scoped_path_resolver.rs

@@ -245,7 +245,7 @@ mod tests {
     fn test_scoped_resolve_invalid() {
         scoped_resolve("./root_is_not_absolute_path", ".").unwrap_err();
         scoped_resolve("C:", ".").unwrap_err();
-        scoped_resolve(r#"\\server\test"#, ".").unwrap_err();
+        scoped_resolve(r"\\server\test", ".").unwrap_err();
         scoped_resolve(r#"http://localhost/test"#, ".").unwrap_err();
         // Chinese Unicode characters
         scoped_resolve(r#"您好"#, ".").unwrap_err();

src/runtime-rs/crates/resource/src/network/utils/address.rs

@@ -148,7 +148,7 @@ mod tests {
 
     #[test]
     fn test_parse_ip_cidr() {
-        let test_cases = vec![
+        let test_cases = [
             ("127.0.0.1/32", ("127.0.0.1", 32u8)),
             ("2001:4860:4860::8888/32", ("2001:4860:4860::8888", 32u8)),
             ("2001:4860:4860::8888/128", ("2001:4860:4860::8888", 128u8)),
@@ -158,7 +158,7 @@ mod tests {
             assert_eq!(ipaddr.to_string(), tc.1 .0);
             assert_eq!(mask, tc.1 .1);
         }
-        let test_cases = vec![
+        let test_cases = [
             "127.0.0.1/33",
             "2001:4860:4860::8888/129",
             "2001:4860:4860::8888/300",

src/runtime-rs/crates/resource/src/volume/block_volume.rs

@@ -181,7 +181,7 @@ impl Volume for BlockVolume {
 }
 
 pub(crate) fn is_block_volume(m: &oci::Mount) -> Result<bool> {
-    let vol_types = vec![KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE];
+    let vol_types = [KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE];
     if !vol_types.contains(&m.r#type.as_str()) {
         return Ok(false);
     }

src/runtime-rs/crates/resource/src/volume/shm_volume.rs

@@ -68,7 +68,7 @@ impl ShmVolume {
                 r#type: "tmpfs".to_string(),
                 destination: m.destination.clone(),
                 source: "shm".to_string(),
-                options: vec![
+                options: [
                     "noexec",
                     "nosuid",
                     "nodev",

src/runtime-rs/crates/resource/src/volume/spdk_volume.rs

@@ -75,7 +75,7 @@ impl SPDKVolume {
 
         let block_driver = get_block_driver(d).await;
 
-        let mut vhu_blk_config = &mut VhostUserConfig {
+        let vhu_blk_config = &mut VhostUserConfig {
             socket_path: device,
             device_type: VhostUserType::Blk("vhost-user-blk-pci".to_owned()),
             driver_option: block_driver,
@@ -183,7 +183,7 @@ impl Volume for SPDKVolume {
 
 pub(crate) fn is_spdk_volume(m: &oci::Mount) -> bool {
     // spdkvol or spoolvol will share the same implementation
-    let vol_types = vec![KATA_SPDK_VOLUME_TYPE, KATA_SPOOL_VOLUME_TYPE];
+    let vol_types = [KATA_SPDK_VOLUME_TYPE, KATA_SPOOL_VOLUME_TYPE];
     if vol_types.contains(&m.r#type.as_str()) {
         return true;
     }

src/runtime/Makefile

@@ -56,6 +56,7 @@ BINLIBEXECLIST :=
 BIN_PREFIX = $(PROJECT_TYPE)
 PROJECT_DIR = $(PROJECT_TAG)
 IMAGENAME = $(PROJECT_TAG).img
+IMAGETDXNAME = $(PROJECT_TAG)-tdx.img
 INITRDNAME = $(PROJECT_TAG)-initrd.img
 INITRDSEVNAME = $(PROJECT_TAG)-initrd-sev.img
 
@@ -102,6 +103,9 @@ GENERATED_VARS = \
 		CONFIG_QEMU_SNP_IN \
 		CONFIG_CLH_IN \
 		CONFIG_FC_IN \
+		CONFIG_CLH_TDX_IN \
+		CONFIG_QEMU_SE_IN \
+		CONFIG_REMOTE_IN \
 		$(USER_VARS)
 SCRIPTS += $(COLLECT_SCRIPT)
 SCRIPTS_DIR := $(BINDIR)
@@ -116,6 +120,7 @@ PKGLIBEXECDIR := $(LIBEXECDIR)/$(PROJECT_DIR)
 KERNELDIR := $(PKGDATADIR)
 
 IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME)
+IMAGETDXPATH := $(PKGDATADIR)/$(IMAGETDXNAME)
 INITRDPATH := $(PKGDATADIR)/$(INITRDNAME)
 INITRDSEVPATH := $(PKGDATADIR)/$(INITRDSEVNAME)
 
@@ -126,16 +131,29 @@ DEFROOTFSTYPE := $(ROOTFSTYPE_EXT4)
 
 FIRMWAREPATH :=
 FIRMWAREVOLUMEPATH :=
+TDSHIMFIRMWAREPATH := ${PREFIXDEPS}/share/td-shim/td-shim.bin
 
-FIRMWARETDVFPATH := $(PREFIXDEPS)/share/tdvf/OVMF.fd
-FIRMWARETDVFVOLUMEPATH :=
-
-FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
-FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
+AGENTCONFIGFILEPATH := /etc/agent-config.toml
+AGENTCONFIGFILEKERNELPARAM := agent.config_file=$(AGENTCONFIGFILEPATH)
 
 ROOTMEASURECONFIG ?= ""
-KERNELPARAMS += $(ROOTMEASURECONFIG)
-KERNELTDXPARAMS += $(ROOTMEASURECONFIG)
+ROOTMEASURECONFIGTDX ?= ""
+AGENT_AA_KBC_PARAMS ?= ""
+AGENT_AA_KBC_PARAMS_TDX ?= ""
+AGENT_AA_KBC_PARAMS_SEV ?= ""
+AGENT_AA_KBC_PARAMS_SNP ?= ""
+TDXKERNELPARAMS := tdx_disable_filter agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_TDX)
+KERNELTDXPARAMS += $(TDXKERNELPARAMS) $(ROOTMEASURECONFIGTDX)
+TDXKERNELPARAMS_CLH += $(TDXKERNELPARAMS) $(ROOTMEASURECONFIG)
+SEVKERNELPARAMS := $(AGENTCONFIGFILEKERNELPARAM) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_SEV)
+SNPKERNELPARAMS := $(AGENTCONFIGFILEKERNELPARAM) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_SNP)
+KERNELPARAMS += $(ROOTMEASURECONFIG) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS)
+
+FIRMWARETDVFPATH := $(PREFIXDEPS)/share/tdvf/OVMF_CODE.fd
+FIRMWARETDVFVOLUMEPATH := $(PREFIXDEPS)/share/tdvf/OVMF_VARS.fd
+
+FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd
+FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
 
 # Name of default configuration file the runtime will use.
 CONFIG_FILE = configuration.toml
@@ -144,16 +162,20 @@ HYPERVISOR_ACRN = acrn
 HYPERVISOR_FC = firecracker
 HYPERVISOR_QEMU = qemu
 HYPERVISOR_CLH = cloud-hypervisor
+HYPERVISOR_REMOTE = remote
 
 # Determines which hypervisor is specified in $(CONFIG_FILE).
 DEFAULT_HYPERVISOR ?= $(HYPERVISOR_QEMU)
 
 # List of hypervisors this build system can generate configuration for.
-HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
+HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_REMOTE)
 
 QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD)
 QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"]
 
+QEMUSNPPATH := $(QEMUBINDIR)/$(QEMUSNPCMD)
+QEMUSNPVALIDHYPERVISORPATHS := [\"$(QEMUSNPPATH)\"]
+
 QEMUTDXPATH := $(QEMUBINDIR)/$(QEMUTDXCMD)
 QEMUTDXVALIDHYPERVISORPATHS := [\"$(QEMUTDXPATH)\"]
 
@@ -253,6 +275,26 @@ DEFBINDMOUNTS := []
 # Image Service Offload
 DEFSERVICEOFFLOAD ?= false
 
+# Image Request Timeout in seconds
+DEFIMAGEREQUESTTIMEOUT ?= 60
+
+# Enable sealed secret or not
+DEFSEALEDSECRETENABLED ?= false
+
+# SEV & SEV-ES Guest Pre-Attestation
+DEFGUESTPREATTESTATION ?= false
+DEFGUESTPREATTESTATIONPROXY ?= localhost:44444
+DEFGUESTPREATTESTATIONMODE ?= online
+DEFGUESTPREATTESTATIONKEYSET ?= KEYSET-1
+DEFSEVCERTCHAIN ?= /opt/sev/cert_chain.cert
+DEFSEVGUESTPOLICY ?= 3
+
+# SNP
+DEFSNPGUEST ?= true
+# Based on SEV Secure Nested Paging Firmware ABI Specification section 4.3
+# unspecified or == 0 --> 0x30000 i.e. Bit#17 is '1' and Bit#16 is '1' (SMT is allowed)
+DEFSNPGUESTPOLICY ?= 0x30000
+
 SED = sed
 
 CLI_DIR = cmd
@@ -315,6 +357,18 @@ ifneq (,$(QEMUCMD))
 
     CONFIGS += $(CONFIG_QEMU_SEV)
 
+    CONFIG_FILE_QEMU_SE = configuration-qemu-se.toml
+    CONFIG_QEMU_SE = config/$(CONFIG_FILE_QEMU_SE)
+    CONFIG_QEMU_SE_IN = $(CONFIG_QEMU_SE).in
+
+    CONFIG_PATH_QEMU_SE = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SE))
+    CONFIG_PATHS += $(CONFIG_PATH_QEMU_SE)
+
+    SYSCONFIG_QEMU_SE = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SE))
+    SYSCONFIG_PATHS += $(SYSCONFIG_QEMU_SE)
+
+    CONFIGS += $(CONFIG_QEMU_SE)
+
     CONFIG_FILE_QEMU_SNP = configuration-qemu-snp.toml
     CONFIG_QEMU_SNP = config/$(CONFIG_FILE_QEMU_SNP)
     CONFIG_QEMU_SNP_IN = $(CONFIG_QEMU_SNP).in
@@ -326,6 +380,18 @@ ifneq (,$(QEMUCMD))
     SYSCONFIG_PATHS_SNP += $(SYSCONFIG_QEMU_SNP)
 
     CONFIGS += $(CONFIG_QEMU_SNP)
+	
+    CONFIG_FILE_REMOTE = configuration-remote.toml
+    CONFIG_REMOTE = config/$(CONFIG_FILE_REMOTE)
+    CONFIG_REMOTE_IN = $(CONFIG_REMOTE).in
+
+    CONFIG_PATH_REMOTE = $(abspath $(CONFDIR)/$(CONFIG_FILE_REMOTE))
+    CONFIG_PATHS += $(CONFIG_PATH_REMOTE)
+
+    SYSCONFIG_REMOTE = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_REMOTE))
+    SYSCONFIG_PATHS += $(SYSCONFIG_REMOTE)
+
+    CONFIGS += $(CONFIG_REMOTE)
 
     CONFIG_FILE_QEMU_NVIDIA_GPU = configuration-qemu-nvidia-gpu.toml
     CONFIG_QEMU_NVIDIA_GPU = config/$(CONFIG_FILE_QEMU_NVIDIA_GPU)
@@ -350,10 +416,16 @@ ifneq (,$(QEMUCMD))
     KERNELTDXNAME = $(call MAKE_KERNEL_TDX_NAME,$(KERNELTDXTYPE))
     KERNELTDXPATH = $(KERNELDIR)/$(KERNELTDXNAME)
 
-    KERNELSNPTYPE = compressed
+    KERNELSEVTYPE = compressed
+    KERNELSEVNAME = $(call MAKE_KERNEL_SEV_NAME,$(KERNELSEVTYPE))
+    KERNELSEVPATH = $(KERNELDIR)/$(KERNELSEVNAME)
+
+	KERNELSNPTYPE = compressed
     KERNELSNPNAME = $(call MAKE_KERNEL_SNP_NAME,$(KERNELSNPTYPE))
     KERNELSNPPATH = $(KERNELDIR)/$(KERNELSNPNAME)
 
+    KERNELSENAME = kata-containers-secure.img
+    KERNELSEPATH = $(KERNELDIR)/$(KERNELSENAME)
 endif
 
 ifneq (,$(CLHCMD))
@@ -371,12 +443,28 @@ ifneq (,$(CLHCMD))
 
     CONFIGS += $(CONFIG_CLH)
 
+    CONFIG_FILE_CLH_TDX = configuration-clh-tdx.toml
+    CONFIG_CLH_TDX = config/$(CONFIG_FILE_CLH_TDX)
+    CONFIG_CLH_TDX_IN = $(CONFIG_CLH_TDX).in
+
+    CONFIG_PATH_CLH_TDX = $(abspath $(CONFDIR)/$(CONFIG_FILE_CLH_TDX))
+    CONFIG_PATHS += $(CONFIG_PATH_CLH_TDX)
+
+    SYSCONFIG_CLH_TDX = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_CLH_TDX))
+    SYSCONFIG_PATHS += $(SYSCONFIG_CLH_TDX)
+
+    CONFIGS += $(CONFIG_CLH_TDX)
+
     # CLH-specific options (all should be suffixed by "_CLH")
     # currently, huge pages are required for virtiofsd support
     DEFNETWORKMODEL_CLH := tcfilter
     KERNELTYPE_CLH = uncompressed
     KERNEL_NAME_CLH = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_CLH))
     KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH)
+
+    KERNELTDXTYPE_CLH = compressed
+    KERNELTDXNAME_CLH = $(call MAKE_KERNEL_TDX_NAME,$(KERNELTDXTYPE_CLH))
+    KERNELTDXPATH_CLH = $(KERNELDIR)/$(KERNELTDXNAME_CLH)
 endif
 
 ifneq (,$(FCCMD))
@@ -479,6 +567,7 @@ USER_VARS += CONFIG_CLH_IN
 USER_VARS += CONFIG_FC_IN
 USER_VARS += CONFIG_PATH
 USER_VARS += CONFIG_QEMU_IN
+USER_VARS += CONFIG_REMOTE_IN
 USER_VARS += DESTDIR
 USER_VARS += DEFAULT_HYPERVISOR
 USER_VARS += ACRNCMD
@@ -498,6 +587,8 @@ USER_VARS += FCVALIDJAILERPATHS
 USER_VARS += SYSCONFIG
 USER_VARS += IMAGENAME
 USER_VARS += IMAGEPATH
+USER_VARS += IMAGETDXNAME
+USER_VARS += IMAGETDXPATH
 USER_VARS += INITRDNAME
 USER_VARS += INITRDSEVNAME
 USER_VARS += INITRDPATH
@@ -513,14 +604,18 @@ USER_VARS += KERNELPATH_ACRN
 USER_VARS += KERNELPATH
 USER_VARS += KERNELSEVPATH
 USER_VARS += KERNELTDXPATH
+USER_VARS += KERNELSEVPATH
+USER_VARS += KERNELSEPATH
 USER_VARS += KERNELSNPPATH
 USER_VARS += KERNELPATH_CLH
+USER_VARS += KERNELTDXPATH_CLH
 USER_VARS += KERNELPATH_FC
 USER_VARS += KERNELVIRTIOFSPATH
 USER_VARS += FIRMWAREPATH
 USER_VARS += FIRMWARESEVPATH
 USER_VARS += FIRMWARETDVFPATH
 USER_VARS += FIRMWAREVOLUMEPATH
+USER_VARS += TDSHIMFIRMWAREPATH
 USER_VARS += FIRMWARETDVFVOLUMEPATH
 USER_VARS += FIRMWARESNPPATH
 USER_VARS += MACHINEACCELERATORS
@@ -528,6 +623,11 @@ USER_VARS += CPUFEATURES
 USER_VARS += TDXCPUFEATURES
 USER_VARS += DEFMACHINETYPE_CLH
 USER_VARS += KERNELPARAMS
+USER_VARS += TDXKERNELPARAMS
+USER_VARS += TDXKERNELPARAMS_QEMU
+USER_VARS += TDXKERNELPARAMS_CLH
+USER_VARS += SEVKERNELPARAMS
+USER_VARS += SNPKERNELPARAMS
 USER_VARS += KERNELTDXPARAMS
 USER_VARS += LIBEXECDIR
 USER_VARS += LOCALSTATEDIR
@@ -554,6 +654,10 @@ USER_VARS += QEMUTDXVALIDHYPERVISORPATHS
 USER_VARS += QEMUSNPVALIDHYPERVISORPATHS
 USER_VARS += QEMUVIRTIOFSCMD
 USER_VARS += QEMUVIRTIOFSPATH
+USER_VARS += QEMUSNPPATH
+USER_VARS += QEMUSNPVALIDHYPERVISORPATHS
+USER_VARS += QEMUTDXPATH
+USER_VARS += QEMUTDXVALIDHYPERVISORPATHS
 USER_VARS += RUNTIME_NAME
 USER_VARS += SHAREDIR
 USER_VARS += SYSCONFDIR
@@ -607,8 +711,19 @@ USER_VARS += DEFSTATICRESOURCEMGMT_FC
 USER_VARS += DEFSTATICRESOURCEMGMT_TEE
 USER_VARS += DEFBINDMOUNTS
 USER_VARS += DEFSERVICEOFFLOAD
+USER_VARS += DEFIMAGEREQUESTTIMEOUT
+USER_VARS += DEFSEALEDSECRETENABLED
 USER_VARS += DEFVFIOMODE
 USER_VARS += BUILDFLAGS
+USER_VARS += DEFSERVICEOFFLOAD
+USER_VARS += DEFGUESTPREATTESTATION
+USER_VARS += DEFGUESTPREATTESTATIONPROXY
+USER_VARS += DEFGUESTPREATTESTATIONMODE
+USER_VARS += DEFGUESTPREATTESTATIONKEYSET
+USER_VARS += DEFSEVCERTCHAIN
+USER_VARS += DEFSEVGUESTPOLICY
+USER_VARS += DEFSNPGUESTPOLICY
+USER_VARS += DEFSNPGUEST
 
 
 V              = @
@@ -687,6 +802,14 @@ define MAKE_KERNEL_NAME
 $(if $(findstring uncompressed,$1),vmlinux.container,vmlinuz.container)
 endef
 
+define MAKE_KERNEL_TDX_NAME
+$(if $(findstring uncompressed,$1),vmlinux-tdx.container,vmlinuz-tdx.container)
+endef
+
+define MAKE_KERNEL_SEV_NAME
+$(if $(findstring uncompressed,$1),vmlinux-sev.container,vmlinuz-sev.container)
+endef
+
 define MAKE_KERNEL_VIRTIOFS_NAME
 $(if $(findstring uncompressed,$1),vmlinux-virtiofs.container,vmlinuz-virtiofs.container)
 endef

src/runtime/arch/amd64-options.mk

@@ -12,8 +12,8 @@ CPUFEATURES := pmu=off
 
 QEMUCMD := qemu-system-x86_64
 QEMUTDXCMD := qemu-system-x86_64-tdx-experimental
-QEMUSNPCMD := qemu-system-x86_64-snp-experimental
 TDXCPUFEATURES := -vmx-rdseed-exit,pmu=off
+QEMUSNPCMD := qemu-system-x86_64-snp-experimental
 
 # Firecracker binary name
 FCCMD := firecracker

src/runtime/cmd/kata-runtime/kata-check_amd64.go

@@ -319,6 +319,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
 		return kvmIsUsable()
 	case vc.AcrnHypervisor:
 		return acrnIsUsable()
+	case vc.RemoteHypervisor:
+		return nil
 	case vc.MockHypervisor:
 		return nil
 	default:

src/runtime/cmd/kata-runtime/kata-check_arm64.go

@@ -86,6 +86,9 @@ func checkKVMExtensions() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	if err := kvmIsUsable(); err != nil {
 		return err
 	}

src/runtime/cmd/kata-runtime/kata-check_ppc64le.go

@@ -61,6 +61,9 @@ func setCPUtype(hypervisorType vc.HypervisorType) error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/cmd/kata-runtime/kata-check_s390x.go

@@ -55,6 +55,9 @@ func kvmIsUsable() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/config/configuration-clh-tdx.toml.in

@@ -0,0 +1,454 @@
+# Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH
+# Copyright (c) 2021 Adobe Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_CLH_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.clh]
+path = "@CLHPATH@"
+kernel = "@KERNELTDXPATH_CLH@"
+image = "@IMAGEPATH@"
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Supported TEEs:
+# * Intel TDX
+#
+# Default false
+confidential_guest = true
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+disable_guest_selinux=@DEFDISABLEGUESTSELINUX@
+
+# Path to the firmware.
+# If you want Cloud Hypervisor to use a specific firmware, set its path below.
+# This is option is only used when confidential_guest is enabled.
+#
+# For more information about firmwared that can be used with specific TEEs,
+# please, refer to:
+# * Intel TDX:
+#   - td-shim: https://github.com/confidential-containers/td-shim
+#
+firmware = "@TDSHIMFIRMWAREPATH@"
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+enable_annotations = @DEFENABLEANNOTATIONS@
+
+# List of valid annotations values for the hypervisor
+# Each member of the list is a path pattern as described by glob(3).
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @CLHVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @CLHVALIDHYPERVISORPATHS@
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@TDXKERNELPARAMS_CLH@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+#memory_slots = @DEFMEMSLOTS@
+
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
+# Shared file system type:
+#   - virtio-fs (default)
+#   - virtio-fs-nydus
+shared_fs = "@DEFSHAREDFS_CLH_VIRTIOFS@"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# List of valid annotations values for the virtiofs daemon
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@
+valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Extra args for virtiofsd daemon
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+# Examples:
+#   Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"]
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - none
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-blk.
+block_device_driver = "virtio-blk"
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+#enable_hugepages = true
+
+# Disable the 'seccomp' feature from Cloud Hypervisor, default false
+# disable_seccomp = true
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available.
+#
+# Default false
+#enable_debug = true
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+#
+# These options are related to network rate limiter at the VMM level, and are
+# based on the Cloud Hypervisor I/O throttling.  Those are disabled by default
+# and we strongly advise users to refer the Cloud Hypervisor official
+# documentation for a better understanding of its internals:
+# https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/docs/io_throttling.md
+# 
+# Bandwidth rate limiter options
+# 
+# net_rate_limiter_bw_max_rate controls network I/O bandwidth (size in bits/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#net_rate_limiter_bw_max_rate = 0
+#
+# net_rate_limiter_bw_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if net_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#net_rate_limiter_bw_one_time_burst = 0
+#
+# Operation rate limiter options
+#
+# net_rate_limiter_ops_max_rate controls network I/O bandwidth (size in ops/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#net_rate_limiter_ops_max_rate = 0
+#
+# net_rate_limiter_ops_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if net_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#net_rate_limiter_ops_one_time_burst = 0
+#
+# These options are related to disk rate limiter at the VMM level, and are
+# based on the Cloud Hypervisor I/O throttling.  Those are disabled by default
+# and we strongly advise users to refer the Cloud Hypervisor official
+# documentation for a better understanding of its internals:
+# https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/docs/io_throttling.md
+# 
+# Bandwidth rate limiter options
+# 
+# disk_rate_limiter_bw_max_rate controls disk I/O bandwidth (size in bits/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#disk_rate_limiter_bw_max_rate = 0
+#
+# disk_rate_limiter_bw_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if disk_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#disk_rate_limiter_bw_one_time_burst = 0
+#
+# Operation rate limiter options
+#
+# disk_rate_limiter_ops_max_rate controls disk I/O bandwidth (size in ops/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#disk_rate_limiter_ops_max_rate = 0
+#
+# disk_rate_limiter_ops_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if disk_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#disk_rate_limiter_ops_one_time_burst = 0
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+#enable_tracing = true
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_CLH@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts=@DEFBINDMOUNTS@
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true
+
+# Sealed secret feature configuration.
+# If true, sealed secret feature will be enabled.
+# (default: false)
+sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
+
+# WARNING: All the options in the following section have not been implemented yet.
+# This section was added as a placeholder. DO NOT USE IT!
+[image]
+# Container image service.
+#
+# Offload the CRI image management service to the Kata agent.
+# (default: false)
+service_offload = @DEFSERVICEOFFLOAD@
+
+# Image request timeout in seconds.
+# If specified, indicates the image request timeout in the guest needed for the workload(s)
+# If unspecified then it will be set @DEFIMAGEREQUESTTIMEOUT@ second(s) 
+# to reduce image pull failures caused by network problems, and quickly obtain request failure information at the same time.
+image_request_timeout = @DEFIMAGEREQUESTTIMEOUT@
+
+# Container image decryption keys provisioning.
+# Applies only if service_offload is true.
+# Keys can be provisioned locally (e.g. through a special command or
+# a local file) or remotely (usually after the guest is remotely attested).
+# The provision setting is a complete URL that lets the Kata agent decide
+# which method to use in order to fetch the keys.
+#
+# Keys can be stored in a local file, in a measured and attested initrd:
+#provision=data:///local/key/file
+#
+# Keys could be fetched through a special command or binary from the
+# initrd (guest) image, e.g. a firmware call:
+#provision=file:///path/to/bin/fetcher/in/guest
+#
+# Keys can be remotely provisioned. The Kata agent fetches them from e.g.
+# a HTTPS URL:
+#provision=https://my-key-broker.foo/tenant/<tenant-id>

src/runtime/config/configuration-clh.toml.in

@@ -434,6 +434,11 @@ experimental=@DEFAULTEXPFEATURES@
 # (default: false)
 # enable_pprof = true
 
+# Sealed secret feature configuration.
+# If true, sealed secret feature will be enabled.
+# (default: false)
+sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
+
 # WARNING: All the options in the following section have not been implemented yet.
 # This section was added as a placeholder. DO NOT USE IT!
 [image]
@@ -441,7 +446,13 @@ experimental=@DEFAULTEXPFEATURES@
 #
 # Offload the CRI image management service to the Kata agent.
 # (default: false)
-#service_offload = true
+service_offload = @DEFSERVICEOFFLOAD@
+
+# Image request timeout in seconds.
+# If specified, indicates the image request timeout in the guest needed for the workload(s)
+# If unspecified then it will be set @DEFIMAGEREQUESTTIMEOUT@ second(s) 
+# to reduce image pull failures caused by network problems, and quickly obtain request failure information at the same time.
+image_request_timeout = @DEFIMAGEREQUESTTIMEOUT@
 
 # Container image decryption keys provisioning.
 # Applies only if service_offload is true.

src/runtime/config/configuration-qemu-nvidia-gpu.toml.in

@@ -669,6 +669,11 @@ experimental=@DEFAULTEXPFEATURES@
 # (default: false)
 # enable_pprof = true
 
+# Sealed secret feature configuration.
+# If true, sealed secret feature will be enabled.
+# (default: false)
+sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
+
 # WARNING: All the options in the following section have not been implemented yet.
 # This section was added as a placeholder. DO NOT USE IT!
 [image]
@@ -678,6 +683,12 @@ experimental=@DEFAULTEXPFEATURES@
 # (default: false)
 #service_offload = true
 
+# Image request timeout in seconds.
+# If specified, indicates the image request timeout in the guest needed for the workload(s)
+# If unspecified then it will be set @DEFIMAGEREQUESTTIMEOUT@ second(s) 
+# to reduce image pull failures caused by network problems, and quickly obtain request failure information at the same time.
+image_request_timeout = @DEFIMAGEREQUESTTIMEOUT@
+
 # Container image decryption keys provisioning.
 # Applies only if service_offload is true.
 # Keys can be provisioned locally (e.g. through a special command or

src/runtime/config/configuration-qemu-se.toml.in

@@ -0,0 +1,684 @@
+# Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_QEMU_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.qemu]
+path = "@QEMUPATH@"
+kernel = "@KERNELSEPATH@"
+machine_type = "@MACHINETYPE@"
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+confidential_guest = true
+
+# Enable running QEMU VMM as a non-root user.
+# By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
+# a non-root random user. See documentation for the limitations of this mode.
+# rootless = true
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+enable_annotations = @DEFENABLEANNOTATIONS@
+
+# List of valid annotations values for the hypervisor
+# Each member of the list is a path pattern as described by glob(3).
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWAREPATH@"
+
+# Path to the firmware volume.
+# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables
+# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables
+# can be customized per each user while UEFI code is kept same.
+firmware_volume = "@FIRMWAREVOLUMEPATH@"
+
+# Machine accelerators
+# comma-separated list of machine accelerators to pass to the hypervisor.
+# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
+machine_accelerators="@MACHINEACCELERATORS@"
+
+# Qemu seccomp sandbox feature
+# comma-separated list of seccomp sandbox features to control the syscall access.
+# For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"`
+# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox
+# Another note: enabling this feature may reduce performance, you may enable
+# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html
+#seccompsandbox="@DEFSECCOMPSANDBOXPARAM@"
+
+# CPU features
+# comma-separated list of cpu features to pass to the cpu
+# For example, `cpu_features = "pmu=off,vmx=off"
+cpu_features="@CPUFEATURES@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+#memory_slots = @DEFMEMSLOTS@
+
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM devie.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+#memory_offset = 0
+
+# Specifies virtio-mem will be enabled or not.
+# Please note that this option should be used with the command
+# "echo 1 > /proc/sys/vm/overcommit_memory".
+# Default false
+#enable_virtio_mem = true
+
+# Disable block device from being used for a container's rootfs.
+# In case of a storage driver like devicemapper where a container's
+# root file system is backed by a block device, the block device is passed
+# directly to the hypervisor for performance reasons.
+# This flag prevents the block device from being passed to the hypervisor,
+# virtio-fs is used instead to pass the rootfs.
+disable_block_device_use = @DEFDISABLEBLOCK@
+
+# Shared file system type:
+#   - virtio-fs (default)
+#   - virtio-9p
+#   - virtio-fs-nydus
+shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# List of valid annotations values for the virtiofs daemon
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@
+valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Default size of virtqueues
+virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@
+
+# Extra args for virtiofsd daemon
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+# Examples:
+#   Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"]
+#
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - never
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@"
+
+# aio is the I/O mechanism used by qemu
+# Options:
+#
+#   - threads
+#     Pthread based disk I/O.
+#
+#   - native
+#     Native Linux I/O.
+#
+#   - io_uring
+#     Linux io_uring API. This provides the fastest I/O operations on Linux, requires kernel>5.1 and
+#     qemu >=5.0.
+block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
+
+# Specifies cache-related options will be set to block devices or not.
+# Default false
+#block_device_cache_set = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
+# Default false
+#block_device_cache_direct = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether flush requests for the device are ignored.
+# Default false
+#block_device_cache_noflush = true
+
+# Enable iothreads (data-plane) to be used. This causes IO to be
+# handled in a separate IO thread. This is currently only implemented
+# for SCSI.
+#
+enable_iothreads = @DEFENABLEIOTHREADS@
+
+# Enable pre allocation of VM RAM, default false
+# Enabling this will result in lower container density
+# as all of the memory will be allocated and locked
+# This is useful when you want to reserve all the memory
+# upfront or in the cases where you want memory latencies
+# to be very predictable
+# Default false
+#enable_mem_prealloc = true
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+# This is useful when you want to use vhost-user network
+# stacks within the container. This will automatically
+# result in memory pre allocation
+#enable_hugepages = true
+
+# Enable vhost-user storage device, default false
+# Enabling this will result in some Linux reserved block type
+# major range 240-254 being chosen to represent vhost-user devices.
+enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
+
+# The base directory specifically used for vhost-user devices.
+# Its sub-path "block" is used for block devices; "block/sockets" is
+# where we expect vhost-user sockets to live; "block/devices" is where
+# simulated block device nodes for vhost-user devices to live.
+vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
+
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: intel_iommu=on,iommu=pt
+#enable_iommu = true
+
+# Enable IOMMU_PLATFORM, default false
+# Enabling this will result in the VM device having iommu_platform=on set
+#enable_iommu_platform = true
+
+# List of valid annotations values for the vhost user store path
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@
+valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@
+
+# Enable file based guest memory support. The default is an empty string which
+# will disable this feature. In the case of virtio-fs, this is enabled
+# automatically and '/dev/shm' is used as the backing folder.
+# This option will be ignored if VM templating is enabled.
+#file_mem_backend = "@DEFFILEMEMBACKEND@"
+
+# List of valid annotations values for the file_mem_backend annotation
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDFILEMEMBACKENDS@
+valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
+
+# -pflash can add image file to VM. The arguments of it should be in format
+# of ["/path/to/flash0.img", "/path/to/flash1.img"]
+pflashes = []
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available.
+#
+# Default false
+#enable_debug = true
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+#
+#disable_nesting_checks = true
+
+# This is the msize used for 9p shares. It is the number of bytes
+# used for 9p packet payload.
+#msize_9p = @DEFMSIZE9P@
+
+# If false and nvdimm is supported, use nvdimm device to plug guest image.
+# Otherwise virtio-block device is used.
+#
+# nvdimm is not supported when `confidential_guest = true`.
+#
+# Default is false
+#disable_image_nvdimm = true
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hotplugging on root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hotplugging on
+# a bridge.
+# Default false
+#hotplug_vfio_on_root_bus = true
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+#pcie_root_port = 2
+
+# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
+# security (vhost-net runs ring0) for network I/O performance.
+#disable_vhost_net = true
+
+#
+# Default entropy source.
+# The path to a host source of entropy (including a real hardware RNG)
+# /dev/urandom and /dev/random are two main options.
+# Be aware that /dev/random is a blocking source of entropy.  If the host
+# runs out of entropy, the VMs boot time will increase leading to get startup
+# timeouts.
+# The source of entropy /dev/urandom is non-blocking and provides a
+# generally acceptable source of entropy. It should work well for pretty much
+# all practical purposes.
+#entropy_source= "@DEFENTROPYSOURCE@"
+
+# List of valid annotations values for entropy_source
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+#
+# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic.
+# Default 0-sized value means unlimited rate.
+#rx_rate_limiter_max_rate = 0
+# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block)
+# to discipline traffic.
+# Default 0-sized value means unlimited rate.
+#tx_rate_limiter_max_rate = 0
+
+# Set where to save the guest memory dump file.
+# If set, when GUEST_PANICKED event occurred,
+# guest memeory will be dumped to host filesystem under guest_memory_dump_path,
+# This directory will be created automatically if it does not exist.
+#
+# The dumped file(also called vmcore) can be processed with crash or gdb.
+#
+# WARNING:
+#   Dump guest’s memory can take very long depending on the amount of guest memory
+#   and use much disk space.
+#guest_memory_dump_path="/var/crash/kata"
+
+# If enable paging.
+# Basically, if you want to use "gdb" rather than "crash",
+# or need the guest-virtual addresses in the ELF vmcore,
+# then you should enable paging.
+#
+# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details
+#guest_memory_dump_paging=false
+
+# Enable swap in the guest. Default false.
+# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device
+# if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness")
+# is bigger than 0.
+# The size of the swap device should be
+# swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes.
+# If swap_in_bytes is not set, the size should be memory_limit_in_bytes.
+# If swap_in_bytes and memory_limit_in_bytes is not set, the size should
+# be default_memory.
+#enable_guest_swap = true
+
+# use legacy serial for guest console if available and implemented for architecture. Default false
+#use_legacy_serial = true
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+disable_guest_selinux=@DEFDISABLEGUESTSELINUX@
+
+
+[factory]
+# VM templating support. Once enabled, new VMs are created from template
+# using vm cloning. They will share the same initial kernel, initramfs and
+# agent memory by mapping it readonly. It helps speeding up new container
+# creation and saves a lot of memory if there are many kata containers running
+# on the same host.
+#
+# When disabled, new VMs are created from scratch.
+#
+# Note: Requires "initrd=" to be set ("image=" is not supported).
+#
+# Default false
+#enable_template = true
+
+# Specifies the path of template.
+#
+# Default "/run/vc/vm/template"
+#template_path = "/run/vc/vm/template"
+
+# The number of caches of VMCache:
+# unspecified or == 0   --> VMCache is disabled
+# > 0                   --> will be set to the specified number
+#
+# VMCache is a function that creates VMs as caches before using it.
+# It helps speed up new container creation.
+# The function consists of a server and some clients communicating
+# through Unix socket.  The protocol is gRPC in protocols/cache/cache.proto.
+# The VMCache server will create some VMs and cache them by factory cache.
+# It will convert the VM to gRPC format and transport it when gets
+# requestion from clients.
+# Factory grpccache is the VMCache client.  It will request gRPC format
+# VM and convert it back to a VM.  If VMCache function is enabled,
+# kata-runtime will request VM from factory grpccache when it creates
+# a new sandbox.
+#
+# Default 0
+#vm_cache_number = 0
+
+# Specify the address of the Unix socket that is used by VMCache.
+#
+# Default /var/run/kata-containers/cache.sock
+#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+#enable_tracing = true
+
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_QEMU@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# vCPUs pinning settings
+# if enabled, each vCPU thread will be scheduled to a fixed CPU
+# qualified condition: num(vCPU threads) == num(CPUs in sandbox's CPUSet)
+# enable_vcpus_pinning = false
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts=@DEFBINDMOUNTS@
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true
+
+# Sealed secret feature configuration.
+# If true, sealed secret feature will be enabled.
+# (default: false)
+sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
+
+# WARNING: All the options in the following section have not been implemented yet.
+# This section was added as a placeholder. DO NOT USE IT!
+[image]
+# Container image service.
+#
+# Offload the CRI image management service to the Kata agent.
+# (default: false)
+service_offload = @DEFSERVICEOFFLOAD@
+
+# Image request timeout in seconds.
+# If specified, indicates the image request timeout in the guest needed for the workload(s)
+# If unspecified then it will be set @DEFIMAGEREQUESTTIMEOUT@ second(s) 
+# to reduce image pull failures caused by network problems and quickly obtain request failure information at the same time.
+image_request_timeout = @DEFIMAGEREQUESTTIMEOUT@
+
+# Container image decryption keys provisioning.
+# Applies only if service_offload is true.
+# Keys can be provisioned locally (e.g. through a special command or
+# a local file) or remotely (usually after the guest is remotely attested).
+# The provision setting is a complete URL that lets the Kata agent decide
+# which method to use in order to fetch the keys.
+#
+# Keys can be stored in a local file, in a measured and attested initrd:
+#provision=data:///local/key/file
+#
+# Keys could be fetched through a special command or binary from the
+# initrd (guest) image, e.g. a firmware call:
+#provision=file:///path/to/bin/fetcher/in/guest
+#
+# Keys can be remotely provisioned. The Kata agent fetches them from e.g.
+# a HTTPS URL:
+#provision=https://my-key-broker.foo/tenant/<tenant-id>

src/runtime/config/configuration-qemu-sev.toml.in

@@ -32,6 +32,28 @@ machine_type = "@MACHINETYPE@"
 # Default false
 confidential_guest = true
 
+# Enable pre-attestation for AMD SEV and SEV-ES guests.
+# Applies only if confidential_guest is true.
+# (default: false)
+guest_pre_attestation = true
+
+# A remote key broker server (KBS) is required to validate the attestation
+# measurement and inject a secret key.  
+# See https://github.com/confidential-containers/simple-kbs
+guest_pre_attestation_kbs_uri = "@DEFGUESTPREATTESTATIONPROXY@"
+
+# The simple-kbs can be run in "online" and "offline" modes
+guest_pre_attestation_kbs_mode="@DEFGUESTPREATTESTATIONMODE@" 
+
+# Keyset ID for injected secrets (offline kbs)
+guest_pre_attestation_keyset = "@DEFGUESTPREATTESTATIONKEYSET@"
+
+# SEV guest policy
+sev_guest_policy = @DEFSEVGUESTPOLICY@
+
+# SEV certificate chain path 
+sev_cert_chain = "@DEFSEVCERTCHAIN@"
+
 # Enable running QEMU VMM as a non-root user.
 # By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
 # a non-root random user. See documentation for the limitations of this mode.
@@ -58,7 +80,7 @@ valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
 # may stop the virtual machine from booting.
 # To see the list of default parameters, enable hypervisor debug, create a
 # container and look for 'default-kernel-parameters' log entries.
-kernel_params = "@KERNELPARAMS@"
+kernel_params = "@SEVKERNELPARAMS@"
 
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
@@ -627,6 +649,11 @@ experimental=@DEFAULTEXPFEATURES@
 # (default: false)
 # enable_pprof = true
 
+# Sealed secret feature configuration.
+# If true, sealed secret feature will be enabled.
+# (default: false)
+sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
+
 # WARNING: All the options in the following section have not been implemented yet.
 # This section was added as a placeholder. DO NOT USE IT!
 [image]
@@ -636,6 +663,12 @@ experimental=@DEFAULTEXPFEATURES@
 # (default: false)
 service_offload = @DEFSERVICEOFFLOAD@
 
+# Image request timeout in seconds.
+# If specified, indicates the image request timeout in the guest needed for the workload(s)
+# If unspecified then it will be set @DEFIMAGEREQUESTTIMEOUT@ second(s) 
+# to reduce image pull failures caused by network problems and quickly obtain request failure information at the same time.
+image_request_timeout = @DEFIMAGEREQUESTTIMEOUT@
+
 # Container image decryption keys provisioning.
 # Applies only if service_offload is true.
 # Keys can be provisioned locally (e.g. through a special command or

src/runtime/config/configuration-qemu-snp.toml.in

@@ -1,5 +1,6 @@
 # Copyright (c) 2017-2019 Intel Corporation
 # Copyright (c) 2021 Adobe Inc.
+# Copyright 2022 Advanced Micro Devices, Inc.
 # Copyright (c) 2023 IBM Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -16,7 +17,7 @@
 path = "@QEMUSNPPATH@"
 kernel = "@KERNELSNPPATH@"
 #image = "@IMAGEPATH@"
-initrd = "@INITRDPATH@"
+initrd = "@INITRDSEVPATH@"
 machine_type = "@MACHINETYPE@"
 
 # rootfs filesystem type:
@@ -42,7 +43,13 @@ rootfs_type=@DEFROOTFSTYPE@
 confidential_guest = true
 
 # enable SEV SNP VMs
-sev_snp_guest = true
+sev_snp_guest = @DEFSNPGUEST@
+
+# SNP guest policy
+# Based on SEV Secure Nested Paging Firmware ABI Specification section 4.3
+# If it is unspecified or 0, it will default to 0x30000 (i.e. Bit#17 is '1' which is reserved and Bit#16 is '1' which means SMT is allowed).
+snp_guest_policy=@DEFSNPGUESTPOLICY@
+
 
 # Enable running QEMU VMM as a non-root user.
 # By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
@@ -70,7 +77,8 @@ valid_hypervisor_paths = @QEMUSNPVALIDHYPERVISORPATHS@
 # may stop the virtual machine from booting.
 # To see the list of default parameters, enable hypervisor debug, create a
 # container and look for 'default-kernel-parameters' log entries.
-kernel_params = "@KERNELPARAMS@"
+
+kernel_params = "@SNPKERNELPARAMS@"
 
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
@@ -80,7 +88,7 @@ firmware = "@FIRMWARESNPPATH@"
 # firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables
 # as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables
 # can be customized per each user while UEFI code is kept same.
-firmware_volume = "@FIRMWARETDVFVOLUMEPATH@"
+firmware_volume = "@FIRMWAREVOLUMEPATH@"
 
 # Machine accelerators
 # comma-separated list of machine accelerators to pass to the hypervisor.
@@ -333,8 +341,8 @@ pflashes = []
 # Disable the customizations done in the runtime when it detects
 # that it is running on top a VMM. This will result in the runtime
 # behaving as it would when running on bare metal.
-#
-#disable_nesting_checks = true
+# Needed for nested-SNP on hyper-V
+disable_nesting_checks = true
 
 # This is the msize used for 9p shares. It is the number of bytes
 # used for 9p packet payload.
@@ -454,7 +462,6 @@ disable_selinux=@DEFDISABLESELINUX@
 # (default: true)
 disable_guest_selinux=@DEFDISABLEGUESTSELINUX@
 
-
 [factory]
 # VM templating support. Once enabled, new VMs are created from template
 # using vm cloning. They will share the same initial kernel, initramfs and
@@ -667,6 +674,11 @@ experimental=@DEFAULTEXPFEATURES@
 # (default: false)
 # enable_pprof = true
 
+# Sealed secret feature configuration.
+# If true, sealed secret feature will be enabled.
+# (default: false)
+sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
+
 # WARNING: All the options in the following section have not been implemented yet.
 # This section was added as a placeholder. DO NOT USE IT!
 [image]
@@ -674,7 +686,13 @@ experimental=@DEFAULTEXPFEATURES@
 #
 # Offload the CRI image management service to the Kata agent.
 # (default: false)
-#service_offload = true
+service_offload = @DEFSERVICEOFFLOAD@
+
+# Image request timeout in seconds.
+# If specified, indicates the image request timeout in the guest needed for the workload(s)
+# If unspecified then it will be set @DEFIMAGEREQUESTTIMEOUT@ second(s) 
+# to reduce image pull failures caused by network problems and quickly obtain request failure information at the same time.
+image_request_timeout = @DEFIMAGEREQUESTTIMEOUT@
 
 # Container image decryption keys provisioning.
 # Applies only if service_offload is true.

src/runtime/config/configuration-qemu-tdx.toml.in

@@ -14,7 +14,7 @@
 [hypervisor.qemu]
 path = "@QEMUTDXPATH@"
 kernel = "@KERNELTDXPATH@"
-image = "@IMAGEPATH@"
+image = "@IMAGETDXPATH@"
 # initrd = "@INITRDPATH@"
 machine_type = "@MACHINETYPE@"
 
@@ -450,7 +450,6 @@ disable_selinux=@DEFDISABLESELINUX@
 # (default: true)
 disable_guest_selinux=@DEFDISABLEGUESTSELINUX@
 
-
 [factory]
 # VM templating support. Once enabled, new VMs are created from template
 # using vm cloning. They will share the same initial kernel, initramfs and
@@ -663,6 +662,11 @@ experimental=@DEFAULTEXPFEATURES@
 # (default: false)
 # enable_pprof = true
 
+# Sealed secret feature configuration.
+# If true, sealed secret feature will be enabled.
+# (default: false)
+sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
+
 # WARNING: All the options in the following section have not been implemented yet.
 # This section was added as a placeholder. DO NOT USE IT!
 [image]
@@ -670,7 +674,13 @@ experimental=@DEFAULTEXPFEATURES@
 #
 # Offload the CRI image management service to the Kata agent.
 # (default: false)
-#service_offload = true
+service_offload = true
+
+# Image request timeout in seconds.
+# If specified, indicates the image request timeout in the guest needed for the workload(s)
+# If unspecified then it will be set @DEFIMAGEREQUESTTIMEOUT@ second(s) 
+# to reduce image pull failures caused by network problems and quickly obtain request failure information at the same time.
+image_request_timeout = @DEFIMAGEREQUESTTIMEOUT@
 
 # Container image decryption keys provisioning.
 # Applies only if service_offload is true.

src/runtime/config/configuration-qemu.toml.in

@@ -40,12 +40,40 @@ rootfs_type=@DEFROOTFSTYPE@
 # Default false
 # confidential_guest = true
 
+# Enable pre-attestation for AMD SEV and SEV-ES guests.
+# Applies only if confidential_guest is true.
+# (default: false)
+#guest_pre_attestation = true
+#
+# A remote key broker server (KBS) is required to validate the attestation
+# measurement and inject a secret key.  
+# See https://github.com/confidential-containers/simple-kbs
+#guest_pre_attestation_kbs_uri = "@DEFGUESTPREATTESTATIONPROXY@"
+#
+# The simple-kbs can be run in "online" and "offline" modes
+#guest_pre_attestation_kbs_mode="@DEFGUESTPREATTESTATIONMODE@" 
+#
+# Keyset ID of the injected secret (offline kbs)
+#guest_pre_attestation_keyset = "@DEFGUESTPREATTESTATIONKEYSET@"
+#
+# SEV guest policy
+#sev_guest_policy = @DEFSEVGUESTPOLICY@
+
+# SEV certificate chain path 
+#sev_cert_chain="@DEFSEVCERTCHAIN@"
+
 # Choose AMD SEV-SNP confidential guests
 # In case of using confidential guests on AMD hardware that supports both SEV
 # and SEV-SNP, the following enables SEV-SNP guests. SEV guests are default.
 # Default false
 # sev_snp_guest = true
 
+# SNP guest policy
+# Refer SEV Secure Nested Paging Firmware ABI Specification (sectin 4.3) to
+# change this value.
+# unspecified or == 0  -->  0x30000 i.e. Bit#17 is '1' and Bit#16 is '1' (SMT is allowed)
+# snp_guest_policy=0x30000
+
 # Enable running QEMU VMM as a non-root user.
 # By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
 # a non-root random user. See documentation for the limitations of this mode.
@@ -681,6 +709,11 @@ experimental=@DEFAULTEXPFEATURES@
 # (default: false)
 # enable_pprof = true
 
+# Sealed secret feature configuration.
+# If true, sealed secret feature will be enabled.
+# (default: false)
+sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
+
 # WARNING: All the options in the following section have not been implemented yet.
 # This section was added as a placeholder. DO NOT USE IT!
 [image]
@@ -688,7 +721,13 @@ experimental=@DEFAULTEXPFEATURES@
 #
 # Offload the CRI image management service to the Kata agent.
 # (default: false)
-#service_offload = true
+service_offload = @DEFSERVICEOFFLOAD@
+
+# Image request timeout in seconds.
+# If specified, indicates the image request timeout in the guest needed for the workload(s)
+# If unspecified then it will be set @DEFIMAGEREQUESTTIMEOUT@ second(s) 
+# to reduce image pull failures caused by network problems and quickly obtain request failure information at the same time.
+image_request_timeout = @DEFIMAGEREQUESTTIMEOUT@
 
 # Container image decryption keys provisioning.
 # Applies only if service_offload is true.

src/runtime/config/configuration-remote.toml.in

@@ -0,0 +1,329 @@
+# Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2023 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_REMOTE_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+
+[hypervisor.remote]
+remote_hypervisor_socket = "/run/peerpod/hypervisor.sock"
+remote_hypervisor_timeout = 600
+
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+# confidential_guest = true
+
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+# Note: Remote hypervisor is only handling the following annotations
+enable_annotations = ["machine_type", "default_memory", "default_vcpus"]
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+# NOTE: kernel_params are not currently passed over in remote hypervisor
+# kernel_params = ""
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWAREPATH@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+# default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+# default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+# default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+#memory_slots = @DEFMEMSLOTS@
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. And Debug also enable the hmp socket.
+#
+# Default false
+#enable_debug = true
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+# Note: The remote hypervisor has a different guest, so currently requires this to be disabled
+disable_guest_selinux = true
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+#enable_tracing = true
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+# Note: The remote hypervisor, uses it's own network, so "none" is required
+internetworking_model="none"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+# Note: The remote hypervisor has a different guest, so currently requires this to be set to true
+disable_guest_seccomp=true
+
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+# Note: The remote hypervisor has a different networking model, which requires true 
+disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+# Note: the remote hypervisor uses the peer pod config to determine the sandbox size, so requires this to be set to true
+static_sandbox_resource_mgmt=true
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+# Note: remote hypervisor has no sharing of emptydir mounts from host to guest
+disable_guest_empty_dir=false
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true
+
+# Sealed secret feature configuration.
+# If true, sealed secret feature will be enabled.
+# (default: false)
+sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
+
+# WARNING: All the options in the following section have not been implemented yet.
+# This section was added as a placeholder. DO NOT USE IT!
+[image]
+# Container image service.
+#
+# Offload the CRI image management service to the Kata agent.
+# (default: false)
+# Note: The remote hypervisor offloads the pulling on images on the peer pod VM, so requries this to be true
+service_offload = true
+
+# Image request timeout in seconds.
+# If specified, indicates the image request timeout in the guest needed for the workload(s)
+# If unspecified then it will be set @DEFIMAGEREQUESTTIMEOUT@ second(s) 
+# to reduce image pull failures caused by network problems and quickly obtain request failure information at the same time.
+image_request_timeout = @DEFIMAGEREQUESTTIMEOUT@
+
+# Container image decryption keys provisioning.
+# Applies only if service_offload is true.
+# Keys can be provisioned locally (e.g. through a special command or
+# a local file) or remotely (usually after the guest is remotely attested).
+# The provision setting is a complete URL that lets the Kata agent decide
+# which method to use in order to fetch the keys.
+#
+# Keys can be stored in a local file, in a measured and attested initrd:
+#provision=data:///local/key/file
+#
+# Keys could be fetched through a special command or binary from the
+# initrd (guest) image, e.g. a firmware call:
+#provision=file:///path/to/bin/fetcher/in/guest
+#
+# Keys can be remotely provisioned. The Kata agent fetches them from e.g.
+# a HTTPS URL:
+#provision=https://my-key-broker.foo/tenant/<tenant-id>

src/runtime/go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/BurntSushi/toml v1.2.0
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/blang/semver/v4 v4.0.0
+	github.com/container-orchestrated-devices/container-device-interface v0.6.0
 	github.com/containerd/cgroups v1.0.5-0.20220625035431-cf7417bca682
 	github.com/containerd/console v1.0.3
 	github.com/containerd/containerd v1.6.8
@@ -31,7 +32,7 @@ require (
 	github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9
 	github.com/mdlayher/vsock v1.1.0
 	github.com/opencontainers/runc v1.1.3
-	github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab
+	github.com/opencontainers/runtime-spec v1.1.0-rc.1
 	github.com/opencontainers/selinux v1.10.1
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/pkg/errors v0.9.1
@@ -51,8 +52,9 @@ require (
 	go.opentelemetry.io/otel/sdk v1.3.0
 	go.opentelemetry.io/otel/trace v1.3.0
 	golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f
+	golang.org/x/sys v0.1.0
 	google.golang.org/grpc v1.47.0
+	google.golang.org/protobuf v1.28.1
 	k8s.io/apimachinery v0.22.5
 	k8s.io/cri-api v0.23.1
 )
@@ -93,23 +95,27 @@ require (
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.8.1-0.20210923151022-86f73c517451 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	go.mongodb.org/mongo-driver v1.7.5 // indirect
 	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace (
+	github.com/containerd/containerd => github.com/confidential-containers/containerd v1.6.7-0.20221123142530-25f68aa818ec
 	github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.2
 	github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.3
 	github.com/uber-go/atomic => go.uber.org/atomic v1.5.1

src/runtime/go.sum

@@ -64,30 +64,23 @@ code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5/go.mod h1:v4VVB
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg=
-github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v66.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
 github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
 github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc=
 github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U=
-github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
 github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
 github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
 github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg=
 github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU=
 github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
 github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E=
-github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -101,35 +94,18 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/Masterminds/semver/v3 v3.0.3/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
-github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
-github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
-github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
-github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
-github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
-github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
-github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
-github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
-github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=
-github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=
-github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00=
-github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
 github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
-github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
 github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0=
-github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
 github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
 github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
 github.com/Microsoft/hcsshim v0.9.4 h1:mnUj0ivWy6UzbB1uLFqKR6F+ZyiDc7j4iGgHTpO+5+I=
 github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
-github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
-github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -142,7 +118,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/ReneKroon/ttlcache/v2 v2.11.0/go.mod h1:mBxvsNY+BT8qLLd6CuAJubbKo6r0jh3nb5et22bbfGY=
-github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -150,7 +125,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -166,7 +140,6 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:o
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
-github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.44.44/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48=
 github.com/aws/aws-sdk-go-v2/config v1.15.11/go.mod h1:mD5tNFciV7YHNjPpFYqJ6KGpoSfY107oZULvTHIxtbI=
@@ -182,30 +155,21 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.7/go.mod h1:lVxTdiiSHY3jb1aeg+BBF
 github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/beeker1121/goque v1.0.3-0.20191103205551-d618510128af/go.mod h1:84CWnaDz4g1tEVnFLnuBigmGK15oPohy0RfvSN8d4eg=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
-github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
-github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
-github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
 github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
-github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
-github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
-github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
-github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
 github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
@@ -223,9 +187,6 @@ github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAc
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
-github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
-github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k=
 github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
@@ -242,127 +203,54 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
-github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
+github.com/confidential-containers/containerd v1.6.7-0.20221123142530-25f68aa818ec h1:eFjq9ReTZ77/j29LoI68rW+utQ/oglOS9liDQAO8aCI=
+github.com/confidential-containers/containerd v1.6.7-0.20221123142530-25f68aa818ec/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0=
 github.com/container-orchestrated-devices/container-device-interface v0.4.0/go.mod h1:E1zcucIkq9P3eyNmY+68dBQsTcsXJh9cgRo2IVNScKQ=
-github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
-github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
-github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
+github.com/container-orchestrated-devices/container-device-interface v0.6.0 h1:aWwcz/Ep0Fd7ZuBjQGjU/jdPloM7ydhMW13h85jZNvk=
+github.com/container-orchestrated-devices/container-device-interface v0.6.0/go.mod h1:OQlgtJtDrOxSQ1BWODC8OZK1tzi9W69wek+Jy17ndzo=
 github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
-github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E=
-github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
 github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
-github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI=
-github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
-github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
-github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
-github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
-github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
 github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
 github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8=
 github.com/containerd/cgroups v1.0.5-0.20220625035431-cf7417bca682 h1:d/YjAAP6A6fT0vpMhbYSDkE+K1ww/DZodOIamD8Pr/E=
 github.com/containerd/cgroups v1.0.5-0.20220625035431-cf7417bca682/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA=
-github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
-github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
-github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
 github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
 github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
 github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ=
-github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
-github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
-github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
-github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
-github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
-github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
-github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ=
-github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE=
-github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0=
-github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs=
-github.com/containerd/containerd v1.6.8/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0=
-github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
-github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
 github.com/containerd/continuity v0.2.2 h1:QSqfxcn8c+12slxwu00AtzXrsami0MJb/MQs9lOLHLA=
 github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk=
 github.com/containerd/cri-containerd v1.19.0 h1:PcTvvl+SHaekCMQZFQkYjn1RKlYrK6khYbuhOeF68k0=
 github.com/containerd/cri-containerd v1.19.0/go.mod h1:wxbGdReWGCalzGOEpifoHeYCK4xAgnj4o/4bVB+9voU=
-github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
-github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
-github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
-github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
-github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
 github.com/containerd/fifo v1.0.0 h1:6PirWBr9/L7GDamKr+XM0IeUFXu5mf3M/BPpH9gaLBU=
 github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
-github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU=
-github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=
-github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
 github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
 github.com/containerd/go-cni v1.1.6/go.mod h1:BWtoWl5ghVymxu6MBjg79W9NZrCRyHIdUtk4cauMe34=
-github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
-github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
-github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g=
-github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
 github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0=
 github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
-github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0=
-github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA=
-github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow=
-github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
-github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4=
 github.com/containerd/imgcrypt v1.1.4/go.mod h1:LorQnPtzL/T0IyCeftcsMEO7AqxUDbdO8j/tSUpgxvo=
-github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
-github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM=
 github.com/containerd/stargz-snapshotter/estargz v0.9.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0=
 github.com/containerd/stargz-snapshotter/estargz v0.10.1/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0=
 github.com/containerd/stargz-snapshotter/estargz v0.11.4/go.mod h1:7vRJIcImfY8bpifnMjt+HTJoQxASq7T28MYbP15/Nf0=
 github.com/containerd/stargz-snapshotter/estargz v0.12.0/go.mod h1:AIQ59TewBFJ4GOPEQXujcrJ/EKxh5xXZegW1rkR1P/M=
-github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
-github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
-github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
-github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
 github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
 github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI=
 github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
-github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
-github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk=
-github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
 github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
 github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
-github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw=
-github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y=
-github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
-github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
 github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
-github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y=
 github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
 github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=
 github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
-github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM=
-github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
-github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE=
 github.com/containernetworking/plugins v1.1.1 h1:+AGfFigZ5TiQH00vhR8qPeSatj53eNGz0C1d3wVYlHE=
 github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8=
 github.com/containers/buildah v1.27.0/go.mod h1:anH3ExvDXRNP9zLQCrOc1vWb5CrhqLF/aYFim4tslvA=
@@ -370,10 +258,6 @@ github.com/containers/common v0.49.1/go.mod h1:ueM5hT0itKqCQvVJDs+EtjornAQtrHYxQ
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
 github.com/containers/image/v5 v5.22.0/go.mod h1:D8Ksv2RNB8qLJ7xe1P3rgJJOSQpahA6amv2Ax++/YO4=
 github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
-github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
-github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=
-github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
-github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
 github.com/containers/ocicrypt v1.1.3/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g=
 github.com/containers/ocicrypt v1.1.5/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc=
 github.com/containers/podman/v4 v4.2.0 h1:mqQ0CtdSOTfsl6IEcO0UiA/Yi/9Yxoe/mSEC4h21CK8=
@@ -385,22 +269,16 @@ github.com/containers/storage v1.42.0/go.mod h1:JiUJwOgOo1dr2DdOUc1MRe2GCAXABYoY
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
-github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
 github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-oidc/v3 v3.2.0/go.mod h1:rEJ/idjfUyfkBit1eI1fvyr+64/g9dcKpAm8MJMesvo=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -416,13 +294,10 @@ github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
-github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
 github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
-github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
@@ -430,11 +305,8 @@ github.com/digitalocean/go-libvirt v0.0.0-20201209184759-e2a69bcd5bd1/go.mod h1:
 github.com/digitalocean/go-qemu v0.0.0-20210326154740-ac9e0b687001/go.mod h1:IetBE52JfFxK46p2n2Rqm+p5Gx1gpu2hRHsrbnPOWZQ=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/disiqueira/gotree/v3 v3.0.2/go.mod h1:ZuyjE4+mUQZlbpkI24AmruZKhg3VHEgPLDY8Qk+uUu8=
-github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/cli v20.10.16+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
-github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
@@ -445,22 +317,17 @@ github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avu
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11/go.mod h1:a6bNUGTbQBsY6VRHTr4h/rkOXjl244DyRD0tx3fgq4Q=
-github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
 github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=
 github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
-github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
 github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
 github.com/docker/go-plugins-helpers v0.0.0-20211224144127-6eecb7beb651/go.mod h1:LFyLie6XcDbyKGeVK6bHe+9aJTYCxWLBg5IrJZOaXKA=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
-github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
-github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dtylman/scp v0.0.0-20181017070807-f3000a34aef4/go.mod h1:jN1ZaUPSNA8jm10nmaRLky84qV/iCeiHmcEf3EbP+dc=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
@@ -476,7 +343,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.5.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA=
@@ -507,8 +373,6 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/fsouza/go-dockerclient v1.7.7/go.mod h1:njNCXvoZj3sLPjf3yO0DPHf1mdLdCPDYPc14GskKA4Y=
 github.com/fsouza/go-dockerclient v1.8.1/go.mod h1:zmA2ogSxRnXmbZcy0Aq7yhRoCdP/bDns/qghCK9SWtM=
-github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
-github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -521,7 +385,6 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gorp/gorp/v3 v3.0.2/go.mod h1:BJ3q1ejpV8cVALtcXvXaXyTOlMmJhWDxTmncaR6rwBY=
-github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ini/ini v1.28.2 h1:drmmYv7psRpoGZkPtPKKTB+ZFSnvmwCMfNj5o1nLh2Y=
 github.com/go-ini/ini v1.28.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -685,9 +548,6 @@ github.com/gobuffalo/tags/v3 v3.0.2/go.mod h1:ZQeN6TCTiwAFnS0dNcbDtSgZDwNKSpqajv
 github.com/gobuffalo/tags/v3 v3.1.0/go.mod h1:ZQeN6TCTiwAFnS0dNcbDtSgZDwNKSpqajvVtt6mlYpA=
 github.com/gobuffalo/validate/v3 v3.0.0/go.mod h1:HFpjq+AIiA2RHoQnQVTFKF/ZpUPXwyw82LgyDPxQ9r0=
 github.com/gobuffalo/validate/v3 v3.1.0/go.mod h1:HFpjq+AIiA2RHoQnQVTFKF/ZpUPXwyw82LgyDPxQ9r0=
-github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -695,19 +555,15 @@ github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
 github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -813,26 +669,20 @@ github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97Dwqy
 github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
-github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -847,7 +697,6 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
@@ -894,9 +743,6 @@ github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Go
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
@@ -904,8 +750,6 @@ github.com/insomniacslk/dhcp v0.0.0-20220119180841-3c283ff8b7dd/go.mod h1:h+MxyH
 github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9 h1:x9HFDMDCsaxTvC4X3o0ZN6mw99dT/wYnTItGwhBRmg0=
 github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4=
 github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
-github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
-github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
 github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
@@ -940,8 +784,6 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
 github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
-github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo=
@@ -980,7 +822,6 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -1035,7 +876,6 @@ github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHh
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
-github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -1045,7 +885,6 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@@ -1054,10 +893,8 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
@@ -1084,7 +921,6 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
-github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -1105,7 +941,6 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
@@ -1120,9 +955,7 @@ github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdx
 github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
 github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
-github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
 github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs=
-github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/moby/vpnkit v0.5.0/go.mod h1:KyjUrL9cb6ZSNNAUwZfqRjhwwgJ3BJN+kXh0t43WTUQ=
@@ -1140,7 +973,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -1150,16 +982,11 @@ github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQ
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
-github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
@@ -1171,15 +998,12 @@ github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3
 github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY=
 github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
-github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
 github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
 github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
@@ -1188,31 +1012,24 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9
 github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q=
 github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
-github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w=
 github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
-github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab h1:YQZXa3elcHgKXAa2GjVFC9M3JeP7ZPyFD1YByDx/dgQ=
 github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
+github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.1.0-rc.1 h1:wHa9jroFfKGQqFHj0I1fMRKLl0pfj+ynAqBxo3v6u9w=
+github.com/opencontainers/runtime-spec v1.1.0-rc.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-tools v0.0.0-20190417131837-cd1349b7c47e/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
 github.com/opencontainers/runtime-tools v0.9.1-0.20220714195903-17b3287fafb7/go.mod h1:/tgP02fPXGHkU3/qKK1Y0Db4yqNyGm03vLq/mzHzcS4=
-github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
-github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
-github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI=
 github.com/opencontainers/selinux v1.8.5/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo=
 github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
@@ -1229,7 +1046,6 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
-github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@@ -1237,7 +1053,6 @@ github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -1248,7 +1063,6 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr
 github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0=
-github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -1259,13 +1073,11 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr
 github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@@ -1276,16 +1088,13 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9
 github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
@@ -1311,7 +1120,6 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
-github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 h1:ZFfeKAhIQiiOrQaI3/znw0gOmYpO28Tcu1YaqMa/jtQ=
 github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -1331,8 +1139,6 @@ github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sigstore/sigstore v1.3.1-0.20220629021053-b95fc0d626c1/go.mod h1:y83NePRM98MJpbGgBgi54UZduhG0aD7lYngAVCx+i/E=
-github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
-github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -1346,7 +1152,6 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs
 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -1372,7 +1177,6 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
@@ -1380,13 +1184,11 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q
 github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
-github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -1399,8 +1201,7 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/sylabs/sif/v2 v2.7.1/go.mod h1:bBse2nEFd3yHkmq6KmAOFEWQg5LdFYiQUdVcgamxlc8=
-github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
@@ -1411,7 +1212,6 @@ github.com/theupdateframework/go-tuf v0.3.1/go.mod h1:lhHZ3Vt2pdAh15h0Cc6gWdlI+O
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -1422,9 +1222,7 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA=
@@ -1435,14 +1233,12 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
 github.com/vbauerster/mpb/v7 v7.4.2/go.mod h1:UmOiIUI8aPqWXIps0ciik3RKMdzx7+ooQpq+fBcXwBA=
-github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netlink v1.1.1-0.20220115184804-dd687eb2f2d4/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
 github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
-github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
@@ -1451,17 +1247,17 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+
 github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
 github.com/weppos/publicsuffix-go v0.15.1-0.20210807195340-dc689ff0bb59/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
 github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
-github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@@ -1477,9 +1273,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
-github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
-github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
 github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
@@ -1489,10 +1282,7 @@ github.com/zmap/zlint/v3 v3.3.1-0.20211019173530-cb17369b4628/go.mod h1:O+4OXRfN
 gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965 h1:EXE1ZsUqiUWGV5Dw2oTYpXx24ffxj0//yhTB0Ppv+4s=
 gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965/go.mod h1:TBB3sR7/jg4RCThC/cgT4fB8mAbbMO307TycfgeR59w=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
-go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
@@ -1563,9 +1353,7 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 goji.io/v3 v3.0.0/go.mod h1:c02FFnNiVNCDo+DpR2IhBQpM9r5G1BG/MkHNTPUJ13U=
-golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -1576,14 +1364,12 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
@@ -1629,13 +1415,13 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1650,13 +1436,11 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -1676,7 +1460,6 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -1770,10 +1553,7 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1782,15 +1562,12 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1802,7 +1579,6 @@ golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1816,12 +1592,10 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1831,11 +1605,8 @@ golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1853,7 +1624,6 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1893,8 +1663,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1911,12 +1681,10 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2007,7 +1775,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -2057,18 +1824,14 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -2115,20 +1878,16 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
@@ -2137,7 +1896,6 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
@@ -2172,33 +1930,14 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
-k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
-k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8=
 k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs=
-k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
 k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
 k8s.io/apimachinery v0.22.5 h1:cIPwldOYm1Slq9VLBRPtEYpyhjIm1C6aAMAoENuvN9s=
 k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U=
-k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
-k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM=
-k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=
 k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ=
-k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
-k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k=
-k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0=
 k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y=
 k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0=
-k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
-k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI=
-k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=
 k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI=
-k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
-k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
-k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
-k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=
 k8s.io/cri-api v0.23.1 h1:0DHL/hpTf4Fp+QkUXFefWcp1fhjXr9OlNdY9X99c+O8=
 k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
@@ -2211,23 +1950,18 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
-k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
-k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

src/runtime/hack/update-generated-hypervisor-proto.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# (C) Copyright IBM Corp. 2022.
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit -o pipefail -o nounset
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+protoc --gogottrpc_out=protocols/hypervisor \
+	--gogottrpc_opt=plugins=ttrpc+fieldpath,paths=source_relative \
+	-Iprotocols/hypervisor \
+	-I../libs/protocols/protos/gogo/protobuf \
+	protocols/hypervisor/hypervisor.proto

src/runtime/pkg/containerd-shim-v2/create.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	containerd_types "github.com/containerd/containerd/api/types"
 	"github.com/containerd/containerd/mount"
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
@@ -51,6 +52,62 @@ var defaultStartManagementServerFunc startManagementServerFunc = func(s *service
 	shimLog.Info("management server started")
 }
 
+// CDI (Container Device Interface), is a specification, for container- runtimes,
+// to support third-party devices.
+// It introduces an abstract notion of a device as a resource. Such devices are
+// uniquely specified by a fully-qualified name that is constructed from a
+// vendor ID, a device class, and a name that is unique per vendor ID-device
+// class pair.
+//
+// vendor.com/class=unique_name
+//
+// The combination of vendor ID and device class (vendor.com/class in the
+// above example) is referred to as the device kind.
+// CDI concerns itself only with enabling containers to be device aware.
+// Areas like resource management are explicitly left out of CDI (and are
+// expected to be handled by the orchestrator). Because of this focus, the CDI
+// specification is simple to implement and allows great flexibility for
+// runtimes and orchestrators.
+func withCDI(annotations map[string]string, cdiSpecDirs []string, spec *specs.Spec) (*specs.Spec, error) {
+	// Add devices from CDI annotations
+	_, devsFromAnnotations, err := cdi.ParseAnnotations(annotations)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI device annotations: %w", err)
+	}
+	if len(devsFromAnnotations) == 0 {
+		// No devices found, skip device injection
+		return spec, nil
+	}
+
+	var registry cdi.Registry
+	if len(cdiSpecDirs) > 0 {
+		// We can override the directories where to search for CDI specs
+		// if needed, the default is /etc/cdi /var/run/cdi
+		registry = cdi.GetRegistry(cdi.WithSpecDirs(cdiSpecDirs...))
+	} else {
+		registry = cdi.GetRegistry()
+	}
+
+	if err = registry.Refresh(); err != nil {
+		// We don't consider registry refresh failure a fatal error.
+		// For instance, a dynamically generated invalid CDI Spec file for
+		// any particular vendor shouldn't prevent injection of devices of
+		// different vendors. CDI itself knows better and it will fail the
+		// injection if necessary.
+		return nil, fmt.Errorf("CDI registry refresh failed: %w", err)
+	}
+
+	if _, err := registry.InjectDevices(spec, devsFromAnnotations...); err != nil {
+		return nil, fmt.Errorf("CDI device injection failed: %w", err)
+	}
+
+	// One crucial thing to keep in mind is that CDI device injection
+	// might add OCI Spec environment variables, hooks, and mounts as
+	// well. Therefore it is important that none of the corresponding
+	// OCI Spec fields are reset up in the call stack once we return.
+	return spec, nil
+}
+
 func copyLayersToMounts(rootFs *vc.RootFs, spec *specs.Spec) error {
 	for _, o := range rootFs.Options {
 		if !strings.HasPrefix(o, annotations.FileSystemLayer) {
@@ -111,6 +168,16 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 		if s.sandbox != nil {
 			return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID())
 		}
+		// We can provide additional directories where to search for
+		// CDI specs if needed. immutable OS's only have specific
+		// directories where applications can write too. For instance /opt/cdi
+		//
+		// _, err = withCDI(ociSpec.Annotations, []string{"/opt/cdi"}, ociSpec)
+		//
+		_, err = withCDI(ociSpec.Annotations, []string{}, ociSpec)
+		if err != nil {
+			return nil, fmt.Errorf("adding CDI devices failed")
+		}
 
 		s.config = runtimeConfig
 
@@ -291,6 +358,11 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string
 }
 
 func checkAndMount(s *service, r *taskAPI.CreateTaskRequest) (bool, error) {
+	// In the confidential computing, there is no Image information on the host,
+	// so there is no Rootfs.
+	if s.config.ServiceOffload && len(r.Rootfs) == 0 {
+		return false, nil
+	}
 	if len(r.Rootfs) == 1 {
 		m := r.Rootfs[0]
 

src/runtime/pkg/containerd-shim-v2/image.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2021 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"time"
+
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/runtime/v2/shim"
+	"github.com/containerd/containerd/runtime/v2/task"
+	"github.com/containerd/ttrpc"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/image"
+)
+
+func init() {
+	plugin.Register(&plugin.Registration{
+		Type:     plugin.TTRPCPlugin,
+		ID:       "image",
+		Requires: []plugin.Type{"*"},
+		InitFn:   initImageService,
+	})
+}
+
+type ImageService struct {
+	s *service
+}
+
+func initImageService(ic *plugin.InitContext) (interface{}, error) {
+	i, err := ic.GetByID(plugin.TTRPCPlugin, "task")
+	if err != nil {
+		return nil, errors.Errorf("get task plugin error. %v", err)
+	}
+	task := i.(shim.TaskService)
+	s := task.TaskService.(*service)
+	is := &ImageService{s: s}
+	return is, nil
+}
+
+func (is *ImageService) RegisterTTRPC(server *ttrpc.Server) error {
+	task.RegisterImageService(server, is)
+	return nil
+}
+
+// Pull image and unbundle ready for container creation
+func (is *ImageService) PullImage(ctx context.Context, req *task.PullImageRequest) (_ *task.PullImageResponse, err error) {
+	shimLog.WithField("image", req.Image).Debug("PullImage() start")
+	defer shimLog.WithField("image", req.Image).Debug("PullImage() end")
+	span, spanCtx := katatrace.Trace(is.s.rootCtx, shimLog, "PullImage", shimTracingTags)
+	defer span.End()
+
+	start := time.Now()
+	defer func() {
+		err = toGRPC(err)
+		rpcDurationsHistogram.WithLabelValues("pullimage").Observe(float64(time.Since(start).Nanoseconds() / int64(time.Millisecond)))
+	}()
+
+	is.s.mu.Lock()
+	defer is.s.mu.Unlock()
+
+	shimLog.WithFields(logrus.Fields{
+		"image": req.Image,
+	}).Debug("Making image pull request")
+
+	r := &image.PullImageReq{
+		Image: req.Image,
+	}
+
+	resp, err := is.s.sandbox.PullImage(spanCtx, r)
+	if err != nil {
+		shimLog.Errorf("kata runtime PullImage err. %v", err)
+		return nil, err
+	}
+	return &task.PullImageResponse{
+		ImageRef: resp.ImageRef,
+	}, err
+}

src/runtime/pkg/govmm/qemu/qemu.go

@@ -294,11 +294,11 @@ type Object struct {
 	FirmwareVolume string
 
 	// CBitPos is the location of the C-bit in a guest page table entry
-	// This is only relevant for sev-guest objects
+	// This is only relevant for sev-guest and sev-snp-guest objects
 	CBitPos uint32
 
 	// ReducedPhysBits is the reduction in the guest physical address space
-	// This is only relevant for sev-guest objects
+	// This is only relevant for sev-guest and sev-snp-guest objects
 	ReducedPhysBits uint32
 
 	// ReadOnly specifies whether `MemPath` is opened read-only or read/write (default)
@@ -306,6 +306,26 @@ type Object struct {
 
 	// Prealloc enables memory preallocation
 	Prealloc bool
+
+	// SevPolicy is the policy for the SEV instance. For more info, see AMD document 55766
+	// This is only relevant for sev-guest objects
+	SevPolicy uint32
+
+	// SnpPolicy is the policy for the SNP instance. For more info, see AMD document 56860
+	// This is only relevant for sev-snp-guest objects
+	SnpPolicy uint64
+
+	// SevCertFilePath is the path to the guest Diffie–Hellman key
+	// This is only relevant for sev-guest objects
+	SevCertFilePath string
+
+	// SevSessionFilePath is the path to the launch blog
+	// This is only relevant for sev-guest objects
+	SevSessionFilePath string
+
+	// SevKernelHashes specifies whether the hashes of the kernel, initrd, & cmdline are included in the measurement
+	// This is only relevant for sev-guest objects
+	SevKernelHashes bool
 }
 
 // Valid returns true if the Object structure is valid and complete.
@@ -363,20 +383,41 @@ func (object Object) QemuParams(config *Config) []string {
 
 	case TDXGuest:
 		objectParams = append(objectParams, string(object.Type))
-		objectParams = append(objectParams, "sept-ve-disable=on")
 		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
 		if object.Debug {
 			objectParams = append(objectParams, "debug=on")
 		}
-		config.Bios = object.File
+		deviceParams = append(deviceParams, string(object.Driver))
+		deviceParams = append(deviceParams, fmt.Sprintf("id=%s", object.DeviceID))
+		deviceParams = append(deviceParams, fmt.Sprintf("file=%s", object.File))
+		if object.FirmwareVolume != "" {
+			deviceParams = append(deviceParams, fmt.Sprintf("config-firmware-volume=%s", object.FirmwareVolume))
+		}
 	case SEVGuest:
-		fallthrough
+		objectParams = append(objectParams, string(object.Type))
+		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
+		objectParams = append(objectParams, fmt.Sprintf("cbitpos=%d", object.CBitPos))
+		objectParams = append(objectParams, fmt.Sprintf("reduced-phys-bits=%d", object.ReducedPhysBits))
+		objectParams = append(objectParams, fmt.Sprintf("policy=%d", object.SevPolicy))
+		if object.SevCertFilePath != "" {
+			objectParams = append(objectParams, fmt.Sprintf("dh-cert-file=%s", object.SevCertFilePath))
+		}
+		if object.SevSessionFilePath != "" {
+			objectParams = append(objectParams, fmt.Sprintf("session-file=%s", object.SevSessionFilePath))
+		}
+		if object.SevKernelHashes {
+			objectParams = append(objectParams, "kernel-hashes=on")
+		}
+		// Add OVMF firmware as pflash drive
+		driveParams = append(driveParams, "if=pflash,format=raw,readonly=on")
+		driveParams = append(driveParams, fmt.Sprintf("file=%s", object.File))
 	case SNPGuest:
 		objectParams = append(objectParams, string(object.Type))
 		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
 		objectParams = append(objectParams, fmt.Sprintf("cbitpos=%d", object.CBitPos))
 		objectParams = append(objectParams, fmt.Sprintf("reduced-phys-bits=%d", object.ReducedPhysBits))
-
+		objectParams = append(objectParams, fmt.Sprintf("policy=0x%x", object.SnpPolicy))
+		// Add OVMF firmware as pflash drive
 		driveParams = append(driveParams, "if=pflash,format=raw,readonly=on")
 		driveParams = append(driveParams, fmt.Sprintf("file=%s", object.File))
 	case SecExecGuest:

src/runtime/pkg/govmm/qemu/qmp.go

@@ -257,6 +257,22 @@ type StatusInfo struct {
 	Status     string `json:"status"`
 }
 
+// SEVInfo represents the SEV guest information
+type SEVInfo struct {
+	State    string `json:"state"`
+	Enabled  bool   `json:"enabled"`
+	APIMajor uint32 `json:"api-major"`
+	APIMinor uint32 `json:"api-minor"`
+	BuildId  uint32 `json:"build-id"`
+	Policy   uint32 `json:"policy"`
+	Handle   uint32 `json:"handle"`
+}
+
+// SEVLaunchMeasurement represents the SEV prelaunch measurement
+type SEVLaunchMeasurement struct {
+	Measurement string `json:"data"`
+}
+
 func (q *QMP) readLoop(fromVMCh chan<- []byte) {
 	scanner := bufio.NewScanner(q.conn)
 	if q.cfg.MaxCapacity > 0 {
@@ -1679,3 +1695,53 @@ func (q *QMP) ExecuteDumpGuestMemory(ctx context.Context, protocol string, pagin
 
 	return q.executeCommand(ctx, "dump-guest-memory", args, nil)
 }
+
+// ExecuteQuerySEV queries SEV hardware details
+func (q *QMP) ExecuteQuerySEV(ctx context.Context) (SEVInfo, error) {
+	response, err := q.executeCommandWithResponse(ctx, "query-sev", nil, nil, nil)
+	if err != nil {
+		return SEVInfo{}, err
+	}
+
+	data, err := json.Marshal(response)
+	if err != nil {
+		return SEVInfo{}, fmt.Errorf("unable to extract SEV information: %v", err)
+	}
+
+	var info SEVInfo
+	if err = json.Unmarshal(data, &info); err != nil {
+		return SEVInfo{}, fmt.Errorf("unable to convert SEV information: %v", err)
+	}
+
+	return info, nil
+}
+
+// ExecuteQuerySEVLaunchMeasure queries SEV launch measurement
+func (q *QMP) ExecuteQuerySEVLaunchMeasure(ctx context.Context) (SEVLaunchMeasurement, error) {
+	response, err := q.executeCommandWithResponse(ctx, "query-sev-launch-measure", nil, nil, nil)
+	if err != nil {
+		return SEVLaunchMeasurement{}, err
+	}
+
+	data, err := json.Marshal(response)
+	if err != nil {
+		return SEVLaunchMeasurement{}, fmt.Errorf("unable to extract launch measurement: %v", err)
+	}
+
+	var measurement SEVLaunchMeasurement
+	if err = json.Unmarshal(data, &measurement); err != nil {
+		return SEVLaunchMeasurement{}, fmt.Errorf("unable to convert launch measurement: %v", err)
+	}
+
+	return measurement, nil
+}
+
+// ExecuteSEVInjectLaunchSecret injects launch secret bundle into SEV guest
+func (q *QMP) ExecuteSEVInjectLaunchSecret(ctx context.Context, packetHeader string, secret string) error {
+	args := map[string]interface{}{
+		"packet-header": packetHeader,
+		"secret":        secret,
+	}
+
+	return q.executeCommand(ctx, "sev-inject-launch-secret", args, nil)
+}

src/runtime/pkg/katautils/config-settings.go.in

@@ -97,6 +97,13 @@ const defaultDisableSeccomp = false
 const defaultDisableGuestSeLinux = true
 const defaultVfioMode = "guest-kernel"
 const defaultLegacySerial = false
+const defaultGuestPreAttestation = false
+const defaultGuestPreAttestationURI string = ""
+const defaultGuestPreAttestationMode string = ""
+const defaultGuestPreAttestationKeyset string = ""
+const defaultSEVCertChainPath string = ""
+const defaultSEVGuestPolicy uint32 = 0
+const defaultSNPGuestPolicy uint64 = 0x30000
 
 var defaultSGXEPCSize = int64(0)
 

src/runtime/pkg/katautils/config.go

@@ -52,6 +52,7 @@ const (
 	qemuHypervisorTableType        = "qemu"
 	acrnHypervisorTableType        = "acrn"
 	dragonballHypervisorTableType  = "dragonball"
+	remoteHypervisorTableType      = "remote"
 
 	// the maximum amount of PCI bridges that can be cold plugged in a VM
 	maxPCIBridges uint32 = 5
@@ -66,8 +67,9 @@ type tomlConfig struct {
 }
 
 type image struct {
-	Provision      string `toml:"provision"`
-	ServiceOffload bool   `toml:"service_offload"`
+	Provision           string `toml:"provision"`
+	ServiceOffload      bool   `toml:"service_offload"`
+	ImageRequestTimeout uint64 `toml:"image_request_timeout"`
 }
 
 type factory struct {
@@ -101,7 +103,12 @@ type hypervisor struct {
 	GuestHookPath                  string          `toml:"guest_hook_path"`
 	GuestMemoryDumpPath            string          `toml:"guest_memory_dump_path"`
 	SeccompSandbox                 string          `toml:"seccompsandbox"`
+	GuestPreAttestationURI         string          `toml:"guest_pre_attestation_kbs_uri"`
+	GuestPreAttestationMode        string          `toml:"guest_pre_attestation_kbs_mode"`
+	GuestPreAttestationKeyset      string          `toml:"guest_pre_attestation_keyset"`
+	SEVCertChainPath               string          `toml:"sev_cert_chain"`
 	BlockDeviceAIO                 string          `toml:"block_device_aio"`
+	RemoteHypervisorSocket         string          `toml:"remote_hypervisor_socket"`
 	HypervisorPathList             []string        `toml:"valid_hypervisor_paths"`
 	JailerPathList                 []string        `toml:"valid_jailer_paths"`
 	CtlPathList                    []string        `toml:"valid_ctlpaths"`
@@ -131,6 +138,10 @@ type hypervisor struct {
 	MemSlots                       uint32          `toml:"memory_slots"`
 	DefaultBridges                 uint32          `toml:"default_bridges"`
 	Msize9p                        uint32          `toml:"msize_9p"`
+	GuestPreAttestationGRPCTimeout uint32          `toml:"guest_pre_attestation_grpc_timeout"`
+	SEVGuestPolicy                 uint32          `toml:"sev_guest_policy"`
+	SNPGuestPolicy                 uint64          `toml:"snp_guest_policy"`
+	RemoteHypervisorTimeout        uint32          `toml:"remote_hypervisor_timeout"`
 	NumVCPUs                       int32           `toml:"default_vcpus"`
 	BlockDeviceCacheSet            bool            `toml:"block_device_cache_set"`
 	BlockDeviceCacheDirect         bool            `toml:"block_device_cache_direct"`
@@ -159,6 +170,7 @@ type hypervisor struct {
 	DisableSeLinux                 bool            `toml:"disable_selinux"`
 	DisableGuestSeLinux            bool            `toml:"disable_guest_selinux"`
 	LegacySerial                   bool            `toml:"use_legacy_serial"`
+	GuestPreAttestation            bool            `toml:"guest_pre_attestation"`
 }
 
 type runtime struct {
@@ -179,6 +191,7 @@ type runtime struct {
 	StaticSandboxResourceMgmt bool     `toml:"static_sandbox_resource_mgmt"`
 	EnablePprof               bool     `toml:"enable_pprof"`
 	DisableGuestEmptyDir      bool     `toml:"disable_guest_empty_dir"`
+	SealedSecretEnabled       bool     `toml:"sealed_secret_enabled"`
 }
 
 type agent struct {
@@ -647,6 +660,13 @@ func (a agent) kernelModules() []string {
 	return a.KernelModules
 }
 
+func (h hypervisor) getSnpGuestPolicy() uint64 {
+	if h.SNPGuestPolicy == 0 { // or unspecified
+		return defaultSNPGuestPolicy
+	}
+	return h.SNPGuestPolicy
+}
+
 func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	hypervisor, err := h.path()
 	if err != nil {
@@ -820,73 +840,80 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	txRateLimiterMaxRate := h.getTxRateLimiterCfg()
 
 	return vc.HypervisorConfig{
-		HypervisorPath:          hypervisor,
-		HypervisorPathList:      h.HypervisorPathList,
-		KernelPath:              kernel,
-		InitrdPath:              initrd,
-		ImagePath:               image,
-		RootfsType:              rootfsType,
-		FirmwarePath:            firmware,
-		FirmwareVolumePath:      firmwareVolume,
-		PFlash:                  pflashes,
-		MachineAccelerators:     machineAccelerators,
-		CPUFeatures:             cpuFeatures,
-		KernelParams:            vc.DeserializeParams(strings.Fields(kernelParams)),
-		HypervisorMachineType:   machineType,
-		NumVCPUs:                h.defaultVCPUs(),
-		DefaultMaxVCPUs:         h.defaultMaxVCPUs(),
-		MemorySize:              h.defaultMemSz(),
-		MemSlots:                h.defaultMemSlots(),
-		MemOffset:               h.defaultMemOffset(),
-		DefaultMaxMemorySize:    h.defaultMaxMemSz(),
-		VirtioMem:               h.VirtioMem,
-		EntropySource:           h.GetEntropySource(),
-		EntropySourceList:       h.EntropySourceList,
-		DefaultBridges:          h.defaultBridges(),
-		DisableBlockDeviceUse:   h.DisableBlockDeviceUse,
-		SharedFS:                sharedFS,
-		VirtioFSDaemon:          h.VirtioFSDaemon,
-		VirtioFSDaemonList:      h.VirtioFSDaemonList,
-		VirtioFSCacheSize:       h.VirtioFSCacheSize,
-		VirtioFSCache:           h.defaultVirtioFSCache(),
-		VirtioFSQueueSize:       h.VirtioFSQueueSize,
-		VirtioFSExtraArgs:       h.VirtioFSExtraArgs,
-		MemPrealloc:             h.MemPrealloc,
-		HugePages:               h.HugePages,
-		IOMMU:                   h.IOMMU,
-		IOMMUPlatform:           h.getIOMMUPlatform(),
-		FileBackedMemRootDir:    h.FileBackedMemRootDir,
-		FileBackedMemRootList:   h.FileBackedMemRootList,
-		Debug:                   h.Debug,
-		DisableNestingChecks:    h.DisableNestingChecks,
-		BlockDeviceDriver:       blockDriver,
-		BlockDeviceAIO:          blockAIO,
-		BlockDeviceCacheSet:     h.BlockDeviceCacheSet,
-		BlockDeviceCacheDirect:  h.BlockDeviceCacheDirect,
-		BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
-		EnableIOThreads:         h.EnableIOThreads,
-		Msize9p:                 h.msize9p(),
-		DisableImageNvdimm:      h.DisableImageNvdimm,
-		HotPlugVFIO:             h.hotPlugVFIO(),
-		ColdPlugVFIO:            h.coldPlugVFIO(),
-		DisableVhostNet:         h.DisableVhostNet,
-		EnableVhostUserStore:    h.EnableVhostUserStore,
-		VhostUserStorePath:      h.vhostUserStorePath(),
-		VhostUserStorePathList:  h.VhostUserStorePathList,
-		SeccompSandbox:          h.SeccompSandbox,
-		GuestHookPath:           h.guestHookPath(),
-		RxRateLimiterMaxRate:    rxRateLimiterMaxRate,
-		TxRateLimiterMaxRate:    txRateLimiterMaxRate,
-		EnableAnnotations:       h.EnableAnnotations,
-		GuestMemoryDumpPath:     h.GuestMemoryDumpPath,
-		GuestMemoryDumpPaging:   h.GuestMemoryDumpPaging,
-		ConfidentialGuest:       h.ConfidentialGuest,
-		SevSnpGuest:             h.SevSnpGuest,
-		GuestSwap:               h.GuestSwap,
-		Rootless:                h.Rootless,
-		LegacySerial:            h.LegacySerial,
-		DisableSeLinux:          h.DisableSeLinux,
-		DisableGuestSeLinux:     h.DisableGuestSeLinux,
+		HypervisorPath:            hypervisor,
+		HypervisorPathList:        h.HypervisorPathList,
+		KernelPath:                kernel,
+		InitrdPath:                initrd,
+		ImagePath:                 image,
+		RootfsType:                rootfsType,
+		FirmwarePath:              firmware,
+		FirmwareVolumePath:        firmwareVolume,
+		PFlash:                    pflashes,
+		MachineAccelerators:       machineAccelerators,
+		CPUFeatures:               cpuFeatures,
+		KernelParams:              vc.DeserializeParams(strings.Fields(kernelParams)),
+		HypervisorMachineType:     machineType,
+		NumVCPUs:                  h.defaultVCPUs(),
+		DefaultMaxVCPUs:           h.defaultMaxVCPUs(),
+		MemorySize:                h.defaultMemSz(),
+		MemSlots:                  h.defaultMemSlots(),
+		MemOffset:                 h.defaultMemOffset(),
+		DefaultMaxMemorySize:      h.defaultMaxMemSz(),
+		VirtioMem:                 h.VirtioMem,
+		EntropySource:             h.GetEntropySource(),
+		EntropySourceList:         h.EntropySourceList,
+		DefaultBridges:            h.defaultBridges(),
+		DisableBlockDeviceUse:     h.DisableBlockDeviceUse,
+		SharedFS:                  sharedFS,
+		VirtioFSDaemon:            h.VirtioFSDaemon,
+		VirtioFSDaemonList:        h.VirtioFSDaemonList,
+		VirtioFSCacheSize:         h.VirtioFSCacheSize,
+		VirtioFSCache:             h.defaultVirtioFSCache(),
+		VirtioFSQueueSize:         h.VirtioFSQueueSize,
+		VirtioFSExtraArgs:         h.VirtioFSExtraArgs,
+		MemPrealloc:               h.MemPrealloc,
+		HugePages:                 h.HugePages,
+		IOMMU:                     h.IOMMU,
+		IOMMUPlatform:             h.getIOMMUPlatform(),
+		FileBackedMemRootDir:      h.FileBackedMemRootDir,
+		FileBackedMemRootList:     h.FileBackedMemRootList,
+		Debug:                     h.Debug,
+		DisableNestingChecks:      h.DisableNestingChecks,
+		BlockDeviceDriver:         blockDriver,
+		BlockDeviceAIO:            blockAIO,
+		BlockDeviceCacheSet:       h.BlockDeviceCacheSet,
+		BlockDeviceCacheDirect:    h.BlockDeviceCacheDirect,
+		BlockDeviceCacheNoflush:   h.BlockDeviceCacheNoflush,
+		EnableIOThreads:           h.EnableIOThreads,
+		Msize9p:                   h.msize9p(),
+		DisableImageNvdimm:        h.DisableImageNvdimm,
+		HotPlugVFIO:               h.hotPlugVFIO(),
+		ColdPlugVFIO:              h.coldPlugVFIO(),
+		DisableVhostNet:           h.DisableVhostNet,
+		EnableVhostUserStore:      h.EnableVhostUserStore,
+		VhostUserStorePath:        h.vhostUserStorePath(),
+		VhostUserStorePathList:    h.VhostUserStorePathList,
+		SeccompSandbox:            h.SeccompSandbox,
+		GuestHookPath:             h.guestHookPath(),
+		RxRateLimiterMaxRate:      rxRateLimiterMaxRate,
+		TxRateLimiterMaxRate:      txRateLimiterMaxRate,
+		EnableAnnotations:         h.EnableAnnotations,
+		GuestMemoryDumpPath:       h.GuestMemoryDumpPath,
+		GuestMemoryDumpPaging:     h.GuestMemoryDumpPaging,
+		ConfidentialGuest:         h.ConfidentialGuest,
+		SevSnpGuest:               h.SevSnpGuest,
+		GuestSwap:                 h.GuestSwap,
+		Rootless:                  h.Rootless,
+		LegacySerial:              h.LegacySerial,
+		DisableSeLinux:            h.DisableSeLinux,
+		GuestPreAttestation:       h.GuestPreAttestation,
+		GuestPreAttestationURI:    h.GuestPreAttestationURI,
+		GuestPreAttestationMode:   h.GuestPreAttestationMode,
+		GuestPreAttestationKeyset: h.GuestPreAttestationKeyset,
+		SEVGuestPolicy:            h.SEVGuestPolicy,
+		SNPGuestPolicy:            h.getSnpGuestPolicy(),
+		SEVCertChainPath:          h.SEVCertChainPath,
+		DisableGuestSeLinux:       h.DisableGuestSeLinux,
 	}, nil
 }
 
@@ -1117,6 +1144,21 @@ func newDragonballHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	}, nil
 }
 
+func newRemoteHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
+
+	return vc.HypervisorConfig{
+		RemoteHypervisorSocket:  h.RemoteHypervisorSocket,
+		RemoteHypervisorTimeout: h.RemoteHypervisorTimeout,
+		DisableGuestSeLinux:     h.DisableGuestSeLinux,
+
+		// No valid value so avoid to append block device to list in kata_agent.appendDevices
+		BlockDeviceDriver: "dummy",
+		EnableAnnotations: h.EnableAnnotations,
+		// Add GuestHookPath
+		GuestHookPath: h.guestHookPath(),
+	}, nil
+}
+
 func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
 	if f.TemplatePath == "" {
 		f.TemplatePath = defaultTemplatePath
@@ -1153,6 +1195,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi
 		case dragonballHypervisorTableType:
 			config.HypervisorType = vc.DragonballHypervisor
 			hConfig, err = newDragonballHypervisorConfig(hypervisor)
+		case remoteHypervisorTableType:
+			config.HypervisorType = vc.RemoteHypervisor
+			hConfig, err = newRemoteHypervisorConfig(hypervisor)
 		}
 
 		if err != nil {
@@ -1260,55 +1305,62 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run
 
 func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 	return vc.HypervisorConfig{
-		HypervisorPath:           defaultHypervisorPath,
-		JailerPath:               defaultJailerPath,
-		KernelPath:               defaultKernelPath,
-		ImagePath:                defaultImagePath,
-		InitrdPath:               defaultInitrdPath,
-		RootfsType:               defaultRootfsType,
-		FirmwarePath:             defaultFirmwarePath,
-		FirmwareVolumePath:       defaultFirmwareVolumePath,
-		MachineAccelerators:      defaultMachineAccelerators,
-		CPUFeatures:              defaultCPUFeatures,
-		HypervisorMachineType:    defaultMachineType,
-		NumVCPUs:                 defaultVCPUCount,
-		DefaultMaxVCPUs:          defaultMaxVCPUCount,
-		MemorySize:               defaultMemSize,
-		MemOffset:                defaultMemOffset,
-		VirtioMem:                defaultVirtioMem,
-		DisableBlockDeviceUse:    defaultDisableBlockDeviceUse,
-		DefaultBridges:           defaultBridgesCount,
-		MemPrealloc:              defaultEnableMemPrealloc,
-		HugePages:                defaultEnableHugePages,
-		IOMMU:                    defaultEnableIOMMU,
-		IOMMUPlatform:            defaultEnableIOMMUPlatform,
-		FileBackedMemRootDir:     defaultFileBackedMemRootDir,
-		Debug:                    defaultEnableDebug,
-		DisableNestingChecks:     defaultDisableNestingChecks,
-		BlockDeviceDriver:        defaultBlockDeviceDriver,
-		BlockDeviceAIO:           defaultBlockDeviceAIO,
-		BlockDeviceCacheSet:      defaultBlockDeviceCacheSet,
-		BlockDeviceCacheDirect:   defaultBlockDeviceCacheDirect,
-		BlockDeviceCacheNoflush:  defaultBlockDeviceCacheNoflush,
-		EnableIOThreads:          defaultEnableIOThreads,
-		Msize9p:                  defaultMsize9p,
-		ColdPlugVFIO:             defaultColdPlugVFIO,
-		HotPlugVFIO:              defaultHotPlugVFIO,
-		GuestHookPath:            defaultGuestHookPath,
-		VhostUserStorePath:       defaultVhostUserStorePath,
-		VhostUserDeviceReconnect: defaultVhostUserDeviceReconnect,
-		VirtioFSCache:            defaultVirtioFSCacheMode,
-		DisableImageNvdimm:       defaultDisableImageNvdimm,
-		RxRateLimiterMaxRate:     defaultRxRateLimiterMaxRate,
-		TxRateLimiterMaxRate:     defaultTxRateLimiterMaxRate,
-		SGXEPCSize:               defaultSGXEPCSize,
-		ConfidentialGuest:        defaultConfidentialGuest,
-		SevSnpGuest:              defaultSevSnpGuest,
-		GuestSwap:                defaultGuestSwap,
-		Rootless:                 defaultRootlessHypervisor,
-		DisableSeccomp:           defaultDisableSeccomp,
-		DisableGuestSeLinux:      defaultDisableGuestSeLinux,
-		LegacySerial:             defaultLegacySerial,
+		HypervisorPath:            defaultHypervisorPath,
+		JailerPath:                defaultJailerPath,
+		KernelPath:                defaultKernelPath,
+		ImagePath:                 defaultImagePath,
+		InitrdPath:                defaultInitrdPath,
+		RootfsType:                defaultRootfsType,
+		FirmwarePath:              defaultFirmwarePath,
+		FirmwareVolumePath:        defaultFirmwareVolumePath,
+		MachineAccelerators:       defaultMachineAccelerators,
+		CPUFeatures:               defaultCPUFeatures,
+		HypervisorMachineType:     defaultMachineType,
+		NumVCPUs:                  defaultVCPUCount,
+		DefaultMaxVCPUs:           defaultMaxVCPUCount,
+		MemorySize:                defaultMemSize,
+		MemOffset:                 defaultMemOffset,
+		VirtioMem:                 defaultVirtioMem,
+		DisableBlockDeviceUse:     defaultDisableBlockDeviceUse,
+		DefaultBridges:            defaultBridgesCount,
+		MemPrealloc:               defaultEnableMemPrealloc,
+		HugePages:                 defaultEnableHugePages,
+		IOMMU:                     defaultEnableIOMMU,
+		IOMMUPlatform:             defaultEnableIOMMUPlatform,
+		FileBackedMemRootDir:      defaultFileBackedMemRootDir,
+		Debug:                     defaultEnableDebug,
+		DisableNestingChecks:      defaultDisableNestingChecks,
+		BlockDeviceDriver:         defaultBlockDeviceDriver,
+		BlockDeviceAIO:            defaultBlockDeviceAIO,
+		BlockDeviceCacheSet:       defaultBlockDeviceCacheSet,
+		BlockDeviceCacheDirect:    defaultBlockDeviceCacheDirect,
+		BlockDeviceCacheNoflush:   defaultBlockDeviceCacheNoflush,
+		EnableIOThreads:           defaultEnableIOThreads,
+		Msize9p:                   defaultMsize9p,
+		ColdPlugVFIO:              defaultColdPlugVFIO,
+		HotPlugVFIO:               defaultHotPlugVFIO,
+		GuestHookPath:             defaultGuestHookPath,
+		VhostUserStorePath:        defaultVhostUserStorePath,
+		VhostUserDeviceReconnect:  defaultVhostUserDeviceReconnect,
+		VirtioFSCache:             defaultVirtioFSCacheMode,
+		DisableImageNvdimm:        defaultDisableImageNvdimm,
+		RxRateLimiterMaxRate:      defaultRxRateLimiterMaxRate,
+		TxRateLimiterMaxRate:      defaultTxRateLimiterMaxRate,
+		SGXEPCSize:                defaultSGXEPCSize,
+		ConfidentialGuest:         defaultConfidentialGuest,
+		SevSnpGuest:               defaultSevSnpGuest,
+		GuestSwap:                 defaultGuestSwap,
+		Rootless:                  defaultRootlessHypervisor,
+		DisableSeccomp:            defaultDisableSeccomp,
+		DisableGuestSeLinux:       defaultDisableGuestSeLinux,
+		LegacySerial:              defaultLegacySerial,
+		GuestPreAttestation:       defaultGuestPreAttestation,
+		GuestPreAttestationURI:    defaultGuestPreAttestationURI,
+		GuestPreAttestationMode:   defaultGuestPreAttestationMode,
+		GuestPreAttestationKeyset: defaultGuestPreAttestationKeyset,
+		SEVGuestPolicy:            defaultSEVGuestPolicy,
+		SNPGuestPolicy:            defaultSNPGuestPolicy,
+		SEVCertChainPath:          defaultSEVCertChainPath,
 	}
 }
 
@@ -1404,6 +1456,9 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 	config.JaegerEndpoint = tomlConf.Runtime.JaegerEndpoint
 	config.JaegerUser = tomlConf.Runtime.JaegerUser
 	config.JaegerPassword = tomlConf.Runtime.JaegerPassword
+	config.SealedSecretEnabled = tomlConf.Runtime.SealedSecretEnabled
+	config.ServiceOffload = tomlConf.Image.ServiceOffload
+	config.ImageRequestTimeout = tomlConf.Image.ImageRequestTimeout
 	for _, f := range tomlConf.Runtime.Experimental {
 		feature := exp.Get(f)
 		if feature == nil {
@@ -1747,6 +1802,11 @@ func checkFactoryConfig(config oci.RuntimeConfig) error {
 // checkHypervisorConfig performs basic "sanity checks" on the hypervisor
 // config.
 func checkHypervisorConfig(config vc.HypervisorConfig) error {
+
+	if config.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	type image struct {
 		path   string
 		initrd bool

src/runtime/pkg/katautils/config_test.go

@@ -182,6 +182,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 		VirtioFSCache:         defaultVirtioFSCacheMode,
 		PFlash:                []string{},
 		SGXEPCSize:            epcSize,
+		SNPGuestPolicy:        defaultSNPGuestPolicy,
 	}
 
 	if goruntime.GOARCH == "arm64" && len(hypervisorConfig.PFlash) == 0 && hypervisorConfig.FirmwarePath == "" {
@@ -566,6 +567,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		VirtioFSCache:         defaultVirtioFSCacheMode,
 		BlockDeviceAIO:        defaultBlockDeviceAIO,
 		DisableGuestSeLinux:   defaultDisableGuestSeLinux,
+		SNPGuestPolicy:        defaultSNPGuestPolicy,
 		HotPlugVFIO:           defaultHotPlugVFIO,
 		ColdPlugVFIO:          defaultColdPlugVFIO,
 	}

src/runtime/pkg/katautils/create.go

@@ -129,7 +129,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo
 	}
 
 	if !rootFs.Mounted && len(sandboxConfig.Containers) == 1 {
-		if rootFs.Source != "" {
+		if rootFs.Source != "" && !vc.HasOptionPrefix(rootFs.Options, vc.VirtualVolumePrefix) {
 			realPath, err := ResolvePath(rootFs.Source)
 			if err != nil {
 				return nil, vc.Process{}, err

src/runtime/pkg/oci/utils.go

@@ -154,6 +154,16 @@ type RuntimeConfig struct {
 
 	// Determines if Kata creates emptyDir on the guest
 	DisableGuestEmptyDir bool
+
+	// Offload the CRI image management service to the Kata agent.
+	ServiceOffload bool
+
+	// Image request timeout which, if provided, indicates the image request timeout
+	// in the guest needed for the workload(s)
+	ImageRequestTimeout uint64
+
+	// Sealed secret enabled configuration
+	SealedSecretEnabled bool
 }
 
 // AddKernelParam allows the addition of new kernel parameters to an existing
@@ -453,6 +463,10 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig,
 		return err
 	}
 
+	if err := addConfidentialComputingOverrides(ocispec, config); err != nil {
+		return err
+	}
+
 	if err := addHypervisorHotColdPlugVfioOverrides(ocispec, config); err != nil {
 		return err
 	}
@@ -902,7 +916,17 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, r
 				value, vcAnnotations.VfioMode)
 		}
 	}
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.ImageRequestTimeout).setUint(func(imageRequestTimeout uint64) {
+		sbConfig.ImageRequestTimeout = imageRequestTimeout
+	}); err != nil {
+		return err
+	}
 
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.SealedSecretEnabled).setBool(func(SealedSecretEnabled bool) {
+		sbConfig.SealedSecretEnabled = SealedSecretEnabled
+	}); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -932,6 +956,29 @@ func addAgentConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error
 	return nil
 }
 
+func addConfidentialComputingOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
+
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.GuestPreAttestation).setBool(func(guestPreAttestation bool) {
+		sbConfig.HypervisorConfig.GuestPreAttestation = guestPreAttestation
+	}); err != nil {
+		return err
+	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.GuestPreAttestationURI]; ok {
+		if value != "" {
+			sbConfig.HypervisorConfig.GuestPreAttestationURI = value
+		}
+	}
+
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.SEVGuestPolicy).setUint(func(sevGuestPolicy uint64) {
+		sbConfig.HypervisorConfig.SEVGuestPolicy = uint32(sevGuestPolicy)
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // SandboxConfig converts an OCI compatible runtime configuration file
 // to a virtcontainers sandbox configuration structure.
 func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid string, detach, systemdCgroup bool) (vc.SandboxConfig, error) {
@@ -991,6 +1038,12 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
 		GuestSeLinuxLabel: runtime.GuestSeLinuxLabel,
 
 		Experimental: runtime.Experimental,
+
+		ServiceOffload: runtime.ServiceOffload,
+
+		ImageRequestTimeout: runtime.ImageRequestTimeout,
+
+		SealedSecretEnabled: runtime.SealedSecretEnabled,
 	}
 
 	if err := addAnnotations(ocispec, &sandboxConfig, runtime); err != nil {

src/runtime/pkg/oci/utils_test.go

@@ -810,12 +810,17 @@ func TestAddRuntimeAnnotations(t *testing.T) {
 	ocispec.Annotations[vcAnnotations.SandboxCgroupOnly] = "true"
 	ocispec.Annotations[vcAnnotations.DisableNewNetNs] = "true"
 	ocispec.Annotations[vcAnnotations.InterNetworkModel] = "macvtap"
+	ocispec.Annotations[vcAnnotations.ImageRequestTimeout] = "100"
+	ocispec.Annotations[vcAnnotations.SealedSecretEnabled] = "true"
 
 	addAnnotations(ocispec, &config, runtimeConfig)
 	assert.Equal(config.DisableGuestSeccomp, true)
 	assert.Equal(config.SandboxCgroupOnly, true)
 	assert.Equal(config.NetworkConfig.DisableNewNetwork, true)
 	assert.Equal(config.NetworkConfig.InterworkingModel, vc.NetXConnectMacVtapModel)
+	assert.Equal(config.ImageRequestTimeout, uint64(100))
+	assert.Equal(config.SealedSecretEnabled, true)
+
 }
 
 func TestRegexpContains(t *testing.T) {
@@ -1209,8 +1214,8 @@ func TestNewMount(t *testing.T) {
 	assert := assert.New(t)
 
 	testCases := []struct {
-		out vc.Mount
 		in  specs.Mount
+		out vc.Mount
 	}{
 		{
 			in: specs.Mount{